diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 0f5da29..9a577b7 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -51,7 +51,6 @@ org.apache.hadoop hadoop-yarn-server-common - test org.apache.hadoop diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index 459fd56..89a04b2 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.ProtoUtils; import org.apache.hadoop.yarn.util.Records; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java deleted file mode 100644 index c7112e3..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerManagerSecurityInfo.java +++ /dev/null @@ -1,60 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.security; - -import java.lang.annotation.Annotation; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.KerberosInfo; -import org.apache.hadoop.security.SecurityInfo; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenInfo; -import org.apache.hadoop.security.token.TokenSelector; -import org.apache.hadoop.yarn.api.ContainerManagerPB; - -public class ContainerManagerSecurityInfo extends SecurityInfo { - - @Override - public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { - return null; - } - - @Override - public TokenInfo getTokenInfo(Class protocol, Configuration conf) { - if (!protocol - .equals(ContainerManagerPB.class)) { - return null; - } - return new TokenInfo() { - - @Override - public Class annotationType() { - return null; - } - - @Override - public Class> - value() { - return ContainerTokenSelector.class; - } - }; - - } - -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java deleted file mode 100644 index b8e75ab..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java +++ /dev/null @@ -1,149 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.security; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.BuilderUtils; - -/** - * TokenIdentifier for a container. Encodes {@link ContainerId}, - * {@link Resource} needed by the container and the target NMs host-address. - * - */ -public class ContainerTokenIdentifier extends TokenIdentifier { - - private static Log LOG = LogFactory.getLog(ContainerTokenIdentifier.class); - - public static final Text KIND = new Text("ContainerToken"); - - private ContainerId containerId; - private String nmHostAddr; - private String appSubmitter; - private Resource resource; - private long expiryTimeStamp; - private int masterKeyId; - - public ContainerTokenIdentifier(ContainerId containerID, String hostName, - String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId) { - this.containerId = containerID; - this.nmHostAddr = hostName; - this.appSubmitter = appSubmitter; - this.resource = r; - this.expiryTimeStamp = expiryTimeStamp; - this.masterKeyId = masterKeyId; - } - - /** - * Default constructor needed by RPC layer/SecretManager. - */ - public ContainerTokenIdentifier() { - } - - public ContainerId getContainerID() { - return this.containerId; - } - - public String getApplicationSubmitter() { - return this.appSubmitter; - } - - public String getNmHostAddress() { - return this.nmHostAddr; - } - - public Resource getResource() { - return this.resource; - } - - public long getExpiryTimeStamp() { - return this.expiryTimeStamp; - } - - public int getMasterKeyId() { - return this.masterKeyId; - } - - @Override - public void write(DataOutput out) throws IOException { - LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this); - ApplicationAttemptId applicationAttemptId = this.containerId - .getApplicationAttemptId(); - ApplicationId applicationId = applicationAttemptId.getApplicationId(); - out.writeLong(applicationId.getClusterTimestamp()); - out.writeInt(applicationId.getId()); - out.writeInt(applicationAttemptId.getAttemptId()); - out.writeInt(this.containerId.getId()); - out.writeUTF(this.nmHostAddr); - out.writeUTF(this.appSubmitter); - out.writeInt(this.resource.getMemory()); - out.writeInt(this.resource.getVirtualCores()); - out.writeLong(this.expiryTimeStamp); - out.writeInt(this.masterKeyId); - } - - @Override - public void readFields(DataInput in) throws IOException { - ApplicationId applicationId = BuilderUtils.newApplicationId( - in.readLong(), in.readInt()); - ApplicationAttemptId applicationAttemptId = BuilderUtils - .newApplicationAttemptId(applicationId, in.readInt()); - this.containerId = BuilderUtils.newContainerId(applicationAttemptId, in - .readInt()); - this.nmHostAddr = in.readUTF(); - this.appSubmitter = in.readUTF(); - int memory = in.readInt(); - int vCores = in.readInt(); - this.resource = BuilderUtils.newResource(memory, vCores); - this.expiryTimeStamp = in.readLong(); - this.masterKeyId = in.readInt(); - } - - @Override - public Text getKind() { - return KIND; - } - - @Override - public UserGroupInformation getUser() { - return UserGroupInformation.createRemoteUser(this.containerId.toString()); - } - - // TODO: Needed? - @InterfaceAudience.Private - public static class Renewer extends Token.TrivialRenewer { - @Override - protected Text getKind() { - return KIND; - } - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java deleted file mode 100644 index 20480ae..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java +++ /dev/null @@ -1,56 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.security; - -import java.util.Collection; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; - -public class ContainerTokenSelector implements - TokenSelector { - - private static final Log LOG = LogFactory - .getLog(ContainerTokenSelector.class); - - @SuppressWarnings("unchecked") - @Override - public Token selectToken(Text service, - Collection> tokens) { - if (service == null) { - return null; - } - for (Token token : tokens) { - if (LOG.isDebugEnabled()) { - LOG.info("Looking for service: " + service + ". Current token is " - + token); - } - if (ContainerTokenIdentifier.KIND.equals(token.getKind()) && - service.equals(token.getService())) { - return (Token) token; - } - } - return null; - } - -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 3e19069..066559a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; /** * Builder utilities to construct various objects. @@ -272,19 +271,6 @@ public static ClientToken newClientToken(byte[] identifier, String kind, return newToken(ClientToken.class, identifier, kind, password, service); } - public static ContainerToken newContainerToken(NodeId nodeId, - byte[] password, ContainerTokenIdentifier tokenIdentifier) { - // RPC layer client expects ip:port as service for tokens - InetSocketAddress addr = - NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); - // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token - ContainerToken containerToken = - newToken(ContainerToken.class, tokenIdentifier.getBytes(), - ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil - .buildTokenService(addr).toString()); - return containerToken; - } - public static ContainerLaunchContext newContainerLaunchContext( String user, Map localResources, Map environment, List commands, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index babc2fb..cbf0e81 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,6 +12,5 @@ # limitations under the License. # org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo org.apache.hadoop.yarn.security.SchedulerSecurityInfo org.apache.hadoop.yarn.security.admin.AdminSecurityInfo diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index 2334040..c7d374a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -11,7 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.ApplicationTokenIdentifier org.apache.hadoop.yarn.security.client.ClientTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index 0e87a7c..2608329 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -12,5 +12,4 @@ # limitations under the License. # org.apache.hadoop.yarn.security.ApplicationTokenIdentifier$Renewer -org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java deleted file mode 100644 index 2c03c35..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn; - -import java.lang.reflect.UndeclaredThrowableException; -import java.net.InetSocketAddress; - -import junit.framework.Assert; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ContainerManager; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnRemoteException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; -import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.util.BuilderUtils; -import org.junit.Test; - -/* - * Test that the container launcher rpc times out properly. This is used - * by both RM to launch an AM as well as an AM to launch containers. - */ -public class TestContainerLaunchRPC { - - static final Log LOG = LogFactory.getLog(TestContainerLaunchRPC.class); - - private static final String EXCEPTION_CAUSE = "java.net.SocketTimeoutException"; - private static final RecordFactory recordFactory = RecordFactoryProvider - .getRecordFactory(null); - - @Test - public void testHadoopProtoRPCTimeout() throws Exception { - testRPCTimeout(HadoopYarnProtoRPC.class.getName()); - } - - private void testRPCTimeout(String rpcClass) throws Exception { - Configuration conf = new Configuration(); - // set timeout low for the test - conf.setInt("yarn.rpc.nm-command-timeout", 3000); - - conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass); - YarnRPC rpc = YarnRPC.create(conf); - String bindAddr = "localhost:0"; - InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); - Server server = rpc.getServer(ContainerManager.class, - new DummyContainerManager(), addr, conf, null, 1); - server.start(); - try { - - ContainerManager proxy = (ContainerManager) rpc.getProxy( - ContainerManager.class, - server.getListenerAddress(), conf); - ContainerLaunchContext containerLaunchContext = recordFactory - .newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser("dummy-user"); - ContainerId containerId = recordFactory - .newRecordInstance(ContainerId.class); - ApplicationId applicationId = recordFactory - .newRecordInstance(ApplicationId.class); - ApplicationAttemptId applicationAttemptId = recordFactory - .newRecordInstance(ApplicationAttemptId.class); - applicationId.setClusterTimestamp(0); - applicationId.setId(0); - applicationAttemptId.setApplicationId(applicationId); - applicationAttemptId.setAttemptId(0); - containerId.setApplicationAttemptId(applicationAttemptId); - containerId.setId(100); - Container container = - BuilderUtils.newContainer(containerId, null, null, recordFactory - .newRecordInstance(Resource.class), null, null, 0); - - StartContainerRequest scRequest = recordFactory - .newRecordInstance(StartContainerRequest.class); - scRequest.setContainerLaunchContext(containerLaunchContext); - scRequest.setContainer(container); - try { - proxy.startContainer(scRequest); - } catch (Exception e) { - LOG.info(StringUtils.stringifyException(e)); - Assert.assertTrue("Error, exception does not contain: " - + EXCEPTION_CAUSE, - e.getCause().getMessage().contains(EXCEPTION_CAUSE)); - - return; - } - } finally { - server.stop(); - } - - Assert.fail("timeout exception should have occurred!"); - } - - public class DummyContainerManager implements ContainerManager { - - private ContainerStatus status = null; - - @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnRemoteException { - GetContainerStatusResponse response = recordFactory - .newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); - return response; - } - - @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnRemoteException { - StartContainerResponse response = recordFactory - .newRecordInstance(StartContainerResponse.class); - status = recordFactory.newRecordInstance(ContainerStatus.class); - try { - // make the thread sleep to look like its not going to respond - Thread.sleep(10000); - } catch (Exception e) { - LOG.error(e); - throw new UndeclaredThrowableException(e); - } - status.setState(ContainerState.RUNNING); - status.setContainerId(request.getContainer().getId()); - status.setExitStatus(0); - return response; - } - - @Override - public StopContainerResponse stopContainer(StopContainerRequest request) - throws YarnRemoteException { - Exception e = new Exception("Dummy function", new Exception( - "Dummy function cause")); - throw new YarnRemoteException(e); - } - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java index 571d566..3f720ec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java @@ -36,9 +36,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; -import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.yarn.util.Records; /** @@ -199,6 +198,6 @@ public ContainerToken createContainerToken(ContainerId containerId, this.readLock.unlock(); } - return BuilderUtils.newContainerToken(nodeId, password, tokenIdentifier); + return YarnServerBuilderUtils.newContainerToken(nodeId, password, tokenIdentifier); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerManagerSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerManagerSecurityInfo.java new file mode 100644 index 0000000..7c0e4aa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerManagerSecurityInfo.java @@ -0,0 +1,60 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.security; + +import java.lang.annotation.Annotation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.yarn.api.ContainerManagerPB; + +public class ContainerManagerSecurityInfo extends SecurityInfo { + + @Override + public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { + return null; + } + + @Override + public TokenInfo getTokenInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ContainerManagerPB.class)) { + return null; + } + return new TokenInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public Class> + value() { + return ContainerTokenSelector.class; + } + }; + + } + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenIdentifier.java new file mode 100644 index 0000000..2186e0a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenIdentifier.java @@ -0,0 +1,149 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.security; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.BuilderUtils; + +/** + * TokenIdentifier for a container. Encodes {@link ContainerId}, + * {@link Resource} needed by the container and the target NMs host-address. + * + */ +public class ContainerTokenIdentifier extends TokenIdentifier { + + private static Log LOG = LogFactory.getLog(ContainerTokenIdentifier.class); + + public static final Text KIND = new Text("ContainerToken"); + + private ContainerId containerId; + private String nmHostAddr; + private String appSubmitter; + private Resource resource; + private long expiryTimeStamp; + private int masterKeyId; + + public ContainerTokenIdentifier(ContainerId containerID, String hostName, + String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId) { + this.containerId = containerID; + this.nmHostAddr = hostName; + this.appSubmitter = appSubmitter; + this.resource = r; + this.expiryTimeStamp = expiryTimeStamp; + this.masterKeyId = masterKeyId; + } + + /** + * Default constructor needed by RPC layer/SecretManager. + */ + public ContainerTokenIdentifier() { + } + + public ContainerId getContainerID() { + return this.containerId; + } + + public String getApplicationSubmitter() { + return this.appSubmitter; + } + + public String getNmHostAddress() { + return this.nmHostAddr; + } + + public Resource getResource() { + return this.resource; + } + + public long getExpiryTimeStamp() { + return this.expiryTimeStamp; + } + + public int getMasterKeyId() { + return this.masterKeyId; + } + + @Override + public void write(DataOutput out) throws IOException { + LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this); + ApplicationAttemptId applicationAttemptId = this.containerId + .getApplicationAttemptId(); + ApplicationId applicationId = applicationAttemptId.getApplicationId(); + out.writeLong(applicationId.getClusterTimestamp()); + out.writeInt(applicationId.getId()); + out.writeInt(applicationAttemptId.getAttemptId()); + out.writeInt(this.containerId.getId()); + out.writeUTF(this.nmHostAddr); + out.writeUTF(this.appSubmitter); + out.writeInt(this.resource.getMemory()); + out.writeInt(this.resource.getVirtualCores()); + out.writeLong(this.expiryTimeStamp); + out.writeInt(this.masterKeyId); + } + + @Override + public void readFields(DataInput in) throws IOException { + ApplicationId applicationId = BuilderUtils.newApplicationId( + in.readLong(), in.readInt()); + ApplicationAttemptId applicationAttemptId = BuilderUtils + .newApplicationAttemptId(applicationId, in.readInt()); + this.containerId = BuilderUtils.newContainerId(applicationAttemptId, in + .readInt()); + this.nmHostAddr = in.readUTF(); + this.appSubmitter = in.readUTF(); + int memory = in.readInt(); + int vCores = in.readInt(); + this.resource = BuilderUtils.newResource(memory, vCores); + this.expiryTimeStamp = in.readLong(); + this.masterKeyId = in.readInt(); + } + + @Override + public Text getKind() { + return KIND; + } + + @Override + public UserGroupInformation getUser() { + return UserGroupInformation.createRemoteUser(this.containerId.toString()); + } + + // TODO: Needed? + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSelector.java new file mode 100644 index 0000000..70d6757 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSelector.java @@ -0,0 +1,56 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.security; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +public class ContainerTokenSelector implements + TokenSelector { + + private static final Log LOG = LogFactory + .getLog(ContainerTokenSelector.class); + + @SuppressWarnings("unchecked") + @Override + public Token selectToken(Text service, + Collection> tokens) { + if (service == null) { + return null; + } + for (Token token : tokens) { + if (LOG.isDebugEnabled()) { + LOG.info("Looking for service: " + service + ". Current token is " + + token); + } + if (ContainerTokenIdentifier.KIND.equals(token.getKind()) && + service.equals(token.getService())) { + return (Token) token; + } + } + return null; + } + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index c31a77b..0eb0f62 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -18,16 +18,23 @@ package org.apache.hadoop.yarn.server.utils; +import java.net.InetSocketAddress; import java.util.List; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.SerializedException; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; /** @@ -63,4 +70,17 @@ public static SerializedException newSerializedException(Throwable e) { se.init(e); return se; } + + public static ContainerToken newContainerToken(NodeId nodeId, + byte[] password, ContainerTokenIdentifier tokenIdentifier) { + // RPC layer client expects ip:port as service for tokens + InetSocketAddress addr = + NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); + // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token + ContainerToken containerToken = BuilderUtils. + newToken(ContainerToken.class, tokenIdentifier.getBytes(), + ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil + .buildTokenService(addr).toString()); + return containerToken; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index 5a2a009..dfd393f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,3 +12,4 @@ # limitations under the License. # org.apache.hadoop.yarn.server.RMNMSecurityInfoClass +org.apache.hadoop.yarn.server.security.ContainerManagerSecurityInfo diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 63cb07e..5d1d732 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -61,7 +61,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; @@ -98,6 +97,7 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.service.ServiceStateChangeListener; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java index bc70f26..7f13cc1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java @@ -31,9 +31,9 @@ import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; /** * The NM maintains only two master-keys. The current key that RM knows and the diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index b5af79b..d5da275 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -53,12 +53,12 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.ProtoUtils; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index fdde3ba..d8bd59a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -77,13 +77,13 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.AfterClass;