diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java index e2e04f3..fed7504 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; /** @@ -56,10 +57,19 @@ private long expiryTimeStamp; private int masterKeyId; private long rmIdentifier; + private Priority priority; + private long createTime; public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier) { + this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, + rmIdentifier, null, 0); + } + + public ContainerTokenIdentifier(ContainerId containerID, + String hostName, String appSubmitter, Resource r, long expiryTimeStamp, + int masterKeyId, long rmIdentifier, Priority priority, long createTime) { this.containerId = containerID; this.nmHostAddr = hostName; this.appSubmitter = appSubmitter; @@ -67,6 +77,8 @@ public ContainerTokenIdentifier(ContainerId containerID, String hostName, this.expiryTimeStamp = expiryTimeStamp; this.masterKeyId = masterKeyId; this.rmIdentifier = rmIdentifier; + this.priority = priority; + this.createTime = createTime; } /** @@ -99,6 +111,13 @@ public int getMasterKeyId() { return this.masterKeyId; } + public Priority getPriority() { + return this.priority; + } + + public long getCreateTime() { + return this.createTime; + } /** * Get the RMIdentifier of RM in which containers are allocated * @return RMIdentifier @@ -124,6 +143,8 @@ public void write(DataOutput out) throws IOException { out.writeLong(this.expiryTimeStamp); out.writeInt(this.masterKeyId); out.writeLong(this.rmIdentifier); + out.writeInt(this.priority.getPriority()); + out.writeLong(this.createTime); } @Override @@ -142,6 +163,8 @@ public void readFields(DataInput in) throws IOException { this.expiryTimeStamp = in.readLong(); this.masterKeyId = in.readInt(); this.rmIdentifier = in.readLong(); + this.priority = Priority.newInstance(in.readInt()); + this.createTime = in.readLong(); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java index d55e5a6..789c6dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java @@ -34,7 +34,8 @@ public static NMContainerStatus newInstance(ContainerId containerId, ContainerState containerState, Resource allocatedResource, - String diagnostics, int containerExitStatus) { + String diagnostics, int containerExitStatus, Priority priority, + long createTime) { NMContainerStatus status = Records.newRecord(NMContainerStatus.class); status.setContainerId(containerId); @@ -42,6 +43,8 @@ public static NMContainerStatus newInstance(ContainerId containerId, status.setAllocatedResource(allocatedResource); status.setDiagnostics(diagnostics); status.setContainerExitStatus(containerExitStatus); + status.setPriority(priority); + status.setCreateTime(createTime); return status; } @@ -95,4 +98,11 @@ public static NMContainerStatus newInstance(ContainerId containerId, public abstract Priority getPriority(); public abstract void setPriority(Priority priority); + + /** + * Get the time when the container is created + */ + public abstract long getCreateTime(); + + public abstract void setCreateTime(long createTime); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java index 785bc5d..f5c9616 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; @@ -196,6 +197,18 @@ public void setPriority(Priority priority) { this.priority = priority; } + @Override + public long getCreateTime() { + NMContainerStatusProtoOrBuilder p = viaProto ? proto : builder; + return p.getCreateTime(); + } + + @Override + public void setCreateTime(long createTime) { + maybeInitBuilder(); + builder.setCreateTime(createTime); + } + private void mergeLocalToBuilder() { if (this.containerId != null && !((ContainerIdPBImpl) containerId).getProto().equals( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index ebd752f..1d94b6e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -66,4 +66,5 @@ message NMContainerStatusProto { optional PriorityProto priority = 4; optional string diagnostics = 5 [default = "N/A"]; optional int32 container_exit_status = 6; + optional int64 create_time = 7; } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java index 2ffc9c9..98bd5e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; @@ -38,7 +39,7 @@ public class TestProtocolRecords { @Test - public void testContainerRecoveryReport() { + public void testNMContainerStatus() { ApplicationId appId = ApplicationId.newInstance(123456789, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newInstance(attemptId, 1); @@ -47,7 +48,7 @@ public void testContainerRecoveryReport() { NMContainerStatus report = NMContainerStatus.newInstance(containerId, ContainerState.COMPLETE, resource, "diagnostics", - ContainerExitStatus.ABORTED); + ContainerExitStatus.ABORTED, Priority.newInstance(10), 1234); NMContainerStatus reportProto = new NMContainerStatusPBImpl( ((NMContainerStatusPBImpl) report).getProto()); @@ -58,15 +59,8 @@ public void testContainerRecoveryReport() { Assert.assertEquals(ContainerState.COMPLETE, reportProto.getContainerState()); Assert.assertEquals(containerId, reportProto.getContainerId()); - } - - public static NMContainerStatus createContainerRecoveryReport( - ApplicationAttemptId appAttemptId, int id, ContainerState containerState) { - ContainerId containerId = ContainerId.newInstance(appAttemptId, id); - NMContainerStatus containerReport = - NMContainerStatus.newInstance(containerId, containerState, - Resource.newInstance(1024, 1), "diagnostics", 0); - return containerReport; + Assert.assertEquals(Priority.newInstance(10), reportProto.getPriority()); + Assert.assertEquals(1234, reportProto.getCreateTime()); } @Test @@ -78,7 +72,7 @@ public void testRegisterNodeManagerRequest() { NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, ContainerState.RUNNING, Resource.newInstance(1024, 1), "diagnostics", - 0); + 0, Priority.newInstance(10), 1234); List reports = Arrays.asList(containerReport); RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 22e47e7..df525fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -391,8 +391,10 @@ public ContainerStatus cloneAndGetContainerStatus() { public NMContainerStatus getNMContainerStatus() { this.readLock.lock(); try { - return NMContainerStatus.newInstance(this.containerId, - getCurrentState(), getResource(), diagnostics.toString(), exitCode); + return NMContainerStatus.newInstance(this.containerId, getCurrentState(), + getResource(), diagnostics.toString(), exitCode, + containerTokenIdentifier.getPriority(), + containerTokenIdentifier.getCreateTime()); } finally { this.readLock.unlock(); } @@ -935,5 +937,4 @@ public String toString() { private boolean hasDefaultExitCode() { return (this.exitCode == ContainerExitStatus.INVALID); } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index 3cbf3d3..ea8f734 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -529,7 +530,8 @@ public static NMContainerStatus createNMContainerStatus(int id, ContainerId containerId = ContainerId.newInstance(applicationAttemptId, id); NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, containerState, - Resource.newInstance(1024, 1), "recover container", 0); + Resource.newInstance(1024, 1), "recover container", 0, + Priority.newInstance(10), 0); return containerReport; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index d02f05f..186788d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -18,11 +18,10 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; -import static org.junit.matchers.JUnitMatchers.*; +import static org.junit.matchers.JUnitMatchers.containsString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -41,7 +40,6 @@ import java.util.List; import java.util.Map; -import org.junit.Assert; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; @@ -59,6 +57,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -66,6 +65,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -87,6 +88,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -480,7 +482,7 @@ public void testContainerEnvVariables() throws Exception { containerLaunchContext.setCommands(commands); StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, - createContainerToken(cId)); + createContainerToken(cId, Priority.newInstance(0), 0)); List list = new ArrayList(); list.add(scRequest); StartContainersRequest allRequests = @@ -679,7 +681,9 @@ private void internalKillTest(boolean delayed) throws Exception { // set up the rest of the container List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - Token containerToken = createContainerToken(cId); + Priority priority = Priority.newInstance(10); + long createTime = 1234; + Token containerToken = createContainerToken(cId, priority, createTime); StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, @@ -698,6 +702,11 @@ private void internalKillTest(boolean delayed) throws Exception { Assert.assertTrue("ProcessStartFile doesn't exist!", processStartFile.exists()); + NMContainerStatus nmContainerStatus = + containerManager.getContext().getContainers().get(cId) + .getNMContainerStatus(); + Assert.assertEquals(priority, nmContainerStatus.getPriority()); + // Now test the stop functionality. List containerIds = new ArrayList(); containerIds.add(cId); @@ -783,11 +792,13 @@ public void handle(Event event) { launch.call(); } - protected Token createContainerToken(ContainerId cId) throws InvalidToken { + protected Token createContainerToken(ContainerId cId, Priority priority, + long createTime) throws InvalidToken { Resource r = BuilderUtils.newResource(1024, 1); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, - r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER); + r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER, + priority, createTime); Token containerToken = BuilderUtils.newContainerToken( context.getNodeId(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 09cf52d..a980cc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -414,7 +414,8 @@ public ContainersAndNMTokensAllocation(List containerList, // create container token and NMToken altogether. container.setContainerToken(rmContext.getContainerTokenSecretManager() .createContainerToken(container.getId(), container.getNodeId(), - getUser(), container.getResource())); + getUser(), container.getResource(), container.getPriority(), + rmContainer.getStartTime())); NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index c62f2ee..13943f8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -34,8 +35,8 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.security.MasterKeyData; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; /** * SecretManager for ContainerTokens. This is RM-specific and rolls the @@ -169,11 +170,13 @@ public void run() { * @param nodeId * @param appSubmitter * @param capability + * @param priority + * @param createTime * @return the container-token */ - public Token - createContainerToken(ContainerId containerId, NodeId nodeId, - String appSubmitter, Resource capability) { + public Token createContainerToken(ContainerId containerId, NodeId nodeId, + String appSubmitter, Resource capability, Priority priority, + long createTime) { byte[] password; ContainerTokenIdentifier tokenIdentifier; long expiryTimeStamp = @@ -185,7 +188,8 @@ public void run() { tokenIdentifier = new ContainerTokenIdentifier(containerId, nodeId.toString(), appSubmitter, capability, expiryTimeStamp, this.currentMasterKey - .getMasterKey().getKeyId(), ResourceManager.getClusterTimeStamp()); + .getMasterKey().getKeyId(), + ResourceManager.getClusterTimeStamp(), priority, createTime); password = this.createPassword(tokenIdentifier); } finally { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 9c2d87e..5c79841 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -1851,7 +1852,8 @@ public static NMContainerStatus createNMContainerStatus( ContainerId containerId = ContainerId.newInstance(appAttemptId, id); NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, containerState, - Resource.newInstance(1024, 1), "recover container", 0); + Resource.newInstance(1024, 1), "recover container", 0, + Priority.newInstance(0), 0); return containerReport; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index e4a9a1c..4360aef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -18,11 +18,14 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -33,12 +36,12 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -57,17 +60,10 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; - import org.junit.After; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - public class TestResourceTrackerService { private final static File TEMP_DIR = new File(System.getProperty( @@ -493,7 +489,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { ContainerId.newInstance( ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), ContainerState.COMPLETE, Resource.newInstance(1024, 1), - "Dummy Completed", 0); + "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report); verify(handler, never()).handle((Event) any()); @@ -504,7 +500,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { report = NMContainerStatus.newInstance( ContainerId.newInstance(currentAttempt.getAppAttemptId(), 0), ContainerState.COMPLETE, Resource.newInstance(1024, 1), - "Dummy Completed", 0); + "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report); verify(handler, never()).handle((Event)any()); @@ -516,7 +512,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { ContainerId.newInstance( ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), ContainerState.COMPLETE, Resource.newInstance(1024, 1), - "Dummy Completed", 0); + "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report); } catch (Exception e) { @@ -531,7 +527,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { report = NMContainerStatus.newInstance( ContainerId.newInstance(currentAttempt.getAppAttemptId(), 0), ContainerState.COMPLETE, Resource.newInstance(1024, 1), - "Dummy Completed", 0); + "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report); } catch (Exception e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 853b4c3..a9bfc2f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -21,8 +21,6 @@ import java.util.ArrayList; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -31,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; @@ -48,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -209,10 +209,11 @@ protected RMContainerTokenSecretManager createContainerTokenSecretManager( @Override public Token createContainerToken(ContainerId containerId, - NodeId nodeId, String appSubmitter, Resource capability) { + NodeId nodeId, String appSubmitter, Resource capability, + Priority priority, long createTime) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, - capability); + capability, priority, createTime); } }; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index d1f1cf7..bc720c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -29,8 +29,6 @@ import java.util.Collection; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -52,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; @@ -60,15 +59,18 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -215,7 +217,11 @@ private void testNMTokens(Configuration conf) throws Exception { org.apache.hadoop.yarn.api.records.Token validContainerToken = containerTokenSecretManager.createContainerToken(validContainerId, - validNode, user, r); + validNode, user, r, Priority.newInstance(10), 1234); + ContainerTokenIdentifier identifier = + BuilderUtils.newContainerTokenIdentifier(validContainerToken); + Assert.assertEquals(Priority.newInstance(10), identifier.getPriority()); + Assert.assertEquals(1234, identifier.getCreateTime()); StringBuilder sb; // testInvalidNMToken ... creating NMToken using different secret manager. @@ -280,8 +286,7 @@ private void testNMTokens(Configuration conf) throws Exception { 4 * 60 * 1000); validContainerToken = containerTokenSecretManager.createContainerToken(validContainerId, - validNode, user, r); - + validNode, user, r, Priority.newInstance(0), 0); testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, validNMToken, false); Assert.assertTrue(nmTokenSecretManagerNM @@ -590,7 +595,7 @@ private void testContainerToken(Configuration conf) throws IOException, // Creating modified containerToken Token containerToken = tamperedContainerTokenSecretManager.createContainerToken(cId, nodeId, - user, r); + user, r, Priority.newInstance(0), 0); Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.create(conf);