diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index 110e9c8..4d92c00 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -150,7 +150,7 @@ public void handle(ContainerAllocatorEvent event) { // Assign the same container ID as the AM ContainerId cID = ContainerId.newInstance(getContext().getApplicationAttemptId(), - this.containerId.getId()); + this.containerId.getContainerId()); Container container = recordFactory.newRecordInstance(Container.class); container.setId(cID); NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index baff0c0..4c63d34 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -244,9 +244,10 @@ public AllocateResponse allocate(AllocateRequest request) getContext().getApplicationAttemptId(), request.getResponseId() + i); containers.add(Container.newInstance(containerId, - NodeId.newInstance("host" + containerId.getId(), 2345), - "host" + containerId.getId() + ":5678", - req.getCapability(), req.getPriority(), null)); + NodeId.newInstance( + "host" + containerId.getContainerId(), 2345), + "host" + containerId.getContainerId() + ":5678", + req.getCapability(), req.getPriority(), null)); } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestCheckpointPreemptionPolicy.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestCheckpointPreemptionPolicy.java index b62c1c9..5e01596 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestCheckpointPreemptionPolicy.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestCheckpointPreemptionPolicy.java @@ -101,7 +101,7 @@ public void setup() { for (Map.Entry ent : assignedContainers.entrySet()) { - System.out.println("cont:" + ent.getKey().getId() + + System.out.println("cont:" + ent.getKey().getContainerId() + " type:" + ent.getValue().getTaskId().getTaskType() + " res:" + contToResourceMap.get(ent.getKey()).getMemory() + "MB" ); } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 5699600..6cf9310 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -619,7 +619,7 @@ public void testContainerRollingLog() throws IOException, if (!foundAppMaster) { final ContainerId cid = ConverterUtils.toContainerId( containerPathComponent.getName()); - foundAppMaster = (cid.getId() == 1); + foundAppMaster = ((cid.getContainerId() & 0xffffffffffL) == 1); } final FileStatus[] sysSiblings = localFs.globStatus(new Path( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index d03a439..9d4303a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -80,6 +80,7 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId, * @return lower 32 bits of identifier of the ContainerId */ @Public + @Deprecated @Stable public abstract int getId(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index f728eb3..da25aa2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -199,8 +199,8 @@ public void testNodeStatusPBImpl() { original.setResponseId(1); NodeStatusPBImpl copy = new NodeStatusPBImpl(original.getProto()); - assertEquals(3, copy.getContainersStatuses().get(1).getContainerId() - .getId()); + assertEquals(3L, copy.getContainersStatuses().get(1).getContainerId() + .getContainerId()); assertEquals(3, copy.getKeepAliveApplications().get(0).getId()); assertEquals(1000, copy.getNodeHealthStatus().getLastHealthReportTime()); assertEquals(9090, copy.getNodeId().getPort()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 1af48bb..81c320d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -225,7 +225,7 @@ private boolean shouldUploadLogs(ContainerId containerId, // AM Container only if (this.retentionPolicy .equals(ContainerLogsRetentionPolicy.APPLICATION_MASTER_ONLY)) { - if (containerId.getId() == 1) { + if ((containerId.getContainerId() & 0xffffffffffL) == 1) { return true; } return false; @@ -234,7 +234,7 @@ private boolean shouldUploadLogs(ContainerId containerId, // AM + Failing containers if (this.retentionPolicy .equals(ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY)) { - if (containerId.getId() == 1) { + if ((containerId.getContainerId() & 0xffffffffffL) == 1) { return true; } else if(!wasContainerSuccessful) { return true; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index f2109b5..317aa73 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -664,13 +664,13 @@ public void testMultipleContainersLaunch() throws Exception { Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size()); for (ContainerId id : response.getSuccessfullyStartedContainers()) { // Containers with odd id should succeed. - Assert.assertEquals(1, id.getId() & 1); + Assert.assertEquals(1, id.getContainerId() & 1); } Assert.assertEquals(5, response.getFailedRequests().size()); for (Map.Entry entry : response .getFailedRequests().entrySet()) { // Containers with even id should fail. - Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertEquals(0, entry.getKey().getContainerId() & 1); Assert.assertTrue(entry.getValue().getMessage() .contains( "Container " + entry.getKey() + " rejected as it is allocated by a previous RM")); @@ -717,13 +717,13 @@ public void testMultipleContainersStopAndGetStatus() throws Exception { Assert.assertEquals(5, statusResponse.getContainerStatuses().size()); for (ContainerStatus status : statusResponse.getContainerStatuses()) { // Containers with odd id should succeed - Assert.assertEquals(1, status.getContainerId().getId() & 1); + Assert.assertEquals(1, status.getContainerId().getContainerId() & 1); } Assert.assertEquals(5, statusResponse.getFailedRequests().size()); for (Map.Entry entry : statusResponse .getFailedRequests().entrySet()) { // Containers with even id should fail. - Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertEquals(0, entry.getKey().getContainerId() & 1); Assert.assertTrue(entry.getValue().getMessage() .contains("Reject this container")); } @@ -737,13 +737,13 @@ public void testMultipleContainersStopAndGetStatus() throws Exception { .size()); for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) { // Containers with odd id should succeed. - Assert.assertEquals(1, id.getId() & 1); + Assert.assertEquals(1, id.getContainerId() & 1); } Assert.assertEquals(5, stopResponse.getFailedRequests().size()); for (Map.Entry entry : stopResponse .getFailedRequests().entrySet()) { // Containers with even id should fail. - Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertEquals(0, entry.getKey().getContainerId() & 1); Assert.assertTrue(entry.getValue().getMessage() .contains("Reject this container")); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java index e42702a..cd85d92 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java @@ -49,7 +49,7 @@ public class TestLocalizedResource { - static ContainerId getMockContainer(int id) { + static ContainerId getMockContainer(long id) { ApplicationId appId = mock(ApplicationId.class); when(appId.getClusterTimestamp()).thenReturn(314159265L); when(appId.getId()).thenReturn(3); @@ -57,7 +57,7 @@ static ContainerId getMockContainer(int id) { when(appAttemptId.getApplicationId()).thenReturn(appId); when(appAttemptId.getAttemptId()).thenReturn(0); ContainerId container = mock(ContainerId.class); - when(container.getId()).thenReturn(id); + when(container.getContainerId()).thenReturn(id); when(container.getApplicationAttemptId()).thenReturn(appAttemptId); return container; } @@ -77,7 +77,7 @@ public void testNotification() throws Exception { // mock resource LocalResource apiRsrc = createMockResource(); - final ContainerId container0 = getMockContainer(0); + final ContainerId container0 = getMockContainer(0L); final Credentials creds0 = new Credentials(); final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE; final LocalizerContext ctxt0 = @@ -96,7 +96,7 @@ public void testNotification() throws Exception { // Register C1, verify request event final Credentials creds1 = new Credentials(); - final ContainerId container1 = getMockContainer(1); + final ContainerId container1 = getMockContainer(1L); final LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1); final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC; @@ -120,13 +120,13 @@ public void testNotification() throws Exception { assertEquals(ResourceState.DOWNLOADING, local.getState()); // Register C2, C3 - final ContainerId container2 = getMockContainer(2); + final ContainerId container2 = getMockContainer(2L); final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE; final Credentials creds2 = new Credentials(); final LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2); - final ContainerId container3 = getMockContainer(3); + final ContainerId container3 = getMockContainer(3L); final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE; final Credentials creds3 = new Credentials(); final LocalizerContext ctxt3 = @@ -159,7 +159,7 @@ public void testNotification() throws Exception { assertEquals(ResourceState.LOCALIZED, local.getState()); // Register C4, verify notification - final ContainerId container4 = getMockContainer(4); + final ContainerId container4 = getMockContainer(4L); final Credentials creds4 = new Credentials(); final LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 5078b2c..5f53805 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -134,7 +134,7 @@ public NodeHeartbeatResponse nodeHeartbeat(boolean isHealthy) throws Exception { } public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId, - int containerId, ContainerState containerState) throws Exception { + long containerId, ContainerState containerState) throws Exception { HashMap> nodeUpdate = new HashMap>(1); ContainerStatus containerStatus = BuilderUtils.newContainerStatus( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index d4ecc89..e0221aa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -169,7 +169,7 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // launch the 2nd and 3rd containers. for (Container c : conts) { nm.nodeHeartbeat(attempt0.getAppAttemptId(), - c.getId().getId(), ContainerState.RUNNING); + c.getId().getContainerId(), ContainerState.RUNNING); rm0.waitForState(nm, c.getId(), RMContainerState.RUNNING); } @@ -185,9 +185,9 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // Stop all non-AM containers for (Container c : conts) { - if (c.getId().getId() == 1) continue; + if (c.getId().getContainerId() == 1) continue; nm.nodeHeartbeat(attempt0.getAppAttemptId(), - c.getId().getId(), ContainerState.COMPLETE); + c.getId().getContainerId(), ContainerState.COMPLETE); rm0.waitForState(nm, c.getId(), RMContainerState.COMPLETED); } @@ -198,7 +198,7 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // that all containers will complete prior to saving. ContainerId cId = ContainerId.newInstance(attempt0.getAppAttemptId(), 1); nm.nodeHeartbeat(attempt0.getAppAttemptId(), - cId.getId(), ContainerState.COMPLETE); + cId.getContainerId(), ContainerState.COMPLETE); rm0.waitForState(nm, cId, RMContainerState.COMPLETED); // Check that the container metrics match those from the app usage report. @@ -280,7 +280,7 @@ private void amRestartTests(boolean keepRunningContainers) ContainerId containerId2 = ContainerId.newInstance(am0.getApplicationAttemptId(), 2); nm.nodeHeartbeat(am0.getApplicationAttemptId(), - containerId2.getId(), ContainerState.RUNNING); + containerId2.getContainerId(), ContainerState.RUNNING); rm.waitForState(nm, containerId2, RMContainerState.RUNNING); // Capture the containers here so the metrics can be calculated after the @@ -295,7 +295,7 @@ private void amRestartTests(boolean keepRunningContainers) ContainerId amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId(); nm.nodeHeartbeat(am0.getApplicationAttemptId(), - amContainerId.getId(), ContainerState.COMPLETE); + amContainerId.getContainerId(), ContainerState.COMPLETE); am0.waitForState(RMAppAttemptState.FAILED); long memorySeconds = 0; @@ -365,7 +365,7 @@ private void amRestartTests(boolean keepRunningContainers) // earlier attempt's attemptId amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId(); nm.nodeHeartbeat(am0.getApplicationAttemptId(), - amContainerId.getId(), ContainerState.COMPLETE); + amContainerId.getContainerId(), ContainerState.COMPLETE); MockRM.finishAMAndVerifyAppState(app, rm, nm, am1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index cd67ebc..42a3a00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -206,7 +206,8 @@ public void testNMTokenSentForNormalContainer() throws Exception { nm1.nodeHeartbeat(true); MockAM am = MockRM.launchAM(app, rm, nm1); // am container Id not equal to 1. - Assert.assertTrue(attempt.getMasterContainer().getId().getId() != 1); + Assert.assertTrue( + attempt.getMasterContainer().getId().getContainerId() != 1); // NMSecretManager doesn't record the node on which the am is allocated. Assert.assertFalse(rm.getRMContext().getNMTokenSecretManager() .isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), @@ -374,11 +375,13 @@ public void testNMToken() throws Exception { am.unregisterAppAttempt(); // marking all the containers as finished. for (Container container : containersReceivedForNM1) { - nm1.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getId(), + nm1.nodeHeartbeat(attempt.getAppAttemptId(), + container.getId().getContainerId(), ContainerState.COMPLETE); } for (Container container : containersReceivedForNM2) { - nm2.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getId(), + nm2.nodeHeartbeat(attempt.getAppAttemptId(), + container.getId().getContainerId(), ContainerState.COMPLETE); } nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1,