diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java index 57e0bce..a3359e9 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.mapreduce.v2.app.rm.preemption; +import com.google.common.collect.ComparisonChain; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -194,7 +195,9 @@ public void preempt(Context ctxt, PreemptionMessage preemptionRequests) { Collections.sort(listOfCont, new Comparator() { @Override public int compare(final Container o1, final Container o2) { - return o2.getId().getId() - o1.getId().getId(); + return ComparisonChain.start().compare( + o2.getId().getContainerId(), o1.getId().getContainerId() + ).result(); } }); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index 73e8085..08d15b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -37,9 +37,9 @@ @Private @Unstable public static ContainerId newInstance(ApplicationAttemptId appAttemptId, - int containerId) { + long containerId) { ContainerId id = Records.newRecord(ContainerId.class); - id.setId(containerId); + id.setContainerId(containerId); id.setApplicationAttemptId(appAttemptId); id.build(); return id; @@ -75,11 +75,16 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId, */ @Public @Stable + @Deprecated public abstract int getId(); + + @Public + @Unstable + public abstract long getContainerId(); @Private @Unstable - protected abstract void setId(int id); + protected abstract void setContainerId(long id); // TODO: fail the app submission if attempts are more than 10 or something diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3f1fa6c..0387a3e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -50,7 +50,7 @@ message ApplicationAttemptIdProto { message ContainerIdProto { optional ApplicationIdProto app_id = 1; optional ApplicationAttemptIdProto app_attempt_id = 2; - optional int32 id = 3; + optional int64 id = 3; } message ResourceProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java index 9be829f..629e95a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java @@ -50,11 +50,17 @@ public ContainerIdProto getProto() { @Override public int getId() { Preconditions.checkNotNull(proto); + return (int) proto.getId(); + } + + @Override + public long getContainerId() { + Preconditions.checkNotNull(proto); return proto.getId(); } @Override - protected void setId(int id) { + protected void setContainerId(long id) { Preconditions.checkNotNull(builder); builder.setId((id)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 64eb428..6a300de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -138,12 +138,12 @@ public static ApplicationId convert(long clustertimestamp, CharSequence id) { } public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, - int containerId) { + long containerId) { return ContainerId.newInstance(appAttemptId, containerId); } public static ContainerId newContainerId(int appId, int appAttemptId, - long timestamp, int id) { + long timestamp, long id) { ApplicationId applicationId = newApplicationId(timestamp, appId); ApplicationAttemptId applicationAttemptId = newApplicationAttemptId( applicationId, appAttemptId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 01d5064..af6ef13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -102,5 +102,5 @@ void setRMApplicationHistoryWriter( boolean isWorkPreservingRecoveryEnabled(); - int getEpoch(); + long getEpoch(); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index f72ef30..d0bb7c9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -82,7 +82,7 @@ private ApplicationMasterService applicationMasterService; private RMApplicationHistoryWriter rmApplicationHistoryWriter; private ConfigurationProvider configurationProvider; - private int epoch; + private long epoch; /** * Default constructor. To be used in conjunction with setter methods for @@ -362,11 +362,11 @@ public void setConfigurationProvider( } @Override - public int getEpoch() { + public long getEpoch() { return this.epoch; } - void setEpoch(int epoch) { + void setEpoch(long epoch) { this.epoch = epoch; } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index cea3d7c..0f48b0c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -602,8 +602,7 @@ public int compare(RMContainer a, RMContainer b) { if (priorityComp != 0) { return priorityComp; } - return b.getContainerId().getId() - - a.getContainerId().getId(); + return b.getContainerId().compareTo(a.getContainerId()); } }); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 0e605a9..bbdb59e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -151,9 +151,9 @@ protected synchronized void storeVersion() throws Exception { } @Override - public synchronized int getAndIncrementEpoch() throws Exception { + public synchronized long getAndIncrementEpoch() throws Exception { Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE); - int currentEpoch = 0; + long currentEpoch = 0; if (fs.exists(epochNodePath)) { // load current epoch FileStatus status = fs.getFileStatus(epochNodePath); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 05cbb09..ebc0367 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -43,7 +43,7 @@ public class MemoryRMStateStore extends RMStateStore { RMState state = new RMState(); - private int epoch = 0; + private long epoch = 0L; @VisibleForTesting public RMState getState() { @@ -55,8 +55,8 @@ public void checkVersion() throws Exception { } @Override - public synchronized int getAndIncrementEpoch() throws Exception { - int currentEpoch = epoch; + public synchronized long getAndIncrementEpoch() throws Exception { + long currentEpoch = epoch; epoch = epoch + 1; return currentEpoch; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index 690f0be..01dbf38 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -48,8 +48,8 @@ protected void closeInternal() throws Exception { } @Override - public synchronized int getAndIncrementEpoch() throws Exception { - return 0; + public synchronized long getAndIncrementEpoch() throws Exception { + return 0L; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 5b75b42..39a1dc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -526,7 +526,7 @@ public void checkVersion() throws Exception { /** * Get the current epoch of RM and increment the value. */ - public abstract int getAndIncrementEpoch() throws Exception; + public abstract long getAndIncrementEpoch() throws Exception; /** * Blocking API diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 68b4632..cb795b9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -399,9 +399,9 @@ protected synchronized RMStateVersion loadVersion() throws Exception { } @Override - public synchronized int getAndIncrementEpoch() throws Exception { + public synchronized long getAndIncrementEpoch() throws Exception { String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE); - int currentEpoch = 0; + long currentEpoch = 0; if (existsWithRetries(epochNodePath, true) != null) { // load current epoch byte[] data = getDataWithRetries(epochNodePath, true); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java index 0668789..73ff6a2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java @@ -32,15 +32,15 @@ @Unstable public abstract class Epoch { - public static Epoch newInstance(int sequenceNumber) { + public static Epoch newInstance(long sequenceNumber) { Epoch epoch = Records.newRecord(Epoch.class); epoch.setEpoch(sequenceNumber); return epoch; } - public abstract int getEpoch(); + public abstract long getEpoch(); - public abstract void setEpoch(int sequenceNumber); + public abstract void setEpoch(long sequenceNumber); public abstract EpochProto getProto(); @@ -52,7 +52,8 @@ public String toString() { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + getEpoch(); + result = prime * result + (int) (getEpoch() & 0xffffffff); + result = prime * result + (int) (getEpoch() >> 32); return result; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java index 4430672..65758c1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java @@ -53,13 +53,13 @@ private void maybeInitBuilder() { } @Override - public int getEpoch() { + public long getEpoch() { EpochProtoOrBuilder p = viaProto ? proto : builder; - return (int) (p.getEpoch() & 0xffffffff); + return p.getEpoch(); } @Override - public void setEpoch(int sequentialNumber) { + public void setEpoch(long sequentialNumber) { maybeInitBuilder(); builder.setEpoch(sequentialNumber); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 581321c..72b83ba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,9 +58,8 @@ Queue queue; final String user; // TODO making containerIdCounter long - private final AtomicInteger containerIdCounter; - private final int EPOCH_BIT_MASK = 0x3ff; - private final int EPOCH_BIT_SHIFT = 22; + private final AtomicLong containerIdCounter; + private final int EPOCH_BIT_SHIFT = 32; final Set priorities = new TreeSet( new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); @@ -77,15 +76,14 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, ActiveUsersManager activeUsersManager, - int epoch) { + long epoch) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; this.queueName = queue.getQueueName(); this.user = user; this.activeUsersManager = activeUsersManager; - this.containerIdCounter = new AtomicInteger( - (epoch & EPOCH_BIT_MASK) << EPOCH_BIT_SHIFT); + this.containerIdCounter = new AtomicLong(epoch << EPOCH_BIT_SHIFT); } public ApplicationId getApplicationId() { @@ -117,7 +115,7 @@ private synchronized void clearRequests() { LOG.info("Application " + applicationId + " requests cleared"); } - public int getNewContainerId() { + public long getNewContainerId() { return this.containerIdCounter.incrementAndGet(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 3a51417..8621343 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -163,7 +163,7 @@ public String getUser() { return appSchedulingInfo.getResourceRequests(priority); } - public int getNewContainerId() { + public long getNewContainerId() { return appSchedulingInfo.getNewContainerId(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index fb5c3a3..e633e50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -219,7 +219,7 @@ public void testSchedulerRecovery() throws Exception { assertEquals(availableResources, schedulerAttempt.getHeadroom()); // *********** check appSchedulingInfo state *********** - assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId()); + assertEquals(0x100000001L, schedulerAttempt.getNewContainerId()); } private void checkCSQueue(MockRM rm, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 6ac23a2..c25891c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -501,13 +501,13 @@ public void testEpoch(RMStateStoreHelper stateStoreHelper) RMStateStore store = stateStoreHelper.getRMStateStore(); store.setRMDispatcher(new TestDispatcher()); - int firstTimeEpoch = store.getAndIncrementEpoch(); + long firstTimeEpoch = store.getAndIncrementEpoch(); Assert.assertEquals(0, firstTimeEpoch); - int secondTimeEpoch = store.getAndIncrementEpoch(); + long secondTimeEpoch = store.getAndIncrementEpoch(); Assert.assertEquals(1, secondTimeEpoch); - int thirdTimeEpoch = store.getAndIncrementEpoch(); + long thirdTimeEpoch = store.getAndIncrementEpoch(); Assert.assertEquals(2, thirdTimeEpoch); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 20a4aa8..712e7dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -63,13 +63,13 @@ public void testMove() { ApplicationAttemptId appAttId = createAppAttemptId(0, 0); RMContext rmContext = mock(RMContext.class); - when(rmContext.getEpoch()).thenReturn(3); + when(rmContext.getEpoch()).thenReturn(3L); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.getActiveUsersManager(), rmContext); oldMetrics.submitApp(user); // confirm that containerId is calculated based on epoch. - assertEquals(app.getNewContainerId(), 0x00c00001); + assertEquals(0x300000001L, app.getNewContainerId()); // Resource request Resource requestedResource = Resource.newInstance(1536, 2); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index db28dca..96891fb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -171,7 +171,9 @@ public static ContainerId getMockContainerId(FiCaSchedulerApp application) { ContainerId containerId = mock(ContainerId.class); doReturn(application.getApplicationAttemptId()). when(containerId).getApplicationAttemptId(); - doReturn(application.getNewContainerId()).when(containerId).getId(); + long id = application.getNewContainerId(); + doReturn((int)id).when(containerId).getId(); + doReturn(id).when(containerId).getContainerId(); return containerId; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java index 2d5a6d4..bdb890b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java @@ -61,7 +61,7 @@ public void testDelayScheduling() { ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); + Mockito.when(rmContext.getEpoch()).thenReturn(0L); FSSchedulerApp schedulerApp = new FSSchedulerApp(applicationAttemptId, "user1", queue , null, rmContext); @@ -123,7 +123,7 @@ public void testDelaySchedulingForContinuousScheduling() long rackLocalityDelayMs = 6 * 1000L; // 6 seconds RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); + Mockito.when(rmContext.getEpoch()).thenReturn(0L); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); FSSchedulerApp schedulerApp = new FSSchedulerApp(applicationAttemptId, "user1", queue, @@ -180,7 +180,7 @@ public void testLocalityLevelWithoutDelays() { Mockito.when(prio.getPriority()).thenReturn(1); RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getEpoch()).thenReturn(0); + Mockito.when(rmContext.getEpoch()).thenReturn(0L); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); FSSchedulerApp schedulerApp = new FSSchedulerApp(applicationAttemptId, "user1", queue , null, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index cc738f5..4e006af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -62,7 +62,7 @@ public void setup() throws Exception { maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); - when(rmContext.getEpoch()).thenReturn(0); + when(rmContext.getEpoch()).thenReturn(0L); } private FSSchedulerApp addApp(FSLeafQueue queue, String user) {