diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index 73e8085..ebd2cd1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -20,6 +20,7 @@ import java.text.NumberFormat; +import java.util.UUID; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -33,14 +34,26 @@ @Public @Stable public abstract class ContainerId implements Comparable{ + + public static final UUID defaultUUID = UUID.nameUUIDFromBytes( + "defaultRMId".getBytes()); @Private @Unstable public static ContainerId newInstance(ApplicationAttemptId appAttemptId, int containerId) { + return newInstance(appAttemptId, containerId, 0L, defaultUUID); + } + + @Private + @Unstable + public static ContainerId newInstance(ApplicationAttemptId appAttemptId, + int containerId, long timestamp, UUID uuid) { ContainerId id = Records.newRecord(ContainerId.class); id.setId(containerId); id.setApplicationAttemptId(appAttemptId); + id.setClusterTimestamp(timestamp); + id.setRMUUID(uuid); id.build(); return id; } @@ -69,8 +82,28 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId, @Unstable protected abstract void setApplicationAttemptId(ApplicationAttemptId atId); + @Private + @Unstable + public abstract long getClusterTimestamp(); + + @Private + @Unstable + protected abstract void setClusterTimestamp(long timestamp); + + @Private + @Unstable + public abstract UUID getRMUUID(); + + @Private + @Unstable + protected abstract void setRMUUID(UUID uuid); + /** * Get the identifier of the ContainerId. + * The identifier is sequence number of containers generated by + * ResourceManager. Note that it's unique only while RM lives. + * After ResourceManager restarts, the sequence number will be reset. + * To distinguish the containers, please use {@link #compareTo} correctly. * @return identifier of the ContainerId */ @Public @@ -129,6 +162,11 @@ public boolean equals(Object obj) { return false; if (this.getId() != other.getId()) return false; + if (this.getClusterTimestamp() != other.getClusterTimestamp()) + return false; + if (!this.getRMUUID().equals(other.getRMUUID())) + return false; + return true; } @@ -136,6 +174,15 @@ public boolean equals(Object obj) { public int compareTo(ContainerId other) { if (this.getApplicationAttemptId().compareTo( other.getApplicationAttemptId()) == 0) { + // transform return type from long to integer + long tsResult = this.getClusterTimestamp() - other.getClusterTimestamp(); + if (tsResult > 0) { + return 1; + } else if (tsResult < 0) { + return -1; + } + + // compare Ids after checking timestamp return this.getId() - other.getId(); } else { return this.getApplicationAttemptId().compareTo( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 48aac9d..46a995e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -51,6 +51,8 @@ message ContainerIdProto { optional ApplicationIdProto app_id = 1; optional ApplicationAttemptIdProto app_attempt_id = 2; optional int32 id = 3; + optional int64 cluster_timestamp = 4; + optional string rm_uuid = 5; } message ResourceProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java index 9be829f..2523a00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import java.util.UUID; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -74,6 +75,31 @@ protected void setApplicationAttemptId(ApplicationAttemptId atId) { this.applicationAttemptId = atId; } + @Override + public long getClusterTimestamp() { + Preconditions.checkNotNull(proto); + return proto.getClusterTimestamp(); + } + + @Override + protected void setClusterTimestamp(long timestamp) { + Preconditions.checkNotNull(builder); + builder.setClusterTimestamp(timestamp); + } + + @Override + public UUID getRMUUID() { + Preconditions.checkNotNull(proto); + return UUID.fromString(proto.getRmUuid()); + } + + @Override + protected void setRMUUID(UUID uuid) { + Preconditions.checkNotNull(builder); + Preconditions.checkNotNull(uuid); + builder.setRmUuid(uuid.toString()); + } + private ApplicationAttemptIdPBImpl convertFromProtoFormat( ApplicationAttemptIdProto p) { return new ApplicationAttemptIdPBImpl(p); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index d31018c..98b0d65 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -227,7 +227,10 @@ public void testMassiveWriteContainerHistoryData() throws IOException { } writeApplicationFinishData(appId); long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb; - Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); + Assert.assertTrue("The diff should be less than 25:" + + "(usedDiskAfter, usedDiskBefore) = " + + "(" + usedDiskAfter + ", " + usedDiskBefore + ")", + (usedDiskAfter - usedDiskBefore) < 25); } @Test