diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2da958a..e5bbbc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -210,6 +210,12 @@
+
+
+
+
+
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f2a8376..3800aa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -227,7 +227,8 @@ void handleNMContainerStatus(NMContainerStatus containerStatus) {
containerStatus.getContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt =
- new RMAppAttemptContainerFinishedEvent(appAttemptId, status);
+ new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
+ masterContainer.getResource());
rmContext.getDispatcher().getEventHandler().handle(evt);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 2b590a0..d1d7ca4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -223,4 +224,27 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
* @return the external user-facing state of ApplicationMaster.
*/
YarnApplicationState createApplicationState();
+
+ /**
+ * Get total resource preempted of the {@link RMApp}.
+ *
+ * @return total resource preempted of the {@link RMApp}
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of task containers preempted of the {@link RMApp}.
+ *
+ * @return total number of task containers preempted of the {@link RMApp}
+ */
+ int getNumNonAMContainersPreempted();
+
+ /**
+ * Get total number of application master containers preempted of the
+ * {@link RMApp}.
+ *
+ * @return total number of application master containers preempted of the
+ * {@link RMApp}
+ */
+ int getNumAMContainersPreempted();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index b6ca684..897d0ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -78,6 +79,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class RMAppImpl implements RMApp, Recoverable {
@@ -1200,4 +1202,37 @@ private RMAppState getRecoveredFinalState() {
public Set getRanNodes() {
return ranNodes;
}
+
+ @Override
+ public Resource getResourcePreempted() {
+ Resource sum = Resource.newInstance(0, 0);
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ Resources.addTo(sum, attempt.getResourcePreempted());
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumNonAMContainersPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ sum += attempt.getNumAMContainersPreempted();
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumAMContainersPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt && attempt.isPreempted()) {
+ sum ++;
+ }
+ }
+ return sum;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
index 42c37a9..df65181 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -201,4 +202,16 @@
* scheduler.
*/
boolean isPreempted();
+
+ /**
+ * Get total resource preempted from the {@link RMAppAttempt}
+ * @return total resource preeempted in this application attempt
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of containers preempted from the {@link RMAppAttempt}
+ * @return total number of containers preempted from the application attempt
+ */
+ int getNumAMContainersPreempted();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 1e7693f..3cc9ca2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -27,6 +27,7 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -53,6 +54,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -96,6 +98,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -162,6 +165,10 @@
private RMAppAttemptState recoveredFinalState;
private RMAppAttemptState stateBeforeFinalSaving;
private Object transitionTodo;
+
+ // preemption info
+ private Resource resourcePreempted = Resource.newInstance(0, 0);
+ private AtomicInteger numNonAMContainersPreempted = new AtomicInteger(0);
private static final StateMachineFactory attempts = rmApp.getAppAttempts().values();
String amString =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 0f0ed50..c323154 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -120,6 +120,12 @@ protected void render(Block html) {
_("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
_("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())).
+ _("Total Resource Preempted:",
+ lqinfo.getResourcePreempted().toString()).
+ _("Num Non-AM Containers Preempted:",
+ lqinfo.getNumNonAMContainersPreempted()).
+ _("Num AM Containers Preempted:",
+ lqinfo.getNumAMContainersPreempted()).
_r("Active users: ", activeUserList.toString());
html._(InfoBlock.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 11f798d..8923247 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -78,6 +78,15 @@
protected int allocatedMB;
protected int allocatedVCores;
protected int runningContainers;
+
+ // preemption info fields
+ protected int preemptedResourceMB;
+ protected int preemptedResourceVCores;
+ protected int numNonAMContainerPreempted;
+ protected int numAMContainerPreempted;
+ protected int currentAttemptPreemptedResourceMB;
+ protected int currentAttemptPreemptedResourceVCores;
+ protected int currentAttemptNumNonAMContainerPreempted;
public AppInfo() {
} // JAXB needs this
@@ -85,6 +94,7 @@ public AppInfo() {
public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
this.schemePrefix = schemePrefix;
if (app != null) {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
String trackingUrl = app.getTrackingUrl();
this.state = app.createApplicationState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
@@ -125,7 +135,6 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
this.elapsedTime = Times.elapsed(app.getStartTime(),
app.getFinishTime());
- RMAppAttempt attempt = app.getCurrentAppAttempt();
if (attempt != null) {
Container masterContainer = attempt.getMasterContainer();
if (masterContainer != null) {
@@ -145,8 +154,21 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
allocatedVCores = usedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
}
+
+ currentAttemptPreemptedResourceVCores =
+ attempt.getResourcePreempted().getVirtualCores();
+ currentAttemptPreemptedResourceMB =
+ attempt.getResourcePreempted().getMemory();
+ currentAttemptNumNonAMContainerPreempted =
+ attempt.getNumAMContainersPreempted();
}
}
+
+ // copy preemption info fields
+ numAMContainerPreempted = app.getNumAMContainersPreempted();
+ preemptedResourceMB = app.getResourcePreempted().getMemory();
+ numNonAMContainerPreempted = app.getNumNonAMContainersPreempted();
+ preemptedResourceVCores = app.getResourcePreempted().getVirtualCores();
}
}
@@ -254,4 +276,31 @@ public int getAllocatedVCores() {
return this.allocatedVCores;
}
+ public int getPreemptedMB() {
+ return preemptedResourceMB;
+ }
+
+ public int getPreemptedVCores() {
+ return preemptedResourceVCores;
+ }
+
+ public int getNumNonAMContainersPreempted() {
+ return numNonAMContainerPreempted;
+ }
+
+ public int getNumAMContainersPreempted() {
+ return numAMContainerPreempted;
+ }
+
+ public int getCurrentAttemptPreemptedMB() {
+ return currentAttemptPreemptedResourceMB;
+ }
+
+ public int getCurrentAttemptPreemptedVCores() {
+ return currentAttemptPreemptedResourceVCores;
+ }
+
+ public int getCurrentAttemptNumNonAMContainersPreempted() {
+ return currentAttemptNumNonAMContainerPreempted;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index d90e963..11efc16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -37,6 +37,9 @@
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected int numNonAMContainerPreempted;
+ protected int numAMContainerPreempted;
+ protected ResourceInfo resourcePreempted;
CapacitySchedulerLeafQueueInfo() {
};
@@ -53,6 +56,9 @@
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsers());
userLimitFactor = q.getUserLimitFactor();
+ numAMContainerPreempted = q.getNumAMContainersPreempted();
+ numNonAMContainerPreempted = q.getNumNonAMContainersPreempted();
+ resourcePreempted = new ResourceInfo(q.getResourcePreempted());
}
public int getNumActiveApplications() {
@@ -95,4 +101,16 @@ public UsersInfo getUsers() {
public float getUserLimitFactor() {
return userLimitFactor;
}
+
+ public ResourceInfo getResourcePreempted() {
+ return resourcePreempted;
+ }
+
+ public int getNumNonAMContainersPreempted() {
+ return numNonAMContainerPreempted;
+ }
+
+ public int getNumAMContainersPreempted() {
+ return numAMContainerPreempted;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index cfd05f9..f1a3bbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -24,7 +24,6 @@
import java.util.List;
import org.junit.Assert;
-
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -34,6 +33,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -251,4 +251,22 @@ public Object run() throws Exception {
public ApplicationAttemptId getApplicationAttemptId() {
return this.attemptId;
}
+
+ public List allocateAndWaitForContainers(int nContainer,
+ int memory, MockNM nm) throws Exception {
+ // AM request for containers
+ allocate("ANY", memory, nContainer, null);
+ // kick the scheduler
+ nm.nodeHeartbeat(true);
+ List conts =
+ allocate(new ArrayList(), null)
+ .getAllocatedContainers();
+ while (conts.size() < nContainer) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(allocate(new ArrayList(),
+ new ArrayList()).getAllocatedContainers());
+ Thread.sleep(500);
+ }
+ return conts;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 4349a23..fc762f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -165,6 +166,20 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ public Resource getResourcePreempted() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
+ public int getNumNonAMContainersPreempted() {
+ return 0;
+ }
+
+ @Override
+ public int getNumAMContainersPreempted() {
+ return 0;
+ }
}
public static RMApp newApplication(int i) {
@@ -183,6 +198,8 @@ public static RMApp newApplication(int i) {
YarnApplicationState[] allStates = YarnApplicationState.values();
final YarnApplicationState state = allStates[i % allStates.length];
final int maxAppAttempts = i % 1000;
+ final Resource preemptedResource = Resource.newInstance(1024, 1);
+ final int preemptedContainers = 1;
return new ApplicationBase() {
@Override
public ApplicationId getApplicationId() {
@@ -252,6 +269,16 @@ public int getMaxAppAttempts() {
public Set getApplicationTags() {
return null;
}
+
+ @Override
+ public Resource getResourcePreempted() {
+ return preemptedResource;
+ }
+
+ @Override
+ public int getNumNonAMContainersPreempted() {
+ return preemptedContainers;
+ }
};
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 8f26d10..a42a193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -238,4 +239,18 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
return null;
}
+
+ public Resource getResourcePreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumNonAMContainersPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumAMContainersPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index c3b1d57..47fbd66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -27,6 +27,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -50,6 +51,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -62,6 +64,7 @@
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -72,6 +75,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -706,7 +710,55 @@ public void testAsyncScheduling() throws Exception {
CapacityScheduler.schedule(cs);
}
}
+
+ private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
+ throws Exception {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ nm.nodeHeartbeat(true);
+ MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
+ am.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ return am;
+ }
+
+ private void waitForQueuePreemptionInfo(LeafQueue queue, Resource preempted,
+ int numAMPreempted, int numTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (queue.getResourcePreempted().equals(preempted)
+ && queue.getNumAMContainersPreempted() == numAMPreempted
+ && queue.getNumNonAMContainersPreempted() == numTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
+ int numAMPreempted, int numTaskPreempted,
+ Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
+ int numLatestAttemptTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (app.getResourcePreempted().equals(preempted)
+ && app.getNumAMContainersPreempted() == numAMPreempted
+ && app.getNumNonAMContainersPreempted() == numTaskPreempted
+ && app.getCurrentAppAttempt().getResourcePreempted().equals(
+ currentAttemptPreempted)
+ && app.getCurrentAppAttempt().isPreempted() == currentAttemptAMPreempted
+ && app.getCurrentAppAttempt().getNumAMContainersPreempted()
+ == numLatestAttemptTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+ private void waitForNewAttemptCreated(RMApp app,
+ ApplicationAttemptId previousAttemptId) throws InterruptedException {
+ while (app.getCurrentAppAttempt().equals(previousAttemptId)) {
+ Thread.sleep(500);
+ }
+ }
+
@Test(timeout = 30000)
public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception {
final YarnConfiguration conf = new YarnConfiguration();
@@ -828,4 +880,85 @@ public void testNumClusterNodes() throws Exception {
cs.stop();
}
+
+ @Test(timeout = 120000)
+ public void testPreemptionInfo() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ int CONTAINER_MEMORY = 1024; // start RM
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+
+ // get scheduler
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+
+ // start NM
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // create app and launch the AM
+ RMApp app0 = rm1.submitApp(CONTAINER_MEMORY);
+ MockAM am0 = launchAM(app0, rm1, nm1);
+
+ // get scheduler app
+ FiCaSchedulerApp schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+ LeafQueue leafQueue = (LeafQueue) schedulerAppAttempt.getQueue();
+
+ // allocate some containers and launch them
+ List allocatedContainers =
+ am0.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1);
+
+ // kill the 3 containers
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ // kill app0-attempt0 AM container
+ cs.killContainer(schedulerAppAttempt.getRMContainer(app0
+ .getCurrentAppAttempt().getMasterContainer().getId()));
+
+ // wait for app0 failed
+ waitForNewAttemptCreated(app0, am0.getApplicationAttemptId());
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3,
+ Resource.newInstance(0, 0), false, 0);
+
+ // launch app0-attempt1
+ MockAM am1 = launchAM(app0, rm1, nm1);
+ schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+
+ // allocate some containers and launch them
+ allocatedContainers =
+ am1.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1);
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ rm1.stop();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index fd14ef6..804740a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -20,6 +20,7 @@
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -242,6 +243,8 @@ public void testSortedQueues() throws Exception {
FiCaSchedulerApp app_0 = getMockApplication(0,user_0);
doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
any(ContainerStatus.class),any(RMContainerEventType.class));
+ doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
+ any(ContainerStatus.class),any(RMContainerEventType.class));
//
Priority priority = TestUtils.createMockPriority(1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 45b3803..d9f59d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1310,33 +1310,56 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
WebServicesTestUtils.getXmlString(element, "amContainerLogs"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
- WebServicesTestUtils.getXmlInt(element, "runningContainers"));
+ WebServicesTestUtils.getXmlInt(element, "runningContainers"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"),
+ WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"),
+ WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"),
+ WebServicesTestUtils.getXmlInt(element,
+ "currentAttemptPreemptedResourceMB"),
+ WebServicesTestUtils
+ .getXmlInt(element, "currentAttemptPreemptedResourceVCores"),
+ WebServicesTestUtils.getXmlInt(element,
+ "currentAttemptNumNonAMContainerPreempted"));
}
}
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 20 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 20, info.length());
+ // 28 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 27, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
- info.getString("name"), info.getString("applicationType"), info.getString("queue"),
- info.getString("state"), info.getString("finalStatus"),
- (float) info.getDouble("progress"), info.getString("trackingUI"),
- info.getString("diagnostics"), info.getLong("clusterId"),
- info.getLong("startedTime"), info.getLong("finishedTime"),
- info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
- info.getString("amContainerLogs"), info.getInt("allocatedMB"),
- info.getInt("allocatedVCores"), info.getInt("runningContainers"));
+ info.getString("name"), info.getString("applicationType"),
+ info.getString("queue"), info.getString("state"),
+ info.getString("finalStatus"), (float) info.getDouble("progress"),
+ info.getString("trackingUI"), info.getString("diagnostics"),
+ info.getLong("clusterId"), info.getLong("startedTime"),
+ info.getLong("finishedTime"), info.getLong("elapsedTime"),
+ info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
+ info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
+ info.getInt("runningContainers"),
+ info.getInt("preemptedResourceMB"),
+ info.getInt("preemptedResourceVCores"),
+ info.getInt("numNonAMContainerPreempted"),
+ info.getInt("numAMContainerPreempted"),
+ info.getInt("currentAttemptPreemptedResourceMB"),
+ info.getInt("currentAttemptPreemptedResourceVCores"),
+ info.getInt("currentAttemptNumNonAMContainerPreempted"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
- String name, String applicationType, String queue, String state, String finalStatus,
- float progress, String trackingUI, String diagnostics, long clusterId,
- long startedTime, long finishedTime, long elapsedTime,
- String amHostHttpAddress, String amContainerLogs, int allocatedMB,
- int allocatedVCores, int numContainers) throws JSONException,
+ String name, String applicationType, String queue, String state,
+ String finalStatus, float progress, String trackingUI,
+ String diagnostics, long clusterId, long startedTime, long finishedTime,
+ long elapsedTime, String amHostHttpAddress, String amContainerLogs,
+ int allocatedMB, int allocatedVCores, int numContainers,
+ int preemptedResourceMB, int preemptedResourceVCores,
+ int numNonAMContainerPreempted, int numAMContainerPreempted,
+ int currentAttemptPreemptedResourceMB,
+ int currentAttemptPreemptedResourceVCores,
+ int currentAttemptNumNonAMContainerPreempted) throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@@ -1371,6 +1394,19 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
assertEquals("numContainers doesn't match", 1, numContainers);
+ assertEquals("preemptedResourceMB doesn't match", 0, preemptedResourceMB);
+ assertEquals("preemptedResourceVCores doesn't match", 0,
+ preemptedResourceVCores);
+ assertEquals("numNonAMContainerPreempted doesn't match", 0,
+ numNonAMContainerPreempted);
+ assertEquals("numAMContainerPreempted doesn't match", 0,
+ numAMContainerPreempted);
+ assertEquals("currentAttemptPreemptedResourceMB doesn't match", 0,
+ currentAttemptPreemptedResourceMB);
+ assertEquals("currentAttemptPreemptedResourceVCores doesn't match", 0,
+ currentAttemptPreemptedResourceVCores);
+ assertEquals("currentAttemptNumAMContainerPreempted doesn't match", 0,
+ currentAttemptNumNonAMContainerPreempted);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index e57e5cf..78aac89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -360,7 +360,7 @@ private void verifySubQueue(JSONObject info, String q,
int numExpectedElements = 11;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 21;
+ numExpectedElements = 24;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());