diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2da958a..e5bbbc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -210,6 +210,12 @@
+
+
+
+
+
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f2a8376..3800aa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -227,7 +227,8 @@ void handleNMContainerStatus(NMContainerStatus containerStatus) {
containerStatus.getContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt =
- new RMAppAttemptContainerFinishedEvent(appAttemptId, status);
+ new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
+ masterContainer.getResource());
rmContext.getDispatcher().getEventHandler().handle(evt);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 2b590a0..9e5f1fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -223,4 +224,52 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
* @return the external user-facing state of ApplicationMaster.
*/
YarnApplicationState createApplicationState();
+
+ /**
+ * Get total resource preempted of the {@link RMApp}.
+ *
+ * @return total resource preempted of the {@link RMApp}
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of task containers preempted of the {@link RMApp}.
+ *
+ * @return total number of task containers preempted of the {@link RMApp}
+ */
+ int getNumberOfTaskContainersPreempted();
+
+ /**
+ * Get total number of application master containers preempted of the
+ * {@link RMApp}.
+ *
+ * @return total number of application master containers preempted of the
+ * {@link RMApp}
+ */
+ int getNumberOfMasterContainerPreempted();
+
+ /**
+ * Get total resource preempted of the latest {@link RMAppAttempt}.
+ *
+ * @return total resource preempted of the latest {@link RMAppAttempt}
+ */
+ Resource getResourcePreemptedFromLatestAttempt();
+
+ /**
+ * Get total number of task containers preempted of the latest
+ * {@link RMAppAttempt}
+ *
+ * @return total number of task containers preempted of the latest
+ * {@link RMAppAttempt}
+ */
+ int getNumberOfTaskContainersPreemptedFromLatestAttempt();
+
+ /**
+ * Get total number of application master containers preempted of the latest
+ * {@link RMAppAttempt}
+ *
+ * @return total number of application master containers preempted of the
+ * latest {@link RMAppAttempt}
+ */
+ boolean isMasterContainersPreemptedFromLatestAttempt();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index b6ca684..9795919 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -78,6 +79,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class RMAppImpl implements RMApp, Recoverable {
@@ -1200,4 +1202,54 @@ private RMAppState getRecoveredFinalState() {
public Set getRanNodes() {
return ranNodes;
}
+
+ public Resource getResourcePreemptedFromLatestAttempt() {
+ return currentAttempt == null ? Resource.newInstance(0, 0) : currentAttempt
+ .getResourcePreempted();
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreemptedFromLatestAttempt() {
+ return currentAttempt == null ? 0 : currentAttempt
+ .getNumberOfTaskContainersPreempted();
+ }
+
+ @Override
+ public boolean isMasterContainersPreemptedFromLatestAttempt() {
+ return currentAttempt == null ? false : currentAttempt
+ .isAMContainerPreempted();
+ }
+
+ @Override
+ public Resource getResourcePreempted() {
+ Resource sum = Resource.newInstance(0, 0);
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ Resources.addTo(sum, attempt.getResourcePreempted());
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ sum += attempt.getNumberOfTaskContainersPreempted();
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt && attempt.isAMContainerPreempted()) {
+ sum ++;
+ }
+ }
+ return sum;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
index 42c37a9..3d9c44f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -201,4 +202,21 @@
* scheduler.
*/
boolean isPreempted();
+
+ /**
+ * Get total resource preempted from the {@link RMAppAttempt}
+ * @return total resource preeempted in this application attempt
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of containers preempted from the {@link RMAppAttempt}
+ * @return total number of containers preempted from the application attempt
+ */
+ int getNumberOfTaskContainersPreempted();
+
+ /**
+ * Get if AM container of this attempt preempted
+ */
+ boolean isAMContainerPreempted();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4ac64ef..4c21517 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -27,6 +27,8 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -53,6 +55,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -96,6 +99,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -162,6 +166,11 @@
private RMAppAttemptState recoveredFinalState;
private RMAppAttemptState stateBeforeFinalSaving;
private Object transitionTodo;
+
+ // preemption info
+ private Resource totalResourcePreempted = Resource.newInstance(0, 0);
+ private AtomicInteger totalTaskContainerPreempted = new AtomicInteger(0);
+ private AtomicBoolean isAMContainerPreempted = new AtomicBoolean(false);
private static final StateMachineFactoryCSQueue to reinsert in childQueues
* @param event event to be sent to the container
+ * @param isPreempted indicates if is this container preempted by scheduler
*/
public void completedContainer(Resource clusterResource,
FiCaSchedulerApp application, FiCaSchedulerNode node,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 74eb196..fd82d45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import com.google.common.base.Preconditions;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -56,10 +55,12 @@
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -92,6 +93,7 @@
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
@LimitedPrivate("yarn")
@Evolving
@@ -1073,12 +1075,59 @@ public void preemptContainer(ApplicationAttemptId aid, RMContainer cont) {
app.addPreemptContainer(cont.getContainerId());
}
}
+
+ private void handlePreemptedContainer(RMContainer container) {
+ ApplicationAttemptId attemptId = container.getApplicationAttemptId();
+ RMApp app = rmContext.getRMApps().get(attemptId.getApplicationId());
+ if (null == app) {
+ LOG.warn("RMApp of given appId=" + attemptId.getApplicationId()
+ + " is null");
+ return;
+ }
+ RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
+ if (null == attempt) {
+ LOG.warn("RMAppAttempt of given appAttemptId=" + attemptId + " is null");
+ return;
+ }
+ FiCaSchedulerApp schedulerApp = getApplicationAttempt(attemptId);
+ LeafQueue queue = null;
+ if (null != schedulerApp) {
+ queue = (LeafQueue) schedulerApp.getQueue();
+ //debug
+ LOG.info("preemption queue=" + queue.getQueueName());
+ }
+ ContainerId containerId = container.getContainerId();
+ if (attempt.getMasterContainer() != null
+ && attempt.getMasterContainer().getId().equals(containerId)) {
+ // container got preempted is a master container
+ LOG.info(String.format("AM container preempted, "
+ + "appAttemptId=%s, containerId=%s, resource=%s", attemptId,
+ containerId, container.getContainer().getResource()));
+ if (null != queue) {
+ queue.incrNumberOfMasterContainerPreempted();
+ }
+ } else {
+ // container got preempted is not an AM container
+ LOG.info(String.format("Non-AM container preempted, appAttemptId=%s, "
+ + "containerId=%s, resource=%s", attemptId, containerId, container
+ .getContainer().getResource()));
+ if (null != queue) {
+ queue.incrNumberOfTaskContainerPreempted();
+ }
+ }
+
+ if (null != queue) {
+ queue.addResourcePreempted(container.getContainer().getResource());
+ }
+ }
@Override
public void killContainer(RMContainer cont) {
if (LOG.isDebugEnabled()) {
LOG.debug("KILL_CONTAINER: container" + cont.toString());
}
+ handlePreemptedContainer(cont);
+
completedContainer(cont, SchedulerUtils.createPreemptedContainerStatus(
cont.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER),
RMContainerEventType.KILL);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 65938aa..93b3b99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -29,6 +29,7 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -129,6 +130,11 @@
private final ResourceCalculator resourceCalculator;
+ // Fields for preemption info
+ Resource preemptedResource = Resource.newInstance(0, 0);
+ AtomicInteger preemptedTaskContainer = new AtomicInteger(0);
+ AtomicInteger preemptedMasterContainer = new AtomicInteger(0);
+
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) {
this.scheduler = cs;
@@ -1403,11 +1409,11 @@ private boolean unreserve(FiCaSchedulerApp application, Priority priority,
}
@Override
- public void completedContainer(Resource clusterResource,
- FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer,
- ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue) {
+ public void completedContainer(Resource clusterResource,
+ FiCaSchedulerApp application, FiCaSchedulerNode node,
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event, CSQueue childQueue) {
if (application != null) {
-
boolean removed = false;
// Careful! Locking order is important!
@@ -1424,7 +1430,8 @@ public void completedContainer(Resource clusterResource,
node, rmContainer);
} else {
removed =
- application.containerCompleted(rmContainer, containerStatus, event);
+ application.containerCompleted(rmContainer, containerStatus,
+ event);
node.releaseContainer(container);
}
@@ -1619,4 +1626,29 @@ public void collectSchedulerApplications(
apps.add(app.getApplicationAttemptId());
}
}
+
+ /* methods for printing preemption info on scheduler page */
+ public Resource getResourcePreempted() {
+ return preemptedResource;
+ }
+
+ public synchronized void addResourcePreempted(Resource res) {
+ Resources.addTo(preemptedResource, res);
+ }
+
+ public int getNumberOfTaskContainerPreempted() {
+ return preemptedTaskContainer.get();
+ }
+
+ public void incrNumberOfTaskContainerPreempted() {
+ preemptedTaskContainer.incrementAndGet();
+ }
+
+ public int getNumberOfMasterContainerPreempted() {
+ return preemptedMasterContainer.get();
+ }
+
+ public void incrNumberOfMasterContainerPreempted() {
+ preemptedMasterContainer.incrementAndGet();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 470cb10..57c42dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -48,8 +48,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
/**
* Represents an application attempt from the viewpoint of the FIFO or Capacity
@@ -69,7 +69,7 @@ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
RMContext rmContext) {
super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
}
-
+
synchronized public boolean containerCompleted(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event) {
@@ -82,14 +82,10 @@ synchronized public boolean containerCompleted(RMContainer rmContainer,
ContainerId containerId = container.getId();
// Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(
- containerId,
- containerStatus,
- event)
- );
- LOG.info("Completed container: " + rmContainer.getContainerId() +
+ LOG.info("Completed container: " + rmContainer.getContainerId() +
" in state: " + rmContainer.getState() + " event:" + event);
+ rmContainer.handle(new RMContainerFinishedEvent(containerId,
+ containerStatus, event));
containersToPreempt.remove(rmContainer.getContainerId());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
index ac8578e..79ed269 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
@@ -110,19 +110,33 @@ protected void render(Block html) {
setTitle(join("Application ", aid));
- info("Application Overview").
- _("User:", app.getUser()).
- _("Name:", app.getName()).
- _("Application Type:", app.getApplicationType()).
- _("Application Tags:", app.getApplicationTags()).
- _("State:", app.getState()).
- _("FinalStatus:", app.getFinalStatus()).
- _("Started:", Times.format(app.getStartTime())).
- _("Elapsed:", StringUtils.formatTime(
- Times.elapsed(app.getStartTime(), app.getFinishTime()))).
- _("Tracking URL:", !app.isTrackingUrlReady() ?
- "#" : app.getTrackingUrlPretty(), app.getTrackingUI()).
- _("Diagnostics:", app.getNote());
+ info("Application Overview")
+ ._("User:", app.getUser())
+ ._("Name:", app.getName())
+ ._("Application Type:", app.getApplicationType())
+ ._("Application Tags:", app.getApplicationTags())
+ ._("State:", app.getState())
+ ._("FinalStatus:", app.getFinalStatus())
+ ._("Started:", Times.format(app.getStartTime()))
+ ._("Elapsed:",
+ StringUtils.formatTime(Times.elapsed(app.getStartTime(),
+ app.getFinishTime())))
+ ._("Tracking URL:",
+ !app.isTrackingUrlReady() ? "#" : app.getTrackingUrlPretty(),
+ app.getTrackingUI())
+ ._("Resource Preempted:", rmApp.getResourcePreempted().toString())
+ ._("Number of Task Containers Preempted:",
+ String.valueOf(rmApp.getNumberOfTaskContainersPreempted()))
+ ._("Number of AM Containers Preempted:",
+ String.valueOf(rmApp.getNumberOfMasterContainerPreempted()))
+ ._("Resource Preempted from Latest Attempt:",
+ rmApp.getResourcePreemptedFromLatestAttempt())
+ ._("Number of Task Containers Preempted from Latest Attempt:",
+ String.valueOf(rmApp
+ .getNumberOfTaskContainersPreemptedFromLatestAttempt()))
+ ._("Did AM Containers Preempted from Latest Attempt:",
+ String.valueOf(rmApp
+ .isMasterContainersPreemptedFromLatestAttempt()));
Collection attempts = rmApp.getAppAttempts().values();
String amString =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 0f0ed50..18063a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -120,6 +120,9 @@ protected void render(Block html) {
_("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
_("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())).
+ _("Resource Preempted:", lqinfo.getResourcePreempted().toString()).
+ _("Number of Task Container Preempted:", lqinfo.getNumberOfTaskContainerPreempted()).
+ _("Number of Master Container Preempted:", lqinfo.getNumberOfMasterContainerPreempted()).
_r("Active users: ", activeUserList.toString());
html._(InfoBlock.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 11f798d..3c974aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -78,6 +78,16 @@
protected int allocatedMB;
protected int allocatedVCores;
protected int runningContainers;
+
+ // preemption info fields
+ protected int preemptedMB;
+ protected int preemptedVCores;
+ protected int preemptedTaskContainerNum;
+ protected int preemptedMasterContainerNum;
+ protected int lastestAttemptPreemptedMB;
+ protected int lastestAttemptPreemptedVCores;
+ protected int lastestAttemptPreemptedTaskContainerNum;
+ protected int lastestAttemptPreemptedMasterContainerNum;
public AppInfo() {
} // JAXB needs this
@@ -147,6 +157,20 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
}
}
}
+
+ // copy preemption info fields
+ preemptedMasterContainerNum = app.getNumberOfMasterContainerPreempted();
+ preemptedMB = app.getResourcePreempted().getMemory();
+ preemptedTaskContainerNum = app.getNumberOfTaskContainersPreempted();
+ preemptedVCores = app.getResourcePreempted().getVirtualCores();
+ lastestAttemptPreemptedMasterContainerNum =
+ app.isMasterContainersPreemptedFromLatestAttempt() ? 1 : 0;
+ lastestAttemptPreemptedMB =
+ app.getResourcePreemptedFromLatestAttempt().getMemory();
+ lastestAttemptPreemptedTaskContainerNum =
+ app.getNumberOfTaskContainersPreemptedFromLatestAttempt();
+ lastestAttemptPreemptedVCores =
+ app.getResourcePreemptedFromLatestAttempt().getVirtualCores();
}
}
@@ -254,4 +278,35 @@ public int getAllocatedVCores() {
return this.allocatedVCores;
}
+ public int getPreemptedMB() {
+ return preemptedMB;
+ }
+
+ public int getPreemptedVCores() {
+ return preemptedVCores;
+ }
+
+ public int getPreemptedTaskContainerNum() {
+ return preemptedTaskContainerNum;
+ }
+
+ public int getPreemptedMasterContainerNum() {
+ return preemptedMasterContainerNum;
+ }
+
+ public int getLastestAttemptPreemptedMB() {
+ return lastestAttemptPreemptedMB;
+ }
+
+ public int getLastestAttemptPreemptedVCores() {
+ return lastestAttemptPreemptedVCores;
+ }
+
+ public int getLastestAttemptPreemptedTaskContainerNum() {
+ return lastestAttemptPreemptedTaskContainerNum;
+ }
+
+ public int getLastestAttemptPreemptedMasterContainerNum() {
+ return lastestAttemptPreemptedMasterContainerNum;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index d90e963..56f1206 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -37,6 +37,9 @@
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected int numberOfTaskContainerPreempted;
+ protected int numberOfMasterContainerPreempted;
+ protected ResourceInfo totalResourcePreempted;
CapacitySchedulerLeafQueueInfo() {
};
@@ -53,6 +56,9 @@
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsers());
userLimitFactor = q.getUserLimitFactor();
+ numberOfMasterContainerPreempted = q.getNumberOfMasterContainerPreempted();
+ numberOfTaskContainerPreempted = q.getNumberOfTaskContainerPreempted();
+ totalResourcePreempted = new ResourceInfo(q.getResourcePreempted());
}
public int getNumActiveApplications() {
@@ -95,4 +101,16 @@ public UsersInfo getUsers() {
public float getUserLimitFactor() {
return userLimitFactor;
}
+
+ public ResourceInfo getResourcePreempted() {
+ return totalResourcePreempted;
+ }
+
+ public int getNumberOfTaskContainerPreempted() {
+ return numberOfTaskContainerPreempted;
+ }
+
+ public int getNumberOfMasterContainerPreempted() {
+ return numberOfMasterContainerPreempted;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 4349a23..93c954d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -165,6 +166,35 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ public Resource getResourcePreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public Resource getResourcePreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public boolean isMasterContainersPreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
}
public static RMApp newApplication(int i) {
@@ -183,6 +213,8 @@ public static RMApp newApplication(int i) {
YarnApplicationState[] allStates = YarnApplicationState.values();
final YarnApplicationState state = allStates[i % allStates.length];
final int maxAppAttempts = i % 1000;
+ final Resource preemptedResource = Resource.newInstance(1024, 1);
+ final int preemptedContainers = 1;
return new ApplicationBase() {
@Override
public ApplicationId getApplicationId() {
@@ -252,6 +284,16 @@ public int getMaxAppAttempts() {
public Set getApplicationTags() {
return null;
}
+
+ @Override
+ public Resource getResourcePreempted() {
+ return preemptedResource;
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ return preemptedContainers;
+ }
};
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 8f26d10..ededc05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -238,4 +239,33 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
return null;
}
+
+ public Resource getResourcePreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public Resource getResourcePreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public boolean isMasterContainersPreemptedFromLatestAttempt() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 6322df3..cf8ebf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -25,6 +25,7 @@
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -36,6 +37,7 @@
import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -47,12 +49,18 @@
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -685,5 +693,148 @@ public void testAsyncScheduling() throws Exception {
CapacityScheduler.schedule(cs);
}
}
+
+ private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
+ throws Exception {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ nm.nodeHeartbeat(true);
+ MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
+ am.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ return am;
+ }
+
+ private List waitForContainerAllocated(MockAM am, int nContainer,
+ int memory, MockNM nm) throws Exception {
+ // AM request for containers
+ am.allocate("ANY", memory, nContainer, null);
+ // kick the scheduler
+ nm.nodeHeartbeat(true);
+ List conts =
+ am.allocate(new ArrayList(), null)
+ .getAllocatedContainers();
+ while (conts.size() < nContainer) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList(),
+ new ArrayList()).getAllocatedContainers());
+ Thread.sleep(500);
+ }
+ return conts;
+ }
+
+ private void waitForQueuePreemptionInfo(LeafQueue queue, Resource preempted,
+ int numAMPreempted, int numTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (queue.getResourcePreempted().equals(preempted)
+ && queue.getNumberOfMasterContainerPreempted() == numAMPreempted
+ && queue.getNumberOfTaskContainerPreempted() == numTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
+ int numAMPreempted, int numTaskPreempted,
+ Resource latestAttemptPreempted, boolean latestAttemptAMPreempted,
+ int numLatestAttemptTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (app.getResourcePreempted().equals(preempted)
+ && app.getNumberOfMasterContainerPreempted() == numAMPreempted
+ && app.getNumberOfTaskContainersPreempted() == numTaskPreempted
+ && app.getResourcePreemptedFromLatestAttempt().equals(
+ latestAttemptPreempted)
+ && app.isMasterContainersPreemptedFromLatestAttempt() == latestAttemptAMPreempted
+ && app.getNumberOfTaskContainersPreemptedFromLatestAttempt() == numLatestAttemptTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void waitForNewAttemptCreated(RMApp app,
+ ApplicationAttemptId previousAttemptId) throws InterruptedException {
+ while (app.getCurrentAppAttempt().equals(previousAttemptId)) {
+ Thread.sleep(500);
+ }
+ }
+
+ @Test (timeout = 120000)
+ public void testPreemptionInfo() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);
+ int CONTAINER_MEMORY = 1024;
+ // start RM
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+
+ // get scheduler
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+
+ // start NM
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // create app and launch the AM
+ RMApp app0 = rm1.submitApp(CONTAINER_MEMORY);
+ MockAM am0 = launchAM(app0, rm1, nm1);
+
+ // get scheduler app
+ FiCaSchedulerApp schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+ LeafQueue leafQueue = (LeafQueue) schedulerAppAttempt.getQueue();
+
+ // allocate some containers and launch them
+ List allocatedContainers = waitForContainerAllocated(am0, 3, CONTAINER_MEMORY, nm1);
+
+ // kill the 3 containers
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ // kill app0-attempt0 AM container
+ cs.killContainer(schedulerAppAttempt.getRMContainer(app0
+ .getCurrentAppAttempt().getMasterContainer().getId()));
+
+ // wait for app0 failed
+ waitForNewAttemptCreated(app0, am0.getApplicationAttemptId());
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3,
+ Resource.newInstance(0, 0), false, 0);
+
+ // launch app0-attempt1
+ MockAM am1 = launchAM(app0, rm1, nm1);
+ schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+
+ // allocate some containers and launch them
+ allocatedContainers = waitForContainerAllocated(am1, 3, CONTAINER_MEMORY, nm1);
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ rm1.stop();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index fd14ef6..804740a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -20,6 +20,7 @@
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -242,6 +243,8 @@ public void testSortedQueues() throws Exception {
FiCaSchedulerApp app_0 = getMockApplication(0,user_0);
doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
any(ContainerStatus.class),any(RMContainerEventType.class));
+ doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
+ any(ContainerStatus.class),any(RMContainerEventType.class));
//
Priority priority = TestUtils.createMockPriority(1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 690fa74..b852c2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 45b3803..5fde3d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1310,33 +1310,58 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
WebServicesTestUtils.getXmlString(element, "amContainerLogs"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
- WebServicesTestUtils.getXmlInt(element, "runningContainers"));
+ WebServicesTestUtils.getXmlInt(element, "runningContainers"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedMB"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedVCores"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedTaskContainerNum"),
+ WebServicesTestUtils
+ .getXmlInt(element, "preemptedMasterContainerNum"),
+ WebServicesTestUtils.getXmlInt(element, "lastestAttemptPreemptedMB"),
+ WebServicesTestUtils.getXmlInt(element,
+ "lastestAttemptPreemptedVCores"), WebServicesTestUtils.getXmlInt(
+ element, "lastestAttemptPreemptedTaskContainerNum"),
+ WebServicesTestUtils.getXmlInt(element,
+ "lastestAttemptPreemptedMasterContainerNum"));
}
}
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 20 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 20, info.length());
+ // 28 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 28, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
- info.getString("name"), info.getString("applicationType"), info.getString("queue"),
- info.getString("state"), info.getString("finalStatus"),
- (float) info.getDouble("progress"), info.getString("trackingUI"),
- info.getString("diagnostics"), info.getLong("clusterId"),
- info.getLong("startedTime"), info.getLong("finishedTime"),
- info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
- info.getString("amContainerLogs"), info.getInt("allocatedMB"),
- info.getInt("allocatedVCores"), info.getInt("runningContainers"));
+ info.getString("name"), info.getString("applicationType"),
+ info.getString("queue"), info.getString("state"),
+ info.getString("finalStatus"), (float) info.getDouble("progress"),
+ info.getString("trackingUI"), info.getString("diagnostics"),
+ info.getLong("clusterId"), info.getLong("startedTime"),
+ info.getLong("finishedTime"), info.getLong("elapsedTime"),
+ info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
+ info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
+ info.getInt("runningContainers"),
+ info.getInt("preemptedMB"),
+ info.getInt("preemptedVCores"),
+ info.getInt("preemptedTaskContainerNum"),
+ info.getInt("preemptedMasterContainerNum"),
+ info.getInt("lastestAttemptPreemptedMB"),
+ info.getInt("lastestAttemptPreemptedVCores"),
+ info.getInt("lastestAttemptPreemptedTaskContainerNum"),
+ info.getInt("lastestAttemptPreemptedTaskContainerNum"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
- String name, String applicationType, String queue, String state, String finalStatus,
- float progress, String trackingUI, String diagnostics, long clusterId,
- long startedTime, long finishedTime, long elapsedTime,
- String amHostHttpAddress, String amContainerLogs, int allocatedMB,
- int allocatedVCores, int numContainers) throws JSONException,
+ String name, String applicationType, String queue, String state,
+ String finalStatus, float progress, String trackingUI,
+ String diagnostics, long clusterId, long startedTime, long finishedTime,
+ long elapsedTime, String amHostHttpAddress, String amContainerLogs,
+ int allocatedMB, int allocatedVCores, int numContainers, int preemptedMB,
+ int preemptedVCores, int preemptedTaskContainerNum,
+ int preemptedMasterContainerNum, int lastestAttemptPreemptedMB,
+ int lastestAttemptPreemptedVCores,
+ int lastestAttemptPreemptedTaskContainerNum,
+ int lastestAttemptPreemptedMasterContainerNum) throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@@ -1371,6 +1396,20 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
assertEquals("numContainers doesn't match", 1, numContainers);
+ assertEquals("preemptedMB doesn't match", 0, preemptedMB);
+ assertEquals("preemptedVCores doesn't match", 0, preemptedVCores);
+ assertEquals("preemptedTaskContainerNum doesn't match", 0,
+ preemptedTaskContainerNum);
+ assertEquals("preemptedMasterContainerNum doesn't match", 0,
+ preemptedMasterContainerNum);
+ assertEquals("lastestAttemptPreemptedMB doesn't match", 0,
+ lastestAttemptPreemptedMB);
+ assertEquals("lastestAttemptPreemptedVCores doesn't match", 0,
+ lastestAttemptPreemptedVCores);
+ assertEquals("lastestAttemptPreemptedTaskContainerNum doesn't match", 0,
+ lastestAttemptPreemptedTaskContainerNum);
+ assertEquals("lastestAttemptPreemptedMasterContainerNum doesn't match", 0,
+ lastestAttemptPreemptedMasterContainerNum);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index e57e5cf..78aac89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -360,7 +360,7 @@ private void verifySubQueue(JSONObject info, String q,
int numExpectedElements = 11;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 21;
+ numExpectedElements = 24;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());