diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2da958a..e5bbbc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -210,6 +210,12 @@
+
+
+
+
+
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f2a8376..3800aa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -227,7 +227,8 @@ void handleNMContainerStatus(NMContainerStatus containerStatus) {
containerStatus.getContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt =
- new RMAppAttemptContainerFinishedEvent(appAttemptId, status);
+ new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
+ masterContainer.getResource());
rmContext.getDispatcher().getEventHandler().handle(evt);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 2b590a0..281c956 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -223,4 +224,27 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
* @return the external user-facing state of ApplicationMaster.
*/
YarnApplicationState createApplicationState();
+
+ /**
+ * Get total resource preempted of the {@link RMApp}.
+ *
+ * @return total resource preempted of the {@link RMApp}
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of task containers preempted of the {@link RMApp}.
+ *
+ * @return total number of task containers preempted of the {@link RMApp}
+ */
+ int getNumberOfTaskContainersPreempted();
+
+ /**
+ * Get total number of application master containers preempted of the
+ * {@link RMApp}.
+ *
+ * @return total number of application master containers preempted of the
+ * {@link RMApp}
+ */
+ int getNumberOfMasterContainerPreempted();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index b6ca684..8ef29a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -78,6 +79,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class RMAppImpl implements RMApp, Recoverable {
@@ -1200,4 +1202,37 @@ private RMAppState getRecoveredFinalState() {
public Set getRanNodes() {
return ranNodes;
}
+
+ @Override
+ public Resource getResourcePreempted() {
+ Resource sum = Resource.newInstance(0, 0);
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ Resources.addTo(sum, attempt.getResourcePreempted());
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ sum += attempt.getNumberOfTaskContainersPreempted();
+ }
+ }
+ return sum;
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ int sum = 0;
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt && attempt.isPreempted()) {
+ sum ++;
+ }
+ }
+ return sum;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
index 42c37a9..9c41521 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -201,4 +202,16 @@
* scheduler.
*/
boolean isPreempted();
+
+ /**
+ * Get total resource preempted from the {@link RMAppAttempt}
+ * @return total resource preeempted in this application attempt
+ */
+ Resource getResourcePreempted();
+
+ /**
+ * Get total number of containers preempted from the {@link RMAppAttempt}
+ * @return total number of containers preempted from the application attempt
+ */
+ int getNumberOfTaskContainersPreempted();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4ac64ef..431a90d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -27,6 +27,8 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -53,6 +55,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -96,6 +99,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -162,6 +166,11 @@
private RMAppAttemptState recoveredFinalState;
private RMAppAttemptState stateBeforeFinalSaving;
private Object transitionTodo;
+
+ // preemption info
+ private Resource totalResourcePreempted = Resource.newInstance(0, 0);
+ private AtomicInteger totalTaskContainerPreempted = new AtomicInteger(0);
+ private AtomicBoolean isAMContainerPreempted = new AtomicBoolean(false);
private static final StateMachineFactory attempts = rmApp.getAppAttempts().values();
String amString =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 0f0ed50..305a5cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -120,6 +120,9 @@ protected void render(Block html) {
_("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
_("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())).
+ _("Total Resource Preempted:", lqinfo.getResourcePreempted().toString()).
+ _("Num Non-AM Containers Preempted:", lqinfo.getNumberOfTaskContainerPreempted()).
+ _("Num AM Containers Preempted:", lqinfo.getNumberOfMasterContainerPreempted()).
_r("Active users: ", activeUserList.toString());
html._(InfoBlock.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 11f798d..5489f11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -78,6 +78,15 @@
protected int allocatedMB;
protected int allocatedVCores;
protected int runningContainers;
+
+ // preemption info fields
+ protected int preemptedMB;
+ protected int preemptedVCores;
+ protected int preemptedTaskContainerNum;
+ protected int preemptedMasterContainerNum;
+ protected int lastestAttemptPreemptedMB;
+ protected int lastestAttemptPreemptedVCores;
+ protected int lastestAttemptPreemptedTaskContainerNum;
public AppInfo() {
} // JAXB needs this
@@ -147,6 +156,18 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
}
}
}
+
+ // copy preemption info fields
+ preemptedMasterContainerNum = app.getNumberOfMasterContainerPreempted();
+ preemptedMB = app.getResourcePreempted().getMemory();
+ preemptedTaskContainerNum = app.getNumberOfTaskContainersPreempted();
+ preemptedVCores = app.getResourcePreempted().getVirtualCores();
+ lastestAttemptPreemptedMB =
+ app.getCurrentAppAttempt().getResourcePreempted().getMemory();
+ lastestAttemptPreemptedTaskContainerNum =
+ app.getCurrentAppAttempt().getNumberOfTaskContainersPreempted();
+ lastestAttemptPreemptedVCores =
+ app.getCurrentAppAttempt().getResourcePreempted().getVirtualCores();
}
}
@@ -254,4 +275,31 @@ public int getAllocatedVCores() {
return this.allocatedVCores;
}
+ public int getPreemptedMB() {
+ return preemptedMB;
+ }
+
+ public int getPreemptedVCores() {
+ return preemptedVCores;
+ }
+
+ public int getPreemptedTaskContainerNum() {
+ return preemptedTaskContainerNum;
+ }
+
+ public int getPreemptedMasterContainerNum() {
+ return preemptedMasterContainerNum;
+ }
+
+ public int getLastestAttemptPreemptedMB() {
+ return lastestAttemptPreemptedMB;
+ }
+
+ public int getLastestAttemptPreemptedVCores() {
+ return lastestAttemptPreemptedVCores;
+ }
+
+ public int getLastestAttemptPreemptedTaskContainerNum() {
+ return lastestAttemptPreemptedTaskContainerNum;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index d90e963..56f1206 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -37,6 +37,9 @@
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected int numberOfTaskContainerPreempted;
+ protected int numberOfMasterContainerPreempted;
+ protected ResourceInfo totalResourcePreempted;
CapacitySchedulerLeafQueueInfo() {
};
@@ -53,6 +56,9 @@
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsers());
userLimitFactor = q.getUserLimitFactor();
+ numberOfMasterContainerPreempted = q.getNumberOfMasterContainerPreempted();
+ numberOfTaskContainerPreempted = q.getNumberOfTaskContainerPreempted();
+ totalResourcePreempted = new ResourceInfo(q.getResourcePreempted());
}
public int getNumActiveApplications() {
@@ -95,4 +101,16 @@ public UsersInfo getUsers() {
public float getUserLimitFactor() {
return userLimitFactor;
}
+
+ public ResourceInfo getResourcePreempted() {
+ return totalResourcePreempted;
+ }
+
+ public int getNumberOfTaskContainerPreempted() {
+ return numberOfTaskContainerPreempted;
+ }
+
+ public int getNumberOfMasterContainerPreempted() {
+ return numberOfMasterContainerPreempted;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 4349a23..f6067a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -165,6 +166,20 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ public Resource getResourcePreempted() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ return 0;
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ return 0;
+ }
}
public static RMApp newApplication(int i) {
@@ -183,6 +198,8 @@ public static RMApp newApplication(int i) {
YarnApplicationState[] allStates = YarnApplicationState.values();
final YarnApplicationState state = allStates[i % allStates.length];
final int maxAppAttempts = i % 1000;
+ final Resource preemptedResource = Resource.newInstance(1024, 1);
+ final int preemptedContainers = 1;
return new ApplicationBase() {
@Override
public ApplicationId getApplicationId() {
@@ -252,6 +269,16 @@ public int getMaxAppAttempts() {
public Set getApplicationTags() {
return null;
}
+
+ @Override
+ public Resource getResourcePreempted() {
+ return preemptedResource;
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ return preemptedContainers;
+ }
};
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 8f26d10..bcadb37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -238,4 +239,18 @@ public YarnApplicationState createApplicationState() {
public Set getRanNodes() {
return null;
}
+
+ public Resource getResourcePreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfTaskContainersPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
+ public int getNumberOfMasterContainerPreempted() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 6322df3..0d81a40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -25,6 +25,7 @@
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -36,6 +37,7 @@
import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -47,12 +49,18 @@
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -685,5 +693,149 @@ public void testAsyncScheduling() throws Exception {
CapacityScheduler.schedule(cs);
}
}
+
+ private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
+ throws Exception {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ nm.nodeHeartbeat(true);
+ MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
+ am.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ return am;
+ }
+
+ private List waitForContainerAllocated(MockAM am, int nContainer,
+ int memory, MockNM nm) throws Exception {
+ // AM request for containers
+ am.allocate("ANY", memory, nContainer, null);
+ // kick the scheduler
+ nm.nodeHeartbeat(true);
+ List conts =
+ am.allocate(new ArrayList(), null)
+ .getAllocatedContainers();
+ while (conts.size() < nContainer) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList(),
+ new ArrayList()).getAllocatedContainers());
+ Thread.sleep(500);
+ }
+ return conts;
+ }
+
+ private void waitForQueuePreemptionInfo(LeafQueue queue, Resource preempted,
+ int numAMPreempted, int numTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (queue.getResourcePreempted().equals(preempted)
+ && queue.getNumberOfMasterContainerPreempted() == numAMPreempted
+ && queue.getNumberOfTaskContainerPreempted() == numTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
+ int numAMPreempted, int numTaskPreempted,
+ Resource latestAttemptPreempted, boolean latestAttemptAMPreempted,
+ int numLatestAttemptTaskPreempted) throws InterruptedException {
+ while (true) {
+ if (app.getResourcePreempted().equals(preempted)
+ && app.getNumberOfMasterContainerPreempted() == numAMPreempted
+ && app.getNumberOfTaskContainersPreempted() == numTaskPreempted
+ && app.getCurrentAppAttempt().getResourcePreempted().equals(
+ latestAttemptPreempted)
+ && app.getCurrentAppAttempt().isPreempted() == latestAttemptAMPreempted
+ && app.getCurrentAppAttempt().getNumberOfTaskContainersPreempted()
+ == numLatestAttemptTaskPreempted) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+
+ private void waitForNewAttemptCreated(RMApp app,
+ ApplicationAttemptId previousAttemptId) throws InterruptedException {
+ while (app.getCurrentAppAttempt().equals(previousAttemptId)) {
+ Thread.sleep(500);
+ }
+ }
+
+ @Test (timeout = 120000)
+ public void testPreemptionInfo() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);
+ int CONTAINER_MEMORY = 1024;
+ // start RM
+ MockRM rm1 = new MockRM(conf);
+ rm1.start();
+
+ // get scheduler
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+
+ // start NM
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // create app and launch the AM
+ RMApp app0 = rm1.submitApp(CONTAINER_MEMORY);
+ MockAM am0 = launchAM(app0, rm1, nm1);
+
+ // get scheduler app
+ FiCaSchedulerApp schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+ LeafQueue leafQueue = (LeafQueue) schedulerAppAttempt.getQueue();
+
+ // allocate some containers and launch them
+ List allocatedContainers = waitForContainerAllocated(am0, 3, CONTAINER_MEMORY, nm1);
+
+ // kill the 3 containers
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ // kill app0-attempt0 AM container
+ cs.killContainer(schedulerAppAttempt.getRMContainer(app0
+ .getCurrentAppAttempt().getMasterContainer().getId()));
+
+ // wait for app0 failed
+ waitForNewAttemptCreated(app0, am0.getApplicationAttemptId());
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3,
+ Resource.newInstance(0, 0), false, 0);
+
+ // launch app0-attempt1
+ MockAM am1 = launchAM(app0, rm1, nm1);
+ schedulerAppAttempt =
+ cs.getSchedulerApplications().get(app0.getApplicationId())
+ .getCurrentAppAttempt();
+
+ // allocate some containers and launch them
+ allocatedContainers = waitForContainerAllocated(am1, 3, CONTAINER_MEMORY, nm1);
+ for (Container c : allocatedContainers) {
+ cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId()));
+ }
+
+ // check values
+ waitForQueuePreemptionInfo(leafQueue,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6);
+ waitForAppPreemptionInfo(app0,
+ Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6,
+ Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
+
+ rm1.stop();
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index fd14ef6..804740a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -20,6 +20,7 @@
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -242,6 +243,8 @@ public void testSortedQueues() throws Exception {
FiCaSchedulerApp app_0 = getMockApplication(0,user_0);
doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
any(ContainerStatus.class),any(RMContainerEventType.class));
+ doReturn(true).when(app_0).containerCompleted(any(RMContainer.class),
+ any(ContainerStatus.class),any(RMContainerEventType.class));
//
Priority priority = TestUtils.createMockPriority(1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 690fa74..b852c2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 45b3803..506f999 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1310,33 +1310,54 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
WebServicesTestUtils.getXmlString(element, "amContainerLogs"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
- WebServicesTestUtils.getXmlInt(element, "runningContainers"));
+ WebServicesTestUtils.getXmlInt(element, "runningContainers"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedMB"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedVCores"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedTaskContainerNum"),
+ WebServicesTestUtils
+ .getXmlInt(element, "preemptedMasterContainerNum"),
+ WebServicesTestUtils.getXmlInt(element, "lastestAttemptPreemptedMB"),
+ WebServicesTestUtils.getXmlInt(element,
+ "lastestAttemptPreemptedVCores"), WebServicesTestUtils.getXmlInt(
+ element, "lastestAttemptPreemptedTaskContainerNum"));
}
}
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 20 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 20, info.length());
+ // 28 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 27, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
- info.getString("name"), info.getString("applicationType"), info.getString("queue"),
- info.getString("state"), info.getString("finalStatus"),
- (float) info.getDouble("progress"), info.getString("trackingUI"),
- info.getString("diagnostics"), info.getLong("clusterId"),
- info.getLong("startedTime"), info.getLong("finishedTime"),
- info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
- info.getString("amContainerLogs"), info.getInt("allocatedMB"),
- info.getInt("allocatedVCores"), info.getInt("runningContainers"));
+ info.getString("name"), info.getString("applicationType"),
+ info.getString("queue"), info.getString("state"),
+ info.getString("finalStatus"), (float) info.getDouble("progress"),
+ info.getString("trackingUI"), info.getString("diagnostics"),
+ info.getLong("clusterId"), info.getLong("startedTime"),
+ info.getLong("finishedTime"), info.getLong("elapsedTime"),
+ info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
+ info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
+ info.getInt("runningContainers"),
+ info.getInt("preemptedMB"),
+ info.getInt("preemptedVCores"),
+ info.getInt("preemptedTaskContainerNum"),
+ info.getInt("preemptedMasterContainerNum"),
+ info.getInt("lastestAttemptPreemptedMB"),
+ info.getInt("lastestAttemptPreemptedVCores"),
+ info.getInt("lastestAttemptPreemptedTaskContainerNum"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
- String name, String applicationType, String queue, String state, String finalStatus,
- float progress, String trackingUI, String diagnostics, long clusterId,
- long startedTime, long finishedTime, long elapsedTime,
- String amHostHttpAddress, String amContainerLogs, int allocatedMB,
- int allocatedVCores, int numContainers) throws JSONException,
+ String name, String applicationType, String queue, String state,
+ String finalStatus, float progress, String trackingUI,
+ String diagnostics, long clusterId, long startedTime, long finishedTime,
+ long elapsedTime, String amHostHttpAddress, String amContainerLogs,
+ int allocatedMB, int allocatedVCores, int numContainers, int preemptedMB,
+ int preemptedVCores, int preemptedTaskContainerNum,
+ int preemptedMasterContainerNum, int lastestAttemptPreemptedMB,
+ int lastestAttemptPreemptedVCores,
+ int lastestAttemptPreemptedTaskContainerNum) throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@@ -1371,6 +1392,18 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
assertEquals("numContainers doesn't match", 1, numContainers);
+ assertEquals("preemptedMB doesn't match", 0, preemptedMB);
+ assertEquals("preemptedVCores doesn't match", 0, preemptedVCores);
+ assertEquals("preemptedTaskContainerNum doesn't match", 0,
+ preemptedTaskContainerNum);
+ assertEquals("preemptedMasterContainerNum doesn't match", 0,
+ preemptedMasterContainerNum);
+ assertEquals("lastestAttemptPreemptedMB doesn't match", 0,
+ lastestAttemptPreemptedMB);
+ assertEquals("lastestAttemptPreemptedVCores doesn't match", 0,
+ lastestAttemptPreemptedVCores);
+ assertEquals("lastestAttemptPreemptedTaskContainerNum doesn't match", 0,
+ lastestAttemptPreemptedTaskContainerNum);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index e57e5cf..78aac89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -360,7 +360,7 @@ private void verifySubQueue(JSONObject info, String q,
int numExpectedElements = 11;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 21;
+ numExpectedElements = 24;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());