diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 2da958a..e5bbbc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -210,6 +210,12 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f2a8376..3800aa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -227,7 +227,8 @@ void handleNMContainerStatus(NMContainerStatus containerStatus) { containerStatus.getContainerExitStatus()); // sending master container finished event. RMAppAttemptContainerFinishedEvent evt = - new RMAppAttemptContainerFinishedEvent(appAttemptId, status); + new RMAppAttemptContainerFinishedEvent(appAttemptId, status, + masterContainer.getResource()); rmContext.getDispatcher().getEventHandler().handle(evt); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 2b590a0..d1d7ca4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -223,4 +224,27 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the external user-facing state of ApplicationMaster. */ YarnApplicationState createApplicationState(); + + /** + * Get total resource preempted of the {@link RMApp}. + * + * @return total resource preempted of the {@link RMApp} + */ + Resource getResourcePreempted(); + + /** + * Get total number of task containers preempted of the {@link RMApp}. + * + * @return total number of task containers preempted of the {@link RMApp} + */ + int getNumNonAMContainersPreempted(); + + /** + * Get total number of application master containers preempted of the + * {@link RMApp}. + * + * @return total number of application master containers preempted of the + * {@link RMApp} + */ + int getNumAMContainersPreempted(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index b6ca684..897d0ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -78,6 +79,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.resource.Resources; @SuppressWarnings({ "rawtypes", "unchecked" }) public class RMAppImpl implements RMApp, Recoverable { @@ -1200,4 +1202,37 @@ private RMAppState getRecoveredFinalState() { public Set getRanNodes() { return ranNodes; } + + @Override + public Resource getResourcePreempted() { + Resource sum = Resource.newInstance(0, 0); + for (RMAppAttempt attempt : attempts.values()) { + if (null != attempt) { + Resources.addTo(sum, attempt.getResourcePreempted()); + } + } + return sum; + } + + @Override + public int getNumNonAMContainersPreempted() { + int sum = 0; + for (RMAppAttempt attempt : attempts.values()) { + if (null != attempt) { + sum += attempt.getNumAMContainersPreempted(); + } + } + return sum; + } + + @Override + public int getNumAMContainersPreempted() { + int sum = 0; + for (RMAppAttempt attempt : attempts.values()) { + if (null != attempt && attempt.isPreempted()) { + sum ++; + } + } + return sum; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index 42c37a9..df65181 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; @@ -201,4 +202,16 @@ * scheduler. */ boolean isPreempted(); + + /** + * Get total resource preempted from the {@link RMAppAttempt} + * @return total resource preeempted in this application attempt + */ + Resource getResourcePreempted(); + + /** + * Get total number of containers preempted from the {@link RMAppAttempt} + * @return total number of containers preempted from the application attempt + */ + int getNumAMContainersPreempted(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1e7693f..3cc9ca2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.event.EventHandler; @@ -96,6 +98,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -162,6 +165,10 @@ private RMAppAttemptState recoveredFinalState; private RMAppAttemptState stateBeforeFinalSaving; private Object transitionTodo; + + // preemption info + private Resource resourcePreempted = Resource.newInstance(0, 0); + private AtomicInteger numNonAMContainersPreempted = new AtomicInteger(0); private static final StateMachineFactory attempts = rmApp.getAppAttempts().values(); String amString = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 0f0ed50..c323154 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -120,6 +120,12 @@ protected void render(Block html) { _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())). + _("Total Resource Preempted:", + lqinfo.getResourcePreempted().toString()). + _("Num Non-AM Containers Preempted:", + lqinfo.getNumNonAMContainersPreempted()). + _("Num AM Containers Preempted:", + lqinfo.getNumAMContainersPreempted()). _r("Active users: ", activeUserList.toString()); html._(InfoBlock.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 11f798d..8923247 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -78,6 +78,15 @@ protected int allocatedMB; protected int allocatedVCores; protected int runningContainers; + + // preemption info fields + protected int preemptedResourceMB; + protected int preemptedResourceVCores; + protected int numNonAMContainerPreempted; + protected int numAMContainerPreempted; + protected int currentAttemptPreemptedResourceMB; + protected int currentAttemptPreemptedResourceVCores; + protected int currentAttemptNumNonAMContainerPreempted; public AppInfo() { } // JAXB needs this @@ -85,6 +94,7 @@ public AppInfo() { public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) { this.schemePrefix = schemePrefix; if (app != null) { + RMAppAttempt attempt = app.getCurrentAppAttempt(); String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() @@ -125,7 +135,6 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) { this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); - RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { @@ -145,8 +154,21 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) { allocatedVCores = usedResources.getVirtualCores(); runningContainers = resourceReport.getNumUsedContainers(); } + + currentAttemptPreemptedResourceVCores = + attempt.getResourcePreempted().getVirtualCores(); + currentAttemptPreemptedResourceMB = + attempt.getResourcePreempted().getMemory(); + currentAttemptNumNonAMContainerPreempted = + attempt.getNumAMContainersPreempted(); } } + + // copy preemption info fields + numAMContainerPreempted = app.getNumAMContainersPreempted(); + preemptedResourceMB = app.getResourcePreempted().getMemory(); + numNonAMContainerPreempted = app.getNumNonAMContainersPreempted(); + preemptedResourceVCores = app.getResourcePreempted().getVirtualCores(); } } @@ -254,4 +276,31 @@ public int getAllocatedVCores() { return this.allocatedVCores; } + public int getPreemptedMB() { + return preemptedResourceMB; + } + + public int getPreemptedVCores() { + return preemptedResourceVCores; + } + + public int getNumNonAMContainersPreempted() { + return numNonAMContainerPreempted; + } + + public int getNumAMContainersPreempted() { + return numAMContainerPreempted; + } + + public int getCurrentAttemptPreemptedMB() { + return currentAttemptPreemptedResourceMB; + } + + public int getCurrentAttemptPreemptedVCores() { + return currentAttemptPreemptedResourceVCores; + } + + public int getCurrentAttemptNumNonAMContainersPreempted() { + return currentAttemptNumNonAMContainerPreempted; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index d90e963..11efc16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -37,6 +37,9 @@ protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected int numNonAMContainerPreempted; + protected int numAMContainerPreempted; + protected ResourceInfo resourcePreempted; CapacitySchedulerLeafQueueInfo() { }; @@ -53,6 +56,9 @@ userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsers()); userLimitFactor = q.getUserLimitFactor(); + numAMContainerPreempted = q.getNumAMContainersPreempted(); + numNonAMContainerPreempted = q.getNumNonAMContainersPreempted(); + resourcePreempted = new ResourceInfo(q.getResourcePreempted()); } public int getNumActiveApplications() { @@ -95,4 +101,16 @@ public UsersInfo getUsers() { public float getUserLimitFactor() { return userLimitFactor; } + + public ResourceInfo getResourcePreempted() { + return resourcePreempted; + } + + public int getNumNonAMContainersPreempted() { + return numNonAMContainerPreempted; + } + + public int getNumAMContainersPreempted() { + return numAMContainerPreempted; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index cfd05f9..f1a3bbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -24,7 +24,6 @@ import java.util.List; import org.junit.Assert; - import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -34,6 +33,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; @@ -251,4 +251,22 @@ public Object run() throws Exception { public ApplicationAttemptId getApplicationAttemptId() { return this.attemptId; } + + public List allocateAndWaitForContainers(int nContainer, + int memory, MockNM nm) throws Exception { + // AM request for containers + allocate("ANY", memory, nContainer, null); + // kick the scheduler + nm.nodeHeartbeat(true); + List conts = + allocate(new ArrayList(), null) + .getAllocatedContainers(); + while (conts.size() < nContainer) { + nm.nodeHeartbeat(true); + conts.addAll(allocate(new ArrayList(), + new ArrayList()).getAllocatedContainers()); + Thread.sleep(500); + } + return conts; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 4349a23..fc762f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -165,6 +166,20 @@ public YarnApplicationState createApplicationState() { public Set getRanNodes() { throw new UnsupportedOperationException("Not supported yet."); } + + public Resource getResourcePreempted() { + return Resource.newInstance(0, 0); + } + + @Override + public int getNumNonAMContainersPreempted() { + return 0; + } + + @Override + public int getNumAMContainersPreempted() { + return 0; + } } public static RMApp newApplication(int i) { @@ -183,6 +198,8 @@ public static RMApp newApplication(int i) { YarnApplicationState[] allStates = YarnApplicationState.values(); final YarnApplicationState state = allStates[i % allStates.length]; final int maxAppAttempts = i % 1000; + final Resource preemptedResource = Resource.newInstance(1024, 1); + final int preemptedContainers = 1; return new ApplicationBase() { @Override public ApplicationId getApplicationId() { @@ -252,6 +269,16 @@ public int getMaxAppAttempts() { public Set getApplicationTags() { return null; } + + @Override + public Resource getResourcePreempted() { + return preemptedResource; + } + + @Override + public int getNumNonAMContainersPreempted() { + return preemptedContainers; + } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 8f26d10..a42a193 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -238,4 +239,18 @@ public YarnApplicationState createApplicationState() { public Set getRanNodes() { return null; } + + public Resource getResourcePreempted() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public int getNumNonAMContainersPreempted() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public int getNumAMContainersPreempted() { + throw new UnsupportedOperationException("Not supported yet."); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index c3b1d57..47fbd66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -62,6 +64,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.resourcemanager.Application; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -72,6 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -706,7 +710,55 @@ public void testAsyncScheduling() throws Exception { CapacityScheduler.schedule(cs); } } + + private MockAM launchAM(RMApp app, MockRM rm, MockNM nm) + throws Exception { + RMAppAttempt attempt = app.getCurrentAppAttempt(); + nm.nodeHeartbeat(true); + MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); + am.registerAppAttempt(); + rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); + return am; + } + + private void waitForQueuePreemptionInfo(LeafQueue queue, Resource preempted, + int numAMPreempted, int numTaskPreempted) throws InterruptedException { + while (true) { + if (queue.getResourcePreempted().equals(preempted) + && queue.getNumAMContainersPreempted() == numAMPreempted + && queue.getNumNonAMContainersPreempted() == numTaskPreempted) { + return; + } + Thread.sleep(500); + } + } + + private void waitForAppPreemptionInfo(RMApp app, Resource preempted, + int numAMPreempted, int numTaskPreempted, + Resource currentAttemptPreempted, boolean currentAttemptAMPreempted, + int numLatestAttemptTaskPreempted) throws InterruptedException { + while (true) { + if (app.getResourcePreempted().equals(preempted) + && app.getNumAMContainersPreempted() == numAMPreempted + && app.getNumNonAMContainersPreempted() == numTaskPreempted + && app.getCurrentAppAttempt().getResourcePreempted().equals( + currentAttemptPreempted) + && app.getCurrentAppAttempt().isPreempted() == currentAttemptAMPreempted + && app.getCurrentAppAttempt().getNumAMContainersPreempted() + == numLatestAttemptTaskPreempted) { + return; + } + Thread.sleep(500); + } + } + private void waitForNewAttemptCreated(RMApp app, + ApplicationAttemptId previousAttemptId) throws InterruptedException { + while (app.getCurrentAppAttempt().equals(previousAttemptId)) { + Thread.sleep(500); + } + } + @Test(timeout = 30000) public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception { final YarnConfiguration conf = new YarnConfiguration(); @@ -828,4 +880,85 @@ public void testNumClusterNodes() throws Exception { cs.stop(); } + + @Test(timeout = 120000) + public void testPreemptionInfo() throws Exception { + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + int CONTAINER_MEMORY = 1024; // start RM + MockRM rm1 = new MockRM(conf); + rm1.start(); + + // get scheduler + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + + // start NM + MockNM nm1 = + new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + + // create app and launch the AM + RMApp app0 = rm1.submitApp(CONTAINER_MEMORY); + MockAM am0 = launchAM(app0, rm1, nm1); + + // get scheduler app + FiCaSchedulerApp schedulerAppAttempt = + cs.getSchedulerApplications().get(app0.getApplicationId()) + .getCurrentAppAttempt(); + LeafQueue leafQueue = (LeafQueue) schedulerAppAttempt.getQueue(); + + // allocate some containers and launch them + List allocatedContainers = + am0.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1); + + // kill the 3 containers + for (Container c : allocatedContainers) { + cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId())); + } + + // check values + waitForQueuePreemptionInfo(leafQueue, + Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3); + waitForAppPreemptionInfo(app0, + Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3, + Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3); + + // kill app0-attempt0 AM container + cs.killContainer(schedulerAppAttempt.getRMContainer(app0 + .getCurrentAppAttempt().getMasterContainer().getId())); + + // wait for app0 failed + waitForNewAttemptCreated(app0, am0.getApplicationAttemptId()); + + // check values + waitForQueuePreemptionInfo(leafQueue, + Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3); + waitForAppPreemptionInfo(app0, + Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3, + Resource.newInstance(0, 0), false, 0); + + // launch app0-attempt1 + MockAM am1 = launchAM(app0, rm1, nm1); + schedulerAppAttempt = + cs.getSchedulerApplications().get(app0.getApplicationId()) + .getCurrentAppAttempt(); + + // allocate some containers and launch them + allocatedContainers = + am1.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1); + for (Container c : allocatedContainers) { + cs.killContainer(schedulerAppAttempt.getRMContainer(c.getId())); + } + + // check values + waitForQueuePreemptionInfo(leafQueue, + Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6); + waitForAppPreemptionInfo(app0, + Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6, + Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3); + + rm1.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index fd14ef6..804740a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -242,6 +243,8 @@ public void testSortedQueues() throws Exception { FiCaSchedulerApp app_0 = getMockApplication(0,user_0); doReturn(true).when(app_0).containerCompleted(any(RMContainer.class), any(ContainerStatus.class),any(RMContainerEventType.class)); + doReturn(true).when(app_0).containerCompleted(any(RMContainer.class), + any(ContainerStatus.class),any(RMContainerEventType.class)); // Priority priority = TestUtils.createMockPriority(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 45b3803..d9f59d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1310,33 +1310,56 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException, WebServicesTestUtils.getXmlString(element, "amContainerLogs"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), WebServicesTestUtils.getXmlInt(element, "allocatedVCores"), - WebServicesTestUtils.getXmlInt(element, "runningContainers")); + WebServicesTestUtils.getXmlInt(element, "runningContainers"), + WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"), + WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), + WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), + WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), + WebServicesTestUtils.getXmlInt(element, + "currentAttemptPreemptedResourceMB"), + WebServicesTestUtils + .getXmlInt(element, "currentAttemptPreemptedResourceVCores"), + WebServicesTestUtils.getXmlInt(element, + "currentAttemptNumNonAMContainerPreempted")); } } public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 20 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 20, info.length()); + // 28 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 27, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), - info.getString("name"), info.getString("applicationType"), info.getString("queue"), - info.getString("state"), info.getString("finalStatus"), - (float) info.getDouble("progress"), info.getString("trackingUI"), - info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs"), info.getInt("allocatedMB"), - info.getInt("allocatedVCores"), info.getInt("runningContainers")); + info.getString("name"), info.getString("applicationType"), + info.getString("queue"), info.getString("state"), + info.getString("finalStatus"), (float) info.getDouble("progress"), + info.getString("trackingUI"), info.getString("diagnostics"), + info.getLong("clusterId"), info.getLong("startedTime"), + info.getLong("finishedTime"), info.getLong("elapsedTime"), + info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), + info.getInt("allocatedMB"), info.getInt("allocatedVCores"), + info.getInt("runningContainers"), + info.getInt("preemptedResourceMB"), + info.getInt("preemptedResourceVCores"), + info.getInt("numNonAMContainerPreempted"), + info.getInt("numAMContainerPreempted"), + info.getInt("currentAttemptPreemptedResourceMB"), + info.getInt("currentAttemptPreemptedResourceVCores"), + info.getInt("currentAttemptNumNonAMContainerPreempted")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, - String name, String applicationType, String queue, String state, String finalStatus, - float progress, String trackingUI, String diagnostics, long clusterId, - long startedTime, long finishedTime, long elapsedTime, - String amHostHttpAddress, String amContainerLogs, int allocatedMB, - int allocatedVCores, int numContainers) throws JSONException, + String name, String applicationType, String queue, String state, + String finalStatus, float progress, String trackingUI, + String diagnostics, long clusterId, long startedTime, long finishedTime, + long elapsedTime, String amHostHttpAddress, String amContainerLogs, + int allocatedMB, int allocatedVCores, int numContainers, + int preemptedResourceMB, int preemptedResourceVCores, + int numNonAMContainerPreempted, int numAMContainerPreempted, + int currentAttemptPreemptedResourceMB, + int currentAttemptPreemptedResourceVCores, + int currentAttemptNumNonAMContainerPreempted) throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() @@ -1371,6 +1394,19 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, assertEquals("allocatedMB doesn't match", 1024, allocatedMB); assertEquals("allocatedVCores doesn't match", 1, allocatedVCores); assertEquals("numContainers doesn't match", 1, numContainers); + assertEquals("preemptedResourceMB doesn't match", 0, preemptedResourceMB); + assertEquals("preemptedResourceVCores doesn't match", 0, + preemptedResourceVCores); + assertEquals("numNonAMContainerPreempted doesn't match", 0, + numNonAMContainerPreempted); + assertEquals("numAMContainerPreempted doesn't match", 0, + numAMContainerPreempted); + assertEquals("currentAttemptPreemptedResourceMB doesn't match", 0, + currentAttemptPreemptedResourceMB); + assertEquals("currentAttemptPreemptedResourceVCores doesn't match", 0, + currentAttemptPreemptedResourceVCores); + assertEquals("currentAttemptNumAMContainerPreempted doesn't match", 0, + currentAttemptNumNonAMContainerPreempted); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index e57e5cf..78aac89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -360,7 +360,7 @@ private void verifySubQueue(JSONObject info, String q, int numExpectedElements = 11; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 21; + numExpectedElements = 24; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());