diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 5f3a7f5..79b9d7a 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -55,7 +55,6 @@ public abstract class RMContainerRequestor extends RMCommunicator { private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class); - static final String ANY = "*"; private int lastResponseID; private Resource availableResources; @@ -279,7 +278,7 @@ protected void addContainerReq(ContainerRequest req) { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } protected void decContainerReq(ContainerRequest req) { @@ -292,7 +291,7 @@ protected void decContainerReq(ContainerRequest req) { decResourceRequest(req.priority, rack, req.capability); } - decResourceRequest(req.priority, ANY, req.capability); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } private void addResourceRequest(Priority priority, String resourceName, diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index 6cc7729..0819631 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -231,7 +231,7 @@ public AllocateResponse allocate(AllocateRequest request) List askList = request.getAskList(); List containers = new ArrayList(); for (ResourceRequest req : askList) { - if (req.getHostName() != "*") { + if (req.getHostName() != ResourceRequest.ANY) { continue; } int numContainers = req.getNumContainers(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index 72d703e..821ef2d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -49,6 +49,13 @@ @Public @Stable public abstract class ResourceRequest implements Comparable { + + /** + * The constant string representing no locality. + * It should be used by all references that want to pass an arbitrary host name in. + */ + public static final String ANY = "*"; + /** * Get the Priority of the request. * @return Priority of the request diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java index 1fa86d2..996d466 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java @@ -35,11 +35,6 @@ public interface AMRMClient extends Service { /** - * Value used to define no locality - */ - static final String ANY = "*"; - - /** * Object to represent container request for resources. * Resources may be localized to nodes and racks. * Resources may be assigned priorities. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java index 42b5adb..96cf00e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java @@ -260,7 +260,8 @@ public synchronized void addContainerRequest(ContainerRequest req) { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability, req.containerCount); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override @@ -278,7 +279,8 @@ public synchronized void removeContainerRequest(ContainerRequest req) { } } - decResourceRequest(req.priority, ANY, req.capability, req.containerCount); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java index 95fae13..9f1ec97 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java @@ -18,18 +18,15 @@ package org.apache.hadoop.yarn.client; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Set; import java.util.TreeSet; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -58,6 +55,11 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.service.Service.STATE; import org.apache.hadoop.yarn.util.Records; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestAMRMClient { Configuration conf = null; @@ -183,7 +185,7 @@ private void testAllocation(final AMRMClientImpl amClient) int containersRequestedRack = amClient.remoteRequestsTable.get(priority) .get(rack).get(capability).getNumContainers(); int containersRequestedAny = amClient.remoteRequestsTable.get(priority) - .get(AMRMClient.ANY).get(capability).getNumContainers(); + .get(ResourceRequest.ANY).get(capability).getNumContainers(); assertTrue(containersRequestedNode == 2); assertTrue(containersRequestedRack == 2); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb3e733..c58613e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -38,9 +38,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -63,10 +63,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -737,7 +737,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // Request a container for the AM. ResourceRequest request = BuilderUtils.newResourceRequest( - AM_CONTAINER_PRIORITY, "*", appAttempt.submissionContext + AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt.submissionContext .getAMContainerSpec().getResource(), 1); Allocation amContainerAllocation = appAttempt.scheduler.allocate( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index dc7295a..93cad7f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -36,8 +36,6 @@ */ public interface RMNode { - public static final String ANY = "*"; - /** * the node id of of this node. * @return the node id of this node. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 452cb9b..645d34b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; /** * This class keeps track of all the consumption of an application. This also @@ -129,7 +127,7 @@ synchronized public void updateResourceRequests( boolean updatePendingResources = false; ResourceRequest lastRequest = null; - if (hostName.equals(RMNode.ANY)) { + if (hostName.equals(ResourceRequest.ANY)) { if (LOG.isDebugEnabled()) { LOG.debug("update:" + " application=" + applicationId + " request=" + request); @@ -195,7 +193,7 @@ synchronized public ResourceRequest getResourceRequest(Priority priority, } public synchronized Resource getResource(Priority priority) { - ResourceRequest request = getResourceRequest(priority, RMNode.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); return request.getCapability(); } @@ -261,7 +259,7 @@ synchronized private void allocateNodeLocal( this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -284,7 +282,7 @@ synchronized private void allocateRackLocal( this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -322,7 +320,7 @@ synchronized private void decrementOutstanding( synchronized private void checkForDeactivation() { boolean deactivate = true; for (Priority priority : getPriorities()) { - ResourceRequest request = getResourceRequest(priority, RMNodeImpl.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); if (request.getNumContainers() > 0) { deactivate = false; break; @@ -351,7 +349,7 @@ synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) { // clear pending resources metrics for the application QueueMetrics metrics = queue.getMetrics(); for (Map asks : requests.values()) { - ResourceRequest request = asks.get(RMNode.ANY); + ResourceRequest request = asks.get(ResourceRequest.ANY); if (request != null) { metrics.decrPendingResources(user, request.getNumContainers(), Resources.multiply(request.getCapability(), request diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 719cf1e..b633a52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -812,7 +811,8 @@ private synchronized FiCaSchedulerApp getApplication( for (Priority priority : application.getPriorities()) { // Required resource Resource required = - application.getResourceRequest(priority, RMNode.ANY).getCapability(); + application.getResourceRequest( + priority, ResourceRequest.ANY).getCapability(); // Do we need containers at this 'priority'? if (!needContainers(application, priority, required)) { @@ -1157,7 +1157,7 @@ private Resource assignOffSwitchContainers(Resource clusterResource, FiCaSchedul FiCaSchedulerApp application, Priority priority, RMContainer reservedContainer) { ResourceRequest request = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { if (canAssign(application, priority, node, NodeType.OFF_SWITCH, reservedContainer)) { @@ -1179,7 +1179,7 @@ boolean canAssign(FiCaSchedulerApp application, Priority priority, // 'Delay' off-switch ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); long missedOpportunities = application.getSchedulingOpportunities(priority); long requiredContainers = offSwitchRequest.getNumContainers(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 317f2a2..1ff6e02 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java index b846075..f6bac00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java @@ -60,8 +60,6 @@ private final RMNode rmNode; - public static final String ANY = "*"; - public FiCaSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 46a6fe9..32328be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -336,7 +335,7 @@ public Resource assignContainer(FSSchedulerNode node, boolean reserved) { } ResourceRequest offSwitchRequest = app.getResourceRequest(priority, - RMNode.ANY); + ResourceRequest.ANY); if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 && allowedLocality.equals(NodeType.OFF_SWITCH)) { return assignContainer(node, app, priority, offSwitchRequest, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 49a52bb..0734a4a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -40,8 +40,8 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index 2833ca6..efc1c9f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -59,8 +59,6 @@ private final RMNode rmNode; - public static final String ANY = "*"; - public FSSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 3f25537..df14356 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -393,7 +393,7 @@ private void assignContainers(FiCaSchedulerNode node) { private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) { ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); int maxContainers = offSwitchRequest.getNumContainers(); if (type == NodeType.OFF_SWITCH) { @@ -483,7 +483,7 @@ private int assignRackLocalContainers(FiCaSchedulerNode node, if (request != null) { // Don't allocate on this rack if the application doens't need containers ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest.getNumContainers() <= 0) { return 0; } @@ -504,7 +504,7 @@ private int assignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; ResourceRequest request = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { assignedContainers = assignContainer(node, application, priority, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index ee6cc99..20ffeab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -43,11 +43,9 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Task.State; @@ -200,9 +198,7 @@ public synchronized void addTask(Task task) { } // Off-switch - addResourceRequest(priority, requests, - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY, - capability); + addResourceRequest(priority, requests, ResourceRequest.ANY, capability); } public synchronized void finishTask(Task task) throws IOException { @@ -377,10 +373,7 @@ private void updateResourceRequests(Map requests, } } - updateResourceRequest( - requests.get( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY) - ); + updateResourceRequest(requests.get(ResourceRequest.ANY)); if(LOG.isDebugEnabled()) { LOG.debug("updateResourceRequests:" + " application=" + applicationId diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index ace5efb..b1ef51e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -28,11 +28,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.AMResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -122,8 +121,8 @@ public AMResponse allocate( reqs.add(rackReq); } - ResourceRequest offRackReq = createResourceReq("*", memory, priority, - containers); + ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, + priority, containers); reqs.add(offRackReq); return reqs; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java index 0fac0cd..eebfa1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; public class Task { @@ -57,8 +58,7 @@ public Task(Application application, Priority priority, String[] hosts) { // Special case: Don't care about locality if (!(hosts.length == 1 && - hosts[0].equals( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY))) { + hosts[0].equals(ResourceRequest.ANY))) { for (String host : hosts) { this.hosts.add(host); this.racks.add(Application.resolve(host)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 9190433..519c1d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -264,14 +264,14 @@ public void testHeadroom() throws Exception { // Ask for a 1 GB container for app 1 List ask1 = new ArrayList(); - ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(GB, 1), 1)); + ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1)); fs.allocate(appAttemptId1, ask1, emptyId); // Ask for a 2 GB container for app 2 List ask2 = new ArrayList(); - ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(2 * GB, 1), 1)); + ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1)); fs.allocate(appAttemptId2, ask2, emptyId); // Trigger container assignment diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index c6ee0d5..cee742f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -122,7 +123,7 @@ public void testResourceAllocation() throws IOException { Task t2 = new Task(application, priority1, new String[] {host1, host2}); application.addTask(t2); - Task t3 = new Task(application, priority0, new String[] {RMNode.ANY}); + Task t3 = new Task(application, priority0, new String[] {ResourceRequest.ANY}); application.addTask(t3); // Send resource requests to the scheduler diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 2889435..5953f84 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.junit.After; @@ -506,7 +505,7 @@ public void testHeadroom() throws Exception { List app_0_0_requests = new ArrayList(); app_0_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_0.updateResourceRequests(app_0_0_requests); @@ -525,7 +524,7 @@ public void testHeadroom() throws Exception { List app_0_1_requests = new ArrayList(); app_0_1_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_1.updateResourceRequests(app_0_1_requests); @@ -544,7 +543,7 @@ public void testHeadroom() throws Exception { List app_1_0_requests = new ArrayList(); app_1_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_1_0.updateResourceRequests(app_1_0_requests); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index d3a7b0c..48717ed 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.Application; @@ -183,7 +184,7 @@ public void testCapacityScheduler() throws Exception { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); application_1.schedule(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index ccf2a47..2a33b1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -24,8 +24,8 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -55,13 +55,12 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -282,7 +281,7 @@ public void testSingleQueueOneUserMetrics() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); // Start testing... @@ -404,11 +403,11 @@ public void testSingleQueueWithOneUser() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Start testing... @@ -537,11 +536,11 @@ public void testUserLimits() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -630,11 +629,11 @@ public void testHeadroomWithMaxCap() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -669,7 +668,7 @@ public void testHeadroomWithMaxCap() throws Exception { // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, priority, recordFactory))); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); @@ -686,7 +685,7 @@ public void testHeadroomWithMaxCap() throws Exception { // Check headroom for app_2 LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 0, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, priority, recordFactory))); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); @@ -747,11 +746,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); /** @@ -781,11 +780,11 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Submit resource requests for other apps now to 'activate' them app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 3*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, priority, recordFactory))); app_3.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Now allocations should goto app_2 since @@ -909,11 +908,11 @@ public void testReservation() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1011,7 +1010,7 @@ public void testStolenReservedContainer() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); // Setup app_1 to request a 4GB container on host_0 and @@ -1022,7 +1021,7 @@ public void testStolenReservedContainer() throws Exception { priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(DEFAULT_RACK, 4*GB, 1, priority, recordFactory)); - appRequests_1.add(TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 2, + appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, priority, recordFactory)); app_1.updateResourceRequests(appRequests_1); @@ -1117,11 +1116,11 @@ public void testReservationExchange() throws Exception { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1243,7 +1242,7 @@ public void testLocalityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1308,7 +1307,7 @@ public void testLocalityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); assertEquals(2, app_0.getTotalRequiredResources(priority)); @@ -1386,7 +1385,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority_1, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); // P2 @@ -1398,7 +1397,7 @@ public void testApplicationPriorityScheduling() throws Exception { TestUtils.createResourceRequest(rack_2, 2*GB, 1, priority_2, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority_2, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1523,7 +1522,7 @@ public void testSchedulingConstraints() throws Exception { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1546,7 +1545,7 @@ public void testSchedulingConstraints() throws Exception { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index a5cfadd..9d0d2da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -141,7 +140,8 @@ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { } - private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) { + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); request.setHostName(host); @@ -156,27 +156,33 @@ private ResourceRequest createResourceRequest(int memory, String host, int prior * Creates a single container priority-1 request and submits to * scheduler. */ - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId) { return createSchedulingRequest(memory, queueId, userId, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers) { return createSchedulingRequest(memory, queueId, userId, numContainers, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers, int priority) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id, queueId, userId); List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, numContainers); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, numContainers); ask.add(request); scheduler.allocate(id, ask, new ArrayList()); return id; } - private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) { + private void createSchedulingRequestExistingApplication(int memory, int priority, + ApplicationAttemptId attId) { List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, 1); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1); ask.add(request); scheduler.allocate(attId, ask, new ArrayList()); } @@ -438,7 +444,8 @@ public void testQueueDemandCalculation() throws Exception { // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); - ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1); + ResourceRequest request1 = + createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList()); @@ -452,7 +459,8 @@ public void testQueueDemandCalculation() throws Exception { // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); - ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1); + ResourceRequest request4 = + createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList()); @@ -1290,7 +1298,7 @@ public void testMultipleNodesSingleRackRequest() throws Exception { asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1)); asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1)); asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1)); - asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2)); + asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2)); scheduler.allocate(appId, asks, new ArrayList()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index c4dbe87..8f45f45 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -200,15 +200,15 @@ public void testFifoScheduler() throws Exception { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); Task task_1_2 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_2); Task task_1_3 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_3); application_1.schedule(); @@ -222,7 +222,7 @@ public void testFifoScheduler() throws Exception { application_0.addTask(task_0_2); Task task_0_3 = new Task(application_0, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_0.addTask(task_0_3); application_0.schedule(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 80567bc..288c415 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -69,11 +68,8 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -88,7 +84,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -480,8 +475,8 @@ private Container requestAndGetContainer(AMRMProtocol scheduler, // Request a container allocation. List ask = new ArrayList(); - ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(1024, 1), 1)); + ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(1024, 1), 1)); AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest( BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask,