diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 014aa89415139f8348382db00726bc4903da1a3a..d83c7bcd3f2c4582fc851e13724799a15ddfbf7b 100644 --- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -978,4 +978,8 @@ protected void decreaseContainer( } + @Override + protected void assignContainers(SchedulerNode node) { + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 64eb7773a024baceb1c0d3240077077db6fdac5c..170f548a91fe61af7560c6e93db35634c1a003fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -94,7 +94,7 @@ protected Resource minimumAllocation; protected RMContext rmContext; - + private volatile Priority maxClusterLevelAppPriority; /* @@ -789,4 +789,21 @@ private SchedContainerChangeRequest createSchedContainerChangeRequest( } return schedulerChangeRequests; } + + /** + * Assigns containers based on the free resources available at the node. + * @param node The node to use for assigning containers + */ + @VisibleForTesting + protected abstract void assignContainers(N node); + + /** + * Method determine whether assignContainers can be called. + * @returns true if the scheduler is cleared to call assignContainer(). + */ + public boolean isReadyToAssignContainers() { + // Determine if work-preserving restart recovery time has not yet passed + return (!rmContext.isWorkPreservingRecoveryEnabled() + || rmContext.isSchedulerReadyForAllocatingContainers()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index ee62a701514304a2743063395368d5fc359d8611..42576ce98f7fa41705d57cdd277fee60dc945ecf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -395,13 +395,15 @@ static void schedule(CapacityScheduler cs) { Collection nodes = cs.nodeTracker.getAllNodes(); int start = random.nextInt(nodes.size()); for (FiCaSchedulerNode node : nodes) { - if (current++ >= start) { - cs.allocateContainersToNode(node); + if (current++ >= start && cs.isReadyToAssignContainers()) { + cs.assignContainers(node); } } // Now, just get everyone to be safe for (FiCaSchedulerNode node : nodes) { - cs.allocateContainersToNode(node); + if (cs.isReadyToAssignContainers()) { + cs.assignContainers(node); + } } try { Thread.sleep(cs.getAsyncScheduleInterval()); @@ -1208,12 +1210,9 @@ private void updateSchedulerHealth(long now, FiCaSchedulerNode node, .getAssignmentInformation().getReserved()); } + @Override @VisibleForTesting - protected synchronized void allocateContainersToNode(FiCaSchedulerNode node) { - if (rmContext.isWorkPreservingRecoveryEnabled() - && !rmContext.isSchedulerReadyForAllocatingContainers()) { - return; - } + protected synchronized void assignContainers(FiCaSchedulerNode node) { // reset allocation and reservation stats before we start doing any work updateSchedulerHealth(lastNodeUpdateTime, node, new CSAssignment(Resources.none(), NodeType.NODE_LOCAL)); @@ -1363,8 +1362,8 @@ public void handle(SchedulerEvent event) { RMNode node = nodeUpdatedEvent.getRMNode(); setLastNodeUpdateTime(Time.now()); nodeUpdate(node); - if (!scheduleAsynchronously) { - allocateContainersToNode(getNode(node.getNodeID())); + if (!scheduleAsynchronously && super.isReadyToAssignContainers()) { + assignContainers(getNode(node.getNodeID())); } } break; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 661caa7895d792220a934fa719c1a207bc51ce2c..fab536d2a5114ac264924aebfdff155e233f6a4c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -53,6 +53,7 @@ import org.w3c.dom.Text; import org.xml.sax.SAXException; +import com.google.common.base.CharMatcher; import com.google.common.annotations.VisibleForTesting; @Public @@ -445,7 +446,8 @@ private void loadQueue(String parentName, Element element, Set reservableQueues, Set nonPreemptableQueues) throws AllocationConfigurationException { - String queueName = element.getAttribute("name").trim(); + String queueName = CharMatcher.WHITESPACE.trimFrom( + element.getAttribute("name")); if (queueName.contains(".")) { throw new AllocationConfigurationException("Bad fair scheduler config " diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index bc953ba37379b64291c664a32d59fe95107a0de0..a55e76855eae1635005844554837d5ebf767cb0f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1055,11 +1055,13 @@ private synchronized void nodeUpdate(RMNode nm) { } if (continuousSchedulingEnabled) { - if (!completedContainers.isEmpty()) { - attemptScheduling(node); + if (!completedContainers.isEmpty() && super.isReadyToAssignContainers()) { + assignContainers(node); } } else { - attemptScheduling(node); + if (super.isReadyToAssignContainers()) { + assignContainers(node); + } } // Updating node resource utilization @@ -1081,7 +1083,7 @@ void continuousSchedulingAttempt() throws InterruptedException { try { if (Resources.fitsIn(minimumAllocation, node.getUnallocatedResource())) { - attemptScheduling(node); + assignContainers(node); } } catch (Throwable ex) { LOG.error("Error while attempting scheduling for node " + node + @@ -1128,13 +1130,9 @@ private boolean shouldContinueAssigning(int containers, } } + @Override @VisibleForTesting - synchronized void attemptScheduling(FSSchedulerNode node) { - if (rmContext.isWorkPreservingRecoveryEnabled() - && !rmContext.isSchedulerReadyForAllocatingContainers()) { - return; - } - + protected synchronized void assignContainers(FSSchedulerNode node) { final NodeId nodeID = node.getNodeID(); if (!nodeTracker.exists(nodeID)) { // The node might have just been removed while this thread was waiting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index aeadcf6b4964bad41b3e58608d477b4963e1ef13..8d308dc9baa2363a0353aa536d0f478e6679ef26 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.xml.sax.SAXException; +import com.google.common.base.CharMatcher; import com.google.common.annotations.VisibleForTesting; /** * Maintains a list of queues as well as scheduling parameters for each queue, @@ -457,6 +458,9 @@ public void updateAllocationConfiguration(AllocationConfiguration queueConf) { */ @VisibleForTesting boolean isQueueNameValid(String node) { - return !node.isEmpty() && node.equals(node.trim()); + // use the same white space trim as in QueueMetrics() otherwise things fail + // guava uses a different definition for whitespace than java. + return !node.isEmpty() && + node.equals(CharMatcher.WHITESPACE.trimFrom(node)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index eaab495a0d01d5e37d0925b74d7e7910b54913ca..5d0d2859b303d4c8509965d4a1a0a871dda9b69e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -493,9 +493,18 @@ private synchronized void doneApplicationAttempt( * * @param node node on which resources are available to be allocated */ - private void assignContainers(FiCaSchedulerNode node) { + @Override + protected void assignContainers(FiCaSchedulerNode node) { + if (Resources.lessThan(resourceCalculator, getClusterResource(), + node.getUnallocatedResource(), minimumAllocation)) { + return; + } + + LOG.debug("Node heartbeat " + node.getNodeID() + + " available resource = " + node.getUnallocatedResource()); + LOG.debug("assignContainers:" + - " node=" + node.getRMNode().getNodeAddress() + + " node=" + node.getRMNode().getNodeAddress() + " #applications=" + applications.size()); // Try to assign containers to applications in fifo order @@ -515,13 +524,13 @@ private void assignContainers(FiCaSchedulerNode node) { } for (Priority priority : application.getPriorities()) { - int maxContainers = - getMaxAllocatableContainers(application, priority, node, - NodeType.OFF_SWITCH); + int maxContainers = + getMaxAllocatableContainers(application, priority, node, + NodeType.OFF_SWITCH); // Ensure the application needs containers of this priority if (maxContainers > 0) { - int assignedContainers = - assignContainersOnNode(node, application, priority); + int assignedContainers = + assignContainersOnNode(node, application, priority); // Do not assign out of order w.r.t priorities if (assignedContainers == 0) { break; @@ -529,7 +538,7 @@ private void assignContainers(FiCaSchedulerNode node) { } } } - + LOG.debug("post-assignContainers"); application.showRequests(); @@ -550,13 +559,16 @@ private void assignContainers(FiCaSchedulerNode node) { } updateAppHeadRoom(attempt); } + + LOG.debug("Node after allocation " + node.getNodeID() + " resource = " + + node.getUnallocatedResource()); } private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) { int maxContainers = 0; - - ResourceRequest offSwitchRequest = + + ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest != null) { maxContainers = offSwitchRequest.getNumContainers(); @@ -567,7 +579,7 @@ private int getMaxAllocatableContainers(FiCaSchedulerApp application, } if (type == NodeType.RACK_LOCAL) { - ResourceRequest rackLocalRequest = + ResourceRequest rackLocalRequest = application.getResourceRequest(priority, node.getRMNode().getRackName()); if (rackLocalRequest == null) { return maxContainers; @@ -577,7 +589,7 @@ private int getMaxAllocatableContainers(FiCaSchedulerApp application, } if (type == NodeType.NODE_LOCAL) { - ResourceRequest nodeLocalRequest = + ResourceRequest nodeLocalRequest = application.getResourceRequest(priority, node.getRMNode().getNodeAddress()); if (nodeLocalRequest != null) { maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers()); @@ -588,63 +600,63 @@ private int getMaxAllocatableContainers(FiCaSchedulerApp application, } - private int assignContainersOnNode(FiCaSchedulerNode node, - FiCaSchedulerApp application, Priority priority + private int assignContainersOnNode(FiCaSchedulerNode node, + FiCaSchedulerApp application, Priority priority ) { // Data-local - int nodeLocalContainers = - assignNodeLocalContainers(node, application, priority); + int nodeLocalContainers = + assignNodeLocalContainers(node, application, priority); // Rack-local - int rackLocalContainers = - assignRackLocalContainers(node, application, priority); + int rackLocalContainers = + assignRackLocalContainers(node, application, priority); // Off-switch int offSwitchContainers = - assignOffSwitchContainers(node, application, priority); + assignOffSwitchContainers(node, application, priority); LOG.debug("assignContainersOnNode:" + - " node=" + node.getRMNode().getNodeAddress() + + " node=" + node.getRMNode().getNodeAddress() + " application=" + application.getApplicationId().getId() + - " priority=" + priority.getPriority() + - " #assigned=" + + " priority=" + priority.getPriority() + + " #assigned=" + (nodeLocalContainers + rackLocalContainers + offSwitchContainers)); return (nodeLocalContainers + rackLocalContainers + offSwitchContainers); } - private int assignNodeLocalContainers(FiCaSchedulerNode node, + private int assignNodeLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; - ResourceRequest request = + ResourceRequest request = application.getResourceRequest(priority, node.getNodeName()); if (request != null) { // Don't allocate on this node if we don't need containers on this rack ResourceRequest rackRequest = - application.getResourceRequest(priority, + application.getResourceRequest(priority, node.getRMNode().getRackName()); if (rackRequest == null || rackRequest.getNumContainers() <= 0) { return 0; } - - int assignableContainers = + + int assignableContainers = Math.min( - getMaxAllocatableContainers(application, priority, node, - NodeType.NODE_LOCAL), + getMaxAllocatableContainers(application, priority, node, + NodeType.NODE_LOCAL), request.getNumContainers()); - assignedContainers = - assignContainer(node, application, priority, + assignedContainers = + assignContainer(node, application, priority, assignableContainers, request, NodeType.NODE_LOCAL); } return assignedContainers; } - private int assignRackLocalContainers(FiCaSchedulerNode node, + private int assignRackLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; - ResourceRequest request = + ResourceRequest request = application.getResourceRequest(priority, node.getRMNode().getRackName()); if (request != null) { // Don't allocate on this rack if the application doens't need containers @@ -653,39 +665,39 @@ private int assignRackLocalContainers(FiCaSchedulerNode node, if (offSwitchRequest.getNumContainers() <= 0) { return 0; } - - int assignableContainers = + + int assignableContainers = Math.min( - getMaxAllocatableContainers(application, priority, node, - NodeType.RACK_LOCAL), + getMaxAllocatableContainers(application, priority, node, + NodeType.RACK_LOCAL), request.getNumContainers()); - assignedContainers = - assignContainer(node, application, priority, + assignedContainers = + assignContainer(node, application, priority, assignableContainers, request, NodeType.RACK_LOCAL); } return assignedContainers; } - private int assignOffSwitchContainers(FiCaSchedulerNode node, + private int assignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; - ResourceRequest request = + ResourceRequest request = application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { - assignedContainers = - assignContainer(node, application, priority, + assignedContainers = + assignContainer(node, application, priority, request.getNumContainers(), request, NodeType.OFF_SWITCH); } return assignedContainers; } private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application, - Priority priority, int assignableContainers, + Priority priority, int assignableContainers, ResourceRequest request, NodeType type) { LOG.debug("assignContainers:" + - " node=" + node.getRMNode().getNodeAddress() + - " application=" + application.getApplicationId().getId() + - " priority=" + priority.getPriority() + + " node=" + node.getRMNode().getNodeAddress() + + " application=" + application.getApplicationId().getId() + + " priority=" + priority.getPriority() + " assignableContainers=" + assignableContainers + " request=" + request + " type=" + type); Resource capability = request.getCapability(); @@ -708,13 +720,13 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); - + // Allocate! - + // Inform the application RMContainer rmContainer = application.allocate(type, node, priority, request, container); - + // Inform the node node.allocateContainer(rmContainer); @@ -723,13 +735,13 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application } } - + return assignedContainers; } private synchronized void nodeUpdate(RMNode rmNode) { FiCaSchedulerNode node = getNode(rmNode.getNodeID()); - + List containerInfoList = rmNode.pullContainerUpdates(); List newlyLaunchedContainers = new ArrayList(); List completedContainers = new ArrayList(); @@ -768,22 +780,9 @@ private synchronized void nodeUpdate(RMNode rmNode) { .getAllocatedResource(), 0))); } - if (rmContext.isWorkPreservingRecoveryEnabled() - && !rmContext.isSchedulerReadyForAllocatingContainers()) { - return; - } - - if (Resources.greaterThanOrEqual(resourceCalculator, getClusterResource(), - node.getUnallocatedResource(), minimumAllocation)) { - LOG.debug("Node heartbeat " + rmNode.getNodeID() + - " available resource = " + node.getUnallocatedResource()); - + if (super.isReadyToAssignContainers()) { assignContainers(node); - - LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = " - + node.getUnallocatedResource()); } - updateAvailableResourcesMetrics(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java index 499e041a14f218dc47a4ba752561cf9925feba3e..83ebddddd698e2c053b35ab19769f48a043acd18 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java @@ -95,13 +95,13 @@ protected void decreaseContainers( } @Override - public synchronized void allocateContainersToNode(FiCaSchedulerNode node) { + public synchronized void assignContainers(FiCaSchedulerNode node) { try { Thread.sleep(1000); } catch(InterruptedException e) { LOG.debug("Thread interrupted."); } - super.allocateContainersToNode(node); + super.assignContainers(node); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index cc91ef9c8ce478176f9c38faf47d2baba959ca64..ceba4d4ab008bfc7002cf6e0af2610bf0c8d2a09 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -581,6 +581,30 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { } /** + * Verify that you can't have the queue name with just a non breaking + * whitespace in the allocations file. + */ + @Test (expected = AllocationConfigurationException.class) + public void testQueueNameContainingNBWhitespace() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + } + + /** * Verify that defaultQueueSchedulingMode can't accept FIFO as a value. */ @Test (expected = AllocationConfigurationException.class) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 43ebe53917477e22707f041623e9e234d95421c1..0624d6b48491828dd7f8837d889cd489f0efdb18 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3965,7 +3965,7 @@ public void testContinuousSchedulingInterruptedException() 1, spyScheduler.getNumClusterNodes()); InterruptedException ie = new InterruptedException(); doThrow(new YarnRuntimeException(ie)).when(spyScheduler). - attemptScheduling(isA(FSSchedulerNode.class)); + assignContainers(isA(FSSchedulerNode.class)); // Invoke the continuous scheduling once try { spyScheduler.continuousSchedulingAttempt(); @@ -4011,7 +4011,7 @@ public void testSchedulingOnRemovedNode() throws Exception { new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeNode1); - scheduler.attemptScheduling(node); + scheduler.assignContainers(node); AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(id11, @@ -4362,6 +4362,17 @@ public void testQueueNameWithTrailingSpace() throws Exception { assertEquals(1, scheduler.getQueueManager().getLeafQueue("B.C", true) .getNumRunnableApps()); assertNotNull(scheduler.getSchedulerApp(appAttemptId3)); + + // submit app with queue name "A\u00a0" (non-breaking space) + ApplicationAttemptId appAttemptId4 = createAppAttemptId(4, 1); + AppAddedSchedulerEvent appAddedEvent4 = new AppAddedSchedulerEvent( + appAttemptId4.getApplicationId(), "A\u00a0", "user1"); + scheduler.handle(appAddedEvent4); + // submission rejected + assertEquals(3, scheduler.getQueueManager().getLeafQueues().size()); + assertNull(scheduler.getSchedulerApplications().get(appAttemptId4. + getApplicationId())); + assertNull(scheduler.getSchedulerApp(appAttemptId4)); } @Test @@ -4473,7 +4484,7 @@ public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { // Tick the time and let the fsApp startTime different from initScheduler // time clock.tickSec(DELAY_THRESHOLD_TIME_MS / 1000); - scheduler.attemptScheduling(node); + scheduler.assignContainers(node); Map lastScheduledContainer = fsAppAttempt.getLastScheduledContainer(); long initSchedulerTime = lastScheduledContainer.get(priority); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index a9b27a1a44e37997c152a50382baa5fa2ab592a6..33d44192528f1d316467622b8e1c009bec2a0bea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -131,6 +131,9 @@ public void testCheckQueueNodeName() { assertFalse(queueManager.isQueueNameValid(" a")); assertFalse(queueManager.isQueueNameValid("a ")); assertFalse(queueManager.isQueueNameValid(" a ")); + assertFalse(queueManager.isQueueNameValid("\u00a0")); + assertFalse(queueManager.isQueueNameValid("a\u00a0")); + assertFalse(queueManager.isQueueNameValid("\u00a0a\u00a0")); assertTrue(queueManager.isQueueNameValid("a b")); assertTrue(queueManager.isQueueNameValid("a")); }