diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DiagnosticsCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DiagnosticsCollector.java new file mode 100644 index 00000000000..ab7264ec053 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DiagnosticsCollector.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +/** + * Generic interface that can be used for collecting diagnostics. + */ +public interface DiagnosticsCollector { + + String getDiagnostics(); + + String getDetails(); + + void clear(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index 61b1a87e5a4..11fe34fb9de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -22,6 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +import java.util.Optional; @Private @Unstable @@ -38,8 +41,21 @@ public int compare(Resource unused, Resource lhs, Resource rhs, @Override public long computeAvailableContainers(Resource available, Resource required) { + return computeAvailableContainers(available, required, Optional.empty()); + } + + @Override + public long computeAvailableContainers(Resource available, Resource required, + Optional diagnosticsCollector) { // Only consider memory - return available.getMemorySize() / required.getMemorySize(); + long result = available.getMemorySize() / required.getMemorySize(); + if (diagnosticsCollector.isPresent() && result == 0) { + ResourceDiagnosticsCollector dc = diagnosticsCollector.get(); + dc.collect(ResourceInformation.MEMORY_URI); + dc.addDetail("available", available); + dc.addDetail("required", required); + } + return result; } @Override @@ -132,7 +148,20 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, @Override public boolean fitsIn(Resource smaller, Resource bigger) { - return smaller.getMemorySize() <= bigger.getMemorySize(); + return fitsIn(smaller, bigger, Optional.empty()); + } + + @Override + public boolean fitsIn(Resource smaller, Resource bigger, + Optional diagnosticsCollector) { + boolean result = smaller.getMemorySize() <= bigger.getMemorySize(); + if (diagnosticsCollector.isPresent() && !result) { + ResourceDiagnosticsCollector dc = diagnosticsCollector.get(); + dc.collect(ResourceInformation.MEMORY_URI); + dc.addDetail("fitsIn_smaller", smaller); + dc.addDetail("fitsIn_bigger", bigger); + } + return result; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 16176ef9303..59f9cf7b846 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -28,6 +28,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.Arrays; +import java.util.Optional; /** * A {@link ResourceCalculator} which uses the concept of @@ -330,6 +331,12 @@ private double compareShares(double[] lhsShares, double[] rhsShares) { @Override public long computeAvailableContainers(Resource available, Resource required) { + return computeAvailableContainers(available, required, Optional.empty()); + } + + @Override + public long computeAvailableContainers(Resource available, Resource required, + Optional diagnosticsCollector) { long min = Long.MAX_VALUE; int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { @@ -338,6 +345,14 @@ public long computeAvailableContainers(Resource available, ResourceInformation requiredResource = required.getResourceInformation(i); if (requiredResource.getValue() != 0) { long tmp = availableResource.getValue() / requiredResource.getValue(); + if (diagnosticsCollector.isPresent() && tmp == 0) { + ResourceDiagnosticsCollector dc = diagnosticsCollector.get(); + dc.collect(availableResource.getName()); + if (!dc.hasDetails()) { + dc.addDetail("available", available); + dc.addDetail("required", required); + } + } min = min < tmp ? min : tmp; } } @@ -529,6 +544,12 @@ private Resource multiplyAndNormalize(Resource r, double by, @Override public boolean fitsIn(Resource smaller, Resource bigger) { + return fitsIn(smaller, bigger, Optional.empty()); + } + + @Override + public boolean fitsIn(Resource smaller, Resource bigger, + Optional diagnosticsCollector) { int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation sResourceInformation = smaller @@ -536,6 +557,12 @@ public boolean fitsIn(Resource smaller, Resource bigger) { ResourceInformation bResourceInformation = bigger .getResourceInformation(i); if (sResourceInformation.getValue() > bResourceInformation.getValue()) { + if (diagnosticsCollector.isPresent()) { + ResourceDiagnosticsCollector dc = diagnosticsCollector.get(); + dc.collect(sResourceInformation.getName()); + dc.addDetail("fitsIn_smaller", smaller); + dc.addDetail("fitsIn_bigger", bigger); + } return false; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 09d5ec18a81..d3f2bf7c1b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import java.util.Optional; + /** * A set of {@link Resource} comparison and manipulation interfaces. */ @@ -132,6 +134,10 @@ public static int roundDown(int a, int b) { public abstract long computeAvailableContainers( Resource available, Resource required); + public abstract long computeAvailableContainers( + Resource available, Resource required, + Optional diagnosticsCollector); + /** * Multiply resource r by factor by * and normalize up using step-factor stepFactor. @@ -255,6 +261,9 @@ public abstract float divide( */ public abstract boolean fitsIn(Resource smaller, Resource bigger); + public abstract boolean fitsIn(Resource smaller, Resource bigger, + Optional diagnosticsCollector); + /** * Check if resource has any major resource types (which are all NodeManagers * included) a zero value or negative value. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceDiagnosticsCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceDiagnosticsCollector.java new file mode 100644 index 00000000000..33eba627cc6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceDiagnosticsCollector.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.resource; + +import org.apache.hadoop.yarn.util.DiagnosticsCollector; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** + * Provides a implementation to collect resource diagnostics. + */ +public class ResourceDiagnosticsCollector implements DiagnosticsCollector { + + private Set insufficientResources = new LinkedHashSet<>(); + + private Map details = new LinkedHashMap<>(); + + public void collect(String insufficientResource) { + insufficientResources.add(insufficientResource); + } + + public void addDetail(String key, Object value) { + details.put(key, value); + } + + public boolean hasDetails() { + return details.size() > 0; + } + + @Override + public String getDiagnostics() { + if (insufficientResources.isEmpty()) { + return null; + } + return "insufficient resources=" + insufficientResources; + } + + @Override + public String getDetails() { + if (details.isEmpty()) { + return null; + } + return details.toString(); + } + + @Override + public void clear() { + insufficientResources.clear(); + details.clear(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 9b96fd72b9a..8baa7b19f2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import java.util.Optional; + /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -510,6 +512,12 @@ public static boolean fitsIn(ResourceCalculator rc, Resource smaller, Resource bigger) { return rc.fitsIn(smaller, bigger); } + + public static boolean fitsIn(ResourceCalculator rc, + Resource smaller, Resource bigger, + Optional diagnosticsCollectorOpt) { + return rc.fitsIn(smaller, bigger, diagnosticsCollectorOpt); + } public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java index bf04557ebbe..5b0f2be2dfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java @@ -20,9 +20,11 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Optional; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Assert; import org.junit.Before; @@ -406,4 +408,131 @@ public void testDivisionByZeroRatioNumeratorAndDenominatorIsZero() { 0)); assertEquals(0.0, ratio, 0.00001); } + + @Test + public void testComputeAvailableContainersDiagnosticsCollector() { + ResourceDiagnosticsCollector diagnosticsCollector = + new ResourceDiagnosticsCollector(); + if (resourceCalculator instanceof DefaultResourceCalculator) { + Assert.assertEquals(0, resourceCalculator + .computeAvailableContainers(newResource(0, 0), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertEquals(0, resourceCalculator + .computeAvailableContainers(newResource(0, 1), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertEquals(1, resourceCalculator + .computeAvailableContainers(newResource(1, 0), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + + diagnosticsCollector.clear(); + Assert.assertEquals(1, resourceCalculator + .computeAvailableContainers(newResource(1, 1), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + Assert.assertEquals(0, resourceCalculator + .computeAvailableContainers(newResource(0, 0), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + ", " + + ResourceInformation.VCORES_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertEquals(0, resourceCalculator + .computeAvailableContainers(newResource(0, 1), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + System.out.println(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertEquals(0, resourceCalculator + .computeAvailableContainers(newResource(1, 0), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.VCORES_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertEquals(1, resourceCalculator + .computeAvailableContainers(newResource(1, 1), newResource(1, 1))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + } + } + + @Test + public void testFitsInDiagnosticsCollector() { + ResourceDiagnosticsCollector diagnosticsCollector = + new ResourceDiagnosticsCollector(); + if (resourceCalculator instanceof DefaultResourceCalculator) { + Assert.assertFalse(resourceCalculator + .fitsIn(newResource(1, 0), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertFalse(resourceCalculator + .fitsIn(newResource(1, 1), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertTrue(resourceCalculator + .fitsIn(newResource(0, 1), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + + diagnosticsCollector.clear(); + Assert.assertTrue(resourceCalculator + .fitsIn(newResource(1, 1), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + Assert.assertFalse(resourceCalculator + .fitsIn(newResource(1, 0), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertFalse(resourceCalculator + .fitsIn(newResource(1, 1), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.MEMORY_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertFalse(resourceCalculator + .fitsIn(newResource(0, 1), newResource(0, 0), + Optional.of(diagnosticsCollector))); + Assert.assertNotNull(diagnosticsCollector.getDiagnostics()); + Assert.assertTrue(diagnosticsCollector.getDiagnostics().contains( + "insufficient resources: [" + ResourceInformation.VCORES_URI + "]")); + + diagnosticsCollector.clear(); + Assert.assertTrue(resourceCalculator + .fitsIn(newResource(1, 1), newResource(1, 1), + Optional.of(diagnosticsCollector))); + Assert.assertNull(diagnosticsCollector.getDiagnostics()); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 943f311c882..51a83571aa9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ApplicationSchedulingConfig; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PendingAskUpdateResult; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SingleConstraintAppPlacementAllocator; @@ -765,16 +767,20 @@ public boolean canDelayTo( * @param schedulerKey schedulerKey * @param schedulerNode schedulerNode * @param schedulingMode schedulingMode + * @param pcDiagnosticsCollector + * optional diagnostics collector for placement constraint * @return can use the node or not. */ public boolean precheckNode(SchedulerRequestKey schedulerKey, - SchedulerNode schedulerNode, SchedulingMode schedulingMode) { + SchedulerNode schedulerNode, SchedulingMode schedulingMode, + Optional + pcDiagnosticsCollector) { this.readLock.lock(); try { AppPlacementAllocator ap = schedulerKeyToAppPlacementAllocator.get(schedulerKey); return (ap != null) && ap.precheckNode(schedulerNode, - schedulingMode); + schedulingMode, pcDiagnosticsCollector); } finally { this.readLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java index 780110968a2..58b6c613c96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; /** * Utility for logging scheduler activities @@ -49,10 +50,11 @@ */ public static void recordSkippedAppActivityWithoutAllocation( ActivitiesManager activitiesManager, SchedulerNode node, - SchedulerApplicationAttempt application, Priority priority, + SchedulerApplicationAttempt application, + SchedulerRequestKey requestKey, String diagnostic) { recordAppActivityWithoutAllocation(activitiesManager, node, application, - priority, diagnostic, ActivityState.SKIPPED); + requestKey, diagnostic, ActivityState.SKIPPED); } /* @@ -83,25 +85,39 @@ public static void recordRejectedAppActivityFromLeafQueue( */ public static void recordAppActivityWithoutAllocation( ActivitiesManager activitiesManager, SchedulerNode node, - SchedulerApplicationAttempt application, Priority priority, + SchedulerApplicationAttempt application, + SchedulerRequestKey schedulerKey, String diagnostic, ActivityState appState) { if (activitiesManager == null) { return; } NodeId nodeId = getRecordingNodeId(activitiesManager, node); if (activitiesManager.shouldRecordThisNode(nodeId)) { - String type = "container"; - // Add application-container activity into specific node allocation. - activitiesManager.addSchedulingActivityForNode(nodeId, - application.getApplicationId().toString(), null, - priority.toString(), ActivityState.SKIPPED, diagnostic, type); - type = "app"; + if (schedulerKey != null) { + String allocationRequestId = + String.valueOf(schedulerKey.getAllocationRequestId()); + String priorityStr = getPriorityStr(schedulerKey); + String requestName = getRequestName(priorityStr, allocationRequestId); + String type = "container"; + // Add application-container activity into specific node allocation. + activitiesManager.addSchedulingActivityForNode(nodeId, + requestName, null, + priorityStr, ActivityState.SKIPPED, diagnostic, type, + null); + type = "request"; + // Add application-container activity into specific node allocation. + activitiesManager.addSchedulingActivityForNode(nodeId, + application.getApplicationId().toString(), requestName, + priorityStr, ActivityState.SKIPPED, + ActivityDiagnosticConstant.EMPTY, type, allocationRequestId); + } // Add queue-application activity into specific node allocation. activitiesManager.addSchedulingActivityForNode(nodeId, application.getQueueName(), application.getApplicationId().toString(), application.getPriority().toString(), ActivityState.SKIPPED, - ActivityDiagnosticConstant.EMPTY, type); + schedulerKey != null ? ActivityDiagnosticConstant.EMPTY : + diagnostic, "app", null); } // Add application-container activity into specific application allocation // Under this condition, it fails to allocate a container to this @@ -110,8 +126,11 @@ public static void recordAppActivityWithoutAllocation( application.getApplicationId())) { String type = "container"; activitiesManager.addSchedulingActivityForApp( - application.getApplicationId(), null, priority.toString(), appState, - diagnostic, type); + application.getApplicationId(), null, + getPriorityStr(schedulerKey), appState, + diagnostic, type, nodeId, + schedulerKey == null ? + null : String.valueOf(schedulerKey.getAllocationRequestId())); } } @@ -127,21 +146,38 @@ public static void recordAppActivityWithAllocation( return; } NodeId nodeId = getRecordingNodeId(activitiesManager, node); + if (nodeId == null || nodeId == ActivitiesManager.EMPTY_NODE_ID) { + nodeId = updatedContainer.getNodeId(); + } if (activitiesManager.shouldRecordThisNode(nodeId)) { + String containerPriorityStr = + updatedContainer.getContainer().getPriority().toString(); + String allocationRequestId = String + .valueOf(updatedContainer.getContainer().getAllocationRequestId()); + String requestName = + getRequestName(containerPriorityStr, allocationRequestId); String type = "container"; + // Add application-container activity into specific node allocation. activitiesManager.addSchedulingActivityForNode(nodeId, - application.getApplicationId().toString(), + requestName, updatedContainer.getContainer().toString(), - updatedContainer.getContainer().getPriority().toString(), - activityState, ActivityDiagnosticConstant.EMPTY, type); + containerPriorityStr, + activityState, ActivityDiagnosticConstant.EMPTY, type, null); + type = "request"; + // Add application-container activity into specific node allocation. + activitiesManager.addSchedulingActivityForNode(nodeId, + application.getApplicationId().toString(), + requestName, containerPriorityStr, + activityState, ActivityDiagnosticConstant.EMPTY, type, + allocationRequestId); type = "app"; // Add queue-application activity into specific node allocation. activitiesManager.addSchedulingActivityForNode(nodeId, application.getQueueName(), application.getApplicationId().toString(), application.getPriority().toString(), ActivityState.ACCEPTED, - ActivityDiagnosticConstant.EMPTY, type); + ActivityDiagnosticConstant.EMPTY, type, null); } // Add application-container activity into specific application allocation if (activitiesManager.shouldRecordThisApp( @@ -151,7 +187,9 @@ public static void recordAppActivityWithAllocation( application.getApplicationId(), updatedContainer.getContainerId(), updatedContainer.getContainer().getPriority().toString(), - activityState, ActivityDiagnosticConstant.EMPTY, type); + activityState, ActivityDiagnosticConstant.EMPTY, type, nodeId, + String.valueOf( + updatedContainer.getContainer().getAllocationRequestId())); } } @@ -286,7 +324,7 @@ private static void recordActivity(ActivitiesManager activitiesManager, Priority priority, ActivityState state, String diagnostic, String type) { activitiesManager.addSchedulingActivityForNode(nodeId, parentName, childName, priority != null ? priority.toString() : null, state, - diagnostic, type); + diagnostic, type, null); } private static NodeId getRecordingNodeId(ActivitiesManager activitiesManager, @@ -294,4 +332,17 @@ private static NodeId getRecordingNodeId(ActivitiesManager activitiesManager, return activitiesManager == null ? null : activitiesManager.getRecordingNodeId(node); } + + private static String getRequestName(String priority, + String allocationRequestId) { + return "request_" + + (priority == null ? "" : priority) + + "_" + (allocationRequestId == null ? "" : allocationRequestId); + } + + private static String getPriorityStr(SchedulerRequestKey schedulerKey) { + Priority priority = schedulerKey == null ? + null : schedulerKey.getPriority(); + return priority == null ? null : priority.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java index 176a95dce98..039992e31ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java @@ -30,9 +30,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; +import org.apache.hadoop.yarn.util.DiagnosticsCollector; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.ResourceDiagnosticsCollector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -52,6 +55,7 @@ // An empty node ID, we use this variable as a placeholder // in the activity records when recording multiple nodes assignments. public static final NodeId EMPTY_NODE_ID = NodeId.newInstance("", 0); + public static final String DIAGNOSTIC_DETAILS_SEPARATOR = "\n"; private ThreadLocal>> recordingNodesAllocation; @VisibleForTesting @@ -69,6 +73,7 @@ private int timeThreshold = 600 * 1000; private final RMContext rmContext; private volatile boolean stopped; + private ThreadLocal diagnosticCollectorManager; public ActivitiesManager(RMContext rmContext) { super(ActivitiesManager.class.getName()); @@ -78,6 +83,8 @@ public ActivitiesManager(RMContext rmContext) { completedAppAllocations = new ConcurrentHashMap<>(); activeRecordedNodes = Collections.newSetFromMap(new ConcurrentHashMap<>()); recordingAppActivitiesUntilSpecifiedTime = new ConcurrentHashMap<>(); + diagnosticCollectorManager = + ThreadLocal.withInitial(() -> new DiagnosticsCollectorManager()); this.rmContext = rmContext; } @@ -191,6 +198,8 @@ void startNodeUpdateRecording(NodeId nodeID) { if (activeRecordedNodes.remove(nodeID)) { List nodeAllocation = new ArrayList<>(); recordingNodesAllocation.get().put(nodeID, nodeAllocation); + // enable diagnostic collector + diagnosticCollectorManager.get().enable(); } } @@ -204,6 +213,8 @@ void startAppAllocationRecording(NodeId nodeID, long currTS, appsAllocation.get().put(applicationId, new AppAllocation(application.getPriority(), nodeID, application.getQueueName())); + // enable diagnostic collector + diagnosticCollectorManager.get().enable(); } else { turnOffActivityMonitoringForApp(applicationId); } @@ -213,11 +224,11 @@ void startAppAllocationRecording(NodeId nodeID, long currTS, // Add queue, application or container activity into specific node allocation. void addSchedulingActivityForNode(NodeId nodeId, String parentName, String childName, String priority, ActivityState state, String diagnostic, - String type) { + String type, String allocationRequestId) { if (shouldRecordThisNode(nodeId)) { NodeAllocation nodeAllocation = getCurrentNodeAllocation(nodeId); nodeAllocation.addAllocationActivity(parentName, childName, priority, - state, diagnostic, type); + state, diagnostic, type, nodeId, allocationRequestId); } } @@ -225,12 +236,14 @@ void addSchedulingActivityForNode(NodeId nodeId, String parentName, // allocation. void addSchedulingActivityForApp(ApplicationId applicationId, ContainerId containerId, String priority, ActivityState state, - String diagnostic, String type) { + String diagnostic, String type, NodeId nodeId, + String allocationRequestId) { if (shouldRecordThisApp(applicationId)) { AppAllocation appAllocation = appsAllocation.get().get(applicationId); appAllocation.addAppAllocationActivity(containerId == null ? "Container-Id-Not-Assigned" : - containerId.toString(), priority, state, diagnostic, type); + containerId.toString(), priority, state, diagnostic, type, nodeId, + allocationRequestId); } } @@ -296,6 +309,8 @@ void finishNodeUpdateRecording(NodeId nodeID) { completedNodeAllocations.put(nodeID, value); } } + // disable diagnostic collector + diagnosticCollectorManager.get().disable(); } boolean shouldRecordThisApp(ApplicationId applicationId) { @@ -368,4 +383,75 @@ public NodeId getRecordingNodeId(SchedulerNode node) { } return null; } + + /** + * Class to manage the diagnostics collector. + */ + public static class DiagnosticsCollectorManager { + private boolean enabled = false; + private ResourceDiagnosticsCollector rdc = + new ResourceDiagnosticsCollector(); + private PlacementConstraintDiagnosticsCollector pcdc = + new PlacementConstraintDiagnosticsCollector(); + + public boolean isEnabled() { + return enabled; + } + + public void enable() { + this.enabled = true; + } + + public void disable() { + this.enabled = false; + } + + public Optional + getResourceDiagnosticsCollector() { + if (enabled) { + rdc.clear(); + return Optional.of(rdc); + } + return Optional.empty(); + } + + public Optional + getPCDiagnosticsCollector() { + if (enabled) { + pcdc.clear(); + return Optional.of(pcdc); + } else { + return Optional.empty(); + } + } + } + + public Optional + getResourceDiagnosticsCollector() { + return diagnosticCollectorManager.get().getResourceDiagnosticsCollector(); + } + + public Optional + getPCDiagnosticsCollector() { + return diagnosticCollectorManager.get().getPCDiagnosticsCollector(); + } + + public String getActivityDiagnostics(Optional diagnosticsCollectorOpt) { + if (diagnosticsCollectorOpt != null + && diagnosticsCollectorOpt.isPresent()) { + DiagnosticsCollector diagnosticsCollector = + (DiagnosticsCollector) diagnosticsCollectorOpt.get(); + if (diagnosticsCollector != null + && diagnosticsCollector.getDiagnostics() != null) { + StringBuilder sb = new StringBuilder(); + sb.append(", ").append(diagnosticsCollector.getDiagnostics()); + if (diagnosticsCollector.getDetails() != null) { + sb.append(DIAGNOSTIC_DETAILS_SEPARATOR) + .append(diagnosticsCollector.getDetails()); + } + return sb.toString(); + } + } + return ""; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java index fc4738ee638..d3d4d9b4d33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java @@ -38,9 +38,6 @@ public final static String PRIORITY_SKIPPED = "Priority skipped"; public final static String PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST = "Priority skipped because off-switch request is null"; - public final static String - PRIORITY_SKIPPED_BECAUSE_NODE_PARTITION_DOES_NOT_MATCH_REQUEST = - "Priority skipped because partition of node doesn't match request"; public final static String SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY = "Priority skipped because of relax locality is not allowed"; public final static String SKIP_IN_IGNORE_EXCLUSIVITY_MODE = @@ -74,4 +71,10 @@ "All priorities are skipped of the app"; public final static String RESPECT_FIFO = "To respect FIFO of applications, " + "skipped following applications in the queue"; + public final static String + NODE_DO_NOT_MATCH_PARTITION_OR_PLACEMENT_CONSTRAINTS = + "Node does not match partition or placement constraints"; + public final static String + NODE_CAN_NOT_FIND_CONTAINER_TO_BE_UNRESERVED_WHEN_NEEDED = + "Node can't find a container to be unreserved when needed"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityNode.java index 0cf9604fc99..3edfddd43a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityNode.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities; +import org.apache.hadoop.yarn.api.records.NodeId; + import java.util.LinkedList; import java.util.List; @@ -33,18 +35,32 @@ private String requestPriority; private ActivityState state; private String diagnostic; + private NodeId nodeId; + private String allocationRequestId; private List childNode; public ActivityNode(String activityNodeName, String parentName, String priority, ActivityState state, String diagnostic, String type) { + this(activityNodeName, parentName, priority, state, diagnostic, type, null, + null); + } + + public ActivityNode(String activityNodeName, String parentName, + String priority, ActivityState state, String diagnostic, String type, + NodeId nodeId, String allocationRequestId) { this.activityNodeName = activityNodeName; this.parentName = parentName; if (type != null) { if (type.equals("app")) { this.appPriority = priority; + } else if (type.equals("request")) { + this.requestPriority = priority; + this.allocationRequestId = allocationRequestId; } else if (type.equals("container")) { this.requestPriority = priority; + this.allocationRequestId = allocationRequestId; + this.nodeId = nodeId; } } this.state = state; @@ -84,6 +100,14 @@ public String getRequestPriority() { return requestPriority; } + public NodeId getNodeId() { + return nodeId; + } + + public String getAllocationRequestId() { + return allocationRequestId; + } + public boolean getType() { if (appPriority != null) { return true; @@ -97,6 +121,9 @@ public String toString() { sb.append(this.activityNodeName + " ") .append(this.appPriority + " ") .append(this.state + " "); + if (this.nodeId != null) { + sb.append(this.nodeId + " "); + } if (!this.diagnostic.equals("")) { sb.append(this.diagnostic + "\n"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AllocationActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AllocationActivity.java index ae947b258c5..a71ec6f5613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AllocationActivity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AllocationActivity.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities; +import org.apache.hadoop.yarn.api.records.NodeId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,19 +34,25 @@ private String requestPriority = null; private ActivityState state; private String diagnostic = null; + private NodeId nodeId; + private String allocationRequestId; private static final Logger LOG = LoggerFactory.getLogger(AllocationActivity.class); public AllocationActivity(String parentName, String queueName, - String priority, ActivityState state, String diagnostic, String type) { + String priority, ActivityState state, String diagnostic, String type, + NodeId nodeId, String allocationRequestId) { this.childName = queueName; this.parentName = parentName; if (type != null) { if (type.equals("app")) { this.appPriority = priority; - } else if (type.equals("container")) { + } else if (type.equals("request")) { this.requestPriority = priority; + this.allocationRequestId = allocationRequestId; + } else if (type.equals("container")) { + this.nodeId = nodeId; } } this.state = state; @@ -58,7 +65,12 @@ public ActivityNode createTreeNode() { this.state, this.diagnostic, "app"); } else if (requestPriority != null) { return new ActivityNode(this.childName, this.parentName, - this.requestPriority, this.state, this.diagnostic, "container"); + this.requestPriority, this.state, this.diagnostic, "request", null, + allocationRequestId); + } else if (nodeId != null) { + return new ActivityNode(this.childName, this.parentName, + this.requestPriority, this.state, this.diagnostic, "container", + this.nodeId, null); } else { return new ActivityNode(this.childName, this.parentName, null, this.state, this.diagnostic, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AppAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AppAllocation.java index 1903ae7843a..57cd3e86464 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AppAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/AppAllocation.java @@ -56,9 +56,10 @@ public void updateAppContainerStateAndTime(ContainerId containerId, } public void addAppAllocationActivity(String containerId, String priority, - ActivityState state, String diagnostic, String type) { + ActivityState state, String diagnose, String type, NodeId nId, + String allocationRequestId) { ActivityNode container = new ActivityNode(containerId, null, priority, - state, diagnostic, type); + state, diagnose, type, nId, allocationRequestId); this.allocationAttempts.add(container); if (state == ActivityState.REJECTED) { this.appState = ActivityState.SKIPPED; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/NodeAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/NodeAllocation.java index 7335ed641ba..df1d00c77e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/NodeAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/NodeAllocation.java @@ -55,9 +55,10 @@ public NodeAllocation(NodeId nodeId) { } public void addAllocationActivity(String parentName, String childName, - String priority, ActivityState state, String diagnostic, String type) { + String priority, ActivityState state, String diagnostic, String type, + NodeId nId, String allocationRequestId) { AllocationActivity allocate = new AllocationActivity(parentName, childName, - priority, state, diagnostic, type); + priority, state, diagnostic, type, nId, allocationRequestId); this.allocationOperations.add(allocate); } @@ -134,7 +135,7 @@ public ActivityNode getRoot() { return root; } - public String getNodeId() { - return nodeId.toString(); + public NodeId getNodeId() { + return nodeId; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 340354495b7..004794e80a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1068,8 +1068,8 @@ public CSAssignment assignContainers(Resource clusterResource, && !accessibleToPartition(candidates.getPartition())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, - ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + candidates - .getPartition()); + ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + " " + + candidates.getPartition()); return CSAssignment.NULL_ASSIGNMENT; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index 9d077b9c07a..4e3863d42c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -21,13 +21,13 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceDiagnosticsCollector; import org.apache.hadoop.yarn.util.resource.Resources; /** @@ -100,14 +102,12 @@ private ContainerAllocation preCheckForNodeCandidateSet( Resource clusterResource, FiCaSchedulerNode node, SchedulingMode schedulingMode, ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey) { - Priority priority = schedulerKey.getPriority(); - PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey, ResourceRequest.ANY); if (offswitchPendingAsk.getCount() <= 0) { ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -118,7 +118,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( // Do we need containers at this 'priority'? if (application.getOutstandingAsksCount(schedulerKey) <= 0) { ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -133,7 +133,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( application.updateAppSkipNodeDiagnostics( "Skipping assigning to Node in Ignore Exclusivity mode. "); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_IN_IGNORE_EXCLUSIVITY_MODE); return ContainerAllocation.APP_SKIPPED; } @@ -141,11 +141,18 @@ private ContainerAllocation preCheckForNodeCandidateSet( // Is the nodePartition of pending request matches the node's partition // If not match, jump to next priority. - if (!appInfo.precheckNode(schedulerKey, node, schedulingMode)) { + Optional + pcDiagnosticsCollectorOpt = activitiesManager == null ? + Optional.empty() : + activitiesManager.getPCDiagnosticsCollector(); + if (!appInfo.precheckNode(schedulerKey, node, schedulingMode, + pcDiagnosticsCollectorOpt)) { ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant. - PRIORITY_SKIPPED_BECAUSE_NODE_PARTITION_DOES_NOT_MATCH_REQUEST); + NODE_DO_NOT_MATCH_PARTITION_OR_PLACEMENT_CONSTRAINTS + + activitiesManager + .getActivityDiagnostics(pcDiagnosticsCollectorOpt)); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -153,7 +160,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) { LOG.debug("doesn't need containers based on reservation algo!"); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.DO_NOT_NEED_ALLOCATIONATTEMPTINFOS); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -164,7 +171,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( LOG.debug("cannot allocate required resource={} because of headroom", required); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.QUEUE_SKIPPED_HEADROOM); return ContainerAllocation.QUEUE_SKIPPED; } @@ -179,7 +186,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( // This is possible when #pending resource decreased by a different // thread. ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -209,7 +216,7 @@ private ContainerAllocation preCheckForNodeCandidateSet( + rmContext.getScheduler().getNumClusterNodes()); } ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.NON_PARTITIONED_PARTITION_FIRST); return ContainerAllocation.APP_SKIPPED; } @@ -220,13 +227,11 @@ private ContainerAllocation preCheckForNodeCandidateSet( private ContainerAllocation checkIfNodeBlackListed(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey) { - Priority priority = schedulerKey.getPriority(); - if (SchedulerAppUtils.isPlaceBlacklisted(application, node, LOG)) { application.updateAppSkipNodeDiagnostics( CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_IN_BLACK_LISTED_NODE); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_BLACK_LISTED_NODE); return ContainerAllocation.APP_SKIPPED; } @@ -366,7 +371,7 @@ private ContainerAllocation assignNodeLocalContainers( // Skip node-local request, go to rack-local request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, schedulerKey.getPriority(), + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_NODE_LOCAL_REQUEST); return ContainerAllocation.LOCALITY_SKIPPED; } @@ -384,7 +389,7 @@ private ContainerAllocation assignRackLocalContainers( // Skip rack-local request, go to off-switch request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, schedulerKey.getPriority(), + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_RACK_LOCAL_REQUEST); return ContainerAllocation.LOCALITY_SKIPPED; } @@ -403,7 +408,7 @@ private ContainerAllocation assignOffSwitchContainers( application.updateAppSkipNodeDiagnostics( CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_DUE_TO_LOCALITY); ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, schedulerKey.getPriority(), + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_OFF_SWITCH_REQUEST); return ContainerAllocation.APP_SKIPPED; } @@ -412,8 +417,6 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer reservedContainer, SchedulingMode schedulingMode, ResourceLimits currentResoureLimits) { - Priority priority = schedulerKey.getPriority(); - ContainerAllocation allocation; NodeType requestLocalityType = null; @@ -439,7 +442,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource, if (rackLocalAsk.getCount() > 0) { if (!appInfo.canDelayTo(schedulerKey, node.getRackName())) { ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -465,7 +468,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource, if (offSwitchAsk.getCount() > 0) { if (!appInfo.canDelayTo(schedulerKey, ResourceRequest.ANY)) { ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -489,7 +492,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource, return allocation; } ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, + activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.PRIORITY_SKIPPED); return ContainerAllocation.PRIORITY_SKIPPED; } @@ -498,7 +501,6 @@ private ContainerAllocation assignContainer(Resource clusterResource, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, PendingAsk pendingAsk, NodeType type, RMContainer rmContainer, SchedulingMode schedulingMode, ResourceLimits currentResoureLimits) { - Priority priority = schedulerKey.getPriority(); if (LOG.isDebugEnabled()) { LOG.debug("assignContainers: node=" + node.getNodeName() @@ -511,15 +513,18 @@ private ContainerAllocation assignContainer(Resource clusterResource, Resource available = node.getUnallocatedResource(); Resource totalResource = node.getTotalResource(); - if (!Resources.lessThanOrEqual(rc, clusterResource, - capability, totalResource)) { + Optional dcOpt = activitiesManager == null ? + Optional.empty() : + activitiesManager.getResourceDiagnosticsCollector(); + if (!Resources.fitsIn(rc, capability, totalResource, dcOpt)) { LOG.warn("Node : " + node.getNodeID() + " does not have sufficient resource for ask : " + pendingAsk + " node total capability : " + node.getTotalResource()); // Skip this locality request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, - ActivityDiagnosticConstant.NOT_SUFFICIENT_RESOURCE); + activitiesManager, node, application, schedulerKey, + ActivityDiagnosticConstant.NOT_SUFFICIENT_RESOURCE + + activitiesManager.getActivityDiagnostics(dcOpt)); return ContainerAllocation.LOCALITY_SKIPPED; } @@ -528,7 +533,7 @@ private ContainerAllocation assignContainer(Resource clusterResource, // Can we allocate a container on this node? long availableContainers = - rc.computeAvailableContainers(available, capability); + rc.computeAvailableContainers(available, capability, dcOpt); // How much need to unreserve equals to: // max(required - headroom, amountNeedUnreserve) @@ -556,7 +561,10 @@ private ContainerAllocation assignContainer(Resource clusterResource, toKillContainers.add(killableContainer); Resources.addTo(availableAndKillable, killableContainer.getAllocatedResource()); - if (Resources.fitsIn(rc, capability, availableAndKillable)) { + if (dcOpt.isPresent()) { + dcOpt.get().clear(); + } + if (Resources.fitsIn(rc, capability, availableAndKillable, dcOpt)) { // Stop if we find enough spaces availableContainers = 1; break; @@ -594,8 +602,9 @@ private ContainerAllocation assignContainer(Resource clusterResource, if (null == unreservedContainer) { // Skip the locality request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, - ActivityDiagnosticConstant.LOCALITY_SKIPPED); + activitiesManager, node, application, schedulerKey, + ActivityDiagnosticConstant. + NODE_CAN_NOT_FIND_CONTAINER_TO_BE_UNRESERVED_WHEN_NEEDED); return ContainerAllocation.LOCALITY_SKIPPED; } } @@ -607,6 +616,8 @@ private ContainerAllocation assignContainer(Resource clusterResource, result.setToKillContainers(toKillContainers); return result; } else { + String diagnostics = dcOpt.isPresent() ? + activitiesManager.getActivityDiagnostics(dcOpt) : ""; // if we are allowed to allocate but this node doesn't have space, reserve // it or if this was an already a reserved container, reserve it again if (shouldAllocOrReserveNewContainer || rmContainer != null) { @@ -619,8 +630,9 @@ private ContainerAllocation assignContainer(Resource clusterResource, // Skip the locality request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, - ActivityDiagnosticConstant.LOCALITY_SKIPPED); + activitiesManager, node, application, schedulerKey, + ActivityDiagnosticConstant.NOT_SUFFICIENT_RESOURCE + + diagnostics); return ContainerAllocation.LOCALITY_SKIPPED; } } @@ -633,8 +645,8 @@ private ContainerAllocation assignContainer(Resource clusterResource, } // Skip the locality request ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, priority, - ActivityDiagnosticConstant.LOCALITY_SKIPPED); + activitiesManager, node, application, schedulerKey, + ActivityDiagnosticConstant.NOT_SUFFICIENT_RESOURCE + diagnostics); return ContainerAllocation.LOCALITY_SKIPPED; } } @@ -708,7 +720,7 @@ private ContainerAllocation handleNewContainerAllocation( new ContainerAllocation(allocationResult.containerToBeUnreserved, null, AllocationState.APP_SKIPPED); ActivitiesLogger.APP.recordAppActivityWithoutAllocation(activitiesManager, - node, application, schedulerKey.getPriority(), + node, application, schedulerKey, ActivityDiagnosticConstant.FAIL_TO_ALLOCATE, ActivityState.REJECTED); return ret; } @@ -730,7 +742,7 @@ ContainerAllocation doAllocation(ContainerAllocation allocationResult, .updateAppSkipNodeDiagnostics("Scheduling of container failed. "); LOG.warn("Couldn't get container for allocation!"); ActivitiesLogger.APP.recordAppActivityWithoutAllocation(activitiesManager, - node, application, schedulerKey.getPriority(), + node, application, schedulerKey, ActivityDiagnosticConstant.COULD_NOT_GET_CONTAINER, ActivityState.REJECTED); return ContainerAllocation.APP_SKIPPED; @@ -753,7 +765,7 @@ ContainerAllocation doAllocation(ContainerAllocation allocationResult, + " schedulerRequestKey=" + schedulerKey); ActivitiesLogger.APP .recordAppActivityWithoutAllocation(activitiesManager, node, - application, schedulerKey.getPriority(), + application, schedulerKey, ActivityDiagnosticConstant. PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST, ActivityState.REJECTED); @@ -815,6 +827,10 @@ private ContainerAllocation allocate(Resource clusterResource, // This could be null when #pending request decreased by another thread. if (schedulingPS == null) { + ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( + activitiesManager, null, application, schedulerKey, + ActivityDiagnosticConstant. + APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE); return new ContainerAllocation(reservedContainer, null, AllocationState.QUEUE_SKIPPED); } @@ -873,7 +889,7 @@ public CSAssignment assignContainers(Resource clusterResource, .getPartition()); } ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, application.getPriority(), + activitiesManager, node, application, null, ActivityDiagnosticConstant.APPLICATION_DO_NOT_NEED_RESOURCE); return CSAssignment.SKIP_ASSIGNMENT; } @@ -893,9 +909,6 @@ public CSAssignment assignContainers(Resource clusterResource, // We will reach here if we skipped all priorities of the app, so we will // skip the app. - ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( - activitiesManager, node, application, application.getPriority(), - ActivityDiagnosticConstant.SKIPPED_ALL_PRIORITIES); return CSAssignment.SKIP_ASSIGNMENT; } else { ContainerAllocation result = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintDiagnosticsCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintDiagnosticsCollector.java new file mode 100644 index 00000000000..f1e92201fb7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintDiagnosticsCollector.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; +import org.apache.hadoop.yarn.util.DiagnosticsCollector; + +/** + * Provides a implementation to collect placement constraint diagnostics. + */ +public class PlacementConstraintDiagnosticsCollector implements + DiagnosticsCollector { + + private PlacementConstraint pc; + private NodeId nodeId; + private TargetType targetType; + + public void collect(PlacementConstraint placementConstraint, NodeId nId, + TargetType type) { + if (placementConstraint != null && nId != null && type != null) { + this.pc = placementConstraint; + this.nodeId = nId; + this.targetType = type; + } + } + + public String getDiagnostics() { + if (pc == null || targetType == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + sb.append("unsatisfied PC expression=\"").append(pc).append("\",") + .append(" target-type=").append(targetType); + return sb.toString(); + } + + @Override + public String getDetails() { + if (nodeId == null) { + return null; + } + return new StringBuilder().append("nodeId=").append(nodeId).toString(); + } + + @Override + public void clear() { + this.pc = null; + this.nodeId = null; + this.targetType = null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index d04cf9c2538..f3883fa6cc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import java.util.Iterator; +import java.util.Optional; import java.util.Set; import org.slf4j.Logger; @@ -213,7 +214,8 @@ private static boolean isOpCodeMatches(NodeAttribute requestAttribute, private static boolean canSatisfySingleConstraint(ApplicationId applicationId, SingleConstraint singleConstraint, SchedulerNode schedulerNode, - AllocationTagsManager tagsManager) + AllocationTagsManager tagsManager, + Optional pcDiagnosticsCollector) throws InvalidAllocationTagsQueryException { // Iterate through TargetExpressions Iterator expIt = @@ -225,12 +227,22 @@ private static boolean canSatisfySingleConstraint(ApplicationId applicationId, // Check if conditions are met if (!canSatisfySingleConstraintExpression(applicationId, singleConstraint, currentExp, schedulerNode, tagsManager)) { + if (pcDiagnosticsCollector.isPresent()) { + pcDiagnosticsCollector.get() + .collect(singleConstraint.build(), schedulerNode.getNodeID(), + TargetType.ALLOCATION_TAG); + } return false; } } else if (currentExp.getTargetType().equals(TargetType.NODE_ATTRIBUTE)) { // This is a node attribute expression, check it. if (!canSatisfyNodeConstraintExpression(singleConstraint, currentExp, schedulerNode)) { + if (pcDiagnosticsCollector.isPresent()) { + pcDiagnosticsCollector.get() + .collect(singleConstraint.build(), schedulerNode.getNodeID(), + TargetType.NODE_ATTRIBUTE); + } return false; } } @@ -249,12 +261,14 @@ private static boolean canSatisfySingleConstraint(ApplicationId applicationId, * @throws InvalidAllocationTagsQueryException */ private static boolean canSatisfyAndConstraint(ApplicationId appId, - And constraint, SchedulerNode node, AllocationTagsManager atm) + And constraint, SchedulerNode node, AllocationTagsManager atm, + Optional pcDiagnosticsCollector) throws InvalidAllocationTagsQueryException { // Iterate over the constraints tree, if found any child constraint // isn't satisfied, return false. for (AbstractConstraint child : constraint.getChildren()) { - if(!canSatisfyConstraints(appId, child.build(), node, atm)) { + if(!canSatisfyConstraints(appId, child.build(), node, atm, + pcDiagnosticsCollector)) { return false; } } @@ -271,10 +285,12 @@ private static boolean canSatisfyAndConstraint(ApplicationId appId, * @throws InvalidAllocationTagsQueryException */ private static boolean canSatisfyOrConstraint(ApplicationId appId, - Or constraint, SchedulerNode node, AllocationTagsManager atm) + Or constraint, SchedulerNode node, AllocationTagsManager atm, + Optional pcDiagnosticsCollector) throws InvalidAllocationTagsQueryException { for (AbstractConstraint child : constraint.getChildren()) { - if (canSatisfyConstraints(appId, child.build(), node, atm)) { + if (canSatisfyConstraints(appId, child.build(), node, atm, + pcDiagnosticsCollector)) { return true; } } @@ -283,7 +299,8 @@ private static boolean canSatisfyOrConstraint(ApplicationId appId, private static boolean canSatisfyConstraints(ApplicationId appId, PlacementConstraint constraint, SchedulerNode node, - AllocationTagsManager atm) + AllocationTagsManager atm, + Optional pcDiagnosticsCollector) throws InvalidAllocationTagsQueryException { if (constraint == null) { LOG.debug("Constraint is found empty during constraint validation for" @@ -300,13 +317,16 @@ private static boolean canSatisfyConstraints(ApplicationId appId, // TODO handle other type of constraints, e.g CompositeConstraint if (sConstraintExpr instanceof SingleConstraint) { SingleConstraint single = (SingleConstraint) sConstraintExpr; - return canSatisfySingleConstraint(appId, single, node, atm); + return canSatisfySingleConstraint(appId, single, node, atm, + pcDiagnosticsCollector); } else if (sConstraintExpr instanceof And) { And and = (And) sConstraintExpr; - return canSatisfyAndConstraint(appId, and, node, atm); + return canSatisfyAndConstraint(appId, and, node, atm, + pcDiagnosticsCollector); } else if (sConstraintExpr instanceof Or) { Or or = (Or) sConstraintExpr; - return canSatisfyOrConstraint(appId, or, node, atm); + return canSatisfyOrConstraint(appId, or, node, atm, + pcDiagnosticsCollector); } else { throw new InvalidAllocationTagsQueryException( "Unsupported type of constraint: " @@ -331,12 +351,15 @@ private static boolean canSatisfyConstraints(ApplicationId appId, * @param schedulerNode node * @param pcm placement constraint manager * @param atm allocation tags manager + * @param pcDiagnosticsCollector + * optional diagnostics collector for placement constraint * @return true if the given node satisfies the constraint of the request * @throws InvalidAllocationTagsQueryException */ public static boolean canSatisfyConstraints(ApplicationId applicationId, SchedulingRequest request, SchedulerNode schedulerNode, - PlacementConstraintManager pcm, AllocationTagsManager atm) + PlacementConstraintManager pcm, AllocationTagsManager atm, + Optional pcDiagnosticsCollector) throws InvalidAllocationTagsQueryException { Set sourceTags = null; PlacementConstraint pc = null; @@ -346,7 +369,15 @@ public static boolean canSatisfyConstraints(ApplicationId applicationId, } return canSatisfyConstraints(applicationId, pcm.getMultilevelConstraint(applicationId, sourceTags, pc), - schedulerNode, atm); + schedulerNode, atm, pcDiagnosticsCollector); + } + + public static boolean canSatisfyConstraints(ApplicationId applicationId, + SchedulingRequest request, SchedulerNode schedulerNode, + PlacementConstraintManager pcm, AllocationTagsManager atm) + throws InvalidAllocationTagsQueryException { + return canSatisfyConstraints(applicationId, request, schedulerNode, pcm, + atm, Optional.empty()); } private static NodeAttribute getNodeConstraintFromRequest(String attrKey, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index 088b3dd9c70..b9fe901b422 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -27,11 +27,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import java.util.Collection; import java.util.Iterator; import java.util.Map; +import java.util.Optional; /** *

@@ -150,8 +152,14 @@ public abstract ContainerRequest allocate(SchedulerRequestKey schedulerKey, * * @param schedulerNode schedulerNode * @param schedulingMode schedulingMode + * @param pcDiagnosticsCollector + * optional diagnostics collector for placement constraint * @return accepted/not */ + public abstract boolean precheckNode(SchedulerNode schedulerNode, + SchedulingMode schedulingMode, + Optional pcDiagnosticsCollector); + public abstract boolean precheckNode(SchedulerNode schedulerNode, SchedulingMode schedulingMode); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index 5c9ce505508..783b553db2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ApplicationSchedulingConfig; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import java.util.ArrayList; @@ -41,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -391,6 +393,15 @@ public boolean canDelayTo(String resourceName) { } + + @Override + public boolean precheckNode(SchedulerNode schedulerNode, + SchedulingMode schedulingMode, + Optional + pcDiagnosticsCollector) { + return precheckNode(schedulerNode, schedulingMode); + } + @Override public boolean precheckNode(SchedulerNode schedulerNode, SchedulingMode schedulingMode) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index f4da4d04c38..bf19bb609ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintDiagnosticsCollector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -344,7 +346,9 @@ public ContainerRequest allocate(SchedulerRequestKey schedulerKey, } } - private boolean checkCardinalityAndPending(SchedulerNode node) { + private boolean checkCardinalityAndPending(SchedulerNode node, + Optional + pcDiagnosticsCollector) { // Do we still have pending resource? if (schedulingRequest.getResourceSizing().getNumAllocations() <= 0) { return false; @@ -354,7 +358,8 @@ private boolean checkCardinalityAndPending(SchedulerNode node) { try { return PlacementConstraintsUtil.canSatisfyConstraints( appSchedulingInfo.getApplicationId(), schedulingRequest, node, - placementConstraintManager, allocationTagsManager); + placementConstraintManager, allocationTagsManager, + pcDiagnosticsCollector); } catch (InvalidAllocationTagsQueryException e) { LOG.warn("Failed to query node cardinality:", e); return false; @@ -365,7 +370,7 @@ private boolean checkCardinalityAndPending(SchedulerNode node) { public boolean canAllocate(NodeType type, SchedulerNode node) { readLock.lock(); try { - return checkCardinalityAndPending(node); + return checkCardinalityAndPending(node, Optional.empty()); } finally { readLock.unlock(); } @@ -379,6 +384,14 @@ public boolean canDelayTo(String resourceName) { @Override public boolean precheckNode(SchedulerNode schedulerNode, SchedulingMode schedulingMode) { + return precheckNode(schedulerNode, schedulingMode, Optional.empty()); + } + + @Override + public boolean precheckNode(SchedulerNode schedulerNode, + SchedulingMode schedulingMode, + Optional + pcDiagnosticsCollector) { // We will only look at node label = nodeLabelToLookAt according to // schedulingMode and partition of node. String nodePartitionToLookAt; @@ -392,7 +405,7 @@ public boolean precheckNode(SchedulerNode schedulerNode, try { // Check node partition as well as cardinality/pending resources. return this.targetNodePartition.equals(nodePartitionToLookAt) - && checkCardinalityAndPending(schedulerNode); + && checkCardinalityAndPending(schedulerNode, pcDiagnosticsCollector); } finally { readLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivitiesInfo.java index e64fee4f650..a78247f71f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivitiesInfo.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.NodeAllocation; import javax.xml.bind.annotation.XmlAccessType; @@ -63,7 +65,11 @@ public ActivitiesInfo(List nodeAllocations, String nodeId) { if (nodeAllocations.size() == 0) { diagnostic = "do not have available resources"; } else { - this.nodeId = nodeAllocations.get(0).getNodeId(); + NodeId rootNodeId = nodeAllocations.get(0).getNodeId(); + if (rootNodeId != null && !Strings + .isNullOrEmpty(rootNodeId.getHost())) { + this.nodeId = nodeAllocations.get(0).getNodeId().toString(); + } Date date = new Date(); date.setTime(nodeAllocations.get(0).getTimeStamp()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivityNodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivityNodeInfo.java index 9553a7204ad..e660fa57276 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivityNodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ActivityNodeInfo.java @@ -18,7 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import com.google.common.base.Strings; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -38,17 +41,30 @@ protected String requestPriority; protected String allocationState; protected String diagnostic; + private String nodeId; + private String allocationRequestId; protected List children; ActivityNodeInfo() { } + public ActivityNodeInfo(String name, ActivityState allocationState, + String diagnostic, NodeId nId) { + this.name = name; + this.allocationState = allocationState.name(); + this.diagnostic = diagnostic; + setNodeId(nId); + } + ActivityNodeInfo(ActivityNode node) { this.name = node.getName(); - getPriority(node); + setPriority(node); + setNodeId(node.getNodeId()); this.allocationState = node.getState().name(); this.diagnostic = node.getDiagnostic(); + this.requestPriority = node.getRequestPriority(); + this.allocationRequestId = node.getAllocationRequestId(); this.children = new ArrayList<>(); for (ActivityNode child : node.getChildren()) { @@ -57,11 +73,25 @@ } } - private void getPriority(ActivityNode node) { + public void setNodeId(NodeId nId) { + if (nId != null && !Strings.isNullOrEmpty(nId.getHost())) { + this.nodeId = nId.toString(); + } + } + + private void setPriority(ActivityNode node) { if (node.getType()) { this.appPriority = node.getAppPriority(); } else { this.requestPriority = node.getRequestPriority(); } } + + public String getNodeId() { + return nodeId; + } + + public String getAllocationRequestId() { + return allocationRequestId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAllocationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAllocationInfo.java index 2a25ec4ff2a..48a209427d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAllocationInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAllocationInfo.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AppAllocation; @@ -29,6 +27,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /* * DAO object to display application allocation detailed information. @@ -36,38 +36,62 @@ @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) public class AppAllocationInfo { - protected String nodeId; - protected String queueName; - protected String appPriority; - protected String allocatedContainerId; - protected String allocationState; - protected String diagnostic; - protected String timeStamp; - protected List allocationAttempt; - - private static final Logger LOG = - LoggerFactory.getLogger(AppAllocationInfo.class); + private String nodeId; + private String queueName; + private String appPriority; + private long timestamp; + private String dateTime; + private String allocationState; + private List requestAllocation; AppAllocationInfo() { } AppAllocationInfo(AppAllocation allocation) { - this.allocationAttempt = new ArrayList<>(); - + this.requestAllocation = new ArrayList<>(); this.nodeId = allocation.getNodeId(); this.queueName = allocation.getQueueName(); this.appPriority = allocation.getPriority(); - this.allocatedContainerId = allocation.getContainerId(); + this.timestamp = allocation.getTime(); + this.dateTime = new Date(allocation.getTime()).toString(); this.allocationState = allocation.getAppState().name(); - this.diagnostic = allocation.getDiagnostic(); + Map> requestToActivityNodes = + allocation.getAllocationAttempts().stream().collect(Collectors + .groupingBy((e) -> e.getRequestPriority() + "_" + e + .getAllocationRequestId(), Collectors.toList())); + for (List requestActivityNodes : requestToActivityNodes + .values()) { + AppRequestAllocationInfo requestAllocationInfo = + new AppRequestAllocationInfo(requestActivityNodes); + this.requestAllocation.add(requestAllocationInfo); + } + } - Date date = new Date(); - date.setTime(allocation.getTime()); - this.timeStamp = date.toString(); + public String getNodeId() { + return nodeId; + } - for (ActivityNode attempt : allocation.getAllocationAttempts()) { - ActivityNodeInfo containerInfo = new ActivityNodeInfo(attempt); - this.allocationAttempt.add(containerInfo); - } + public String getQueueName() { + return queueName; + } + + public String getAppPriority() { + return appPriority; + } + + public long getTimestamp() { + return timestamp; + } + + public String getDateTime() { + return dateTime; + } + + public String getAllocationState() { + return allocationState; + } + + public List getRequestAllocation() { + return requestAllocation; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppRequestAllocationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppRequestAllocationInfo.java new file mode 100644 index 00000000000..4b20c9038f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppRequestAllocationInfo.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import com.google.common.collect.Iterables; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityNode; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * DAO object to display request allocation detailed information. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AppRequestAllocationInfo { + private String requestPriority; + private String allocationRequestId; + private String allocationState; + private List allocationAttempt; + + AppRequestAllocationInfo() { + } + + AppRequestAllocationInfo(List activityNodes) { + this.allocationAttempt = new ArrayList<>(); + ActivityNode lastActivityNode = Iterables.getLast(activityNodes); + this.requestPriority = lastActivityNode.getRequestPriority(); + this.allocationRequestId = lastActivityNode.getAllocationRequestId(); + this.allocationState = lastActivityNode.getState().name(); + for (ActivityNode attempt : activityNodes) { + ActivityNodeInfo containerInfo = + new ActivityNodeInfo(attempt.getName(), attempt.getState(), + attempt.getDiagnostic(), attempt.getNodeId()); + this.allocationAttempt.add(containerInfo); + } + } + + public String getRequestPriority() { + return requestPriority; + } + + public String getAllocationRequestId() { + return allocationRequestId; + } + + public String getAllocationState() { + return allocationState; + } + + public List getAllocationAttempt() { + return allocationAttempt; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java index bc81e615d54..026edc34669 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.Assert; import org.junit.Before; @@ -132,7 +133,8 @@ public void testRecordingDifferentNodeActivitiesInMultiThreads() .startNodeUpdateRecording(activitiesManager, node.getNodeID()); ActivitiesLogger.APP .recordAppActivityWithoutAllocation(activitiesManager, node, - randomApp, Priority.newInstance(0), + randomApp, + new SchedulerRequestKey(Priority.newInstance(0), 0, null), ActivityDiagnosticConstant.FAIL_TO_ALLOCATE, ActivityState.REJECTED); ActivitiesLogger.NODE @@ -176,7 +178,8 @@ public void testRecordingSchedulerActivitiesForMultiNodesInMultiThreads() ActivitiesManager.EMPTY_NODE_ID); ActivitiesLogger.APP .recordAppActivityWithoutAllocation(activitiesManager, node, - randomApp, Priority.newInstance(0), + randomApp, + new SchedulerRequestKey(Priority.newInstance(0), 0, null), ActivityDiagnosticConstant.FAIL_TO_ALLOCATE, ActivityState.REJECTED); ActivitiesLogger.NODE.finishNodeUpdateRecording(activitiesManager, @@ -216,7 +219,8 @@ public void testRecordingAppActivitiesInMultiThreads() for (SchedulerNode node : nodes) { ActivitiesLogger.APP .recordAppActivityWithoutAllocation(activitiesManager, node, - randomApp, Priority.newInstance(0), + randomApp, + new SchedulerRequestKey(Priority.newInstance(0), 0, null), ActivityDiagnosticConstant.FAIL_TO_ALLOCATE, ActivityState.REJECTED); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java index 2aa9971b675..274e2a88f5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -231,6 +232,15 @@ public void testNodeAffinityAssignment() createSchedulingRequest(sourceTag1), schedulerNode2, pcm, tm)); Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode3, pcm, tm)); + + // Test PC diagnostics collector + PlacementConstraintDiagnosticsCollector collector = + new PlacementConstraintDiagnosticsCollector(); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, + createSchedulingRequest(sourceTag1), schedulerNode1, pcm, tm, + Optional.of(collector))); + Assert.assertNotNull(collector.getDiagnostics()); + Assert.assertTrue(collector.getDiagnostics().contains("ALLOCATION_TAG")); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java new file mode 100644 index 00000000000..d5c91ac22f6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceSizing; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.function.Predicate; + +import static org.junit.Assert.assertEquals; + +/** + * Some Utils for activities tests. + */ +public final class ActivitiesTestUtils { + + public static final String INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX = + ActivityDiagnosticConstant.NOT_SUFFICIENT_RESOURCE + + ", insufficient resources="; + + public static final String UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX = + ActivityDiagnosticConstant. + NODE_DO_NOT_MATCH_PARTITION_OR_PLACEMENT_CONSTRAINTS + + ", unsatisfied PC expression="; + + private ActivitiesTestUtils(){} + + public static List findInAllocations(JSONObject allocationObj, + Predicate p) throws JSONException { + List target = new ArrayList<>(); + recursiveFindObj(allocationObj.getJSONObject("root"), p, target); + return target; + } + + private static void recursiveFindObj(JSONObject obj, Predicate p, + List target) throws JSONException { + if (p.test(obj)) { + target.add(obj); + } + if (obj.has("children")) { + JSONArray childrenObjs = obj.optJSONArray("children"); + if (childrenObjs != null) { + for (int i = 0; i < childrenObjs.length(); i++) { + recursiveFindObj(childrenObjs.getJSONObject(i), p, target); + } + } else { + JSONObject childrenObj = obj.optJSONObject("children"); + recursiveFindObj(childrenObj, p, target); + } + } + } + + public static SchedulingRequest schedulingRequest(int numContainers, + int priority, long allocReqId, int cores, int mem, + PlacementConstraint placementConstraintExpression, String... tags) { + return SchedulingRequest.newBuilder() + .priority(Priority.newInstance(priority)) + .allocationRequestId(allocReqId) + .allocationTags(new HashSet<>(Arrays.asList(tags))).executionType( + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true)) + .resourceSizing(ResourceSizing + .newInstance(numContainers, Resource.newInstance(mem, cores))) + .placementConstraintExpression(placementConstraintExpression).build(); + } + + + public static void verifyNumberOfNodes(JSONObject allocation, int expectValue) + throws Exception { + if (allocation.isNull("root")) { + assertEquals("State of allocation is wrong", expectValue, 0); + } else { + assertEquals("State of allocation is wrong", expectValue, + 1 + getNumberOfNodes(allocation.getJSONObject("root"))); + } + } + + public static int getNumberOfNodes(JSONObject allocation) throws Exception { + if (!allocation.isNull("children")) { + Object object = allocation.get("children"); + if (object.getClass() == JSONObject.class) { + return 1 + getNumberOfNodes((JSONObject) object); + } else { + int count = 0; + for (int i = 0; i < ((JSONArray) object).length(); i++) { + count += (1 + getNumberOfNodes( + ((JSONArray) object).getJSONObject(i))); + } + return count; + } + } else { + return 0; + } + } + + public static void verifyStateOfAllocations(JSONObject allocation, + String nameToCheck, String expectState) throws Exception { + assertEquals("State of allocation is wrong", expectState, + allocation.get(nameToCheck)); + } + + public static void verifyNumberOfAllocations(JSONObject json, int expectValue) + throws Exception { + if (json.isNull("allocations")) { + assertEquals("Number of allocations is wrong", expectValue, 0); + } else { + Object object = json.get("allocations"); + if (object.getClass() == JSONObject.class) { + assertEquals("Number of allocations is wrong", expectValue, 1); + } else if (object.getClass() == JSONArray.class) { + assertEquals("Number of allocations is wrong in: " + object, + expectValue, ((JSONArray) object).length()); + } + } + } + + public static void verifyQueueOrder(JSONObject json, String expectOrder) + throws Exception { + String order = ""; + if (!json.isNull("root")) { + JSONObject root = json.getJSONObject("root"); + order = root.getString("name") + "-" + getQueueOrder(root); + } + assertEquals("Order of queue is wrong", expectOrder, + order.substring(0, order.length() - 1)); + } + + public static String getQueueOrder(JSONObject node) throws Exception { + if (!node.isNull("children")) { + Object children = node.get("children"); + if (children.getClass() == JSONObject.class) { + if (!((JSONObject) children).isNull("appPriority")) { + return ""; + } + return ((JSONObject) children).getString("name") + "-" + getQueueOrder( + (JSONObject) children); + } else if (children.getClass() == JSONArray.class) { + String order = ""; + for (int i = 0; i < ((JSONArray) children).length(); i++) { + JSONObject child = (JSONObject) ((JSONArray) children).get(i); + if (!child.isNull("appPriority")) { + return ""; + } + order += (child.getString("name") + "-" + getQueueOrder(child)); + } + return order; + } + } + return ""; + } + + public static void verifyNumberOfAllocationAttempts(JSONObject allocation, + int expectValue) throws Exception { + if (allocation.isNull("allocationAttempt")) { + assertEquals("Number of allocation attempts is wrong", expectValue, 0); + } else { + Object object = allocation.get("allocationAttempt"); + if (object.getClass() == JSONObject.class) { + assertEquals("Number of allocations attempts is wrong", expectValue, 1); + } else if (object.getClass() == JSONArray.class) { + assertEquals("Number of allocations attempts is wrong", expectValue, + ((JSONArray) object).length()); + } + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 46d0a6614fa..d66130f6b15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -100,6 +100,8 @@ protected void configureServlets() { conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); + conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 932f58d3093..7a21edce8a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -21,6 +21,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.http.JettyUtils; @@ -29,12 +30,16 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; @@ -42,9 +47,23 @@ import javax.ws.rs.core.MediaType; +import java.util.ArrayList; import java.util.Arrays; - +import java.util.List; +import java.util.function.Predicate; + +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfAllocationAttempts; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfAllocations; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class TestRMWebServicesSchedulerActivities extends TestRMWebServicesCapacitySched { @@ -101,8 +120,7 @@ public void testAssignMultipleContainersPerNodeHeartbeat() verifyStateOfAllocations(json.getJSONObject("allocations"), "finalAllocationState", "ALLOCATED"); verifyQueueOrder(json.getJSONObject("allocations"), "root-a-b-b2-b3-b1"); - } - finally { + } finally { rm.stop(); } } @@ -150,8 +168,7 @@ public void testAssignWithoutAvailableResource() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 0); - } - finally { + } finally { rm.stop(); } } @@ -184,8 +201,7 @@ public void testNoNM() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 0); - } - finally { + } finally { rm.stop(); } } @@ -233,8 +249,7 @@ public void testWrongNodeId() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 0); - } - finally { + } finally { rm.stop(); } } @@ -355,8 +370,7 @@ public void testReserveNewContainer() throws Exception { allocations = json.getJSONObject("allocations"); verifyStateOfAllocations(allocations, "finalAllocationState", "ALLOCATED_FROM_RESERVED"); - } - finally { + } finally { rm.stop(); } } @@ -401,114 +415,15 @@ public void testActivityJSON() throws Exception { verifyStateOfAllocations(allocations, "finalAllocationState", "ALLOCATED"); - verifyNumberOfNodes(allocations, 5); + // Increase number of nodes to 6 since request node has been added + verifyNumberOfNodes(allocations, 6); verifyQueueOrder(json.getJSONObject("allocations"), "root-b-b1"); - } - finally { + } finally { rm.stop(); } } - private void verifyNumberOfNodes(JSONObject allocation, int realValue) - throws Exception { - if (allocation.isNull("root")) { - assertEquals("State of allocation is wrong", 0, realValue); - } else { - assertEquals("State of allocation is wrong", - 1 + getNumberOfNodes(allocation.getJSONObject("root")), realValue); - } - } - - private int getNumberOfNodes(JSONObject allocation) throws Exception { - if (!allocation.isNull("children")) { - Object object = allocation.get("children"); - if (object.getClass() == JSONObject.class) { - return 1 + getNumberOfNodes((JSONObject) object); - } else { - int count = 0; - for (int i = 0; i < ((JSONArray) object).length(); i++) { - count += (1 + getNumberOfNodes( - ((JSONArray) object).getJSONObject(i))); - } - return count; - } - } else { - return 0; - } - } - - private void verifyStateOfAllocations(JSONObject allocation, - String nameToCheck, String realState) throws Exception { - assertEquals("State of allocation is wrong", allocation.get(nameToCheck), - realState); - } - - private void verifyNumberOfAllocations(JSONObject json, int realValue) - throws Exception { - if (json.isNull("allocations")) { - assertEquals("Number of allocations is wrong", 0, realValue); - } else { - Object object = json.get("allocations"); - if (object.getClass() == JSONObject.class) { - assertEquals("Number of allocations is wrong", 1, realValue); - } else if (object.getClass() == JSONArray.class) { - assertEquals("Number of allocations is wrong in: " + object, - ((JSONArray) object).length(), realValue); - } - } - } - - private void verifyQueueOrder(JSONObject json, String realOrder) - throws Exception { - String order = ""; - if (!json.isNull("root")) { - JSONObject root = json.getJSONObject("root"); - order = root.getString("name") + "-" + getQueueOrder(root); - } - assertEquals("Order of queue is wrong", - order.substring(0, order.length() - 1), realOrder); - } - - private String getQueueOrder(JSONObject node) throws Exception { - if (!node.isNull("children")) { - Object children = node.get("children"); - if (children.getClass() == JSONObject.class) { - if (!((JSONObject) children).isNull("appPriority")) { - return ""; - } - return ((JSONObject) children).getString("name") + "-" + getQueueOrder( - (JSONObject) children); - } else if (children.getClass() == JSONArray.class) { - String order = ""; - for (int i = 0; i < ((JSONArray) children).length(); i++) { - JSONObject child = (JSONObject) ((JSONArray) children).get(i); - if (!child.isNull("appPriority")) { - return ""; - } - order += (child.getString("name") + "-" + getQueueOrder(child)); - } - return order; - } - } - return ""; - } - - private void verifyNumberOfAllocationAttempts(JSONObject allocation, - int realValue) throws Exception { - if (allocation.isNull("allocationAttempt")) { - assertEquals("Number of allocation attempts is wrong", 0, realValue); - } else { - Object object = allocation.get("allocationAttempt"); - if (object.getClass() == JSONObject.class) { - assertEquals("Number of allocations attempts is wrong", 1, realValue); - } else if (object.getClass() == JSONArray.class) { - assertEquals("Number of allocations attempts is wrong", - ((JSONArray) object).length(), realValue); - } - } - } - @Test public void testAppActivityJSON() throws Exception { //Start RM so that it accepts app submissions @@ -542,14 +457,25 @@ public void testAppActivityJSON() throws Exception { response.getType().toString()); json = response.getEntity(JSONObject.class); + //Check app activities verifyNumberOfAllocations(json, 1); - JSONObject allocations = json.getJSONObject("allocations"); verifyStateOfAllocations(allocations, "allocationState", "ACCEPTED"); - - verifyNumberOfAllocationAttempts(allocations, 1); - } - finally { + //Check request allocation + JSONObject requestAllocationObj = + allocations.getJSONObject("requestAllocation"); + verifyStateOfAllocations(requestAllocationObj, "allocationState", + "ALLOCATED"); + assertEquals("0", requestAllocationObj.optString("requestPriority")); + assertEquals("-1", requestAllocationObj.optString("allocationRequestId")); + //Check allocation attempts + verifyNumberOfAllocationAttempts(requestAllocationObj, 1); + JSONObject allocationAttemptObj = + requestAllocationObj.getJSONObject("allocationAttempt"); + verifyStateOfAllocations(allocationAttemptObj, "allocationState", + "ALLOCATED"); + assertNotNull(allocationAttemptObj.get("nodeId")); + } finally { rm.stop(); } } @@ -603,8 +529,7 @@ public void testAppAssignMultipleContainersPerNodeHeartbeat() verifyStateOfAllocations(allocations.getJSONObject(i), "allocationState", "ACCEPTED"); } - } - finally { + } finally { rm.stop(); } } @@ -651,8 +576,7 @@ public void testAppAssignWithoutAvailableResource() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 0); - } - finally { + } finally { rm.stop(); } } @@ -685,8 +609,7 @@ public void testAppNoNM() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 0); - } - finally { + } finally { rm.stop(); } } @@ -791,10 +714,243 @@ public void testAppReserveNewContainer() throws Exception { json = response.getEntity(JSONObject.class); verifyNumberOfAllocations(json, 3); + } finally { + rm.stop(); + } + } + + @Test (timeout=30000) + public void testInsufficientResourceDiagnostic() + throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); + + try { + RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + WebResource r = resource(); + + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for next allocation", + json.getString("diagnostic")); + + am1.allocate(Arrays.asList(ResourceRequest + .newInstance(Priority.UNDEFINED, "*", + Resources.createResource(5 * 1024), 1)), null); + + //will reserve a container on nm1 + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + verifyNumberOfAllocations(json, 1); + JSONObject allocationObj = json.getJSONObject("allocations"); + // check diagnostics + Predicate findReqPred = + (obj) -> obj.optString("name").equals("request_-1_-1"); + List app2ReqObjs = + ActivitiesTestUtils.findInAllocations(allocationObj, findReqPred); + assertEquals(1, app2ReqObjs.size()); + JSONObject reqChild = app2ReqObjs.get(0).getJSONObject("children"); + assertTrue(reqChild.getString("diagnostic") + .contains(INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX)); + } finally { + rm.stop(); } - finally { + } + + @Test (timeout=30000) + public void testPlacementConstraintDiagnostic() + throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler)rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + + try { + RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + // init scheduling request + PlacementConstraint pcExpression = PlacementConstraints + .build(PlacementConstraints.targetIn(NODE, allocationTag("foo"))); + List schedulingRequests = new ArrayList<>(); + schedulingRequests.add(ActivitiesTestUtils + .schedulingRequest(5, 1, 1, 1, 512, pcExpression, "foo")); + AllocateRequest allocateReq = + AllocateRequest.newBuilder().schedulingRequests(schedulingRequests) + .build(); + am1.allocate(allocateReq); + + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for next allocation", + json.getString("diagnostic")); + + // trigger scheduling + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + verifyNumberOfAllocations(json, 1); + JSONObject allocationObj = json.getJSONObject("allocations"); + // check diagnostics + Predicate findReqPred = + (obj) -> obj.optString("name").equals("request_1_1"); + List reqObjs = + ActivitiesTestUtils.findInAllocations(allocationObj, findReqPred); + assertEquals(1, reqObjs.size()); + JSONObject reqChild = reqObjs.get(0).getJSONObject("children"); + assertTrue(reqChild.getString("diagnostic") + .contains(UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX)); + } finally { rm.stop(); } } + @Test (timeout=30000) + public void testAppInsufficientResourceDiagnostic() + throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); + + try { + RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("appId", app1.getApplicationId().toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler/app-activities").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for display", + json.getString("diagnostic")); + + // am1 asks for 1 * 5GB container + am1.allocate(Arrays.asList(ResourceRequest + .newInstance(Priority.UNDEFINED, "*", + Resources.createResource(5 * 1024), 1)), null); + // trigger scheduling + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = + r.path("ws").path("v1").path("cluster") + .path("scheduler/app-activities").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + verifyNumberOfAllocations(json, 1); + JSONObject allocationObj = json.getJSONObject("allocations"); + JSONObject requestAllocationObj = + allocationObj.getJSONObject("requestAllocation"); + verifyNumberOfAllocationAttempts(requestAllocationObj, 1); + JSONObject allocationAttemptObj = + requestAllocationObj.getJSONObject("allocationAttempt"); + verifyStateOfAllocations(allocationAttemptObj, "allocationState", + "SKIPPED"); + assertTrue(allocationAttemptObj.optString("diagnostic") + .contains(INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX)); + } finally { + rm.stop(); + } + } + + @Test (timeout=30000) + public void testAppPlacementConstraintDiagnostic() + throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); + + try { + RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("appId", app1.getApplicationId().toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler/app-activities").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for display", + json.getString("diagnostic")); + + // am1 asks for 1 * 5GB container with PC expression: in,node,foo + PlacementConstraint pcExpression = PlacementConstraints + .build(PlacementConstraints.targetIn(NODE, allocationTag("foo"))); + List schedulingRequests = new ArrayList<>(); + schedulingRequests.add(ActivitiesTestUtils + .schedulingRequest(5, 1, 1, 1, 512, pcExpression, "foo")); + AllocateRequest allocateReq = + AllocateRequest.newBuilder().schedulingRequests(schedulingRequests) + .build(); + am1.allocate(allocateReq); + // trigger scheduling + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = + r.path("ws").path("v1").path("cluster") + .path("scheduler/app-activities").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + verifyNumberOfAllocations(json, 1); + JSONObject allocationObj = json.getJSONObject("allocations"); + JSONObject requestAllocationObj = + allocationObj.getJSONObject("requestAllocation"); + verifyNumberOfAllocationAttempts(requestAllocationObj, 1); + JSONObject allocationAttemptObj = + requestAllocationObj.getJSONObject("allocationAttempt"); + verifyStateOfAllocations(allocationAttemptObj, "allocationState", + "SKIPPED"); + assertTrue(allocationAttemptObj.optString("diagnostic") + .contains(UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX)); + } finally { + rm.stop(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java index 7bc86342156..6157d0beab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -49,9 +50,18 @@ import org.junit.Test; import javax.ws.rs.core.MediaType; +import java.util.ArrayList; import java.util.Arrays; - +import java.util.List; +import java.util.function.Predicate; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.findInAllocations; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfAllocationAttempts; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfAllocations; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Tests for scheduler/app activities when multi-nodes enabled. @@ -97,6 +107,8 @@ protected void configureServlets() { conf.set(policyConfPrefix + ".class", ResourceUsageMultiNodeLookupPolicy.class.getName()); conf.set(policyConfPrefix + ".sorting-interval.ms", "0"); + conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -115,6 +127,7 @@ private static void setupQueueConfiguration( final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(queueB, 89.5f); + config.setMaximumApplicationMasterResourcePerQueuePercent(queueB, 100); } @Before @@ -217,7 +230,7 @@ public void testSchedulingWithoutPendingRequests() } } - @Test + @Test (timeout=30000) public void testAppAssignContainer() throws Exception { rm.start(); @@ -260,34 +273,175 @@ public void testAppAssignContainer() throws Exception { verifyNumberOfAllocations(json, 1); - JSONObject allocations = json.getJSONObject("allocations"); - verifyStateOfAllocations(allocations, "allocationState", "ACCEPTED"); - JSONArray allocationAttempts = - allocations.getJSONArray("allocationAttempt"); - assertEquals(2, allocationAttempts.length()); + JSONObject allocationObj = json.getJSONObject("allocations"); + verifyStateOfAllocations(allocationObj, "allocationState", "ACCEPTED"); + JSONObject requestAllocationObj = + allocationObj.getJSONObject("requestAllocation"); + verifyNumberOfAllocationAttempts(requestAllocationObj, 2); + verifyStateOfAllocations(requestAllocationObj, "allocationState", + "ALLOCATED"); + JSONArray allocationAttemptArray = + requestAllocationObj.getJSONArray("allocationAttempt"); + JSONObject allocationAttempt1 = allocationAttemptArray.getJSONObject(0); + verifyStateOfAllocations(allocationAttempt1, "allocationState", + "SKIPPED"); + assertTrue(allocationAttempt1.optString("diagnostic") + .contains(INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX)); + JSONObject allocationAttempt2 = allocationAttemptArray.getJSONObject(1); + verifyStateOfAllocations(allocationAttempt2, "allocationState", + "ALLOCATED"); } finally { rm.stop(); } } - private void verifyNumberOfAllocations(JSONObject json, int realValue) - throws Exception { - if (json.isNull("allocations")) { - assertEquals("Number of allocations is wrong", 0, realValue); - } else { - Object object = json.get("allocations"); - if (object.getClass() == JSONObject.class) { - assertEquals("Number of allocations is wrong", 1, realValue); - } else if (object.getClass() == JSONArray.class) { - assertEquals("Number of allocations is wrong in: " + object, - ((JSONArray) object).length(), realValue); + @Test (timeout=30000) + public void testInsufficientResourceDiagnostic() throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 2 * 1024); + MockNM nm3 = rm.registerNode("127.0.0.3:1234", 2 * 1024); + MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); + + try { + RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + RMApp app2 = rm.submitApp(1024, "app2", "user1", null, "b"); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); + + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for next allocation", json.getString("diagnostic")); + + //Request a container for am2, will reserve a container on nm1 + am2.allocate("*", 4096, 1, new ArrayList<>()); + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = + r.path("ws").path("v1").path("cluster").path("scheduler/activities") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + //Check app activities + verifyNumberOfAllocations(json, 1); + JSONObject allocationObj = json.getJSONObject("allocations"); + //Check diagnostic for request of app1 + Predicate findApp1Pred = (obj) -> obj.optString("name") + .equals(app1.getApplicationId().toString()); + JSONObject app1Obj = + findInAllocations(allocationObj, findApp1Pred).get(0); + assertEquals("SKIPPED", app1Obj.optString("allocationState")); + assertEquals(ActivityDiagnosticConstant.APPLICATION_DO_NOT_NEED_RESOURCE, + app1Obj.optString("diagnostic")); + //Check diagnostic for request of app2 + Predicate findApp2ReqPred = + (obj) -> obj.optString("name").equals("request_1_-1"); + List app2ReqObjs = + findInAllocations(allocationObj, findApp2ReqPred); + assertEquals(1, app2ReqObjs.size()); + JSONArray app2ReqChildren = app2ReqObjs.get(0).getJSONArray("children"); + assertEquals(4, app2ReqChildren.length()); + for (int i = 0; i < app2ReqChildren.length(); i++) { + JSONObject reqChild = app2ReqChildren.getJSONObject(i); + if (reqChild.getString("allocationState").equals("SKIPPED")) { + String diagnostic = reqChild.getString("diagnostic"); + assertTrue( + diagnostic.contains(INSUFFICIENT_RESOURCE_DIAGNOSTIC_PREFIX)); + } } + } finally { + rm.stop(); } } - private void verifyStateOfAllocations(JSONObject allocation, - String nameToCheck, String realState) throws Exception { - assertEquals("State of allocation is wrong", allocation.get(nameToCheck), - realState); + @Test (timeout=30000) + public void testAppInsufficientResourceDiagnostic() throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler)rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 2 * 1024); + MockNM nm3 = rm.registerNode("127.0.0.3:1234", 2 * 1024); + MockNM nm4 = rm.registerNode("127.0.0.4:1234", 2 * 1024); + + try { + RMApp app1 = rm.submitApp(3072, "app1", "user1", null, "b"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add(RMWSConsts.APP_ID, app1.getApplicationId().toString()); + + ClientResponse response = r.path("ws").path("v1").path("cluster").path( + "scheduler/app-activities").queryParams(params).accept( + MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("waiting for display", json.getString("diagnostic")); + + //Request two containers with different priority for am1 + am1.allocate(Arrays.asList(ResourceRequest + .newInstance(Priority.newInstance(0), "*", + Resources.createResource(1024), 1), ResourceRequest + .newInstance(Priority.newInstance(1), "*", + Resources.createResource(4096), 1)), null); + + //Trigger scheduling, will allocate a container with priority 0 + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + //Trigger scheduling, will reserve a container with priority 1 on nm1 + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + response = r.path("ws").path("v1").path("cluster").path( + "scheduler/app-activities").queryParams(params).accept( + MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + json = response.getEntity(JSONObject.class); + + //Check app activities + verifyNumberOfAllocations(json, 2); + JSONArray allocationArray = json.getJSONArray("allocations"); + //Check first activity is for second allocation with RESERVED state + JSONObject allocationObj = allocationArray.getJSONObject(0); + verifyStateOfAllocations(allocationObj, "allocationState", "RESERVED"); + JSONObject requestAllocationObj = + allocationObj.getJSONObject("requestAllocation"); + verifyNumberOfAllocationAttempts(requestAllocationObj, 4); + JSONArray allocationAttemptArray = + requestAllocationObj.getJSONArray("allocationAttempt"); + for (int i=0; i