diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index b352752def4..b0693cb2e13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -199,9 +199,23 @@ static void validateNameOfResourceNameAndThrowException(String resourceName) } } - @VisibleForTesting - static void initializeResourcesMap(Configuration conf) { + /** + * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA* + * @param conf config + * @return maximum allocation + */ + public static Resource fetchMaximumAllocationFromConfig(Configuration conf) { + Map resourceInformationMap = + getResourceInformationMapFromConfig(conf); + Resource ret = Resource.newInstance(0, 0); + for (ResourceInformation entry : resourceInformationMap.values()) { + setMaximumAllocationByConfiguredResourceInformation(ret, entry); + } + return ret; + } + private static Map getResourceInformationMapFromConfig( + Configuration conf) { Map resourceInformationMap = new HashMap<>(); String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); @@ -247,6 +261,13 @@ static void initializeResourcesMap(Configuration conf) { setAllocationForMandatoryResources(resourceInformationMap, conf); + return resourceInformationMap; + } + + @VisibleForTesting + static void initializeResourcesMap(Configuration conf) { + Map resourceInformationMap = + getResourceInformationMapFromConfig(conf); initializeResourcesFromResourceInformationMap(resourceInformationMap); } @@ -537,6 +558,23 @@ public static Resource getResourceTypesMinimumAllocation() { return ret; } + private static void setMaximumAllocationByConfiguredResourceInformation( + Resource ret, ResourceInformation resourceInformationFromConfig) { + String name = resourceInformationFromConfig.getName(); + if (name.equals(ResourceInformation.MEMORY_MB.getName())) { + ret.setMemorySize(resourceInformationFromConfig.getMaximumAllocation()); + } else if (name.equals(ResourceInformation.VCORES.getName())) { + Long tmp = resourceInformationFromConfig.getMaximumAllocation(); + if (tmp > Integer.MAX_VALUE) { + tmp = (long) Integer.MAX_VALUE; + } + ret.setVirtualCores(tmp.intValue()); + } else{ + ret.setResourceValue(name, + resourceInformationFromConfig.getMaximumAllocation()); + } + } + /** * Get a Resource object with for the maximum allocation possible. * @return a Resource object with the maximum allocation for the scheduler @@ -544,19 +582,7 @@ public static Resource getResourceTypesMinimumAllocation() { public static Resource getResourceTypesMaximumAllocation() { Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceTypesArray) { - String name = entry.getName(); - if (name.equals(ResourceInformation.MEMORY_MB.getName())) { - ret.setMemorySize(entry.getMaximumAllocation()); - } else if (name.equals(ResourceInformation.VCORES.getName())) { - Long tmp = entry.getMaximumAllocation(); - if (tmp > Integer.MAX_VALUE) { - tmp = (long) Integer.MAX_VALUE; - } - ret.setVirtualCores(tmp.intValue()); - continue; - } else { - ret.setResourceValue(name, entry.getMaximumAllocation()); - } + setMaximumAllocationByConfiguredResourceInformation(ret, entry); } return ret; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index accf901eb99..3c117bc4b07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -404,7 +404,8 @@ protected Configuration loadNewConfiguration() throws IOException, YarnException { // Retrieve yarn-site.xml in order to refresh scheduling monitor properties. Configuration conf = getConfiguration(new Configuration(false), - YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); // The reason we call Configuration#size() is because when getConfiguration // been called, it invokes Configuration#addResouce, which invokes // Configuration#reloadConfiguration which triggers the reload process in a diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 7e2a623594b..ba2f85a4fbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -143,6 +143,7 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @@ -439,12 +440,15 @@ public void reinitialize(Configuration newConf, RMContext rmContext) validateConf(this.conf); try { LOG.info("Re-initializing queues..."); - refreshMaximumAllocation(this.conf.getMaximumAllocation()); + refreshMaximumAllocation( + ResourceUtils.fetchMaximumAllocationFromConfig(this.conf)); reinitializeQueues(this.conf); } catch (Throwable t) { this.conf = oldConf; - refreshMaximumAllocation(this.conf.getMaximumAllocation()); - throw new IOException("Failed to re-init queues : "+ t.getMessage(), t); + refreshMaximumAllocation( + ResourceUtils.fetchMaximumAllocationFromConfig(this.conf)); + throw new IOException("Failed to re-init queues : " + t.getMessage(), + t); } // update lazy preemption diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8aa41ee4a72..95b288643a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; @@ -835,16 +836,6 @@ public Resource getMinimumAllocation() { return Resources.createResource(minimumMemory, minimumCores); } - public Resource getMaximumAllocation() { - int maximumMemory = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - int maximumCores = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - return Resources.createResource(maximumMemory, maximumCores); - } - @Private public Priority getQueuePriority(String queue) { String queuePolicyPrefix = getQueuePrefix(queue); @@ -879,7 +870,7 @@ public Resource getMaximumAllocationPerQueue(String queue) { LOG.debug("max alloc vcores per queue for " + queue + " is " + maxAllocationVcoresPerQueue); } - Resource clusterMax = getMaximumAllocation(); + Resource clusterMax = ResourceUtils.fetchMaximumAllocationFromConfig(this); if (maxAllocationMbPerQueue == (int)UNDEFINED) { LOG.info("max alloc mb per queue for " + queue + " is undefined"); maxAllocationMbPerQueue = clusterMax.getMemorySize(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index e91f7341a0a..3c3f7c1db0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -154,6 +154,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; @@ -2942,7 +2943,7 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { conf.getMaximumAllocationPerQueue(A1).getMemorySize()); assertEquals("max allocation", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf)); CSQueue rootQueue = cs.getRootQueue(); CSQueue queueA = findQueue(rootQueue, A); @@ -3043,10 +3044,10 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { conf.getMaximumAllocationPerQueue(A1).getVirtualCores()); assertEquals("cluster max allocation MB", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); assertEquals("cluster max allocation vcores", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - conf.getMaximumAllocation().getVirtualCores()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); CSQueue rootQueue = cs.getRootQueue(); CSQueue queueA = findQueue(rootQueue, A); @@ -3065,10 +3066,10 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { conf.getMaximumAllocationPerQueue(A1).getVirtualCores()); assertEquals("max allocation MB cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); assertEquals("max allocation vcores cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - conf.getMaximumAllocation().getVirtualCores()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); assertEquals("queue max allocation MB", 6144, ((LeafQueue) queueA1).getMaximumAllocation().getMemorySize()); assertEquals("queue max allocation vcores", 3,