From e29c980d8859d6cf86d9f725a9a8d19bdddd8053 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 10 Oct 2017 22:23:36 +0530 Subject: [PATCH] YARN-7307 --- .../yarn/api/records/ResourceInformation.java | 6 +++ .../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++++ .../hadoop/yarn/util/resource/ResourceUtils.java | 47 ++++++++++++++++++---- .../yarn/client/api/impl/YarnClientImpl.java | 13 ++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 2a04094..e7d846c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -205,6 +205,12 @@ public static ResourceInformation newInstance(String name, String units) { Long.MAX_VALUE); } + public static ResourceInformation newInstance(String name, String units, + ResourceTypes resourceType) { + return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, + Long.MAX_VALUE); + } + public static ResourceInformation newInstance(String name, long value) { return ResourceInformation .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ac8ff12..fd5b239 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -969,6 +969,17 @@ public static boolean isAclEnabled(Configuration conf) { "resource-profiles.json"; /** + * Enable/disable loading resource-types.xml at client side. + */ + @Public + @Unstable + public static final String RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT = RM_PREFIX + + "load.resource-types.config-file.for-client"; + @Public + @Unstable + public static final boolean DEFAULT_RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT = true; + + /** * Timeout in seconds for YARN node graceful decommission. * This is the maximal time to wait for running containers and applications * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 0564d74..499f354 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -22,8 +22,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; @@ -63,6 +61,7 @@ private static final Map RESOURCE_NAME_TO_INDEX = new ConcurrentHashMap(); private static volatile Map resourceTypes; + private static volatile Map dynamicResourceTypes; private static volatile ResourceInformation[] resourceTypesArray; private static volatile boolean initializedNodeResources = false; private static volatile Map readOnlyNodeResources; @@ -242,6 +241,11 @@ static void initializeResourcesMap(Configuration conf) { } } + // load from dynamic resource types list if its updated from server. + if (dynamicResourceTypes != null && !dynamicResourceTypes.isEmpty()) { + resourceInformationMap.putAll(dynamicResourceTypes); + } + checkMandatoryResources(resourceInformationMap); addMandatoryResources(resourceInformationMap); @@ -340,11 +344,18 @@ private static void initializeResourceTypesIfNeeded(Configuration conf, if (conf == null) { conf = new YarnConfiguration(); } - try { - addResourcesFileToConf(resourceFile, conf); - LOG.debug("Found " + resourceFile + ", adding to configuration"); - } catch (FileNotFoundException fe) { - LOG.debug("Unable to find '" + resourceFile + "'."); + + boolean loadResourceFileConfForClient = conf.getBoolean( + YarnConfiguration.RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT, + YarnConfiguration.DEFAULT_RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT); + + if (loadResourceFileConfForClient) { + try { + addResourcesFileToConf(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + } catch (FileNotFoundException fe) { + LOG.debug("Unable to find '" + resourceFile + "'."); + } } initializeResourcesMap(conf); @@ -573,4 +584,26 @@ public static String getDefaultUnit(String resourceType) { } return array; } + + /** + * Pre-load supported resource types from server given config file is not + * used. + * + * @param resourceTypes Resource Information + */ + public static void addToDynamicListOfResourceTypes( + List resourceTypes) { + if (dynamicResourceTypes == null) { + dynamicResourceTypes = new HashMap<>(); + } + + for (ResourceTypeInfo resourceType : resourceTypes) { + dynamicResourceTypes.put(resourceType.getName(), + ResourceInformation.newInstance(resourceType.getName(), + resourceType.getDefaultUnit(), resourceType.getResourceType())); + } + + // Reset resource type initialize flag to reload from updated resources. + resetResourceTypes(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 3c30023..6927490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -125,6 +125,7 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -149,6 +150,8 @@ private boolean timelineV1ServiceEnabled; protected boolean timelineServiceBestEffort; + private boolean loadResourceFileConfForClient; + private static final String ROOT = "root"; public YarnClientImpl() { @@ -199,6 +202,10 @@ protected void serviceInit(Configuration conf) throws Exception { timelineServiceBestEffort = conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT); + + loadResourceFileConfForClient = conf.getBoolean( + YarnConfiguration.RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT, + YarnConfiguration.DEFAULT_RM_LOAD_RESOURCETYPES_CONFIG_FOR_CLIENT); super.serviceInit(conf); } @@ -217,6 +224,12 @@ protected void serviceStart() throws Exception { } catch (IOException e) { throw new YarnRuntimeException(e); } + + if (!loadResourceFileConfForClient) { + List resourceTypes = getResourceTypeInfo(); + ResourceUtils.addToDynamicListOfResourceTypes(resourceTypes); + } + super.serviceStart(); } -- 2.10.1 (Apple Git-78)