diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java index dbd9c37ceec..84a312ebaf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java @@ -19,9 +19,9 @@ package org.apache.hadoop.yarn.api.protocolrecords; /** - * Enum which represents the resource type. Currently, the only type allowed is - * COUNTABLE. + * Enum which represents the resource type. Currently, the only types allowed + * are COUNTABLE and BOOLEAN. */ public enum ResourceTypes { - COUNTABLE + COUNTABLE, BOOLEAN } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BooleanResourceValidations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BooleanResourceValidations.java new file mode 100644 index 00000000000..119d18d7df2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/BooleanResourceValidations.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +public class BooleanResourceValidations { + public static void checkValue(String name, long value) { + if (value != 0 && value != 1) { + throw new IllegalArgumentException(String.format( + "Invalid value for boolean resource '%s': %d, " + + "valid values are 0 or 1!", + name, value)); + } + } + + public static void checkUnit(String name, String unit) { + if (!unit.equals("")) { + throw new IllegalArgumentException(String.format( + "Invalid unit for boolean resource '%s': %s, " + + "should be \"\"!", + name, unit)); + } + } + + private static void checkMinAllocation(String name, long minAllocation) { + if (minAllocation != 0) { + throw new IllegalArgumentException(String.format( + "Invalid minimum allocation for boolean resource '%s': %d, " + + "should be 0!", + name, minAllocation)); + } + } + + private static void checkMaxAllocation(String name, long maxAllocation) { + if (maxAllocation != 1) { + throw new IllegalArgumentException(String.format( + "Invalid maximum allocation for boolean resource '%s': %d, " + + "should be 1!", + name, maxAllocation)); + } + } + + public static void runValidations(String name, String units, long value, long minimumAllocation, long maximumAllocation) { + checkValue(name, value); + checkUnit(name, units); + checkMinAllocation(name, minimumAllocation); + checkMaxAllocation(name, maximumAllocation); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 173d4c9e777..9d4464c3341 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -498,7 +498,7 @@ public String toString() { for (int i = 2; i < resources.length; i++) { ResourceInformation ri = resources[i]; - if (ri.getValue() == 0) { + if (ri.getValue() == 0 && ri.getResourceType() == ResourceTypes.COUNTABLE) { continue; } sb.append(", "); @@ -539,7 +539,7 @@ protected static int castToIntSafely(long value) { * Create ResourceInformation with basic fields. * @param name Resource Type Name * @param unit Default unit of provided resource type - * @param value Value associated with giveb resource + * @param value Value associated with given resource * @return ResourceInformation object */ protected static ResourceInformation newDefaultInformation(String name, @@ -553,4 +553,14 @@ protected static ResourceInformation newDefaultInformation(String name, ri.setMaximumAllocation(Long.MAX_VALUE); return ri; } + + public static boolean shouldIgnoreBooleanResource(ResourceInformation + lhsResourceInformation, ResourceInformation rhsResourceInformation) { + return shouldIgnoreBooleanResource(lhsResourceInformation) && + shouldIgnoreBooleanResource(rhsResourceInformation); + } + + public static boolean shouldIgnoreBooleanResource(ResourceInformation ri) { + return ri.getResourceType() == ResourceTypes.BOOLEAN; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 904ff4b4083..76284295a7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -199,6 +199,10 @@ public static ResourceInformation newInstance(ResourceInformation other) { public static ResourceInformation newInstance(String name, String units, long value, ResourceTypes type, long minimumAllocation, long maximumAllocation) { + if (type == ResourceTypes.BOOLEAN) { + BooleanResourceValidations.runValidations(name, units, value, minimumAllocation, maximumAllocation); + } + ResourceInformation ret = new ResourceInformation(); ret.setName(name); ret.setResourceType(type); @@ -224,6 +228,11 @@ public static ResourceInformation newInstance(String name, String units) { public static ResourceInformation newInstance(String name, String units, ResourceTypes resourceType) { + if (resourceType == ResourceTypes.BOOLEAN) { + BooleanResourceValidations.checkUnit(name, units); + return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, + 1L); + } return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, Long.MAX_VALUE); } @@ -244,6 +253,11 @@ public static ResourceInformation newInstance(String name) { return ResourceInformation.newInstance(name, ""); } + public static ResourceInformation newBooleanInstance(String name, boolean value) { + return ResourceInformation.newInstance(name, "", value ? 1 : 0, + ResourceTypes.BOOLEAN, 0L, 1L); + } + /** * Copies the content of the source ResourceInformation object to the * destination object, overwriting all properties of the destination object. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java index 3562f173acb..55b99856e71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java @@ -46,7 +46,7 @@ YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS); } catch (Exception e) { throw new YarnRuntimeException( - "Invalid default configuration provider class" + "Invalid default configuration provider class " + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS, e); } ConfigurationProvider configurationProvider = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9156c2dff05..b232125c310 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -773,7 +773,7 @@ public static boolean isAclEnabled(Configuration conf) { public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX + "configuration.provider-class"; - public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = + public static String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 3dbd609b534..f1b0b476ce5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.BooleanResourceValidations; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; @@ -37,6 +38,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -228,19 +230,24 @@ public static Resource fetchMaximumAllocationFromConfig(Configuration conf) { String resourceTypeName = conf.get( YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE, ResourceTypes.COUNTABLE.toString()); - Long minimumAllocation = conf.getLong( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName - + MINIMUM_ALLOCATION, 0L); - Long maximumAllocation = conf.getLong( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName - + MAXIMUM_ALLOCATION, Long.MAX_VALUE); if (resourceName == null || resourceName.isEmpty() || resourceUnits == null || resourceTypeName == null) { throw new YarnRuntimeException( "Incomplete configuration for resource type '" + resourceName + "'. One of name, units or type is configured incorrectly."); } - ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); + ResourceTypes resourceType = parseResourceType(resourceName, + resourceTypeName); + + Long minimumAllocation = conf.getLong( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + + MINIMUM_ALLOCATION, + getDefaultMinimumAllocation(resourceType)); + Long maximumAllocation = conf.getLong( + YarnConfiguration.RESOURCE_TYPES + "." + resourceName + + MAXIMUM_ALLOCATION, + getDefaultMaximumAllocation(resourceType)); + LOG.info("Adding resource type - name = " + resourceName + ", units = " + resourceUnits + ", type = " + resourceTypeName); if (resourceInformationMap.containsKey(resourceName)) { @@ -266,6 +273,15 @@ public static Resource fetchMaximumAllocationFromConfig(Configuration conf) { return resourceInformationMap; } + private static ResourceTypes parseResourceType(String resourceName, String resourceTypeName) { + try { + return ResourceTypes.valueOf(resourceTypeName.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new YarnRuntimeException("Invalid configuration for resource type '" + resourceName + + "'. Value of type should be one of: " + Arrays.toString(ResourceTypes.values())); + } + } + @VisibleForTesting static void initializeResourcesMap(Configuration conf) { Map resourceInformationMap = @@ -517,21 +533,52 @@ public static long getValue(String resourceValue) { private static void addResourceTypeInformation(String prop, String value, Map nodeResources) { if (prop.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { - LOG.info("Found resource entry " + prop); - String resourceType = prop.substring( - YarnConfiguration.NM_RESOURCES_PREFIX.length()); - if (!nodeResources.containsKey(resourceType)) { - nodeResources - .put(resourceType, ResourceInformation.newInstance(resourceType)); - } - String units = getUnits(value); - Long resourceValue = - Long.valueOf(value.substring(0, value.length() - units.length())); - nodeResources.get(resourceType).setValue(resourceValue); - nodeResources.get(resourceType).setUnits(units); - if (LOG.isDebugEnabled()) { - LOG.debug("Setting value for resource type " + resourceType + " to " - + resourceValue + " with units " + units); + if (prop.contains(TYPE)) { + LOG.info("Found resource type entry " + prop); + String resourceNameWithType = + prop.substring(YarnConfiguration.NM_RESOURCES_PREFIX.length()); + String resourceName = resourceNameWithType.split("\\.")[0]; + ResourceTypes resourceType = parseResourceType(resourceName, value); + if (!nodeResources.containsKey(resourceName)) { + if (resourceType == ResourceTypes.BOOLEAN) { + nodeResources.put(resourceName, + ResourceInformation.newBooleanInstance(resourceName, false)); + } else { + nodeResources.put(resourceName, + ResourceInformation.newInstance(resourceName)); + } + } + } else { + LOG.info("Found resource entry " + prop); + String resourceType = + prop.substring(YarnConfiguration.NM_RESOURCES_PREFIX.length()); + if (!nodeResources.containsKey(resourceType)) { + nodeResources.put(resourceType, + ResourceInformation.newInstance(resourceType)); + } + final long resourceValue; + final String units; + if (ResourceTypes.BOOLEAN == nodeResources.get(resourceType) + .getResourceType()) { + try { + resourceValue = + parseStringResourceValueAsBoolean(resourceType, value) ? 1L + : 0L; + units = ""; + } catch (Exception e) { + throw new YarnRuntimeException(e); + } + } else { + units = getUnits(value); + resourceValue = + Long.parseLong(value.substring(0, value.length() - units.length())); + } + nodeResources.get(resourceType).setValue(resourceValue); + nodeResources.get(resourceType).setUnits(units); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting value for resource type " + resourceType + " to " + + resourceValue + " with units " + units); + } } } } @@ -628,6 +675,7 @@ public static void reinitializeResources( * From a given configuration get all entries representing requested * resources: entries that match the {prefix}{resourceName}={value}[{units}] * pattern. + * * @param configuration The configuration * @param prefix Keys with this prefix are considered from the configuration * @return The list of requested resources as described by the configuration @@ -636,28 +684,36 @@ public static void reinitializeResources( Configuration configuration, String prefix) { List result = new ArrayList<>(); Map customResourcesMap = configuration - .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$"); + .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$"); for (Entry resource : customResourcesMap.entrySet()) { String resourceName = resource.getKey().substring(prefix.length()); Matcher matcher = RESOURCE_REQUEST_VALUE_PATTERN.matcher(resource.getValue()); - if (!matcher.matches()) { + if (matcher.matches()) { + long value = Long.parseLong(matcher.group(1)); + String unit = matcher.group(2); + if (unit.isEmpty()) { + unit = ResourceUtils.getDefaultUnit(resourceName); + } + ResourceInformation resourceInformation = new ResourceInformation(); + resourceInformation.setName(resourceName); + resourceInformation.setValue(value); + resourceInformation.setUnits(unit); + resourceInformation.setResourceType(ResourceTypes.COUNTABLE); + result.add(resourceInformation); + } else if ("true".equalsIgnoreCase(resource.getValue()) + || "false".equalsIgnoreCase(resource.getValue())) { + boolean value = Boolean.valueOf(resource.getValue()); + ResourceInformation resourceInformation = ResourceInformation.newBooleanInstance(resourceName, value); + result.add(resourceInformation); + } else { String errorMsg = "Invalid resource request specified for property " + resource.getKey() + ": \"" + resource.getValue() + "\", expected format is: value[ ][units]"; LOG.error(errorMsg); throw new IllegalArgumentException(errorMsg); } - long value = Long.parseLong(matcher.group(1)); - String unit = matcher.group(2); - if (unit.isEmpty()) { - unit = ResourceUtils.getDefaultUnit(resourceName); - } - ResourceInformation resourceInformation = new ResourceInformation(); - resourceInformation.setName(resourceName); - resourceInformation.setValue(value); - resourceInformation.setUnits(unit); - result.add(resourceInformation); + } return result; } @@ -736,4 +792,40 @@ public static void areMandatoryResourcesAvailable(Resource res) { return info; } + + private static long getDefaultMinimumAllocation(ResourceTypes rt) { + if (ResourceTypes.COUNTABLE == rt || ResourceTypes.BOOLEAN == rt) { + return 0L; + } + throw new IllegalArgumentException("Maximum allocation value is unknown for resource type: " + rt); + } + + private static long getDefaultMaximumAllocation(ResourceTypes rt) { + if (ResourceTypes.COUNTABLE == rt) { + return Long.MAX_VALUE; + } else if (ResourceTypes.BOOLEAN == rt) { + return 1L; + } + throw new IllegalArgumentException("Maximum allocation value is unknown for resource type: " + rt); + } + + public static boolean parseStringResourceValueAsBoolean(String resName, String + resValue) { + boolean booleanValue; + final long longValue; + try { + longValue = Long.parseLong(resValue); + BooleanResourceValidations.checkValue(resName, longValue); + booleanValue = (longValue == 1); + } catch (NumberFormatException e) { + if ("true".equalsIgnoreCase(resValue) + || "false".equalsIgnoreCase(resValue)) { + booleanValue = Boolean.valueOf(resValue); + } else { + throw new IllegalArgumentException("Invalid resource value: " + resValue + + " for boolean resource: " + resName); + } + } + return booleanValue; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index d6138e865ff..96e998c7015 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -55,6 +55,7 @@ message ContainerIdProto { enum ResourceTypesProto { COUNTABLE = 0; + BOOLEAN = 1; } message ResourceInformationProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestResource.java index e0ec3703552..78a13c4967b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestResource.java @@ -17,8 +17,22 @@ */ package org.apache.hadoop.yarn.api.records; +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + import static org.junit.Assert.assertEquals; /** @@ -26,6 +40,85 @@ */ public class TestResource { + private static class LocalConfigurationProvider extends ConfigurationProvider { + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws IOException, YarnException { + if (name == null || name.isEmpty()) { + throw new YarnException( + "Illegal argument! The parameter should not be null or empty"); + } else if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name) || + YarnConfiguration.NM_CONFIGURATION_FILES.contains(name)) { + return bootstrapConf.getConfResourceAsInputStream(name); + } + return new FileInputStream(name); + } + + @Override + public void initInternal(Configuration bootstrapConf) throws Exception { + // Do nothing + } + + @Override + public void closeInternal() throws Exception { + // Do nothing + } + } + + private static class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream( + ("\n" + + " \n" + + " yarn.resource-types\n" + + " custom-resource-1," + + "custom-resource-2,custom-resource-3\n" + + " \n" + + " \n" + + " yarn.resource-types" + + ".custom-resource-1.units\n" + + " G\n" + + " \n" + + " \n" + + " yarn.resource-types" + + ".custom-resource-2.units\n" + + " G\n" + + " \n" + + " \n" + + " yarn.resource-types" + + ".custom-resource-3.type\n" + + " BOOLEAN\n" + + " \n" + + "\n").getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + } + + private void initResourceTypes() { + Configuration yarnConf = new Configuration(); + yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(yarnConf); + } + + @BeforeClass + public static void setUpClass() { + YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = CustomResourceTypesConfigurationProvider.class.getName(); + } + + @Before + public void setUp() { + initResourceTypes(); + } + @Test public void testCastToIntSafely() { assertEquals(0, Resource.castToIntSafely(0)); @@ -40,4 +133,91 @@ public void testCastToIntSafely() { + "Integer.MAX_VALUE", Integer.MAX_VALUE, Resource.castToIntSafely(Long.MAX_VALUE)); } + + @Test + public void testCreateNewInstanceIntInt() { + Resource resource = Resource.newInstance(2000, 4); + assertEquals(2000, resource.getMemorySize()); + assertEquals(4, resource.getVirtualCores()); + + ResourceInformation memory = resource.getResourceInformation(Resource.MEMORY_INDEX); + assertEquals(2000, memory.getValue()); + assertEquals(ResourceTypes.COUNTABLE, memory.getResourceType()); + + ResourceInformation vCores = resource.getResourceInformation(Resource.VCORES_INDEX); + assertEquals(4, vCores.getValue()); + assertEquals(ResourceTypes.COUNTABLE, vCores.getResourceType()); + } + + @Test + public void testCreateNewInstanceLongInt() { + Resource resource = Resource.newInstance((long) 2000, 4); + assertEquals(2000, resource.getMemorySize()); + assertEquals(4, resource.getVirtualCores()); + + ResourceInformation memory = resource.getResourceInformation(Resource.MEMORY_INDEX); + assertEquals(2000, memory.getValue()); + assertEquals(ResourceTypes.COUNTABLE, memory.getResourceType()); + + ResourceInformation vCores = resource.getResourceInformation(Resource.VCORES_INDEX); + assertEquals(4, vCores.getValue()); + assertEquals(ResourceTypes.COUNTABLE, vCores.getResourceType()); + } + + @Test + public void testCreateNewInstanceFromExistingResource() { + Resource resource = Resource.newInstance((long) 2000, 4); + resource.setResourceInformation("custom-resource-1", ResourceInformation.newInstance("custom-resource-1", 5)); + resource.setResourceInformation("custom-resource-3", ResourceInformation.newBooleanInstance("custom-resource-3", true)); + + assertEquals(2000, resource.getMemorySize()); + assertEquals(4, resource.getVirtualCores()); + + ResourceInformation memory = resource.getResourceInformation(Resource.MEMORY_INDEX); + assertEquals(2000, memory.getValue()); + assertEquals(ResourceTypes.COUNTABLE, memory.getResourceType()); + + ResourceInformation vCores = resource.getResourceInformation(Resource.VCORES_INDEX); + assertEquals(4, vCores.getValue()); + assertEquals(ResourceTypes.COUNTABLE, vCores.getResourceType()); + + ResourceInformation customResource1 = resource.getResourceInformation("custom-resource-1"); + assertEquals(5, customResource1.getValue()); + assertEquals(ResourceTypes.COUNTABLE, customResource1.getResourceType()); + + ResourceInformation booleanResource = resource.getResourceInformation("custom-resource-3"); + assertEquals(1, booleanResource.getValue()); + assertEquals(ResourceTypes.BOOLEAN, booleanResource.getResourceType()); + } + + @Test + public void testCreateNewInstanceFromMapOfResources() { + Resource resource = Resource.newInstance((long) 2000, 4, + ImmutableMap.of("custom-resource-1", 1100L, + "custom-resource-2", 2200L, + "custom-resource-3", 1L)); + + assertEquals(2000, resource.getMemorySize()); + assertEquals(4, resource.getVirtualCores()); + + ResourceInformation memory = resource.getResourceInformation(Resource.MEMORY_INDEX); + assertEquals(2000, memory.getValue()); + assertEquals(ResourceTypes.COUNTABLE, memory.getResourceType()); + + ResourceInformation vCores = resource.getResourceInformation(Resource.VCORES_INDEX); + assertEquals(4, vCores.getValue()); + assertEquals(ResourceTypes.COUNTABLE, vCores.getResourceType()); + + ResourceInformation customResource1 = resource.getResourceInformation("custom-resource-1"); + assertEquals(1100L, customResource1.getValue()); + assertEquals(ResourceTypes.COUNTABLE, customResource1.getResourceType()); + + ResourceInformation customResource2 = resource.getResourceInformation("custom-resource-2"); + assertEquals(2200L, customResource2.getValue()); + assertEquals(ResourceTypes.COUNTABLE, customResource2.getResourceType()); + + ResourceInformation booleanResource = resource.getResourceInformation("custom-resource-3"); + assertEquals(1, booleanResource.getValue()); + assertEquals(ResourceTypes.BOOLEAN, booleanResource.getResourceType()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java index 66bf3204bf6..338a1f8d459 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -18,20 +18,27 @@ package org.apache.hadoop.yarn.conf; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; /** * Test class to verify various resource informations in a given resource. */ public class TestResourceInformation { + @Rule + public ExpectedException exception = ExpectedException.none(); + @Test public void testName() { String name = "yarn.io/test"; ResourceInformation ri = ResourceInformation.newInstance(name); Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.COUNTABLE, ri.getResourceType()); } @Test @@ -48,6 +55,7 @@ public void testUnits() { } catch (IllegalArgumentException ie) { // do nothing } + Assert.assertEquals("Resource type is incorrect", ResourceTypes.COUNTABLE, ri.getResourceType()); } @Test @@ -57,6 +65,7 @@ public void testValue() { ResourceInformation ri = ResourceInformation.newInstance(name, value); Assert.assertEquals("Resource name incorrect", name, ri.getName()); Assert.assertEquals("Resource value incorrect", value, ri.getValue()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.COUNTABLE, ri.getResourceType()); } @Test @@ -69,5 +78,120 @@ public void testResourceInformation() { Assert.assertEquals("Resource name incorrect", name, ri.getName()); Assert.assertEquals("Resource value incorrect", value, ri.getValue()); Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.COUNTABLE, ri.getResourceType()); + } + + @Test + public void testBooleanResourceInformationTrueValue() { + String name = "yarn.io/test"; + boolean value = true; + String units = ""; + ResourceInformation ri = + ResourceInformation.newBooleanInstance(name, value); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource value incorrect", 1L, ri.getValue()); + Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.BOOLEAN, ri.getResourceType()); + } + + @Test + public void testBooleanResourceInformationFalseValue() { + String name = "yarn.io/test"; + boolean value = false; + String units = ""; + ResourceInformation ri = + ResourceInformation.newBooleanInstance(name, value); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource value incorrect", 0L, ri.getValue()); + Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.BOOLEAN, ri.getResourceType()); + } + + @Test + public void testCopyCountable() { + ResourceInformation src = ResourceInformation.newInstance("res1", "M", 5, + ResourceTypes.COUNTABLE, 0, Long.MAX_VALUE); + ResourceInformation dst = new ResourceInformation(); + + ResourceInformation.copy(src, dst); + Assert.assertEquals("Resource name incorrect", "res1", dst.getName()); + Assert.assertEquals("Resource value incorrect", 5L, dst.getValue()); + Assert.assertEquals("Resource units incorrect", "M", dst.getUnits()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.COUNTABLE, dst.getResourceType()); + } + + @Test + public void testCopyBoolean() { + ResourceInformation src = ResourceInformation.newInstance("res1", "", 1, + ResourceTypes.BOOLEAN, 0, 1); + ResourceInformation dst = new ResourceInformation(); + + ResourceInformation.copy(src, dst); + Assert.assertEquals("Resource name incorrect", "res1", dst.getName()); + Assert.assertEquals("Resource value incorrect", 1L, dst.getValue()); + Assert.assertEquals("Resource units incorrect", "", dst.getUnits()); + Assert.assertEquals("Resource type is incorrect", ResourceTypes.BOOLEAN, dst.getResourceType()); + } + + @Test + public void testBooleanResourceValueBoundsChecked() { + String name = "yarn.io/testBoolean"; + String units = ""; + int value = 5; + exception.expect(IllegalArgumentException.class); + String error = "Invalid value for boolean resource '" + name + "': " + value + ", valid values are 0 or 1!"; + exception.expectMessage(error); + + ResourceInformation ri = ResourceInformation.newInstance(name, units, value, + ResourceTypes.BOOLEAN, 0, 1); + } + + @Test + public void testBooleanResourceUnitIsChecked() { + String name = "yarn.io/testBoolean"; + String units = "G"; + int value = 1; + exception.expect(IllegalArgumentException.class); + String error = "Invalid unit for boolean resource '" + name + "': " + units + ", should be \"\"!"; + exception.expectMessage(error); + + ResourceInformation ri = ResourceInformation.newInstance(name, units, value, + ResourceTypes.BOOLEAN, 0, 1); + } + + @Test + public void testBooleanResourceMinAllocationIsChecked() { + String name = "yarn.io/testBoolean"; + String units = ""; + int value = 1; + int minAllocation = 5; + int maxAllocation = 1; + + exception.expect(IllegalArgumentException.class); + String error = "Invalid minimum allocation for boolean resource '" + name + "': " + minAllocation + + ", should be 0!"; + exception.expectMessage(error); + + ResourceInformation ri = ResourceInformation.newInstance(name, units, value, + ResourceTypes.BOOLEAN, minAllocation, maxAllocation); + } + + @Test + public void testBooleanResourceMaxAllocationIsChecked() { + String name = "yarn.io/testBoolean"; + String units = ""; + int value = 1; + int minAllocation = 0; + int maxAllocation = 5; + + exception.expect(IllegalArgumentException.class); + String error = "Invalid maximum allocation for boolean resource '" + name + "': " + maxAllocation + + ", should be 1!"; + exception.expectMessage(error); + + ResourceInformation ri = ResourceInformation.newInstance(name, units, value, + ResourceTypes.BOOLEAN, minAllocation, maxAllocation); } } + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 8d1d56b3f93..150087ecf62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -46,6 +46,8 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.BooleanResourceValidations; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; @@ -984,35 +986,49 @@ private Resource parseCommandAndCreateResource(String resourceTypes) { Map resourceTypesFromRM = ResourceUtils.getResourceTypes(); String[] resourceTypesArr = resourceTypes.split(","); - for (int k = 0; k < resourceTypesArr.length; k++) { - String resourceType = resourceTypesArr[k]; + for (String resourceType : resourceTypesArr) { String[] resourceTypeArray = resourceType.split("="); - if (resourceTypeArray.length == 2) { - String resName = StringUtils.trim(resourceTypeArray[0]); - String resValue = StringUtils.trim(resourceTypeArray[1]); - if (resourceTypesFromRM.containsKey(resName)) { - String[] resourceValue = ResourceUtils.parseResourceValue(resValue); - if (resourceValue.length == 2) { - ResourceInformation ri = ResourceInformation.newInstance(resName, - resourceValue[0], Long.parseLong(resourceValue[1])); - resource.setResourceInformation(resName, ri); - } else { - throw new IllegalArgumentException("Invalid resource value: " + - resValue + ". Unable to extract unit and actual value."); - } - } else { - throw new IllegalArgumentException("Invalid resource type: " + - resName + ". Not allowed."); - } + if (resourceTypeArray.length != 2) { + throw new IllegalArgumentException("Invalid resource type value: " + "(" + + resourceType + ") for updateNodeResource. " + + "It should be key value pairs separated using '=' symbol."); + } + String resName = StringUtils.trim(resourceTypeArray[0]); + String resValue = StringUtils.trim(resourceTypeArray[1]); + + if (!resourceTypesFromRM.containsKey(resName)) { + throw new IllegalArgumentException( + "Invalid resource type: " + resName + ". Not allowed."); + } + final ResourceInformation ri; + if (ResourceTypes.BOOLEAN == resourceTypesFromRM.get(resName) + .getResourceType()) { + ri = createBooleanResourceInformation(resName, resValue); } else { - throw new IllegalArgumentException("Invalid resource type value: " + - "("+ resourceType + ") for updateNodeResource. " - + "It should be key value pairs separated using '=' symbol."); + ri = createNormalResourceInformation(resName, resValue); } + resource.setResourceInformation(resName, ri); } return resource; } + private ResourceInformation createNormalResourceInformation(String resName, + String resValue) { + String[] resourceValue = ResourceUtils.parseResourceValue(resValue); + if (resourceValue.length == 2) { + return ResourceInformation.newInstance(resName, resourceValue[0], + Long.parseLong(resourceValue[1])); + } else { + throw new IllegalArgumentException("Invalid resource value: " + resValue + + ". Unable to extract unit and actual value."); + } + } + + private ResourceInformation createBooleanResourceInformation(String resName, + String resValue) { + return ResourceInformation.newBooleanInstance(resName, ResourceUtils.parseStringResourceValueAsBoolean(resName, resValue)); + } + private int validateTimeout(String strTimeout) { int timeout; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 1f4b493f457..ba80e818b25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -104,7 +104,7 @@ public void setup() throws Exception { ResourceUtils.resetResourceTypes(); Configuration yarnConf = new YarnConfiguration(); - String resourceTypesFile = "resource-types-4.xml"; + String resourceTypesFile = "resource-types-boolean-1.xml"; InputStream source = yarnConf.getClassLoader().getResourceAsStream(resourceTypesFile); dest = new File(yarnConf.getClassLoader(). @@ -325,7 +325,8 @@ public void testUpdateNodeResourceWithInvalidValue() throws Exception { public void testUpdateNodeResourceTypes() throws Exception { String nodeIdStr = "0.0.0.0:0"; String resourceTypes = - "memory-mb=1024Mi,vcores=1,resource1=3Gi,resource2=2m"; + "memory-mb=1024Mi,vcores=1,resource1=3Gi,resource2=2m," + + "booleanResource=true"; String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; assertEquals(0, rmAdminCLI.run(args)); ArgumentCaptor argument = @@ -340,6 +341,8 @@ public void testUpdateNodeResourceTypes() throws Exception { ResourceInformation.newInstance("resource1", "Gi", 3)); expectedResource.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2)); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); ResourceOption resource = resourceMap.get(nodeId); assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", @@ -354,7 +357,8 @@ public void testUpdateNodeResourceTypesWithOverCommitTimeout() throws Exception { String nodeIdStr = "0.0.0.0:0"; String resourceTypes = - "memory-mb=1024Mi,vcores=1,resource1=3Gi,resource2=2m"; + "memory-mb=1024Mi,vcores=1,resource1=3Gi,resource2=2m," + + "booleanResource=true"; int timeout = 1000; String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes, Integer.toString(timeout)}; @@ -371,6 +375,8 @@ public void testUpdateNodeResourceTypesWithOverCommitTimeout() ResourceInformation.newInstance("resource1", "Gi", 3)); expectedResource.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2)); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); ResourceOption resource = resourceMap.get(nodeId); assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", @@ -420,6 +426,168 @@ public void testUpdateNodeResourceTypesWithInvalidResourceValue() any(UpdateNodeResourceRequest.class)); } + @Test + public void testUpdateNodeResourceTypesWithBooleanResourceValueLowerCase() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=false,resource2=2m"; + int timeout = -1; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + + assertEquals(0, rmAdminCLI.run(args)); + ArgumentCaptor argument = + ArgumentCaptor.forClass(UpdateNodeResourceRequest.class); + verify(admin).updateNodeResource(argument.capture()); + UpdateNodeResourceRequest request = argument.getValue(); + Map resourceMap = request.getNodeResourceMap(); + NodeId nodeId = NodeId.fromString(nodeIdStr); + + Resource expectedResource = Resource.newInstance(1024, 1); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", false)); + expectedResource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2)); + + ResourceOption resource = resourceMap.get(nodeId); + assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", + resource); + assertEquals("resource value for " + nodeIdStr + " is not as expected.", + ResourceOption.newInstance(expectedResource, timeout), resource); + } + + @Test + public void testUpdateNodeResourceTypesWithBooleanResourceValueUpperCase() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=TRUE,resource2=2m"; + int timeout = -1; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + + assertEquals(0, rmAdminCLI.run(args)); + ArgumentCaptor argument = + ArgumentCaptor.forClass(UpdateNodeResourceRequest.class); + verify(admin).updateNodeResource(argument.capture()); + UpdateNodeResourceRequest request = argument.getValue(); + Map resourceMap = request.getNodeResourceMap(); + NodeId nodeId = NodeId.fromString(nodeIdStr); + + Resource expectedResource = Resource.newInstance(1024, 1); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); + expectedResource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2)); + + ResourceOption resource = resourceMap.get(nodeId); + assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", + resource); + assertEquals("resource value for " + nodeIdStr + " is not as expected.", + ResourceOption.newInstance(expectedResource, timeout), resource); + } + + @Test + public void testUpdateNodeResourceTypesWithBooleanResourceValueNumericZero() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=0,resource2=2m"; + int timeout = -1; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + + assertEquals(0, rmAdminCLI.run(args)); + ArgumentCaptor argument = + ArgumentCaptor.forClass(UpdateNodeResourceRequest.class); + verify(admin).updateNodeResource(argument.capture()); + UpdateNodeResourceRequest request = argument.getValue(); + Map resourceMap = request.getNodeResourceMap(); + NodeId nodeId = NodeId.fromString(nodeIdStr); + + Resource expectedResource = Resource.newInstance(1024, 1); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", false)); + expectedResource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2)); + + ResourceOption resource = resourceMap.get(nodeId); + assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", + resource); + assertEquals("resource value for " + nodeIdStr + " is not as expected.", + ResourceOption.newInstance(expectedResource, timeout), resource); + } + + @Test + public void testUpdateNodeResourceTypesWithBooleanResourceValueNumericOne() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=1,resource2=2m"; + int timeout = -1; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + + assertEquals(0, rmAdminCLI.run(args)); + ArgumentCaptor argument = + ArgumentCaptor.forClass(UpdateNodeResourceRequest.class); + verify(admin).updateNodeResource(argument.capture()); + UpdateNodeResourceRequest request = argument.getValue(); + Map resourceMap = request.getNodeResourceMap(); + NodeId nodeId = NodeId.fromString(nodeIdStr); + + Resource expectedResource = Resource.newInstance(1024, 1); + expectedResource.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); + expectedResource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2)); + + ResourceOption resource = resourceMap.get(nodeId); + assertNotNull("resource for " + nodeIdStr + " shouldn't be null.", + resource); + assertEquals("resource value for " + nodeIdStr + " is not as expected.", + ResourceOption.newInstance(expectedResource, timeout), resource); + } + + @Test + public void testUpdateNodeResourceTypesWithInvalidBooleanResourceNumericValue() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=5,resource2=2m"; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + // execution of command line is expected to be failed + assertEquals(-1, rmAdminCLI.run(args)); + // verify admin protocol never calls. + verify(admin, times(0)).updateNodeResource( + any(UpdateNodeResourceRequest.class)); + } + + @Test + public void testUpdateNodeResourceTypesWithInvalidBooleanResourceStringValue() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,booleanResource=fals,resource2=2m"; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + // execution of command line is expected to be failed + assertEquals(-1, rmAdminCLI.run(args)); + // verify admin protocol never calls. + verify(admin, times(0)).updateNodeResource( + any(UpdateNodeResourceRequest.class)); + } + + @Test + public void testUpdateNodeResourceTypesWithNonBooleanResourceWithBooleanValue() + throws Exception { + String nodeIdStr = "0.0.0.0:0"; + String resourceTypes = + "memory-mb=1024Mi,vcores=1,resource2=false"; + String[] args = {"-updateNodeResource", nodeIdStr, resourceTypes}; + // execution of command line is expected to be failed + assertEquals(-1, rmAdminCLI.run(args)); + // verify admin protocol never calls. + verify(admin, times(0)).updateNodeResource( + any(UpdateNodeResourceRequest.class)); + } + @Test public void testUpdateNodeResourceTypesWithInvalidResourceUnit() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 2e85ebca860..89efc6a9721 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -54,14 +55,14 @@ @Private @Unstable public class DominantResourceCalculator extends ResourceCalculator { - static final Log LOG = LogFactory.getLog(DominantResourceCalculator.class); + private static final Log LOG = LogFactory.getLog(DominantResourceCalculator.class); public DominantResourceCalculator() { } /** * Compare two resources - if the value for every resource type for the lhs - * is greater than that of the rhs, return 1. If the value for every resource + * is greater than that of the rhs, return 1. If the value for every countable resource * type in the lhs is less than the rhs, return -1. Otherwise, return 0 * * @param lhs resource to be compared @@ -79,6 +80,7 @@ private int compare(Resource lhs, Resource rhs) { .getResourceInformation(i); ResourceInformation rhsResourceInformation = rhs .getResourceInformation(i); + int diff = lhsResourceInformation.compareTo(rhsResourceInformation); if (diff >= 1) { lhsGreater = true; @@ -340,6 +342,7 @@ public long computeAvailableContainers(Resource available, ResourceInformation availableResource = available .getResourceInformation(i); ResourceInformation requiredResource = required.getResourceInformation(i); + long requiredResourceValue = UnitsConversionUtil.convert( requiredResource.getUnits(), availableResource.getUnits(), requiredResource.getValue()); @@ -387,6 +390,11 @@ public float ratio(Resource a, Resource b) { for (int i = 0; i < maxLength; i++) { ResourceInformation aResourceInformation = a.getResourceInformation(i); ResourceInformation bResourceInformation = b.getResourceInformation(i); + + if (Resource.shouldIgnoreBooleanResource(aResourceInformation, bResourceInformation)) { + continue; + } + long bResourceValue = UnitsConversionUtil.convert( bResourceInformation.getUnits(), aResourceInformation.getUnits(), bResourceInformation.getValue()); @@ -407,6 +415,11 @@ public Resource divideAndCeil(Resource numerator, long denominator) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = ret.getResourceInformation(i); + + if (Resource.shouldIgnoreBooleanResource(resourceInformation)) { + continue; + } + resourceInformation .setValue(divideAndCeil(resourceInformation.getValue(), denominator)); } @@ -432,6 +445,11 @@ public Resource normalize(Resource r, Resource minimumResource, .getResourceInformation(i); ResourceInformation maximumResourceInformation = maximumResource .getResourceInformation(i); + + if (Resource.shouldIgnoreBooleanResource(minimumResourceInformation, maximumResourceInformation)) { + continue; + } + ResourceInformation stepFactorResourceInformation = stepFactor .getResourceInformation(i); ResourceInformation tmp = ret.getResourceInformation(i); @@ -477,6 +495,10 @@ private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { ResourceInformation stepFactorResourceInformation = stepFactor .getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(rResourceInformation)) { + continue; + } + long rValue = rResourceInformation.getValue(); long stepFactorValue = UnitsConversionUtil.convert( stepFactorResourceInformation.getUnits(), @@ -505,6 +527,10 @@ public Resource multiplyAndNormalizeUp(Resource r, double[] by, ResourceInformation stepFactorResourceInformation = stepFactor .getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(rResourceInformation)) { + continue; + } + long rValue = rResourceInformation.getValue(); long stepFactorValue = UnitsConversionUtil.convert( stepFactorResourceInformation.getUnits(), @@ -538,6 +564,10 @@ private Resource multiplyAndNormalize(Resource r, double by, .getResourceInformation(i); ResourceInformation tmp = ret.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(rResourceInformation)) { + continue; + } + long rValue = rResourceInformation.getValue(); long stepFactorValue = UnitsConversionUtil.convert( stepFactorResourceInformation.getUnits(), @@ -586,6 +616,10 @@ public Resource normalizeDown(Resource r, Resource stepFactor) { .getResourceInformation(i); ResourceInformation tmp = ret.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(rResourceInformation)) { + continue; + } + long rValue = rResourceInformation.getValue(); long stepFactorValue = UnitsConversionUtil.convert( stepFactorResourceInformation.getUnits(), @@ -607,6 +641,11 @@ public boolean isAnyMajorResourceZeroOrNegative(Resource resource) { for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = resource.getResourceInformation( i); + + if (shouldIgnoreBooleanResource(resourceInformation)) { + continue; + } + if (resourceInformation.getValue() <= 0L) { return true; } @@ -620,10 +659,19 @@ public boolean isAnyMajorResourceAboveZero(Resource resource) { for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = resource.getResourceInformation( i); + + if (shouldIgnoreBooleanResource(resourceInformation)) { + continue; + } + if (resourceInformation.getValue() > 0) { return true; } } return false; } + + private boolean shouldIgnoreBooleanResource(ResourceInformation ri) { + return ri.getResourceType() == ResourceTypes.BOOLEAN; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 27394f73a7a..0a9c6d08c84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -30,7 +30,7 @@ /** * On a cluster with capacity {@code clusterResource}, compare {@code lhs} - * and {@code rhs}. Consider all resources unless {@code singleType} is set + * and {@code rhs}. Consider all countable resources unless {@code singleType} is set * to true. When {@code singleType} is set to true, consider only one * resource as per the {@link ResourceCalculator} implementation; the * {@link DefaultResourceCalculator} considers memory and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 7826f51cd4d..fae6a592579 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -22,13 +22,14 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; /** - * Resources is a computation class which provides a set of apis to do + * Resources is a computation class which provides a set of APIs to do * mathematical operations on Resource object. */ @InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @@ -182,8 +183,18 @@ private void initResourceMap() { if (types != null) { resources = new ResourceInformation[types.length]; for (int index = 0; index < types.length; index++) { - resources[index] = ResourceInformation.newInstance(types[index]); - resources[index].setValue(resourceValue); + ResourceInformation type = types[index]; + resources[index] = ResourceInformation.newInstance(type); + + if (ResourceTypes.BOOLEAN == type.getResourceType()) { + if (resourceValue == 0L) { + resources[index].setValue(0L); + } else if (resourceValue == Long.MAX_VALUE) { + resources[index].setValue(1L); + } + } else { + resources[index].setValue(resourceValue); + } } } } @@ -239,6 +250,10 @@ public static Resource addTo(Resource lhs, Resource rhs) { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(lhsValue, rhsValue)) { + continue; + } + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) ? rhsValue.getValue() : UnitsConversionUtil.convert(rhsValue.getUnits(), @@ -246,7 +261,6 @@ public static Resource addTo(Resource lhs, Resource rhs) { lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return lhs; @@ -263,6 +277,10 @@ public static Resource subtractFrom(Resource lhs, Resource rhs) { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(lhsValue, rhsValue)) { + continue; + } + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) ? rhsValue.getValue() : UnitsConversionUtil.convert(rhsValue.getUnits(), @@ -270,7 +288,6 @@ public static Resource subtractFrom(Resource lhs, Resource rhs) { lhs.setResourceValue(i, lhsValue.getValue() - convertedRhs); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return lhs; @@ -307,10 +324,14 @@ public static Resource multiplyTo(Resource lhs, double by) { for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); + + if (Resource.shouldIgnoreBooleanResource(lhsValue)) { + continue; + } + lhs.setResourceValue(i, (long) (lhsValue.getValue() * by)); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return lhs; @@ -332,6 +353,10 @@ public static Resource multiplyAndAddTo( ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(lhsValue, rhsValue)) { + continue; + } + long convertedRhs = (long) (((rhsValue.getUnits() .equals(lhsValue.getUnits())) ? rhsValue.getValue() @@ -341,7 +366,6 @@ public static Resource multiplyAndAddTo( lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return lhs; @@ -368,15 +392,19 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) { for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(lhsValue)) { + continue; + } + out.setResourceValue(i, (long) (lhsValue.getValue() * by)); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return out; } + //TODO why this method ignores other resources? public static Resource multiplyAndRoundUp(Resource lhs, double by) { Resource out = clone(lhs); out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); @@ -488,7 +516,6 @@ public static boolean fitsIn(Resource smaller, Resource bigger) { } } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return true; @@ -517,7 +544,6 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) { ret.setResourceInformation(i, outInfo); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return ret; @@ -541,7 +567,6 @@ public static Resource componentwiseMax(Resource lhs, Resource rhs) { ret.setResourceInformation(i, outInfo); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } return ret; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java index 4887b50e2c2..b7431addaf4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java @@ -18,19 +18,36 @@ package org.apache.hadoop.yarn.api; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** * Test class to handle various proto related tests for resources. */ public class TestResourcePBImpl { + + public static void addBooleanResourceToResources(String booleanResource) { + Map riMap = new HashMap<>(); + riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB); + riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES); + + riMap.put(booleanResource, ResourceInformation.newInstance(booleanResource, + "", 0, ResourceTypes.BOOLEAN, 0, 1)); + + ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); + } + @Test public void testEmptyResourcePBInit() throws Exception { Resource res = new ResourcePBImpl(); @@ -61,6 +78,37 @@ public void testResourcePBInitFromOldPB() throws Exception { .getUnits()); } + @Test + public void testResourcePBInitFromOldPBBooleanResource() throws Exception { + addBooleanResourceToResources("booleanResource"); + + YarnProtos.ResourceProto proto = + YarnProtos.ResourceProto.newBuilder() + .setMemory(1024) + .setVirtualCores(3) + .addResourceValueMap(0, + YarnProtos.ResourceInformationProto.newBuilder() + .setKey("booleanResource") + .setType(YarnProtos.ResourceTypesProto.BOOLEAN) + .setValue(1) + .setUnits("") + .build()) + .build(); + // Assert to check it sets resource value and unit to default. + Resource res = new ResourcePBImpl(proto); + Assert.assertEquals(1024, res.getMemorySize()); + Assert.assertEquals(3, res.getVirtualCores()); + Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(), + res.getResourceInformation(ResourceInformation.MEMORY_MB.getName()) + .getUnits()); + Assert.assertEquals(ResourceInformation.VCORES.getUnits(), + res.getResourceInformation(ResourceInformation.VCORES.getName()) + .getUnits()); + + ResourceInformation booleanResource = res.getResourceInformation(2); + Assert.assertEquals(ResourceTypes.BOOLEAN, booleanResource.getResourceType()); + } + @Test @SuppressWarnings("deprecation") public void testGetMemory() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java index 5f3ed196048..99e9147b37c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java @@ -22,6 +22,7 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Assert; @@ -30,10 +31,13 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.apache.hadoop.yarn.util.resource.ResourceUtils + .MAXIMUM_ALLOCATION; import static org.junit.Assert.assertEquals; @RunWith(Parameterized.class) public class TestResourceCalculator { + private static final String BOOLEAN_RESOURCE_NAME = "booleanResource"; private final ResourceCalculator resourceCalculator; @Parameterized.Parameters @@ -50,20 +54,31 @@ public void setupNoExtraResource() { ResourceUtils.resetResourceTypes(new Configuration()); } - private static void setupExtraResource() { + private static void setupExtraResources() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES, "test"); ResourceUtils.resetResourceTypes(conf); } + private static void setupExtraResources2() { + Configuration conf = new Configuration(); + + conf.set(YarnConfiguration.RESOURCE_TYPES, "test"); + conf.set(YarnConfiguration.RESOURCE_TYPES, BOOLEAN_RESOURCE_NAME); + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + BOOLEAN_RESOURCE_NAME + + ResourceUtils.TYPE, ResourceTypes.BOOLEAN.toString()); + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + BOOLEAN_RESOURCE_NAME + + MAXIMUM_ALLOCATION, "1"); + ResourceUtils.resetResourceTypes(conf); + } + public TestResourceCalculator(ResourceCalculator rs) { this.resourceCalculator = rs; } @Test(timeout = 10000) public void testFitsIn() { - if (resourceCalculator instanceof DefaultResourceCalculator) { Assert.assertTrue(resourceCalculator.fitsIn( Resource.newInstance(1, 2), Resource.newInstance(2, 1))); @@ -89,10 +104,34 @@ public void testFitsIn() { } } - private Resource newResource(long memory, int cpu) { - Resource res = Resource.newInstance(memory, cpu); + @Test + public void testFitsInBooleanResources() { + setupExtraResources2(); + if (resourceCalculator instanceof DefaultResourceCalculator) { + Assert.assertTrue(resourceCalculator.fitsIn( + Resource.newInstance(1, 2), Resource.newInstance(2, 1))); + Assert.assertTrue(resourceCalculator.fitsIn( + Resource.newInstance(1, 2), Resource.newInstance(2, 2))); + Assert.assertTrue(resourceCalculator.fitsIn( + Resource.newInstance(1, 2), Resource.newInstance(1, 2))); + Assert.assertTrue(resourceCalculator.fitsIn( + Resource.newInstance(1, 2), Resource.newInstance(1, 1))); + Assert.assertFalse(resourceCalculator.fitsIn( + Resource.newInstance(2, 1), Resource.newInstance(1, 2))); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + Assert.assertTrue(resourceCalculator.fitsIn( + newResourceWithBoolean(1, 2, false), newResourceWithBoolean(1, 2, true))); + Assert.assertTrue(resourceCalculator.fitsIn( + newResourceWithBoolean(1, 2, false), newResourceWithBoolean(1, 2, false))); + Assert.assertTrue(resourceCalculator.fitsIn( + newResourceWithBoolean(1, 2, true), newResourceWithBoolean(1, 2, true))); + Assert.assertFalse(resourceCalculator.fitsIn( + newResourceWithBoolean(1, 2, true), newResourceWithBoolean(1, 2, false))); + } + } - return res; + private Resource newResource(long memory, int cpu) { + return Resource.newInstance(memory, cpu); } private Resource newResource(long memory, int cpu, int test) { @@ -103,6 +142,12 @@ private Resource newResource(long memory, int cpu, int test) { return res; } + private Resource newResourceWithBoolean(long memory, int cpu, boolean booleanValue) { + Resource res = Resource.newInstance(memory, cpu); + res.setResourceValue(BOOLEAN_RESOURCE_NAME, booleanValue ? 1: 0); + return res; + } + /** * Test that the compare() method returns the expected result (0, -1, or 1). * If the expected result is not 0, this method will also test the resources @@ -115,10 +160,25 @@ private Resource newResource(long memory, int cpu, int test) { */ private void assertComparison(Resource cluster, Resource res1, Resource res2, int expected) { - int actual = resourceCalculator.compare(cluster, res1, res2); + assertComparisonInternal(cluster, res1, res2, expected, false); + } + + private void assertComparisonSingleType(Resource cluster, Resource res1, Resource res2, + int expected) { + assertComparisonInternal(cluster, res1, res2, expected, true); + } + + private void assertComparisonInternal(Resource cluster, Resource res1, + Resource res2, int expected, boolean singleType) { + int actual; + if (singleType) { + actual = resourceCalculator.compare(cluster, res1, res2, singleType); + } else { + actual = resourceCalculator.compare(cluster, res1, res2); + } assertEquals(String.format("Resource comparison did not give the expected " - + "result for %s v/s %s", res1.toString(), res2.toString()), + + "result for %s v/s %s with calculator: %s", res1.toString(), res2.toString(), resourceCalculator.getClass()), expected, actual); if (expected != 0) { @@ -126,7 +186,7 @@ private void assertComparison(Resource cluster, Resource res1, Resource res2, // expected result. actual = resourceCalculator.compare(cluster, res2, res1); assertEquals(String.format("Resource comparison did not give the " - + "expected result for %s v/s %s", res2.toString(), res1.toString()), + + "expected result for %s v/s %s with calculator: %s", res2.toString(), res1.toString(), resourceCalculator.getClass()), expected * -1, actual); } } @@ -151,6 +211,56 @@ public void testCompareWithOnlyMandatory() { } } + @Test + public void testCompareBooleanResources() { + setupExtraResources2(); + // Keep cluster resources even so that the numbers are easy to understand + Resource cluster = newResource(4, 4); + + if (resourceCalculator instanceof DefaultResourceCalculator) { + testCompareDefaultWithOnlyMandatory(cluster); + assertComparison(cluster, newResourceWithBoolean(1, 1, true), newResourceWithBoolean(1, 1, true), 0); + assertComparison(cluster, newResourceWithBoolean(0, 0, true), newResourceWithBoolean(0, 0, true), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, false), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, true), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, true), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, false), 0); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + testCompareDominantWithOnlyMandatory(cluster); + assertComparison(cluster, newResourceWithBoolean(1, 1, true), newResourceWithBoolean(1, 1, true), 0); + assertComparison(cluster, newResourceWithBoolean(0, 0, true), newResourceWithBoolean(0, 0, true), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, false), 1); + assertComparison(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, true), -1); + assertComparison(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, true), 0); + assertComparison(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, false), 0); + } + } + + @Test + public void testCompareBooleanResourcesSingleType() { + setupExtraResources2(); + // Keep cluster resources even so that the numbers are easy to understand + Resource cluster = newResource(4, 4); + + if (resourceCalculator instanceof DefaultResourceCalculator) { + testCompareDefaultWithOnlyMandatory(cluster); + assertComparisonSingleType(cluster, newResourceWithBoolean(1, 1, true), newResourceWithBoolean(1, 1, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(0, 0, true), newResourceWithBoolean(0, 0, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, false), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, false), 0); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + testCompareDominantWithOnlyMandatory(cluster); + assertComparisonSingleType(cluster, newResourceWithBoolean(1, 1, true), newResourceWithBoolean(1, 1, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(0, 0, true), newResourceWithBoolean(0, 0, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, false), 1); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, true), -1); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, true), newResourceWithBoolean(2, 2, true), 0); + assertComparisonSingleType(cluster, newResourceWithBoolean(2, 2, false), newResourceWithBoolean(2, 2, false), 0); + } + } + private void testCompareDefaultWithOnlyMandatory(Resource cluster) { assertComparison(cluster, newResource(1, 1), newResource(1, 1), 0); assertComparison(cluster, newResource(1, 2), newResource(1, 1), 0); @@ -172,7 +282,7 @@ private void testCompareDominantWithOnlyMandatory(Resource cluster) { @Test public void testCompare() { // Test with 3 resources - setupExtraResource(); + setupExtraResources(); // Keep cluster resources even so that the numbers are easy to understand Resource cluster = newResource(4L, 4, 4); @@ -392,4 +502,76 @@ public void testNormalize() { assertEquals(2, result.getVirtualCores()); } } + + @Test + public void testInvalidDivisorBooleanResource() { + setupExtraResources2(); + if (resourceCalculator instanceof DefaultResourceCalculator) { + assertEquals(false, resourceCalculator.isInvalidDivisor( + newResourceWithBoolean(1, 2, false))); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + assertEquals(true, resourceCalculator.isInvalidDivisor( + newResourceWithBoolean(1, 2, false))); + } + } + + @Test + public void testRatioBooleanResource() { + setupExtraResources2(); + + // true/false --> 1/0, division by zero would produce an infinite result if boolean resources were considered + assertEquals(1, (int)resourceCalculator.ratio(newResourceWithBoolean(4, 4, true), newResourceWithBoolean(4, 4, false))); + + assertEquals(1, (int)resourceCalculator.ratio(newResourceWithBoolean(4, 4, false), newResourceWithBoolean(4, 4, false))); + assertEquals(1, (int)resourceCalculator.ratio(newResourceWithBoolean(4, 4, true), newResourceWithBoolean(4, 4, true))); + assertEquals(1, (int)resourceCalculator.ratio(newResourceWithBoolean(4, 4, false), newResourceWithBoolean(4, 4, true))); + } + + @Test + public void testDivideAndCeilBooleanResource() { + setupExtraResources2(); + + Resource resource = resourceCalculator.divideAndCeil(newResourceWithBoolean(4, 4, true), 5); + + if (resourceCalculator instanceof DefaultResourceCalculator) { + assertEquals(0L, resource.getResourceInformation(BOOLEAN_RESOURCE_NAME).getValue()); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + assertEquals(1L, resource.getResourceInformation(BOOLEAN_RESOURCE_NAME).getValue()); + } + } + + @Test + public void testComputeAvailableContainers() { + setupExtraResources2(); + + Resource available = newResource(1024, 2); + Resource required = newResource(1024, 2); + assertEquals(1, resourceCalculator.computeAvailableContainers(available, required)); + + available = newResource(1024, 2); + required = newResource(2048, 4); + assertEquals(0, resourceCalculator.computeAvailableContainers(available, required)); + + available = newResource(4096, 8); + required = newResource(1024, 2); + assertEquals(4, resourceCalculator.computeAvailableContainers(available, required)); + } + + @Test + public void testComputeAvailableContainerBoolean() { + setupExtraResources2(); + + Resource available = newResourceWithBoolean(1024, 2, false); + Resource required = newResourceWithBoolean(512, 1, true); + + if (resourceCalculator instanceof DefaultResourceCalculator) { + assertEquals(2, resourceCalculator.computeAvailableContainers(available, required)); + } else if (resourceCalculator instanceof DominantResourceCalculator) { + assertEquals(0, resourceCalculator.computeAvailableContainers(available, required)); + } + + available = newResourceWithBoolean(1024, 2, true); + required = newResourceWithBoolean(512, 1, false); + assertEquals(2, resourceCalculator.computeAvailableContainers(available, required)); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 2671de85057..686e27aacc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -18,11 +18,15 @@ package org.apache.hadoop.yarn.util.resource; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.junit.After; @@ -31,8 +35,13 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Test class to verify all resource utility methods. @@ -101,9 +110,72 @@ private void testMemoryAndVcores(Map res) { res.get(vcores).getResourceType()); } - @Test - public void testGetResourceTypes() throws Exception { + private File overwriteResourceTypesXml(Configuration conf, + String resourceFile) throws IOException { + File source = new File(conf.getClassLoader() + .getResource(resourceFile) + .getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + return dest; + } + + private void assertBooleanResource(Map + resourceTypes) { + Assert.assertEquals(3, resourceTypes.size()); + ResourceInformation booleanResource = + resourceTypes.get("booleanResource"); + Assert.assertNotNull(booleanResource); + Assert.assertEquals(ResourceTypes.BOOLEAN, + booleanResource.getResourceType()); + Assert.assertEquals(0L, booleanResource.getValue()); + Assert.assertEquals("", booleanResource.getUnits()); + Assert.assertEquals(0, booleanResource.getMinimumAllocation()); + Assert.assertEquals(1, booleanResource.getMaximumAllocation()); + } + + private String generateResourceNamesString(String[][] test) { + String resSt = ""; + for (String[] resources : test) { + resSt += (resources[0] + ","); + } + resSt = resSt.substring(0, resSt.length() - 1); + return resSt; + } + + private String generateResourceNamesString(List resourceNames) { + return resourceNames.stream().collect(Collectors.joining(",")); + } + + private Map generateResourceUnitsConfigs(List resourceNames, Map units) { + return resourceNames.stream().collect(Collectors.toMap( + res -> YarnConfiguration.RESOURCE_TYPES + "." + res + ".units", + units::get)); + } + + public static String setupResourceTypes(Configuration conf, String filename) + throws Exception { + File source = new File( + conf.getClassLoader().getResource(filename).getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + FileUtils.copyFile(source, dest); + ResourceUtils.getResourceTypes(); + return dest.getAbsolutePath(); + } + + private void assertBooleanResource(ResourceInformation ri, String + resourceName, boolean value) { + Assert.assertEquals(resourceName, ri.getName()); + Assert.assertEquals(value ? 1L : 0L, ri.getValue()); + Assert.assertEquals("", ri.getUnits()); + Assert.assertEquals(0L, ri.getMinimumAllocation()); + Assert.assertEquals(1L, ri.getMaximumAllocation()); + Assert.assertEquals(ResourceTypes.BOOLEAN, ri.getResourceType()); + } + + @Test + public void testGetResourceTypes() { Map res = ResourceUtils.getResourceTypes(); Assert.assertEquals(2, res.size()); testMemoryAndVcores(res); @@ -111,7 +183,6 @@ public void testGetResourceTypes() throws Exception { @Test public void testGetResourceTypesConfigs() throws Exception { - Configuration conf = new YarnConfiguration(); ResourceFileInformation testFile1 = @@ -153,7 +224,7 @@ public void testGetResourceTypesConfigs() throws Exception { } @Test - public void testGetResourceTypesConfigErrors() throws Exception { + public void testGetResourceTypesConfigErrors() { Configuration conf = new YarnConfiguration(); String[] resourceFiles = {"resource-types-error-1.xml", @@ -163,10 +234,7 @@ public void testGetResourceTypesConfigErrors() throws Exception { ResourceUtils.resetResourceTypes(); File dest = null; try { - File source = - new File(conf.getClassLoader().getResource(resourceFile).getFile()); - dest = new File(source.getParent(), "resource-types.xml"); - FileUtils.copyFile(source, dest); + dest = overwriteResourceTypesXml(conf, resourceFile); ResourceUtils.getResourceTypes(); Assert.fail("Expected error with file " + resourceFile); } catch (NullPointerException ne) { @@ -179,6 +247,103 @@ public void testGetResourceTypesConfigErrors() throws Exception { } } + @Test + public void testGetResourceTypesBooleanResourceInvalidUnits() throws IOException { + Configuration conf = new YarnConfiguration(); + + ResourceUtils.resetResourceTypes(); + File dest = null; + try { + String resourceFile = "resource-types-boolean-error-invalid-units.xml"; + dest = overwriteResourceTypesXml(conf, resourceFile); + ResourceUtils.getResourceTypes(); + Assert.fail("Expected error with file " + resourceFile); + } catch (IllegalArgumentException e) { + String message = e.getMessage(); + Assert.assertEquals("Invalid unit for boolean resource " + + "'booleanResource': m, should be \"\"!", message); + if (dest != null) { + dest.delete(); + } + } + } + + @Test + public void testGetResourceTypesBooleanResourceInvalidMinimum() throws IOException { + Configuration conf = new YarnConfiguration(); + + ResourceUtils.resetResourceTypes(); + File dest = null; + try { + String resourceFile = "resource-types-boolean-error-invalid-minimum.xml"; + dest = overwriteResourceTypesXml(conf, resourceFile); + ResourceUtils.getResourceTypes(); + Assert.fail("Expected error with file " + resourceFile); + } catch (IllegalArgumentException e) { + String message = e.getMessage(); + Assert.assertEquals("Invalid minimum allocation for boolean resource 'booleanResource': 1, should be 0!", message); + if (dest != null) { + dest.delete(); + } + } + } + + @Test + public void testGetResourceTypesBooleanResourceInvalidMaximum() throws IOException { + Configuration conf = new YarnConfiguration(); + + ResourceUtils.resetResourceTypes(); + File dest = null; + try { + String resourceFile = "resource-types-boolean-error-invalid-maximum.xml"; + dest = overwriteResourceTypesXml(conf, resourceFile); + ResourceUtils.getResourceTypes(); + Assert.fail("Expected error with file " + resourceFile); + } catch (IllegalArgumentException e) { + String message = e.getMessage(); + Assert.assertEquals("Invalid maximum allocation for boolean resource 'booleanResource': 2, should be 1!", message); + if (dest != null) { + dest.delete(); + } + } + } + + @Test + public void testGetResourceTypesBooleanResourceValid1() throws IOException { + Configuration conf = new YarnConfiguration(); + + File dest = null; + try { + ResourceUtils.resetResourceTypes(); + String resourceFile = "resource-types-boolean-valid-1.xml"; + dest = overwriteResourceTypesXml(conf, resourceFile); + Map resourceTypes = + ResourceUtils.getResourceTypes(); + + assertBooleanResource(resourceTypes); + } finally { + dest.delete(); + } + } + + @Test + public void testGetResourceTypesBooleanResourceValid2() throws IOException { + Configuration conf = new YarnConfiguration(); + + File dest = null; + try { + ResourceUtils.resetResourceTypes(); + String resourceFile = "resource-types-boolean-valid-2.xml"; + dest = overwriteResourceTypesXml(conf, resourceFile); + Map resourceTypes = + ResourceUtils.getResourceTypes(); + + assertBooleanResource(resourceTypes); + } finally { + dest.delete(); + } + } + @Test public void testInitializeResourcesMap() throws Exception { String[] empty = {"", ""}; @@ -192,7 +357,6 @@ public void testInitializeResourcesMap() throws Exception { String[][][] allTests = {test1, test2, test3, test4}; for (String[][] test : allTests) { - Configuration conf = new YarnConfiguration(); String resSt = ""; for (String[] resources : test) { @@ -202,11 +366,11 @@ public void testInitializeResourcesMap() throws Exception { conf.set(YarnConfiguration.RESOURCE_TYPES, resSt); for (String[] resources : test) { String name = - YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units"; + YarnConfiguration.RESOURCE_TYPES + "." + resources[0] + ".units"; conf.set(name, resources[1]); } Map ret = - ResourceUtils.resetResourceTypes(conf); + ResourceUtils.resetResourceTypes(conf); // for test1, 4 - length will be 1, 4 // for the others, len will be 3 @@ -236,12 +400,70 @@ public void testInitializeResourcesMap() throws Exception { ResourceInformation vcoresInfo = ret.get("vcores"); Assert.assertEquals("", vcoresInfo.getUnits()); Assert - .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType()); + .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType()); } } @Test - public void testInitializeResourcesMapErrors() throws Exception { + public void testInitializeResourcesMapBooleanResource() { + String booleanResource1 = "booleanResource1"; + String booleanResource2 = "booleanResource2"; + Set booleanResourceNames = ImmutableSet.of(booleanResource1, booleanResource2); + + Map units = ImmutableMap.of("resource1", "m", "resource2", "G", booleanResource1, "", booleanResource2, ""); + List resourceTypes = ImmutableList.of("resource1", "resource2", booleanResource1, booleanResource2); + + Configuration conf = new YarnConfiguration(); + //set resource names in config + String resourceNames = generateResourceNamesString(resourceTypes); + conf.set(YarnConfiguration.RESOURCE_TYPES, resourceNames); + + //set units in config + Map resourceUnitConfigs = generateResourceUnitsConfigs(resourceTypes, units); + resourceUnitConfigs.forEach(conf::set); + + //set boolean resource type in config + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource1 + ResourceUtils.TYPE, + ResourceTypes.BOOLEAN.toString()); + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource2 + ResourceUtils.TYPE, + "boolean"); + + + Map resources = + ResourceUtils.resetResourceTypes(conf); + + Assert.assertEquals(6, resources.size()); + + for (String resourceName: resourceTypes) { + Assert.assertTrue(resources.containsKey(resourceName)); + ResourceInformation resInfo = resources.get(resourceName); + Assert.assertEquals(units.get(resourceName), resInfo.getUnits()); + + if (booleanResourceNames.contains(resourceName)) { + Assert.assertEquals(ResourceTypes.BOOLEAN, resInfo.getResourceType()); + Assert.assertEquals(0L, resInfo.getValue()); + Assert.assertEquals(0L, resInfo.getMinimumAllocation()); + Assert.assertEquals(1L, resInfo.getMaximumAllocation()); + } else { + Assert.assertEquals(ResourceTypes.COUNTABLE, resInfo.getResourceType()); + } + } + // we must always have memory and vcores with their fixed units + Assert.assertTrue(resources.containsKey("memory-mb")); + ResourceInformation memInfo = resources.get("memory-mb"); + Assert.assertEquals("Mi", memInfo.getUnits()); + Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType()); + Assert.assertTrue(resources.containsKey("vcores")); + ResourceInformation vcoresInfo = resources.get("vcores"); + Assert.assertEquals("", vcoresInfo.getUnits()); + Assert + .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType()); + } + + @Test + public void testInitializeResourcesMapErrors() { String[] mem1 = {"memory-mb", ""}; String[] vcores1 = {"vcores", "M"}; @@ -260,11 +482,7 @@ public void testInitializeResourcesMapErrors() throws Exception { for (String[][] test : allTests) { Configuration conf = new YarnConfiguration(); - String resSt = ""; - for (String[] resources : test) { - resSt += (resources[0] + ","); - } - resSt = resSt.substring(0, resSt.length() - 1); + String resSt = generateResourceNamesString(test); conf.set(YarnConfiguration.RESOURCE_TYPES, resSt); for (String[] resources : test) { String name = @@ -281,12 +499,132 @@ public void testInitializeResourcesMapErrors() throws Exception { } @Test - public void testGetResourceInformation() throws Exception { + public void testInitializeResourcesMapBooleanResourceErrorInvalidUnit() { + String booleanResource = "booleanResource"; + Map units = ImmutableMap.of(booleanResource, "G"); + List resourceTypes = ImmutableList.of(booleanResource); + + Configuration conf = new YarnConfiguration(); + //set resource names in config + String resourceNames = generateResourceNamesString(resourceTypes); + conf.set(YarnConfiguration.RESOURCE_TYPES, resourceNames); + + //set units in config + Map resourceUnitConfigs = generateResourceUnitsConfigs(resourceTypes, units); + resourceUnitConfigs.forEach(conf::set); + + //set boolean resource type in config + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + ResourceUtils.TYPE, + ResourceTypes.BOOLEAN.toString()); + + try { + ResourceUtils.initializeResourcesMap(conf); + Assert.fail("resource map initialization should fail"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Invalid unit for boolean resource " + + "'booleanResource': G, should be \"\"!", e.getMessage()); + } + } + + @Test + public void testInitializeResourcesMapBooleanResourceErrorInvalidMinimum() { + String booleanResource = "booleanResource"; + Map units = ImmutableMap.of(booleanResource, ""); + List resourceTypes = ImmutableList.of(booleanResource); + + Configuration conf = new YarnConfiguration(); + //set resource names in config + String resourceNames = generateResourceNamesString(resourceTypes); + conf.set(YarnConfiguration.RESOURCE_TYPES, resourceNames); + + //set units in config + Map resourceUnitConfigs = generateResourceUnitsConfigs(resourceTypes, units); + resourceUnitConfigs.forEach(conf::set); + + //set boolean resource type in config + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + ResourceUtils.TYPE, + ResourceTypes.BOOLEAN.toString()); + + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + + ResourceUtils.MINIMUM_ALLOCATION, "11"); + + try { + ResourceUtils.initializeResourcesMap(conf); + Assert.fail("resource map initialization should fail"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Invalid minimum allocation for boolean resource 'booleanResource': 11, should be 0!", e.getMessage()); + } + } + + @Test + public void testInitializeResourcesMapBooleanResourceErrorInvalidMaximum() { + String booleanResource = "booleanResource"; + Map units = ImmutableMap.of(booleanResource, ""); + List resourceTypes = ImmutableList.of(booleanResource); + + Configuration conf = new YarnConfiguration(); + //set resource names in config + String resourceNames = generateResourceNamesString(resourceTypes); + conf.set(YarnConfiguration.RESOURCE_TYPES, resourceNames); + + //set units in config + Map resourceUnitConfigs = generateResourceUnitsConfigs(resourceTypes, units); + resourceUnitConfigs.forEach(conf::set); + + //set boolean resource type in config + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + ResourceUtils.TYPE, + ResourceTypes.BOOLEAN.toString()); + + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + + ResourceUtils.MAXIMUM_ALLOCATION, "11"); + + try { + ResourceUtils.initializeResourcesMap(conf); + Assert.fail("resource map initialization should fail"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Invalid maximum allocation for boolean resource 'booleanResource': 11, should be 1!", e.getMessage()); + } + } + + @Test + public void testInitializeResourcesMapBooleanResourceErrorInvalidTypeValue() { + String booleanResource = "booleanResource"; + Map units = ImmutableMap.of(booleanResource, ""); + List resourceTypes = ImmutableList.of(booleanResource); + + Configuration conf = new YarnConfiguration(); + //set resource names in config + String resourceNames = generateResourceNamesString(resourceTypes); + conf.set(YarnConfiguration.RESOURCE_TYPES, resourceNames); + + //set units in config + Map resourceUnitConfigs = generateResourceUnitsConfigs(resourceTypes, units); + resourceUnitConfigs.forEach(conf::set); + + //set boolean resource type in config + conf.set( + YarnConfiguration.RESOURCE_TYPES + "." + booleanResource + ResourceUtils.TYPE, + "BOOLEE"); + + try { + ResourceUtils.initializeResourcesMap(conf); + Assert.fail("resource map initialization should fail"); + } catch (YarnRuntimeException e) { + Assert.assertEquals("Invalid configuration for resource type " + + "'booleanResource'. Value of type should be one of: " + + "[COUNTABLE, BOOLEAN]", e.getMessage()); + } + } + @Test + public void testGetNodeResourceInformation() throws Exception { Configuration conf = new YarnConfiguration(); Map testRun = new HashMap<>(); setupResourceTypes(conf, "resource-types-4.xml"); - // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); + Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); @@ -299,10 +637,9 @@ public void testGetResourceInformation() throws Exception { for (Map.Entry entry : testRun.entrySet()) { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); - File dest; File source = new File( conf.getClassLoader().getResource(resourceFile).getFile()); - dest = new File(source.getParent(), "node-resources.xml"); + File dest = new File(source.getParent(), "node-resources.xml"); FileUtils.copyFile(source, dest); Map actual = ResourceUtils .getNodeResourceInformation(conf); @@ -316,9 +653,125 @@ public void testGetResourceInformation() throws Exception { } @Test - public void testGetNodeResourcesConfigErrors() throws Exception { + public void testGetNodeResourceInformationBooleanResourceNumericValue() throws Exception { + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-boolean-1.xml"); + + Resource test3Resources = Resource.newInstance(0, 0); + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "Gi", 5L)); + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2L)); + test3Resources.setResourceInformation("yarn.io/gpu", + ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + test3Resources.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); + testRun.put("node-resources-boolean-1.xml", test3Resources); + + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File source = new File( + conf.getClassLoader().getResource(resourceFile).getFile()); + File dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = ResourceUtils + .getNodeResourceInformation(conf); + Assert.assertEquals(actual.size(), + entry.getValue().getResources().length); + for (ResourceInformation resInfo : entry.getValue().getResources()) { + Assert.assertEquals(resInfo, actual.get(resInfo.getName())); + } + dest.delete(); + } + } + + @Test + public void testGetNodeResourceInformationBooleanResourceStringValue() throws Exception { + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-boolean-1.xml"); + + Resource test3Resources = Resource.newInstance(0, 0); + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "Gi", 5L)); + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2L)); + test3Resources.setResourceInformation("yarn.io/gpu", + ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + test3Resources.setResourceInformation("booleanResource", + ResourceInformation.newBooleanInstance("booleanResource", true)); + testRun.put("node-resources-boolean-2.xml", test3Resources); + + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File source = new File( + conf.getClassLoader().getResource(resourceFile).getFile()); + File dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = ResourceUtils + .getNodeResourceInformation(conf); + Assert.assertEquals(actual.size(), + entry.getValue().getResources().length); + for (ResourceInformation resInfo : entry.getValue().getResources()) { + Assert.assertEquals(resInfo, actual.get(resInfo.getName())); + } + dest.delete(); + } + } + + @Test + public void testGetNodeResourceInformationBooleanResourceInvalidValue() throws Exception { Configuration conf = new YarnConfiguration(); Map testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-boolean-1.xml"); + + Resource test3Resources = Resource.newInstance(0, 0); + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "Gi", 5L)); + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2L)); + test3Resources.setResourceInformation("yarn.io/gpu", + ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + test3Resources.setResourceInformation("booleanResource", ResourceInformation.newBooleanInstance("booleanResource", true)); + testRun.put("node-resources-boolean-error.xml", test3Resources); + + try { + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File source = new File( + conf.getClassLoader().getResource(resourceFile).getFile()); + File dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = ResourceUtils + .getNodeResourceInformation(conf); + Assert.assertEquals(actual.size(), + entry.getValue().getResources().length); + for (ResourceInformation resInfo : entry.getValue().getResources()) { + Assert.assertEquals(resInfo, actual.get(resInfo.getName())); + } + dest.delete(); + } + Assert.fail("Should throw exception for invalid boolean resource value"); + } catch (YarnRuntimeException e) { + Throwable cause = e.getCause(); + Assert.assertTrue( + "Cause of YarnRuntimeException is not of type " + + "IllegalArgumentException!", + cause instanceof IllegalArgumentException); + Assert.assertEquals( + "Invalid value for boolean resource 'booleanResource': 2, " + + "valid values are 0 or 1!", + cause.getMessage()); + } + } + + @Test + public void testGetNodeResourcesConfigErrors() throws Exception { + Configuration conf = new YarnConfiguration(); setupResourceTypes(conf, "resource-types-4.xml"); String invalidNodeResFiles[] = { "node-resources-error-1.xml"}; @@ -342,7 +795,7 @@ public void testGetNodeResourcesConfigErrors() throws Exception { } @Test - public void testResourceNameFormatValidation() throws Exception { + public void testResourceNameFormatValidation() { String[] validNames = new String[] { "yarn.io/gpu", "gpu", @@ -377,13 +830,81 @@ public void testResourceNameFormatValidation() throws Exception { } } - public static String setupResourceTypes(Configuration conf, String filename) - throws Exception { - File source = new File( - conf.getClassLoader().getResource(filename).getFile()); - File dest = new File(source.getParent(), "resource-types.xml"); - FileUtils.copyFile(source, dest); - ResourceUtils.getResourceTypes(); - return dest.getAbsolutePath(); + @Test + public void testReinitializeResources() { + List resourceTypeInfos = new ArrayList<>( + ResourceUtils.getResourcesTypeInfo()); + resourceTypeInfos.add(ResourceTypeInfo + .newInstance("resource-1", "", ResourceTypes.COUNTABLE)); + resourceTypeInfos.add(ResourceTypeInfo + .newInstance("resource-2", "", ResourceTypes.BOOLEAN)); + // Reinitialize resource types + ResourceUtils.reinitializeResources(resourceTypeInfos); + Map resourceTypes = ResourceUtils + .getResourceTypes(); + Assert.assertEquals("Resource types size should be 4!", 4, resourceTypes.size()); + + ResourceInformation customResource1 = resourceTypes.get("resource-1"); + Assert.assertEquals("resource-1 type should be COUNTABLE", ResourceTypes.COUNTABLE, customResource1.getResourceType()); + + ResourceInformation customResource2 = resourceTypes.get("resource-2"); + Assert.assertEquals("resource-2 type should be BOOLEAN", ResourceTypes.BOOLEAN, customResource2.getResourceType()); } + + @Test + public void testGetRequestedResourcesFromConfig() { + Configuration conf = new Configuration(); + String prefix = "yarn.app.mapreduce.am.resource."; + conf.set(prefix + "resource1", "100k"); + + List resources = ResourceUtils + .getRequestedResourcesFromConfig(conf, prefix); + + Assert.assertEquals(1, resources.size()); + ResourceInformation resourceInfo = resources.get(0); + Assert.assertEquals("resource1", resourceInfo.getName()); + Assert.assertEquals(100L, resourceInfo.getValue()); + Assert.assertEquals("k", resourceInfo.getUnits()); + Assert.assertEquals(0L, resourceInfo.getMinimumAllocation()); + Assert.assertEquals(0L, resourceInfo.getMaximumAllocation()); + Assert.assertEquals(ResourceTypes.COUNTABLE, resourceInfo.getResourceType()); + } + + @Test + public void testGetRequestedResourcesFromConfigInvalidBooleanResource() { + Configuration conf = new Configuration(); + String prefix = "yarn.app.mapreduce.am.resource."; + String resourceValue = "fals"; + conf.set(prefix + "resource1", resourceValue); + + try { + ResourceUtils.getRequestedResourcesFromConfig(conf, prefix); + Assert.fail("Expected exception because specified resource value is invalid: " + resourceValue); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Invalid resource request specified for property " + + "yarn.app.mapreduce.am.resource.resource1: \"fals\", expected " + + "format is: value[ ][units]", e.getMessage()); + } + } + + @Test + public void testGetRequestedResourcesFromConfigValidBooleanResources() { + Configuration conf = new Configuration(); + String prefix = "yarn.app.mapreduce.am.resource."; + conf.set(prefix + "resource1", "false"); + conf.set(prefix + "resource2", "true"); + conf.set(prefix + "resource3", "FALSe"); + conf.set(prefix + "resource4", "TRUe"); + + List resources = + ResourceUtils.getRequestedResourcesFromConfig(conf, prefix); + + Assert.assertEquals(4, resources.size()); + + assertBooleanResource(resources.get(0), "resource1", false); + assertBooleanResource(resources.get(1), "resource2", true); + assertBooleanResource(resources.get(2), "resource3", false); + assertBooleanResource(resources.get(3), "resource4", true); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index a8404fbaee7..97f7a95f71a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -28,9 +29,12 @@ import java.io.File; +import static org.apache.hadoop.yarn.util.resource.ResourceUtils + .MAXIMUM_ALLOCATION; import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMin; import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMax; import static org.apache.hadoop.yarn.util.resource.Resources.add; +import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundUp; import static org.apache.hadoop.yarn.util.resource.Resources.subtract; import static org.apache.hadoop.yarn.util.resource.Resources.multiply; import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndAddTo; @@ -61,6 +65,18 @@ private void setupExtraResourceType() throws Exception { TestResourceUtils.setupResourceTypes(conf, "resource-types-3.xml"); } + private static void setupExtraResourcesWithBooleanResource() { + Configuration conf = new Configuration(); + + conf.set(YarnConfiguration.RESOURCE_TYPES, BOOLEAN_RESOURCE_NAME + "," + "resource2"); + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + BOOLEAN_RESOURCE_NAME + + ResourceUtils.TYPE, ResourceTypes.BOOLEAN.toString()); + conf.set(YarnConfiguration.RESOURCE_TYPES + "." + BOOLEAN_RESOURCE_NAME + + MAXIMUM_ALLOCATION, "1"); + ResourceUtils.resetResourceTypes(conf); + } + private static final String BOOLEAN_RESOURCE_NAME = "booleanResource"; + private void unsetExtraResourceType() { deleteResourceTypesFile(); ResourceUtils.resetResourceTypes(); @@ -94,6 +110,18 @@ public Resource createResource(long memory, int vCores, long resource2) { return ret; } + private Resource createResourceWithBoolean(long memory, int vCores, boolean booleanValue) { + Resource res = Resource.newInstance(memory, vCores); + res.setResourceValue(BOOLEAN_RESOURCE_NAME, booleanValue ? 1: 0); + return res; + } + + private Resource createResourceWithBoolean(long memory, int vCores, long resource2, boolean booleanValue) { + Resource res = createResource(memory, vCores, resource2); + res.setResourceValue(BOOLEAN_RESOURCE_NAME, booleanValue ? 1: 0); + return res; + } + @Test(timeout = 10000) public void testCompareToWithUnboundedResource() { unsetExtraResourceType(); @@ -149,6 +177,19 @@ public void testFitsIn() { assertTrue(fitsIn(createResource(1, 1, 1), createResource(2, 2, 2))); } + @Test + public void testFitsInBooleanResources() { + setupExtraResourcesWithBooleanResource(); + assertTrue(fitsIn( + createResourceWithBoolean(1, 2, false), createResourceWithBoolean(1, 2, true))); + assertTrue(fitsIn( + createResourceWithBoolean(1, 2, false), createResourceWithBoolean(1, 2, false))); + assertTrue(fitsIn( + createResourceWithBoolean(1, 2, true), createResourceWithBoolean(1, 2, true))); + assertFalse(fitsIn( + createResourceWithBoolean(1, 2, true), createResourceWithBoolean(1, 2, false))); + } + @Test(timeout = 1000) public void testComponentwiseMin() { assertEquals(createResource(1, 1), @@ -165,6 +206,23 @@ public void testComponentwiseMin() { componentwiseMin(createResource(1, 2, 2), createResource(2, 1, 3))); } + @Test + public void testComponentwiseMinBooleanResources() { + setupExtraResourcesWithBooleanResource(); + assertEquals(createResourceWithBoolean(1, 1, false), + componentwiseMin(createResourceWithBoolean(1, 1, false), createResourceWithBoolean(2, 2, true))); + assertEquals(createResourceWithBoolean(1, 1, false), + componentwiseMin(createResourceWithBoolean(2, 2, true), createResourceWithBoolean(1, 1, false))); + assertEquals(createResourceWithBoolean(1, 1, false), + componentwiseMin(createResourceWithBoolean(1, 2, true), createResourceWithBoolean(2, 1, false))); + assertEquals(createResourceWithBoolean(1, 1, 1, true), + componentwiseMin(createResourceWithBoolean(1, 1, 1, true), createResourceWithBoolean(2, 2, 2, true))); + assertEquals(createResourceWithBoolean(1, 1, 0, true), + componentwiseMin(createResourceWithBoolean(2, 2, 2, true), createResourceWithBoolean(1, 1, true))); + assertEquals(createResourceWithBoolean(1, 1, 2, true), + componentwiseMin(createResourceWithBoolean(1, 2, 2, true), createResourceWithBoolean(2, 1, 3, true))); + } + @Test public void testComponentwiseMax() { assertEquals(createResource(2, 2), @@ -183,6 +241,26 @@ public void testComponentwiseMax() { componentwiseMax(createResource(2, 2, 0), createResource(2, 1, 1))); } + @Test + public void testComponentwiseMaxBooleanResources() { + setupExtraResourcesWithBooleanResource(); + + assertEquals(createResourceWithBoolean(2, 2, true), + componentwiseMax(createResourceWithBoolean(1, 1, true), createResourceWithBoolean(2, 2, true))); + assertEquals(createResourceWithBoolean(2, 2, true), + componentwiseMax(createResourceWithBoolean(2, 2, true), createResourceWithBoolean(1, 1, false))); + assertEquals(createResourceWithBoolean(2, 2, true), + componentwiseMax(createResourceWithBoolean(1, 2, false), createResourceWithBoolean(2, 1, true))); + assertEquals(createResource(2, 2, 2), + componentwiseMax(createResourceWithBoolean(1, 1, 1, false), createResourceWithBoolean(2, 2, 2, false))); + assertEquals(createResourceWithBoolean(2, 2, 2, true), + componentwiseMax(createResourceWithBoolean(2, 2, 2, true), createResourceWithBoolean(1, 1, false))); + assertEquals(createResourceWithBoolean(2, 2, 3, true), + componentwiseMax(createResourceWithBoolean(1, 2, 2, false), createResourceWithBoolean(2, 1, 3, true))); + assertEquals(createResourceWithBoolean(2, 2, 1, true), + componentwiseMax(createResourceWithBoolean(2, 2, 0, true), createResourceWithBoolean(2, 1, 1, true))); + } + @Test public void testAdd() { assertEquals(createResource(2, 3), @@ -195,6 +273,19 @@ public void testAdd() { add(createResource(1, 1, 1), createResource(1, 1, 2))); } + @Test + public void testAddBooleanResources() { + setupExtraResourcesWithBooleanResource(); + assertEquals(createResource(2, 3), + add(createResource(1, 1), createResourceWithBoolean(1, 2, true))); + assertEquals(createResource(3, 2), + add(createResource(1, 1), createResourceWithBoolean(2, 1, true))); + assertEquals(createResource(2, 2, 0), + add(createResource(1, 1, 0), createResourceWithBoolean(1, 1, 0, true))); + assertEquals(createResource(2, 2, 3), + add(createResource(1, 1, 1), createResourceWithBoolean(1, 1, 2, true))); + } + @Test public void testSubtract() { assertEquals(createResource(1, 0), @@ -207,6 +298,19 @@ public void testSubtract() { subtract(createResource(2, 2, 3), createResource(1, 1, 1))); } + @Test + public void testSubtractBooleanResources() { + setupExtraResourcesWithBooleanResource(); + assertEquals(createResource(1, 0), + subtract(createResource(2, 1), createResourceWithBoolean(1, 1, true))); + assertEquals(createResource(0, 1), + subtract(createResource(1, 2), createResourceWithBoolean(1, 1, true))); + assertEquals(createResource(2, 2, 0), + subtract(createResource(3, 3, 0), createResourceWithBoolean(1, 1, 0, true))); + assertEquals(createResource(1, 1, 2), + subtract(createResource(2, 2, 3), createResourceWithBoolean(1, 1, 1, true))); + } + @Test public void testClone() { assertEquals(createResource(1, 1), Resources.clone(createResource(1, 1))); @@ -228,6 +332,24 @@ public void testMultiply() { assertEquals(createResource(4, 4, 6), multiply(createResource(2, 2, 3), 2)); } + @Test + public void testMultiplyBooleanResources() { + setupExtraResourcesWithBooleanResource(); + assertEquals(createResource(4, 2), multiply(createResourceWithBoolean(2, 1, false), 2)); + assertEquals(createResource(4, 2, 0), multiply(createResourceWithBoolean(2, 1, false), 2)); + assertEquals(createResource(2, 4), multiply(createResourceWithBoolean(1, 2, false), 2)); + assertEquals(createResource(2, 4, 0), multiply(createResourceWithBoolean(1, 2, false), 2)); + assertEquals(createResource(6, 6, 0), multiply(createResourceWithBoolean(3, 3, 0, false), 2)); + assertEquals(createResource(4, 4, 6), multiply(createResourceWithBoolean(2, 2, 3, false), 2)); + + assertEquals(createResourceWithBoolean(4, 2, true), multiply(createResourceWithBoolean(2, 1, true), 2)); + assertEquals(createResourceWithBoolean(4, 2, 0, true), multiply(createResourceWithBoolean(2, 1, true), 2)); + assertEquals(createResourceWithBoolean(2, 4, true), multiply(createResourceWithBoolean(1, 2, true), 2)); + assertEquals(createResourceWithBoolean(2, 4, 0, true), multiply(createResourceWithBoolean(1, 2, true), 2)); + assertEquals(createResourceWithBoolean(6, 6, 0, true), multiply(createResourceWithBoolean(3, 3, 0, true), 2)); + assertEquals(createResourceWithBoolean(4, 4, 6, true), multiply(createResourceWithBoolean(2, 2, 3, true), 2)); + } + @Test public void testMultiplyAndRoundDown() { assertEquals(createResource(4, 1), @@ -244,6 +366,84 @@ public void testMultiplyAndRoundDown() { multiplyAndRoundDown(createResource(1, 1, 3), 2.5)); } + @Test + public void testMultiplyAndRoundDownBooleanResources() { + setupExtraResourcesWithBooleanResource(); + + assertEquals(createResource(4, 1), + multiplyAndRoundDown(createResourceWithBoolean(3, 1, false), 1.5)); + assertEquals(createResource(4, 1, 0), + multiplyAndRoundDown(createResourceWithBoolean(3, 1, false), 1.5)); + assertEquals(createResource(1, 4), + multiplyAndRoundDown(createResourceWithBoolean(1, 3, false), 1.5)); + assertEquals(createResource(1, 4, 0), + multiplyAndRoundDown(createResourceWithBoolean(1, 3, false), 1.5)); + assertEquals(createResource(7, 7, 0), + multiplyAndRoundDown(createResourceWithBoolean(3, 3, 0, false), 2.5)); + assertEquals(createResource(2, 2, 7), + multiplyAndRoundDown(createResourceWithBoolean(1, 1, 3, false), 2.5)); + + assertEquals(createResourceWithBoolean(4, 1, true), + multiplyAndRoundDown(createResourceWithBoolean(3, 1, true), 1.5)); + assertEquals(createResourceWithBoolean(4, 1, 0, true), + multiplyAndRoundDown(createResourceWithBoolean(3, 1, true), 1.5)); + assertEquals(createResourceWithBoolean(1, 4, true), + multiplyAndRoundDown(createResourceWithBoolean(1, 3, true), 1.5)); + assertEquals(createResourceWithBoolean(1, 4, 0, true), + multiplyAndRoundDown(createResourceWithBoolean(1, 3, true), 1.5)); + assertEquals(createResourceWithBoolean(7, 7, 0, true), + multiplyAndRoundDown(createResourceWithBoolean(3, 3, 0, true), 2.5)); + assertEquals(createResourceWithBoolean(2, 2, 7, true), + multiplyAndRoundDown(createResourceWithBoolean(1, 1, 3, true), 2.5)); + } + + @Test + public void testMultiplyAndRoundUp() { + assertEquals(createResource(5, 2), + multiplyAndRoundUp(createResource(3, 1), 1.5)); + assertEquals(createResource(5, 2, 0), + multiplyAndRoundUp(createResource(3, 1), 1.5)); + assertEquals(createResource(2, 5), + multiplyAndRoundUp(createResource(1, 3), 1.5)); + assertEquals(createResource(2, 5, 0), + multiplyAndRoundUp(createResource(1, 3), 1.5)); + assertEquals(createResource(8, 8, 0), + multiplyAndRoundUp(createResource(3, 3, 0), 2.5)); + assertEquals(createResource(3, 3, 3), + multiplyAndRoundUp(createResource(1, 1, 3), 2.5)); + } + + @Test + public void testMultiplyAndRoundUpBooleanResources() { + setupExtraResourcesWithBooleanResource(); + + assertEquals(createResource(5, 2), + multiplyAndRoundUp(createResourceWithBoolean(3, 1, false), 1.5)); + assertEquals(createResource(5, 2, 0), + multiplyAndRoundUp(createResourceWithBoolean(3, 1, false), 1.5)); + assertEquals(createResource(2, 5), + multiplyAndRoundUp(createResourceWithBoolean(1, 3, false), 1.5)); + assertEquals(createResource(2, 5, 0), + multiplyAndRoundUp(createResourceWithBoolean(1, 3, false), 1.5)); + assertEquals(createResource(8, 8, 0), + multiplyAndRoundUp(createResourceWithBoolean(3, 3, 0, false), 2.5)); + assertEquals(createResource(3, 3, 3), + multiplyAndRoundUp(createResourceWithBoolean(1, 1, 3, false), 2.5)); + + assertEquals(createResourceWithBoolean(5, 2, true), + multiplyAndRoundUp(createResourceWithBoolean(3, 1, true), 1.5)); + assertEquals(createResourceWithBoolean(5, 2, 0, true), + multiplyAndRoundUp(createResourceWithBoolean(3, 1, true), 1.5)); + assertEquals(createResourceWithBoolean(2, 5, true), + multiplyAndRoundUp(createResourceWithBoolean(1, 3, true), 1.5)); + assertEquals(createResourceWithBoolean(2, 5, 0, true), + multiplyAndRoundUp(createResourceWithBoolean(1, 3, true), 1.5)); + assertEquals(createResourceWithBoolean(8, 8, 0, true), + multiplyAndRoundUp(createResourceWithBoolean(3, 3, 0, true), 2.5)); + assertEquals(createResourceWithBoolean(3, 3, 3, true), + multiplyAndRoundUp(createResourceWithBoolean(1, 1, 3, true), 2.5)); + } + @Test public void testMultiplyAndAddTo() throws Exception { unsetExtraResourceType(); @@ -263,4 +463,47 @@ public void testMultiplyAndAddTo() throws Exception { multiplyAndAddTo(createResource(3, 1, 2), createResource(2, 2, 3), 1.5)); } + + @Test + public void testMultiplyAndAddToBooleanResources() { + unsetExtraResourceType(); + setupExtraResourcesWithBooleanResource(); + assertEquals(createResource(6, 4), + multiplyAndAddTo(createResource(3, 1), createResourceWithBoolean(2, 2, true), 1.5)); + assertEquals(createResource(6, 4, 0), + multiplyAndAddTo(createResource(3, 1), createResourceWithBoolean(2, 2, true), 1.5)); + assertEquals(createResource(4, 7), + multiplyAndAddTo(createResource(1, 1), createResourceWithBoolean(2, 4, true), 1.5)); + assertEquals(createResource(4, 7, 0), + multiplyAndAddTo(createResource(1, 1), createResourceWithBoolean(2, 4, true), 1.5)); + assertEquals(createResource(6, 4, 0), + multiplyAndAddTo(createResource(3, 1, 0), createResourceWithBoolean(2, 2, 0, true), + 1.5)); + assertEquals(createResource(6, 4, 6), + multiplyAndAddTo(createResource(3, 1, 2), createResourceWithBoolean(2, 2, 3, true), + 1.5)); + } + + @Test + public void testUnboundedBooleanResourceMaxValueIsOverriden() { + unsetExtraResourceType(); + setupExtraResourcesWithBooleanResource(); + Resource unbounded = ExtendedResources.unbounded(); + assertEquals(Long.MAX_VALUE, unbounded.getMemorySize()); + assertEquals(Integer.MAX_VALUE, unbounded.getVirtualCores()); + assertEquals(Long.MAX_VALUE, unbounded.getResourceValue("resource2")); + assertEquals(1L, unbounded.getResourceValue("booleanResource")); + } + + @Test + public void testNoneResourceBooleanValue() { + unsetExtraResourceType(); + setupExtraResourcesWithBooleanResource(); + Resource none = ExtendedResources.none(); + assertEquals(0L, none.getMemorySize()); + assertEquals(0, none.getVirtualCores()); + assertEquals(0L, none.getResourceValue("resource2")); + assertEquals(0L, none.getResourceValue("booleanResource")); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-1.xml new file mode 100644 index 00000000000..d09ad4bacb3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-1.xml @@ -0,0 +1,59 @@ + + + + + + + + + yarn.nodemanager.resource-type.resource1 + 5Gi + + + + yarn.nodemanager.resource-type.resource2 + 2m + + + + yarn.nodemanager.resource-type.yarn.io/gpu + 1 + + + + yarn.nodemanager.resource-type.booleanResource + 1 + + + + yarn.nodemanager.resource-type.booleanResource.type + BOOLEAN + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-2.xml new file mode 100644 index 00000000000..7f96e8a1c90 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-2.xml @@ -0,0 +1,59 @@ + + + + + + + + + yarn.nodemanager.resource-type.resource1 + 5Gi + + + + yarn.nodemanager.resource-type.resource2 + 2m + + + + yarn.nodemanager.resource-type.yarn.io/gpu + 1 + + + + yarn.nodemanager.resource-type.booleanResource + true + + + + yarn.nodemanager.resource-type.booleanResource.type + BOOLEAN + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-error.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-error.xml new file mode 100644 index 00000000000..0a073918a3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-boolean-error.xml @@ -0,0 +1,59 @@ + + + + + + + + + yarn.nodemanager.resource-type.resource1 + 5Gi + + + + yarn.nodemanager.resource-type.resource2 + 2m + + + + yarn.nodemanager.resource-type.yarn.io/gpu + 1 + + + + yarn.nodemanager.resource-type.booleanResource + 2 + + + + yarn.nodemanager.resource-type.booleanResource.type + BOOLEAN + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-1.xml new file mode 100644 index 00000000000..aeb4399e7ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-1.xml @@ -0,0 +1,60 @@ + + + + + + + + + + yarn.resource-types + resource1,resource2,yarn.io/gpu,booleanResource + + + + yarn.resource-types.resource1.units + G + + + + yarn.resource-types.resource2.units + m + + + + yarn.resource-types.yarn.io/gpu.units + + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-maximum.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-maximum.xml new file mode 100644 index 00000000000..955a7a87ef6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-maximum.xml @@ -0,0 +1,49 @@ + + + + + + + + + + yarn.resource-types + booleanResource + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + yarn.resource-types.booleanResource.maximum-allocation + 2 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-minimum.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-minimum.xml new file mode 100644 index 00000000000..47dd03cb01a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-minimum.xml @@ -0,0 +1,49 @@ + + + + + + + + + + yarn.resource-types + booleanResource + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + yarn.resource-types.booleanResource.minimum-allocation + 1 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-units.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-units.xml new file mode 100644 index 00000000000..6b5ecaa585e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-error-invalid-units.xml @@ -0,0 +1,49 @@ + + + + + + + + + + yarn.resource-types + booleanResource + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + yarn.resource-types.booleanResource.units + m + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-1.xml new file mode 100644 index 00000000000..f1946a12600 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-1.xml @@ -0,0 +1,57 @@ + + + + + + + + + + yarn.resource-types + booleanResource + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + yarn.resource-types.booleanResource.units + + + + yarn.resource-types.booleanResource.minimum-allocation + 0 + + + yarn.resource-types.booleanResource.maximum-allocation + 1 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-2.xml new file mode 100644 index 00000000000..f51a12e7044 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-boolean-valid-2.xml @@ -0,0 +1,45 @@ + + + + + + + + + + yarn.resource-types + booleanResource + + + + yarn.resource-types.booleanResource.type + BOOLEAN + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java index 5b8360a1c9e..b4c38010005 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java @@ -281,6 +281,10 @@ private void resetCapacity(Resource clusterResource, ResourceInformation dResourceInformation = activeCap .getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(nResourceInformation, dResourceInformation)) { + continue; + } + long nValue = nResourceInformation.getValue(); long dValue = UnitsConversionUtil.convert( dResourceInformation.getUnits(), nResourceInformation.getUnits(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 844057ea142..c1a0e59af99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -414,6 +414,10 @@ static void checkResourceRequestAgainstAvailableResource(Resource reqResource, reqResource.getResourceInformation(i); final String reqResourceName = requestedRI.getName(); + if (Resource.shouldIgnoreBooleanResource(requestedRI)) { + continue; + } + if (resourcesWithZeroAmount.containsKey(reqResourceName) && requestedRI.getValue() > 0) { invalidResources.add(requestedRI); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 2363b8809e8..15f65c80f48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1028,6 +1028,10 @@ private Resource getMinResourceNormalized(String name, Map effect ResourceInformation nResourceInformation = minResource .getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(nResourceInformation)) { + continue; + } + Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); if (ratio != null) { ret.setResourceValue(i, @@ -1054,6 +1058,10 @@ private Resource getMinResourceNormalized(String name, Map effect ResourceInformation dResourceInformation = configuredMinResources .getResourceInformation(i); + if (Resource.shouldIgnoreBooleanResource(nResourceInformation, dResourceInformation)) { + continue; + } + long nValue = nResourceInformation.getValue(); long dValue = UnitsConversionUtil.convert( dResourceInformation.getUnits(), nResourceInformation.getUnits(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index eb9f6af7101..e87f1c9e329 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -39,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; @@ -94,6 +96,7 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.io.IOException; @@ -243,7 +246,8 @@ private void validateConf(FairSchedulerConfiguration config) { + "the minimum allocation value."); } - long incrementMem = config.getIncrementAllocation().getMemorySize(); + Resource incrementAllocation = config.getIncrementAllocation(); + long incrementMem = incrementAllocation.getMemorySize(); if (incrementMem <= 0) { throw new YarnRuntimeException("Invalid resource scheduler memory" + " allocation configuration: " @@ -270,13 +274,29 @@ private void validateConf(FairSchedulerConfiguration config) { + "the minimum allocation value."); } - int incrementVcore = config.getIncrementAllocation().getVirtualCores(); + int incrementVcore = incrementAllocation.getVirtualCores(); if (incrementVcore <= 0) { throw new YarnRuntimeException("Invalid resource scheduler vcores" + " allocation configuration: " + FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES + "=" + incrementVcore + ". Values must be greater than 0."); } + + validateBooleanResources(incrementAllocation); + } + + private void validateBooleanResources(Resource res) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation resourceInformation = res.getResourceInformation(i); + if (resourceInformation.getResourceType() == + ResourceTypes.BOOLEAN && resourceInformation.getValue() > 0) { + throw new YarnRuntimeException("Invalid resource scheduler " + + "boolean resource configuration for resource " + + resourceInformation.getName() + ", should not specify " + + "increment allocation for boolean resources!"); + } + } } public FairSchedulerConfiguration getConf() { @@ -972,11 +992,11 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, resourceRequest, queue.getMaxShare()); if (!validationResult.isValid()) { validationResults.add(validationResult); - LOG.warn(String.format("Queue %s cannot handle resource request" + + LOG.warn(String.format("Queue %s cannot handle resource request " + "because it has zero available amount of resource " + "for a requested resource type, " + - "so the resource request is ignored!" - + " Requested resources: %s, " + + "so the resource request is ignored! " + + "Requested resources: %s, " + "maximum queue resources: %s", queue.getName(), resourceRequest.getCapability(), queue.getMaxShare())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8c4932bfe67..4adf645fb61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 15cfdb01e7b..acbe6294a8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -49,6 +49,8 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; + +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -105,6 +107,8 @@ private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); private static Resource configuredMaxAllocation; + private static final String BOOLEAN_RESOURCE = "booleanResource"; + private static class CustomResourceTypesConfigurationProvider extends LocalConfigurationProvider { @@ -117,7 +121,7 @@ public InputStream getConfigurationInputStream(Configuration bootstrapConf, " \n" + " yarn.resource-types\n" + " custom-resource-1," + - "custom-resource-2,custom-resource-3\n" + + "custom-resource-2,custom-resource-3," + BOOLEAN_RESOURCE + "\n" + " \n" + " \n" + " yarn.resource-types" + @@ -129,6 +133,10 @@ public InputStream getConfigurationInputStream(Configuration bootstrapConf, ".custom-resource-2.units\n" + " G\n" + " \n" + + " \n" + + " yarn.resource-types." + BOOLEAN_RESOURCE + ".type\n" + + " " + ResourceTypes.BOOLEAN.toString() + "\n" + + " \n" + "\n").getBytes()); } else { return super.getConfigurationInputStream(bootstrapConf, name); @@ -159,6 +167,7 @@ public void setUp() { .put("custom-resource-1", Long.MAX_VALUE) .put("custom-resource-2", Long.MAX_VALUE) .put("custom-resource-3", Long.MAX_VALUE) + .put("booleanResource", 1L) .build()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 0b54010c276..4b404cd0710 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -27,7 +27,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.ArrayList; @@ -57,6 +59,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -181,6 +184,33 @@ private ResourceManager resourceManager = null; private RMContext mockContext; + private static final String BOOLEAN_RESOURCE = "booleanResource"; + + private static class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream(( + "\n" + + " \n" + + " yarn.resource-types\n" + + " " + BOOLEAN_RESOURCE + "\n" + + " \n" + + " \n" + + " yarn.resource-types." + BOOLEAN_RESOURCE + ".type\n" + + " " + ResourceTypes.BOOLEAN.toString() + "\n" + + " \n" + + "\n") + .getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + } + @Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @@ -197,6 +227,9 @@ protected RMNodeLabelsManager createNodeLabelManager() { YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9120d3a6cc1..7eea2aaeed4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -28,10 +28,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -53,7 +56,9 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.LocalConfigurationProvider; import org.apache.hadoop.yarn.MockApps; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -136,10 +141,44 @@ private final static String ALLOC_FILE = new File(TEST_DIR, "test-queues").getAbsolutePath(); + private static final String BOOLEAN_RESOURCE = "booleanResource"; + + private static class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream(( + "\n" + + " \n" + + " yarn.resource-types\n" + + " " + BOOLEAN_RESOURCE + "\n" + + " \n" + + " \n" + + " yarn.resource-types." + BOOLEAN_RESOURCE + ".type\n" + + " " + ResourceTypes.BOOLEAN.toString() + "\n" + + " \n" + + "\n") + .getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + } + @Before public void setUp() throws IOException { + //need to run this as FairScheduler's nodeTracker will initialize + // its maxAllocation with 2 resources as default + initResourceTypes(); scheduler = new FairScheduler(); conf = createConfiguration(); + //need to specify resource types once again as MockRM uses the conf above + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + resourceManager = new MockRM(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); @@ -166,6 +205,13 @@ public void tearDown() { YarnAuthorizationProvider.destroy(); } + private void initResourceTypes() { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + } + @Test (timeout = 30000) public void testConfValidation() throws Exception { @@ -198,6 +244,25 @@ public void testConfValidation() throws Exception { } } + @Test (timeout = 30000) + public void testConfValidationBooleanResource() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RESOURCE_TYPES + ".booleanResource" + + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "10"); + try { + scheduler.serviceInit(conf); + fail("Exception is expected because the increment allocation should" + + " not be specified for boolean resources."); + } catch (YarnRuntimeException e) { + // Exception is expected. + assertTrue("The thrown exception is not the expected one.", + e.getMessage().startsWith( + "Invalid resource scheduler " + + "boolean resource configuration for resource " + + "booleanResource")); + } + } + // TESTS @SuppressWarnings("deprecation") @@ -5347,18 +5412,18 @@ public void testDumpState() throws IOException { String childQueueString = "{Name: root.parent.child1," + " Weight: 1.0," + " Policy: fair," - + " FairShare: ," - + " SteadyFairShare: ," - + " MaxShare: ," - + " MinShare: ," - + " ResourceUsage: ," - + " Demand: ," + + " FairShare: ," + + " SteadyFairShare: ," + + " MaxShare: ," + + " MinShare: ," + + " ResourceUsage: ," + + " Demand: ," + " Runnable: 1," + " NumPendingApps: 0," + " NonRunnable: 0," + " MaxAMShare: 0.5," - + " MaxAMResource: ," - + " AMResourceUsage: ," + + " MaxAMResource: ," + + " AMResourceUsage: ," + " LastTimeAtMinShare: " + clock.getTime() + "}"; @@ -5372,12 +5437,12 @@ public void testDumpState() throws IOException { String parentQueueString = "{Name: root.parent," + " Weight: 1.0," + " Policy: fair," - + " FairShare: ," - + " SteadyFairShare: ," - + " MaxShare: ," - + " MinShare: ," - + " ResourceUsage: ," - + " Demand: ," + + " FairShare: ," + + " SteadyFairShare: ," + + " MaxShare: ," + + " MinShare: ," + + " ResourceUsage: ," + + " Demand: ," + " MaxAMShare: 0.5," + " Runnable: 0}"; @@ -5642,4 +5707,49 @@ private void testSchedulingAllowedToQueueZeroCapacityOfResource( createSchedulingRequest(memory, vCores, "queueA", "user1", 1, 2); } + + @Test + public void testQueueGetMaxShareReturnsCorrectValue() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0"); + out.println("0.5"); + out.println("4096 mb 4 vcores"); + out.println(""); + out.println(""); + out.println("0.0"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("1"); + out.println("0.5"); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); + + scheduler.init(conf); + + FSLeafQueue queue = + scheduler.getQueueManager().getLeafQueue("queueFSZeroWithMax", true); + + Field maxShareField = + queue.getClass().getSuperclass().getDeclaredField("maxShare"); + maxShareField.setAccessible(true); + ConfigurableResource maxShare = + (ConfigurableResource) maxShareField.get(queue); + maxShare.getResource().setResourceInformation(BOOLEAN_RESOURCE, + ResourceInformation.newBooleanInstance(BOOLEAN_RESOURCE, true)); + + Resource expectedMaxShare = + maxShare.getResource(scheduler.getClusterResource()); + + // queue.getMaxShare() should return the same resource as expectedMaxShare + // (with boolean resource set to 1) + assertEquals(expectedMaxShare, queue.getMaxShare()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 70f83ab3095..40b5079e48c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; @@ -54,6 +55,7 @@ public class TestFairSchedulerConfiguration { private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; + private static final String BOOLEAN_RESOURCE = "booleanResource"; private static class CustomResourceTypesConfigurationProvider extends LocalConfigurationProvider { @@ -66,7 +68,11 @@ public InputStream getConfigurationInputStream(Configuration bootstrapConf, "\n" + " \n" + " yarn.resource-types\n" + - " " + A_CUSTOM_RESOURCE + "\n" + + " " + A_CUSTOM_RESOURCE + "," + BOOLEAN_RESOURCE + "\n" + + " \n" + + " \n" + + " yarn.resource-types." + BOOLEAN_RESOURCE + ".type\n" + + " " + ResourceTypes.BOOLEAN.toString() + "\n" + " \n" + " \n" + " yarn.resource-types.a-custom-resource.units\n" + @@ -430,9 +436,27 @@ public void testAllocationIncrementCustomResource() throws Exception { } } + @Test + public void testAllocationIncrementCustomBooleanResource() { + initResourceTypes(); + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RESOURCE_TYPES + ".booleanResource" + + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "10"); + FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); + Resource increment = fsc.getIncrementAllocation(); + + ResourceInformation booleanResource = increment.getResourceInformation + (BOOLEAN_RESOURCE); + assertNotNull(booleanResource); + //getIncrementAllocation leaves value as is, FairScheduler.validateConf() does validation + assertEquals(10L, booleanResource.getValue()); + assertEquals(0L, booleanResource.getMinimumAllocation()); + assertEquals(1L, booleanResource.getMaximumAllocation()); + } + private Resource customResource(long value, String units) { return new LightWeightResource(0L, 0, new ResourceInformation[] { - null, null, customResourceInformation(value, units) }); + null, null, customBooleanResourceInformation(true), customResourceInformation(value, units) }); } private ResourceInformation customResourceInformation(long value, @@ -441,6 +465,11 @@ private ResourceInformation customResourceInformation(long value, ResourceTypes.COUNTABLE, 0L, Long.MAX_VALUE); } + private ResourceInformation customBooleanResourceInformation(boolean value) { + return ResourceInformation.newBooleanInstance(BOOLEAN_RESOURCE, value); + } + + private void initResourceTypes() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,