diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 47641475455..3febf698991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import java.util.function.BiFunction; +import java.util.function.Function; + /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -305,21 +308,11 @@ public static Resource negate(Resource resource) { } public static Resource multiplyTo(Resource lhs, double by) { - int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); - for (int i = 0; i < maxLength; i++) { - try { - ResourceInformation lhsValue = lhs.getResourceInformation(i); - lhs.setResourceValue(i, (long) (lhsValue.getValue() * by)); - } catch (ResourceNotFoundException ye) { - LOG.warn("Resource is missing:" + ye.getMessage()); - continue; - } - } - return lhs; + return applyFunctionOnValues(clone(lhs), (value) -> (long) (value * by)); } public static Resource multiply(Resource lhs, double by) { - return multiplyTo(clone(lhs), by); + return multiplyTo(lhs, by); } /** @@ -328,20 +321,8 @@ public static Resource multiply(Resource lhs, double by) { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); - for (int i = 0; i < maxLength; i++) { - try { - ResourceInformation rhsValue = rhs.getResourceInformation(i); - ResourceInformation lhsValue = lhs.getResourceInformation(i); - - long convertedRhs = (long) (rhsValue.getValue() * by); - lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); - } catch (ResourceNotFoundException ye) { - LOG.warn("Resource is missing:" + ye.getMessage()); - continue; - } - } - return lhs; + return applyFunctionOnValues(lhs, rhs, + (lhsValue, rhsValue) -> lhsValue + (long) (rhsValue * by)); } public static Resource multiplyAndNormalizeUp(ResourceCalculator calculator, @@ -360,27 +341,64 @@ public static Resource multiplyAndNormalizeDown( } public static Resource multiplyAndRoundDown(Resource lhs, double by) { - Resource out = clone(lhs); - int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); - for (int i = 0; i < maxLength; i++) { + return multiplyTo(lhs, by); + } + + public static Resource multiplyAndRoundUp(Resource lhs, double by) { + return applyFunctionOnValues(clone(lhs), + (value) -> (long) Math.ceil(value * by)); + } + + /** + * Applies a value function on all resources of {@code lhs} and sets + * the resulted value to the same {@code lhs} object. + * @param lhs Resource to apply the function on. + * @param valueFunction Function that performs a calculation on the value + * and gives back a Long value. + * @return The same {@code Resource} object as received, with updated values. + */ + private static Resource applyFunctionOnValues(Resource lhs, + Function valueFunction) { + int numResources = ResourceUtils.getNumberOfCountableResourceTypes(); + for (int i = 0; i < numResources; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); - out.setResourceValue(i, (long) (lhsValue.getValue() * by)); + Long modifiedValue = valueFunction.apply(lhsValue.getValue()); + lhs.setResourceValue(i, modifiedValue); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); - continue; } } - return out; + return lhs; } - public static Resource multiplyAndRoundUp(Resource lhs, double by) { - Resource out = clone(lhs); - out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); - out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by)); - return out; + /** + * Applies a value function on all resources of {@code lhs} and {@code rhs} + * and sets the resulted value to the {@code lhs} object. + * @param lhs Resource to apply the function on. + * @param rhs Resource to apply the function on. + * @param valueFunction Function that performs a calculation on the values of + * {@code lhs} and {@code rhs} and gives back a Long value. + * @return The same {@code lhs} Resource object as received, + * with updated values. + */ + private static Resource applyFunctionOnValues(Resource lhs, Resource rhs, + BiFunction valueFunction) { + int numResources = ResourceUtils.getNumberOfCountableResourceTypes(); + for (int i = 0; i < numResources; i++) { + try { + ResourceInformation lhsValue = lhs.getResourceInformation(i); + ResourceInformation rhsValue = rhs.getResourceInformation(i); + Long modifiedValue = valueFunction.apply( + lhsValue.getValue(), rhsValue.getValue()); + lhs.setResourceValue(i, modifiedValue); + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); + } + } + return lhs; } - + public static Resource normalize( ResourceCalculator calculator, Resource lhs, Resource min, Resource max, Resource increment) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index ecd940ea98b..58005b9ac54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMin; import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMax; import static org.apache.hadoop.yarn.util.resource.Resources.add; +import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundUp; import static org.apache.hadoop.yarn.util.resource.Resources.subtract; import static org.apache.hadoop.yarn.util.resource.Resources.multiply; import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndAddTo; @@ -116,27 +117,6 @@ public void testCompareToWithNoneResource() { assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0); } - @Test(timeout=10000) - public void testMultipleRoundUp() { - final double by = 0.5; - final String memoryErrorMsg = "Invalid memory size."; - final String vcoreErrorMsg = "Invalid virtual core number."; - Resource resource = Resources.createResource(1, 1); - Resource result = Resources.multiplyAndRoundUp(resource, by); - assertEquals(memoryErrorMsg, result.getMemorySize(), 1); - assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); - - resource = Resources.createResource(2, 2); - result = Resources.multiplyAndRoundUp(resource, by); - assertEquals(memoryErrorMsg, result.getMemorySize(), 1); - assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); - - resource = Resources.createResource(0, 0); - result = Resources.multiplyAndRoundUp(resource, by); - assertEquals(memoryErrorMsg, result.getMemorySize(), 0); - assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); - } - @Test(timeout = 1000) public void testFitsIn() { assertTrue(fitsIn(createResource(1, 1), createResource(2, 2))); @@ -228,6 +208,43 @@ public void testMultiply() { assertEquals(createResource(4, 4, 6), multiply(createResource(2, 2, 3), 2)); } + @Test(timeout=10000) + public void testMultipleRoundUp() { + final double by = 0.5; + final String memoryErrorMsg = "Invalid memory size."; + final String vcoreErrorMsg = "Invalid virtual core number."; + Resource resource = Resources.createResource(1, 1); + Resource result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(2, 2); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(0, 0); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 0); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); + } + + @Test + public void testMultiplyAndRoundUpCustomResources() { + assertEquals(createResource(5, 2, 8), + multiplyAndRoundUp(createResource(3, 1, 5), 1.5)); + assertEquals(createResource(5, 2, 0), + multiplyAndRoundUp(createResource(3, 1, 0), 1.5)); + assertEquals(createResource(5, 5, 0), + multiplyAndRoundUp(createResource(3, 3, 0), 1.5)); + assertEquals(createResource(8, 3, 13), + multiplyAndRoundUp(createResource(3, 1, 5), 2.5)); + assertEquals(createResource(8, 3, 0), + multiplyAndRoundUp(createResource(3, 1, 0), 2.5)); + assertEquals(createResource(8, 8, 0), + multiplyAndRoundUp(createResource(3, 3, 0), 2.5)); + } + @Test public void testMultiplyAndRoundDown() { assertEquals(createResource(4, 1),