diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 232cd49b03c..2f7779dee64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -54,6 +54,12 @@ public float divide(Resource unused,
Resource numerator, Resource denominator) {
return ratio(numerator, denominator);
}
+
+ @Override
+ public float[] divideMultiTypes(Resource unused,
+ Resource numerator, Resource denominator) {
+ return ratioMultiType(numerator, denominator);
+ }
public boolean isInvalidDivisor(Resource r) {
if (r.getMemorySize() == 0.0f) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 0a6df09de99..e5412c40f33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -378,6 +378,12 @@ public float divide(Resource clusterResource,
return (float) (max[0] / max[1]);
}
+ @Override
+ public float[] divideMultiTypes(Resource clusterResource,
+ Resource numerator, Resource denominator) {
+ return ratioMultiType(numerator, denominator);
+ }
+
@Override
public boolean isInvalidDivisor(Resource r) {
for (ResourceInformation res : r.getResources()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 91cbdd7e7e9..d51e625a081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import java.util.Set;
@@ -123,6 +124,26 @@ public static int roundDown(int a, int b) {
return (a / b) * b;
}
+ /**
+ * Ratio of resource a to resource b.
+ * with multi type ratio.
+ * @param a resource
+ * @param b resource
+ * @return ratio of resource a to resource b
+ */
+ public float[] ratioMultiType(Resource a, Resource b) {
+ int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
+ float res[] = new float[maxLength];
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation aResourceInformation = a.getResourceInformation(i);
+ ResourceInformation bResourceInformation = b.getResourceInformation(i);
+ final float tmp = divideSafelyAsFloat(aResourceInformation.getValue(),
+ bResourceInformation.getValue());
+ res[i] = tmp;
+ }
+ return res;
+ }
+
/**
* Compute the number of containers which can be allocated given
* available and required resources.
@@ -215,6 +236,22 @@ public abstract Resource normalize(Resource r, Resource minimumResource,
*/
public abstract float divide(
Resource clusterResource, Resource numerator, Resource denominator);
+
+
+ /**
+ * Divide resource numerator by resource denominator
+ * based multi types.
+ * using specified policy (domination, average, fairness etc.); hence overall
+ * clusterResource is provided for context.
+ *
+ * @param clusterResource cluster resources
+ * @param numerator numerator
+ * @param denominator denominator
+ * @return numerator/denominator
+ * using specific policy
+ */
+ public abstract float[] divideMultiTypes(Resource unused,
+ Resource numerator, Resource denominator);
/**
* Determine if a resource is not suitable for use as a divisor
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 9b96fd72b9a..598bccb6057 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -314,6 +314,10 @@ public static Resource multiply(Resource lhs, double by) {
return multiplyTo(clone(lhs), by);
}
+ public static Resource multiplyToVec(Resource lhs, float[] by) {
+ return multiplyAndRoundVec(lhs, by, RoundingDirection.DOWN);
+ }
+
/**
* Multiply {@code rhs} by {@code by}, and add the result to {@code lhs}
* without creating any new {@link Resource} object
@@ -401,6 +405,35 @@ private static Resource multiplyAndRound(Resource lhs, double by,
return lhs;
}
+ /**
+ * Multiply {@code lhs} by {@code by}, and set the result according to
+ * the rounding direction to {@code lhs}
+ * without creating any new {@link Resource} object.
+ * @param lhs Resource object
+ * @param by Multiply values vector by this vector.
+ * @return Returns {@code lhs} itself (without cloning) with updated values
+ */
+ private static Resource multiplyAndRoundVec(Resource lhs, float[] by,
+ RoundingDirection roundingDirection) {
+ int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ try {
+ ResourceInformation lhsValue = lhs.getResourceInformation(i);
+
+ final long value;
+ if (roundingDirection == RoundingDirection.DOWN) {
+ value = (long) (lhsValue.getValue() * by[i]);
+ } else {
+ value = (long) Math.ceil(lhsValue.getValue() * by[i]);
+ }
+ lhs.setResourceValue(i, value);
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
+ }
+ }
+ return lhs;
+ }
+
public static Resource normalize(
ResourceCalculator calculator, Resource lhs, Resource min,
Resource max, Resource increment) {
@@ -432,6 +465,13 @@ public static float divide(
Resource clusterResource, Resource lhs, Resource rhs) {
return resourceCalculator.divide(clusterResource, lhs, rhs);
}
+
+ public static float[] divideMultiTypes(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource, Resource lhs, Resource rhs) {
+ return resourceCalculator.
+ divideMultiTypes(clusterResource, lhs, rhs);
+ }
public static Resource divideAndCeil(
ResourceCalculator resourceCalculator, Resource lhs, int rhs) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index bc3ff2294ff..e7e020a3309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1545,6 +1545,31 @@ private void deriveCapacityFromAbsoluteConfigurations(String label,
.getCapacity(label) + ", MaxCap : " + queueCapacities
.getMaximumCapacity(label));
}
+
+ // 1. Update vector capacity as float[] vector based on parent's minResource
+ float[] f_vector = rc.divideMultiTypes(clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ parent.getQueueResourceQuotas().getEffectiveMinResource(label));
+ queueCapacities.setVectorCapacity(label, f_vector);
+
+ // 2. Update vector max-capacity as float[] based on parent's maxResource
+ f_vector = rc.divideMultiTypes(clusterResource,
+ queueResourceQuotas.getEffectiveMaxResource(label),
+ parent.getQueueResourceQuotas().getEffectiveMaxResource(label));
+ queueCapacities.setVectorMaximumCapacity(label, f_vector);
+
+ // Set absolute cap vector
+ int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
+ float capVec[] = queueCapacities.getVectorCapacity(label);
+ float absVec[] = new float[maxLength];
+ for (int i = 0; i < maxLength; i++) {
+ if(capVec[i] > 0) {
+ absVec[i] = capVec[i] * (
+ parent.getQueueCapacities() == null ? 1 :
+ parent.getQueueCapacities().getVectorAbsoluteCapacity(label)[i]);
+ }
+ }
+ queueCapacities.setVectorAbsoluteCapacity(label, absVec);
}
void updateEffectiveResources(Resource clusterResource) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index dd77a8088c0..ce9306efe43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -99,14 +99,26 @@ public void mergeCapacities(QueueCapacities capacities) {
queueCapacities.setAbsoluteMaximumCapacity(nodeLabel, capacities
.getAbsoluteMaximumCapacity(nodeLabel));
- Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
- csContext.getClusterResource());
- getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
- Resources.multiply(resourceByLabel,
- queueCapacities.getAbsoluteCapacity(nodeLabel)));
- getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel,
- Resources.multiply(resourceByLabel, queueCapacities
- .getAbsoluteMaximumCapacity(nodeLabel)));
+ queueCapacities.setVectorCapacity(nodeLabel,
+ capacities.getVectorCapacity(nodeLabel));
+ queueCapacities.setVectorAbsoluteCapacity(nodeLabel,
+ capacities.getVectorAbsoluteCapacity(nodeLabel));
+ queueCapacities.setVectorAbsoluteMaximumCapacity(nodeLabel,
+ capacities.getVectorAbsoluteMaximumCapacity(nodeLabel));
+ queueCapacities.setVectorMaximumCapacity(nodeLabel,
+ capacities.getVectorMaximumCapacity(nodeLabel));
+
+
+ if (this.capacityConfigType != CapacityConfigType.ABSOLUTE_RESOURCE) {
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+ csContext.getClusterResource());
+ getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+ Resources.multiply(resourceByLabel,
+ queueCapacities.getAbsoluteCapacity(nodeLabel)));
+ getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel,
+ Resources.multiply(resourceByLabel, queueCapacities
+ .getAbsoluteMaximumCapacity(nodeLabel)));
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index a4034768387..ea8180bf3a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -20,12 +20,14 @@
import java.util.Set;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 0a2f0820070..11b47bc4535 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -19,13 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
@@ -1291,6 +1285,11 @@ private void calculateEffectiveResourcesAndCapacity(String label,
if (rootQueue) {
queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ float[] rootVec = new float[ResourceUtils.getNumberOfCountableResourceTypes()];
+ Arrays.fill(rootVec, 1.0f);
+ queueCapacities.setVectorCapacity(label, rootVec);
+ queueCapacities.setVectorAbsoluteCapacity(label, rootVec);
+ queueCapacities.setVectorMaximumCapacity(label, rootVec);
} else{
super.updateEffectiveResources(clusterResource);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 86d35d6cdd5..cd77d7f8562 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -18,10 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -29,11 +26,18 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
public class QueueCapacities {
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
private static final float LABEL_DOESNT_EXIST_CAP = 0f;
+ private static final int resourceTypeNum =
+ ResourceUtils.getNumberOfCountableResourceTypes();
+ private static final float[] LABEL_DOESNT_EXIST_VECTOR_CAP
+ = new float[resourceTypeNum];
private Map capacitiesMap;
+ // Used for vector capacity
+ private Map vectorCapacitiesMap;
private ReadLock readLock;
private WriteLock writeLock;
private final boolean isRoot;
@@ -44,6 +48,8 @@ public QueueCapacities(boolean isRoot) {
writeLock = lock.writeLock();
capacitiesMap = new HashMap();
+ vectorCapacitiesMap = new HashMap();
+ Arrays.fill(LABEL_DOESNT_EXIST_VECTOR_CAP, 0);
this.isRoot = isRoot;
}
@@ -59,16 +65,27 @@ private CapacityType(int idx) {
}
}
+ // Usage enum here to make implement cleaner
+ private enum VectorCapacityType {
+ VECTOR_CAP(0),VECTOR_ABS_CAP(1), VECTOR_MAX_CAP(2), VECTOR_ABS_MAX_CAP(3);
+
+ private int idx;
+
+ private VectorCapacityType(int idx) {
+ this.idx = idx;
+ }
+ }
+
private static class Capacities {
private float[] capacitiesArr;
-
+
public Capacities() {
capacitiesArr = new float[CapacityType.values().length];
// Set weight to -1 by default (means not set)
capacitiesArr[CapacityType.WEIGHT.idx] = -1;
}
-
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -86,6 +103,51 @@ public String toString() {
return sb.toString();
}
}
+
+ private static class VectorCapacities {
+ private float[][] resTypeToCapacitiesArr;
+ public VectorCapacities() {
+ resTypeToCapacitiesArr = new float[VectorCapacityType.values().length]
+ [ResourceUtils.getNumberOfCountableResourceTypes()];
+ }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{vector_cap=" + Arrays.toString(resTypeToCapacitiesArr[0]))
+ .append("{vector_abs_cap=" + Arrays.toString(resTypeToCapacitiesArr[1]));
+ return sb.toString();
+ }
+ }
+
+ private float[] _getVector(String label, VectorCapacityType type) {
+ readLock.lock();
+ try {
+ VectorCapacities vectorCap = vectorCapacitiesMap.get(label);
+ if (null == vectorCap) {
+ // Now not exist vector weight mode
+ return LABEL_DOESNT_EXIST_VECTOR_CAP;
+ }
+ return vectorCap.resTypeToCapacitiesArr[type.idx];
+
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private void _setVector(String label, VectorCapacityType type, float[] valueVec) {
+ readLock.lock();
+ try {
+ VectorCapacities vectorCap = vectorCapacitiesMap.get(label);
+ if (null == vectorCap) {
+ vectorCap = new VectorCapacities();
+ vectorCapacitiesMap.put(label, vectorCap);
+ }
+ vectorCap.resTypeToCapacitiesArr[type.idx] = valueVec;
+
+ } finally {
+ readLock.unlock();
+ }
+ }
private float _get(String label, CapacityType type) {
readLock.lock();
@@ -173,6 +235,74 @@ public void setCapacity(String label, float value) {
_set(label, CapacityType.CAP, value);
}
+ /* Vector Capacity Getter and Setter */
+ public float[] getVectorCapacity() {
+ return _getVector(NL, VectorCapacityType.VECTOR_CAP);
+ }
+
+ public float[] getVectorCapacity(String label) {
+ return _getVector(label, VectorCapacityType.VECTOR_CAP);
+ }
+
+ public void setVectorCapacity(float[] valueVec) {
+ _setVector(NL, VectorCapacityType.VECTOR_CAP, valueVec);
+ }
+
+ public void setVectorCapacity(String label, float[] valueVec) {
+ _setVector(label, VectorCapacityType.VECTOR_CAP, valueVec);
+ }
+
+ /* Vector Absolute Capacity Getter and Setter */
+ public float[] getVectorAbsoluteCapacity() {
+ return _getVector(NL, VectorCapacityType.VECTOR_ABS_CAP);
+ }
+
+ public float[] getVectorAbsoluteCapacity(String label) {
+ return _getVector(label, VectorCapacityType.VECTOR_ABS_CAP);
+ }
+
+ public void setVectorAbsoluteCapacity(float[] valueVec) {
+ _setVector(NL, VectorCapacityType.VECTOR_ABS_CAP, valueVec);
+ }
+
+ public void setVectorAbsoluteCapacity(String label, float[] valueVec) {
+ _setVector(label, VectorCapacityType.VECTOR_ABS_CAP, valueVec);
+ }
+
+ /* Vector Capacity Max Getter and Setter */
+ public float[] getVectorMaximumCapacity() {
+ return _getVector(NL, VectorCapacityType.VECTOR_MAX_CAP);
+ }
+
+ public float[] getVectorMaximumCapacity(String label) {
+ return _getVector(label, VectorCapacityType.VECTOR_MAX_CAP);
+ }
+
+ public void setVectorMaximumCapacity(float[] valueVec) {
+ _setVector(NL, VectorCapacityType.VECTOR_MAX_CAP, valueVec);
+ }
+
+ public void setVectorMaximumCapacity(String label, float[] valueVec) {
+ _setVector(label, VectorCapacityType.VECTOR_MAX_CAP, valueVec);
+ }
+
+ /* Vector Absolute MaxCapacity Getter and Setter */
+ public float[] getVectorAbsoluteMaximumCapacity() {
+ return _getVector(NL, VectorCapacityType.VECTOR_ABS_MAX_CAP);
+ }
+
+ public float[] getVectorAbsoluteMaximumCapacity(String label) {
+ return _getVector(label, VectorCapacityType.VECTOR_ABS_MAX_CAP);
+ }
+
+ public void setVectorAbsoluteMaximumCapacity(float[] valueVec) {
+ _setVector(NL, VectorCapacityType.VECTOR_ABS_MAX_CAP, valueVec);
+ }
+
+ public void setVectorAbsoluteMaximumCapacity(String label, float[] valueVec) {
+ _setVector(label, VectorCapacityType.VECTOR_ABS_MAX_CAP, valueVec);
+ }
+
/* Absolute Capacity Getter and Setter */
public float getAbsoluteCapacity() {
return _get(NL, CapacityType.ABS_CAP);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
index ab99317888d..6fba252496c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -19,9 +19,11 @@
.queuemanagement;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.QueueManagementDynamicEditPolicy;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -56,16 +58,7 @@
import org.apache.hadoop.yarn.util.resource.Resources;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -624,7 +617,7 @@ public boolean isActive(final AutoCreatedLeafQueue leafQueue,
QueueCapacities capacities = leafQueueEntitlements.get(
leafQueue.getQueuePath());
- updateToZeroCapacity(capacities, nodeLabel);
+ updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue);
deactivatedQueues.put(leafQueue.getQueuePath(),
leafQueueTemplateCapacities);
}
@@ -822,7 +815,7 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
updateCapacityFromTemplate(capacities, nodeLabel);
activate(leafQueue, nodeLabel);
} else{
- updateToZeroCapacity(capacities, nodeLabel);
+ updateToZeroCapacity(capacities, nodeLabel, leafQueue);
}
}
@@ -834,10 +827,20 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
}
private void updateToZeroCapacity(QueueCapacities capacities,
- String nodeLabel) {
+ String nodeLabel, LeafQueue leafQueue) {
capacities.setCapacity(nodeLabel, 0.0f);
capacities.setMaximumCapacity(nodeLabel,
leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel));
+ float[] rootVec = new float[ResourceUtils.getNumberOfCountableResourceTypes()];
+ Arrays.fill(rootVec, 0.0f);
+ capacities.setVectorCapacity(nodeLabel, rootVec);
+ capacities.setVectorMaximumCapacity(nodeLabel, rootVec);
+ // related to absolute caculated
+ leafQueue.getQueueResourceQuotas().
+ setConfiguredMinResource(nodeLabel, Resource.newInstance(0, 0));
+ leafQueue.getQueueCapacities().setVectorCapacity(nodeLabel, rootVec);
+ leafQueue.getQueueResourceQuotas().
+ setEffectiveMinResource(nodeLabel, Resource.newInstance(0, 0));
}
private void updateCapacityFromTemplate(QueueCapacities capacities,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index da13e18cfc7..97f9a652c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -231,8 +231,10 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
d1.queueResourceQuotas.getEffectiveMaxResource());
/**
- * After adding child queue d2, adjustment happens and both d1 and d2 shares
- * resource of Managed Parent Queue
+ * After adding child queue d2, d1 + d2 > resource
+ * of Managed Parent queue, d2 will change to 0.
+ * d1 will occupy all entire resource
+ * of Managed Parent queue.
*/
AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue);
cs.addQueue(d2);
@@ -240,9 +242,9 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
cs.getRootQueue().updateClusterResource(cs.getClusterResource(),
new ResourceLimits(cs.getClusterResource()));
- Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
+ Assert.assertEquals(Resource.newInstance(0, 0),
d2.queueResourceQuotas.getConfiguredMinResource());
- Assert.assertEquals(Resource.newInstance(12800, 2),
+ Assert.assertEquals(Resource.newInstance(0, 0),
d2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
d2.queueResourceQuotas.getConfiguredMaxResource());
@@ -251,7 +253,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
d1.queueResourceQuotas.getConfiguredMinResource());
- Assert.assertEquals(Resource.newInstance(12800, 2),
+ Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
d1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
d1.queueResourceQuotas.getConfiguredMaxResource());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
index f9b494ece80..2bad8b7447b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
@@ -235,12 +235,12 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
3, 1);
final CSQueue autoCreatedLeafQueue2 = cs.getQueue(TEST_GROUPUSER2);
- validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2,
- 0.33332032f,
- 0.03333203f, 1f, 0.6f);
- validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1,
- 0.33332032f,
- 0.03333203f, 1f, 0.6f);
+ validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, 0.0f,
+ 0.0f, 1f, 0.6f);
+ validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1, 0.4f,
+ 0.04f, 1f, 0.6f);
+ validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue, 0.4f,
+ 0.04f, 1f, 0.6f);
GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue) parentQueue)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index 8320d8222fa..6732f48a3f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -751,17 +751,17 @@ protected void validateEffectiveMinResource(ResourceManager rm,
* parentQueue.getQueueCapacities().getAbsoluteCapacity(label));
assertEquals(effMinCapacity, Resources.multiply(resourceByLabel,
leafQueue.getQueueCapacities().getAbsoluteCapacity(label)));
- // TODO: Wangda, I think this is a wrong test, it doesn't consider rounding
- // loss of multiplication, the right value should be <10240, 2>, but the
- // test expects <10240, 1>
- // fixme, address this in the future patch (auto queue creation).
-// if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) {
-// assertEquals(Resource.newInstance(10 * GB, 2),
-// leafQueue.getEffectiveCapacity(label));
-// } else {
-// assertEquals(Resource.newInstance(0, 0),
-// leafQueue.getEffectiveCapacity(label));
-// }
+
+ if (leafQueue.getCapacityConfigType()
+ == AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE) {
+ // Vector absolute resource based check
+ Resource effMinCapacityVec = Resources.multiplyToVec(resourceByLabel,
+ leafQueue.getQueueCapacities().getVectorAbsoluteCapacity(label));
+
+ assertEquals(effMinCapacityVec, leafQueue.getEffectiveCapacity(label));
+ } else {
+ assertEquals(effMinCapacity, leafQueue.getEffectiveCapacity(label));
+ }
if (leafQueue.getQueueCapacities().getAbsoluteCapacity(label) > 0) {
assertTrue(Resources.greaterThan(cs.getResourceCalculator(),