diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
index c1549c54db4..fb4fc5e58b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
@@ -50,7 +50,7 @@ private PlacementConstraints() {
public static final String RACK = PlacementConstraint.RACK_SCOPE;
public static final String NODE_PARTITION = "yarn_node_partition/";
- private static final String APPLICATION_LABEL_PREFIX =
+ public static final String APPLICATION_LABEL_PREFIX =
"yarn_application_label/";
@InterfaceAudience.Private
@@ -223,6 +223,30 @@ public static TargetExpression allocationTag(String... allocationTags) {
allocationTags);
}
+ /**
+ * Constructs a target expression on a set of allocation tags under
+ * a certain namespace. A valid namespace could be an application ID,
+ * or a regex that matches a set of application IDs.
+ * For example,
+ *
+ * - application_123456 represents that tags are targeting to
+ * application whose ID is "application_123456"
+ * - application_123* represents that tags are targeting to
+ * applications whose ID has prefix "application_123"
+ * - application_* represents the tags are targeting to all
+ * the applications in this cluster
+ *
+ *
+ * @param namespace namespace of the allocation tags
+ * @param allocationTags allocation tags
+ * @return a target expression
+ */
+ public static TargetExpression allocationTagWithNamespace(String namespace,
+ String... allocationTags) {
+ return new TargetExpression(TargetType.ALLOCATION_TAG,
+ APPLICATION_LABEL_PREFIX + namespace, allocationTags);
+ }
+
/**
* Constructs a target expression on an allocation tag. It is satisfied if
* there are allocations with one of the given tags. Comparing to
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index ab0bbd7f779..09c7a089f63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
@@ -39,6 +40,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.APPLICATION_LABEL_PREFIX;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE_PARTITION;
/**
@@ -56,6 +58,37 @@
private PlacementConstraintsUtil() {
}
+ /**
+ * Try to get application IDs from a target key, if the target key is
+ * malformed or it doesn't contain any valid application ID, null is
+ * returned.
+ *
+ * @param targetKey
+ * @return an application ID or null.
+ */
+ private static Set getTargetApplication(String targetKey) {
+ if (targetKey == null ||
+ !targetKey.startsWith(APPLICATION_LABEL_PREFIX)) {
+ return null;
+ }
+
+ Set applicationIds = new HashSet<>();
+ targetKey = targetKey.replaceFirst(APPLICATION_LABEL_PREFIX, "");
+
+ ApplicationId applicationId;
+ try {
+ applicationId = ApplicationId.fromString(targetKey);
+ applicationIds.add(applicationId);
+ } catch (IllegalArgumentException e2) {
+ // Ignore the parsing error
+ applicationIds = null;
+ }
+
+ // TODO support application regex string
+
+ return applicationIds;
+ }
+
/**
* Returns true if single placement constraint with associated
* allocationTags and scope is satisfied by a specific scheduler Node.
@@ -74,6 +107,14 @@ private static boolean canSatisfySingleConstraintExpression(
ApplicationId targetApplicationId, SingleConstraint sc,
TargetExpression te, SchedulerNode node, AllocationTagsManager tm)
throws InvalidAllocationTagsQueryException {
+ // Check if this is a valid target key
+ Set targetAppIds = getTargetApplication(te.getTargetKey());
+ if (targetAppIds != null) {
+ // Override this applications' ID
+ // TODO support multiple apps and all apps.
+ targetApplicationId = targetAppIds.iterator().next();
+ }
+
long minScopeCardinality = 0;
long maxScopeCardinality = 0;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
index 5135f636dc2..eb45d066c3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
@@ -25,9 +25,11 @@
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTagWithNamespace;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.HashMap;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.HashSet;
@@ -508,4 +510,78 @@ public void testANDConstraintAssignment()
Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
createSchedulingRequest(sourceTag1), schedulerNode3, pcm, tm));
}
+
+ @Test
+ public void testInterAppConstraints()
+ throws InvalidAllocationTagsQueryException {
+ AllocationTagsManager tm = new AllocationTagsManager(rmContext);
+ PlacementConstraintManagerService pcm =
+ new MemoryPlacementConstraintManager();
+ rmContext.setAllocationTagsManager(tm);
+ rmContext.setPlacementConstraintManager(pcm);
+
+ long ts = System.currentTimeMillis();
+ ApplicationId application1 = BuilderUtils.newApplicationId(ts, 123);
+
+ // Register App1 with anti-affinity constraint map.
+ RMNode n0r1 = rmNodes.get(0);
+ RMNode n1r1 = rmNodes.get(1);
+ RMNode n2r2 = rmNodes.get(2);
+ RMNode n3r2 = rmNodes.get(3);
+
+ /**
+ * Place container:
+ * n0: hbase-m(1)
+ * n1: ""
+ * n2: hbase-m(1)
+ * n3: ""
+ */
+ tm.addContainer(n0r1.getNodeID(),
+ newContainerId(application1), ImmutableSet.of("hbase-m"));
+ tm.addContainer(n2r2.getNodeID(),
+ newContainerId(application1), ImmutableSet.of("hbase-m"));
+ Assert.assertEquals(1L, tm.getAllocationTagsWithCount(n0r1.getNodeID())
+ .get("hbase-m").longValue());
+ Assert.assertEquals(1L, tm.getAllocationTagsWithCount(n2r2.getNodeID())
+ .get("hbase-m").longValue());
+
+ SchedulerNode schedulerNode0 = newSchedulerNode(n0r1.getHostName(),
+ n0r1.getRackName(), n0r1.getNodeID());
+ SchedulerNode schedulerNode1 = newSchedulerNode(n1r1.getHostName(),
+ n1r1.getRackName(), n1r1.getNodeID());
+ SchedulerNode schedulerNode2 = newSchedulerNode(n2r2.getHostName(),
+ n2r2.getRackName(), n2r2.getNodeID());
+ SchedulerNode schedulerNode3 = newSchedulerNode(n3r2.getHostName(),
+ n3r2.getRackName(), n3r2.getNodeID());
+
+
+ Map, PlacementConstraint> constraintMap = new HashMap<>();
+ // Set anti-affinity to application 1
+ PlacementConstraint constraint2 = PlacementConstraints
+ .targetNotIn(NODE, allocationTagWithNamespace(
+ application1.toString(), "hbase-m"))
+ .build();
+ Set srcTags2 = new HashSet<>();
+ srcTags2.add("spark");
+ constraintMap.put(srcTags2, constraint2);
+
+ ts = System.currentTimeMillis();
+ ApplicationId application2 = BuilderUtils.newApplicationId(ts, 124);
+ pcm.registerApplication(application2, constraintMap);
+
+ // Anti-affinity with hbase-m so it should not be able to be placed
+ // onto n0 and n2 as they already have hbase-m allocated.
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(
+ application2, createSchedulingRequest(srcTags2),
+ schedulerNode0, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(
+ application2, createSchedulingRequest(srcTags2),
+ schedulerNode1, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(
+ application2, createSchedulingRequest(srcTags2),
+ schedulerNode2, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(
+ application2, createSchedulingRequest(srcTags2),
+ schedulerNode3, pcm, tm));
+ }
}
\ No newline at end of file