diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 2202380..8af79c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Records;
-import java.util.Set;
-
/**
*
ApplicationSubmissionContext represents all of the
* information needed by the ResourceManager to launch
@@ -72,22 +73,35 @@ public static ApplicationSubmissionContext newInstance(
Priority priority, ContainerLaunchContext amContainer,
boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
int maxAppAttempts, Resource resource, String applicationType,
- boolean keepContainers) {
+ boolean keepContainers, String appLabelExpression,
+ String amContainerLabelExpression) {
ApplicationSubmissionContext context =
Records.newRecord(ApplicationSubmissionContext.class);
context.setApplicationId(applicationId);
context.setApplicationName(applicationName);
context.setQueue(queue);
- context.setPriority(priority);
context.setAMContainerSpec(amContainer);
context.setUnmanagedAM(isUnmanagedAM);
context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
context.setMaxAppAttempts(maxAppAttempts);
- context.setResource(resource);
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
+ context.setAppLabelExpression(appLabelExpression);
+ context.setAMContainerResourceRequest(ResourceRequest.newInstance(priority,
+ ResourceRequest.ANY, resource, 1, true, amContainerLabelExpression));
return context;
}
+
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ Priority priority, ContainerLaunchContext amContainer,
+ boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
+ int maxAppAttempts, Resource resource, String applicationType,
+ boolean keepContainers) {
+ return newInstance(applicationId, applicationName, queue, priority,
+ amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
+ resource, applicationType, keepContainers, null, null);
+ }
@Public
@Stable
@@ -98,7 +112,7 @@ public static ApplicationSubmissionContext newInstance(
int maxAppAttempts, Resource resource, String applicationType) {
return newInstance(applicationId, applicationName, queue, priority,
amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
- resource, applicationType, false);
+ resource, applicationType, false, null, null);
}
@Public
@@ -112,6 +126,29 @@ public static ApplicationSubmissionContext newInstance(
amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
resource, null);
}
+
+ @Public
+ @Stable
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ ContainerLaunchContext amContainer, boolean isUnmanagedAM,
+ boolean cancelTokensWhenComplete, int maxAppAttempts,
+ String applicationType, boolean keepContainers,
+ String appLabelExpression, ResourceRequest resourceRequest) {
+ ApplicationSubmissionContext context =
+ Records.newRecord(ApplicationSubmissionContext.class);
+ context.setApplicationId(applicationId);
+ context.setApplicationName(applicationName);
+ context.setQueue(queue);
+ context.setAMContainerSpec(amContainer);
+ context.setUnmanagedAM(isUnmanagedAM);
+ context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
+ context.setMaxAppAttempts(maxAppAttempts);
+ context.setApplicationType(applicationType);
+ context.setKeepContainersAcrossApplicationAttempts(keepContainers);
+ context.setAMContainerResourceRequest(resourceRequest);
+ return context;
+ }
@Public
@Stable
@@ -194,7 +231,11 @@ public static ApplicationSubmissionContext newInstance(
public abstract void setQueue(String queue);
/**
+ * Please note this is DEPRECATED, please use getPriority in
+ * getAMContainerResourceRequest instead.
+ *
* Get the Priority of the application.
+ *
* @return Priority of the application
*/
@Public
@@ -205,8 +246,8 @@ public static ApplicationSubmissionContext newInstance(
* Set the Priority of the application.
* @param priority Priority of the application
*/
- @Private
- @Unstable
+ @Public
+ @Stable
public abstract void setPriority(Priority priority);
/**
@@ -289,6 +330,9 @@ public static ApplicationSubmissionContext newInstance(
public abstract void setMaxAppAttempts(int maxAppAttempts);
/**
+ * Please note this is DEPRECATED, please use getResource in
+ * getAMContainerResourceRequest instead.
+ *
* Get the resource required by the ApplicationMaster for this
* application.
*
@@ -296,7 +340,7 @@ public static ApplicationSubmissionContext newInstance(
* this application.
*/
@Public
- @Stable
+ @Deprecated
public abstract Resource getResource();
/**
@@ -307,7 +351,7 @@ public static ApplicationSubmissionContext newInstance(
* for this application.
*/
@Public
- @Stable
+ @Deprecated
public abstract void setResource(Resource resource);
/**
@@ -379,6 +423,57 @@ public abstract void setKeepContainersAcrossApplicationAttempts(
@Public
@Stable
public abstract void setApplicationTags(Set tags);
+
+ /**
+ * Get label expression for this app
+ *
+ * @return label expression for this app
+ */
+ @Public
+ @Evolving
+ public abstract String getAppLabelExpression();
+
+ /**
+ * Set label expression for the APP
+ *
+ * By default, APP label expression is empty. This field can be overwrite by
+ * resource request label expression and AM container label expression
+ *
+ * e.g.
+ * - APP label expression = "GPU && LARGE_MEM"
+ * - Resource Request label expression = "", it will be set "GPU && LARGE_MEM"
+ * - Resource Request label expression = "GPU && INFINI_BAND",
+ * it will be "GPU && INFINI_BAND"
+ *
+ * As same as label expression AM container Resource Request
+ */
+ @Public
+ @Stable
+ public abstract void setAppLabelExpression(String labelExpression);
+
+ /**
+ * Get ResourceRequest of AM container, if this is not null, scheduler will
+ * use this to acquire resource for AM container.
+ *
+ * If this is null, scheduler will assemble a ResourceRequest by using
+ * getResource and getPriority of
+ * ApplicationSubmissionContext.
+ *
+ * Number of containers and Priority will be ignore.
+ *
+ * @return ResourceRequest of AM container
+ */
+ @Public
+ @Stable
+ public abstract ResourceRequest getAMContainerResourceRequest();
+
+ /**
+ * Set ResourceRequest of AM container
+ * @param request of AM container
+ */
+ @Public
+ @Evolving
+ public abstract void setAMContainerResourceRequest(ResourceRequest request);
/**
* Get the attemptFailuresValidityInterval in milliseconds for the application
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 86b55d1..95494c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -70,12 +70,22 @@ public static ResourceRequest newInstance(Priority priority, String hostName,
@Stable
public static ResourceRequest newInstance(Priority priority, String hostName,
Resource capability, int numContainers, boolean relaxLocality) {
+ return newInstance(priority, hostName, capability, numContainers,
+ relaxLocality, null);
+ }
+
+ @Public
+ @Stable
+ public static ResourceRequest newInstance(Priority priority, String hostName,
+ Resource capability, int numContainers, boolean relaxLocality,
+ String labelExpression) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
request.setRelaxLocality(relaxLocality);
+ request.setLabelExpression(labelExpression);
return request;
}
@@ -239,6 +249,29 @@ public static boolean isAnyLocation(String hostName) {
@Stable
public abstract void setRelaxLocality(boolean relaxLocality);
+ /**
+ * Get Label expression for this Resource Request
+ *
+ * @return label expression
+ */
+ @Public
+ @Stable
+ public abstract String getLabelExpression();
+
+ /**
+ * Set label expression associated with this resource request. Now only
+ * support AND(&&), in the future will provide support for OR(||), NOT(!).
+ *
+ * Examples:
+ * - GPU && LARGE_MEM, ask for node has label GPU and LARGE_MEM together
+ * - "" (empty) means ask for node doesn't have label on it, this is default
+ *
+ * @param labelExpression
+ */
+ @Public
+ @Stable
+ public abstract void setLabelExpression(String labelExpression);
+
@Override
public int hashCode() {
final int prime = 2153;
@@ -283,6 +316,20 @@ public boolean equals(Object obj) {
return false;
} else if (!priority.equals(other.getPriority()))
return false;
+ if (getLabelExpression() == null) {
+ if (other.getLabelExpression() != null) {
+ return false;
+ }
+ } else {
+ // do normalize on label expression before compare
+ String label1 = getLabelExpression().replaceAll("[\\t ]", "");
+ String label2 =
+ other.getLabelExpression() == null ? null : other
+ .getLabelExpression().replaceAll("[\\t ]", "");
+ if (!label1.equals(label2)) {
+ return false;
+ }
+ }
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b368746..5b8bcd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -241,6 +241,7 @@ message ResourceRequestProto {
optional ResourceProto capability = 3;
optional int32 num_containers = 4;
optional bool relax_locality = 5 [default = true];
+ optional string label_expression = 6;
}
enum AMCommandProto {
@@ -293,6 +294,8 @@ message ApplicationSubmissionContextProto {
repeated string applicationTags = 12;
optional int64 attempt_failures_validity_interval = 13 [default = -1];
optional LogAggregationContextProto log_aggregation_context = 14;
+ optional string app_label_expression = 15;
+ optional ResourceRequestProto am_container_resource_request = 16;
}
message LogAggregationContextProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index e4f183b..d5650a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-import com.google.common.base.CharMatcher;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -28,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
@@ -36,12 +38,11 @@
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
+import com.google.common.base.CharMatcher;
import com.google.protobuf.TextFormat;
-import java.util.HashSet;
-import java.util.Set;
-
@Private
@Unstable
public class ApplicationSubmissionContextPBImpl
@@ -56,6 +57,7 @@
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
private Set applicationTags = null;
+ private ResourceRequest amResourceRequest = null;
private LogAggregationContext logAggregationContext = null;
public ApplicationSubmissionContextPBImpl() {
@@ -114,6 +116,10 @@ private void mergeLocalToBuilder() {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
+ if (this.amResourceRequest != null) {
+ builder.setAmContainerResourceRequest(
+ convertToProtoFormat(this.amResourceRequest));
+ }
if (this.logAggregationContext != null) {
builder.setLogAggregationContext(
convertToProtoFormat(this.logAggregationContext));
@@ -134,8 +140,8 @@ private void maybeInitBuilder() {
}
viaProto = false;
}
-
-
+
+ @Deprecated
@Override
public Priority getPriority() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
@@ -148,7 +154,8 @@ public Priority getPriority() {
this.priority = convertFromProtoFormat(p.getPriority());
return this.priority;
}
-
+
+ @Deprecated
@Override
public void setPriority(Priority priority) {
maybeInitBuilder();
@@ -343,6 +350,7 @@ public void setMaxAppAttempts(int maxAppAttempts) {
builder.setMaxAppAttempts(maxAppAttempts);
}
+ @Deprecated
@Override
public Resource getResource() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
@@ -356,6 +364,7 @@ public Resource getResource() {
return this.resource;
}
+ @Deprecated
@Override
public void setResource(Resource resource) {
maybeInitBuilder();
@@ -385,6 +394,14 @@ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
+
+ private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
+ return new ResourceRequestPBImpl(p);
+ }
+
+ private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
+ return ((ResourceRequestPBImpl)t).getProto();
+ }
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p);
@@ -412,6 +429,47 @@ private ResourceProto convertToProtoFormat(Resource t) {
}
@Override
+ public String getAppLabelExpression() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAppLabelExpression()) {
+ return null;
+ }
+ return p.getAppLabelExpression();
+ }
+
+ @Override
+ public void setAppLabelExpression(String labelExpression) {
+ maybeInitBuilder();
+ if (labelExpression == null) {
+ builder.clearAppLabelExpression();
+ return;
+ }
+ builder.setAppLabelExpression(labelExpression);
+ }
+
+ @Override
+ public ResourceRequest getAMContainerResourceRequest() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.amResourceRequest != null) {
+ return amResourceRequest;
+ } // Else via proto
+ if (!p.hasAmContainerResourceRequest()) {
+ return null;
+ }
+ amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest());
+ return amResourceRequest;
+ }
+
+ @Override
+ public void setAMContainerResourceRequest(ResourceRequest request) {
+ maybeInitBuilder();
+ if (request == null) {
+ builder.clearAmContainerResourceRequest();
+ }
+ this.amResourceRequest = request;
+ }
+
+ @Override
public long getAttemptFailuresValidityInterval() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
return p.getAttemptFailuresValidityInterval();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 22863ac..ca052c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -186,4 +186,23 @@ public String toString() {
+ ", Location: " + getResourceName()
+ ", Relax Locality: " + getRelaxLocality() + "}";
}
+
+ @Override
+ public String getLabelExpression() {
+ ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasLabelExpression()) {
+ return null;
+ }
+ return (p.getLabelExpression());
+ }
+
+ @Override
+ public void setLabelExpression(String labelExpression) {
+ maybeInitBuilder();
+ if (labelExpression == null) {
+ builder.clearLabelExpression();
+ return;
+ }
+ builder.setLabelExpression(labelExpression);
+ }
}
\ No newline at end of file