diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 1ee04f0..28c21db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -18,18 +18,18 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Records;
-import java.util.Set;
-
/**
*
ApplicationSubmissionContext represents all of the
* information needed by the ResourceManager to launch
@@ -62,7 +62,7 @@ public static ApplicationSubmissionContext newInstance(
Priority priority, ContainerLaunchContext amContainer,
boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
int maxAppAttempts, Resource resource, String applicationType,
- boolean keepContainers) {
+ boolean keepContainers, String appLabelExpression) {
ApplicationSubmissionContext context =
Records.newRecord(ApplicationSubmissionContext.class);
context.setApplicationId(applicationId);
@@ -78,6 +78,17 @@ public static ApplicationSubmissionContext newInstance(
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
return context;
}
+
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ Priority priority, ContainerLaunchContext amContainer,
+ boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
+ int maxAppAttempts, Resource resource, String applicationType,
+ boolean keepContainers) {
+ return newInstance(applicationId, applicationName, queue, priority,
+ amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
+ resource, applicationType, keepContainers, null);
+ }
@Public
@Stable
@@ -88,7 +99,7 @@ public static ApplicationSubmissionContext newInstance(
int maxAppAttempts, Resource resource, String applicationType) {
return newInstance(applicationId, applicationName, queue, priority,
amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
- resource, applicationType, false);
+ resource, applicationType, false, null);
}
@Public
@@ -153,19 +164,21 @@ public static ApplicationSubmissionContext newInstance(
public abstract void setQueue(String queue);
/**
+ * Please note this is DEPRECATED, please use getPriority in
+ * getAMContainerResourceRequest instead.
+ *
* Get the Priority of the application.
+ *
* @return Priority of the application
*/
- @Public
- @Stable
+ @Deprecated
public abstract Priority getPriority();
/**
* Set the Priority of the application.
* @param priority Priority of the application
*/
- @Private
- @Unstable
+ @Deprecated
public abstract void setPriority(Priority priority);
/**
@@ -248,14 +261,16 @@ public static ApplicationSubmissionContext newInstance(
public abstract void setMaxAppAttempts(int maxAppAttempts);
/**
+ * Please note this is DEPRECATED, please use getResource in
+ * getAMContainerResourceRequest instead.
+ *
* Get the resource required by the ApplicationMaster for this
* application.
*
* @return the resource required by the ApplicationMaster for
* this application.
*/
- @Public
- @Stable
+ @Deprecated
public abstract Resource getResource();
/**
@@ -265,8 +280,7 @@ public static ApplicationSubmissionContext newInstance(
* @param resource the resource required by the ApplicationMaster
* for this application.
*/
- @Public
- @Stable
+ @Deprecated
public abstract void setResource(Resource resource);
/**
@@ -338,4 +352,55 @@ public abstract void setKeepContainersAcrossApplicationAttempts(
@Public
@Stable
public abstract void setApplicationTags(Set tags);
+
+ /**
+ * Get label expression for this app
+ *
+ * @return label expression for this app
+ */
+ @Public
+ @Stable
+ public abstract String getAppLabelExpression();
+
+ /**
+ * Set label expression for the APP
+ *
+ * By default, APP label expression is empty. This field can be overwrite by
+ * resource request label expression and AM container label expression
+ *
+ * e.g.
+ * - APP label expression = "GPU && LARGE_MEM"
+ * - Resource Request label expression = "", it will be set "GPU && LARGE_MEM"
+ * - Resource Request label expression = "GPU && INFINI_BAND",
+ * it will be "GPU && INFINI_BAND"
+ *
+ * As same as label expression AM container Resource Request
+ */
+ @Public
+ @Stable
+ public abstract void setAppLabelExpression(String labelExpression);
+
+ /**
+ * Get ResourceRequest of AM container, if this is not null, scheduler will
+ * use this to acquire resource for AM container.
+ *
+ * If this is null, scheduler will assemble a ResourceRequest by using
+ * getResource and getPriority of
+ * ApplicationSubmissionContext.
+ *
+ * Number of containers will be ignore.
+ *
+ * @return ResourceRequest of AM container
+ */
+ @Public
+ @Stable
+ public abstract ResourceRequest getAMContainerResourceRequest();
+
+ /**
+ * Set ResourceRequest of AM container
+ * @param request of AM container
+ */
+ @Public
+ @Stable
+ public abstract void setAMContainerResourceRequest(ResourceRequest request);
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 86b55d1..95494c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -70,12 +70,22 @@ public static ResourceRequest newInstance(Priority priority, String hostName,
@Stable
public static ResourceRequest newInstance(Priority priority, String hostName,
Resource capability, int numContainers, boolean relaxLocality) {
+ return newInstance(priority, hostName, capability, numContainers,
+ relaxLocality, null);
+ }
+
+ @Public
+ @Stable
+ public static ResourceRequest newInstance(Priority priority, String hostName,
+ Resource capability, int numContainers, boolean relaxLocality,
+ String labelExpression) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
request.setRelaxLocality(relaxLocality);
+ request.setLabelExpression(labelExpression);
return request;
}
@@ -239,6 +249,29 @@ public static boolean isAnyLocation(String hostName) {
@Stable
public abstract void setRelaxLocality(boolean relaxLocality);
+ /**
+ * Get Label expression for this Resource Request
+ *
+ * @return label expression
+ */
+ @Public
+ @Stable
+ public abstract String getLabelExpression();
+
+ /**
+ * Set label expression associated with this resource request. Now only
+ * support AND(&&), in the future will provide support for OR(||), NOT(!).
+ *
+ * Examples:
+ * - GPU && LARGE_MEM, ask for node has label GPU and LARGE_MEM together
+ * - "" (empty) means ask for node doesn't have label on it, this is default
+ *
+ * @param labelExpression
+ */
+ @Public
+ @Stable
+ public abstract void setLabelExpression(String labelExpression);
+
@Override
public int hashCode() {
final int prime = 2153;
@@ -283,6 +316,20 @@ public boolean equals(Object obj) {
return false;
} else if (!priority.equals(other.getPriority()))
return false;
+ if (getLabelExpression() == null) {
+ if (other.getLabelExpression() != null) {
+ return false;
+ }
+ } else {
+ // do normalize on label expression before compare
+ String label1 = getLabelExpression().replaceAll("[\\t ]", "");
+ String label2 =
+ other.getLabelExpression() == null ? null : other
+ .getLabelExpression().replaceAll("[\\t ]", "");
+ if (!label1.equals(label2)) {
+ return false;
+ }
+ }
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3f1fa6c..de40dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -238,6 +238,7 @@ message ResourceRequestProto {
optional ResourceProto capability = 3;
optional int32 num_containers = 4;
optional bool relax_locality = 5 [default = true];
+ optional string label_expression = 6;
}
enum AMCommandProto {
@@ -288,6 +289,8 @@ message ApplicationSubmissionContextProto {
optional string applicationType = 10 [default = "YARN"];
optional bool keep_containers_across_application_attempts = 11 [default = false];
repeated string applicationTags = 12;
+ optional string app_label_expression = 13;
+ optional ResourceRequestProto am_container_resource_request = 14;
}
enum ApplicationAccessTypeProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index c2f3268..c4b2032 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-import com.google.common.base.CharMatcher;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -26,6 +28,7 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
@@ -33,12 +36,11 @@
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
+import com.google.common.base.CharMatcher;
import com.google.protobuf.TextFormat;
-import java.util.HashSet;
-import java.util.Set;
-
@Private
@Unstable
public class ApplicationSubmissionContextPBImpl
@@ -53,6 +55,7 @@
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
private Set applicationTags = null;
+ private ResourceRequest amResourceRequest = null;
public ApplicationSubmissionContextPBImpl() {
builder = ApplicationSubmissionContextProto.newBuilder();
@@ -110,6 +113,10 @@ private void mergeLocalToBuilder() {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
+ if (this.amResourceRequest != null) {
+ builder.setAmContainerResourceRequest(
+ convertToProtoFormat(this.amResourceRequest));
+ }
}
private void mergeLocalToProto() {
@@ -377,6 +384,14 @@ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
+
+ private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
+ return new ResourceRequestPBImpl(p);
+ }
+
+ private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
+ return ((ResourceRequestPBImpl)t).getProto();
+ }
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p);
@@ -402,4 +417,45 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
private ResourceProto convertToProtoFormat(Resource t) {
return ((ResourcePBImpl)t).getProto();
}
+
+ @Override
+ public String getAppLabelExpression() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAppLabelExpression()) {
+ return null;
+ }
+ return p.getAppLabelExpression();
+ }
+
+ @Override
+ public void setAppLabelExpression(String labelExpression) {
+ maybeInitBuilder();
+ if (labelExpression == null) {
+ builder.clearAppLabelExpression();
+ return;
+ }
+ builder.setAppLabelExpression(labelExpression);
+ }
+
+ @Override
+ public ResourceRequest getAMContainerResourceRequest() {
+ ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.amResourceRequest != null) {
+ return amResourceRequest;
+ } // Else via proto
+ if (!p.hasAmContainerResourceRequest()) {
+ return null;
+ }
+ amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest());
+ return amResourceRequest;
+ }
+
+ @Override
+ public void setAMContainerResourceRequest(ResourceRequest request) {
+ maybeInitBuilder();
+ if (request == null) {
+ builder.clearAmContainerResourceRequest();
+ }
+ this.amResourceRequest = request;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 22863ac..ca052c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -186,4 +186,23 @@ public String toString() {
+ ", Location: " + getResourceName()
+ ", Relax Locality: " + getRelaxLocality() + "}";
}
+
+ @Override
+ public String getLabelExpression() {
+ ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasLabelExpression()) {
+ return null;
+ }
+ return (p.getLabelExpression());
+ }
+
+ @Override
+ public void setLabelExpression(String labelExpression) {
+ maybeInitBuilder();
+ if (labelExpression == null) {
+ builder.clearLabelExpression();
+ return;
+ }
+ builder.setLabelExpression(labelExpression);
+ }
}
\ No newline at end of file