diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 95254b2..c53d060 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -257,4 +257,23 @@ public abstract void setStartRange(long begin, long end) @Private @Unstable public abstract void setFinishRange(long begin, long end); + + /** + * Get the tags to filter applications on + * + * @return list of tags to filter on + */ + @Private + @Unstable + public abstract Set getTags(); + + /** + * Set the list of tags to filter applications on + * + * @param tags list of tags to filter on + */ + @Private + @Unstable + public abstract void setTags(Set tags); + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 0854fdb..7b6f17c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationReport is a report of an application.

* @@ -321,6 +323,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setApplicationType(String applicationType); + /** + * Get all tags corresponding to the application + * @return Application's tags + */ + @Public + @Stable + public abstract Set getTags(); + + @Private + @Unstable + public abstract void setTags(Set tags); + @Private @Stable public abstract void setAMRMToken(Token amRmToken); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index eaea13e..c20b4b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Set; + /** *

ApplicationSubmissionContext represents all of the * information needed by the ResourceManager to launch @@ -268,4 +270,22 @@ public static ApplicationSubmissionContext newInstance( @Public @Stable public abstract void setApplicationType(String applicationType); + + + /** + * Get tags for the application + * + * @return the application type + */ + @Public + @Stable + public abstract Set getTags(); + + /** + * Set tags for the application + * @param tags + */ + @Public + @Stable + public abstract void setTags(Set tags); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 7dc62fc..8093e44 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -162,6 +162,7 @@ message ApplicationReportProto { optional float progress = 17; optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; + repeated string tags = 20; } enum NodeStateProto { @@ -248,6 +249,7 @@ message ApplicationSubmissionContextProto { optional int32 maxAppAttempts = 8 [default = 0]; optional ResourceProto resource = 9; optional string applicationType = 10 [default = "YARN"]; + repeated string tags = 11; } enum ApplicationAccessTypeProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index a4631d1..25608e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -136,6 +136,7 @@ message GetApplicationsRequestProto { optional int64 start_end = 7; optional int64 finish_begin = 8; optional int64 finish_end = 9; + repeated string tags = 10; } message GetApplicationsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 10513a3..0f9edce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -49,6 +49,7 @@ Set queues = null; long limit = Long.MAX_VALUE; LongRange start = null, finish = null; + private Set tags; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -112,6 +113,9 @@ public void remove() { }; builder.addAllApplicationStates(iterable); } + if (this.tags != null && !this.tags.isEmpty()) { + builder.addAllTags(this.tags); + } } private void addLocalApplicationTypesToProto() { @@ -187,6 +191,36 @@ public void setApplicationTypes(Set applicationTypes) { this.applicationTypes = applicationTypes; } + private void initTags() { + if (this.tags != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + this.tags = new HashSet(); + this.tags.addAll(p.getTagsList()); + } + + @Override + public Set getTags() { + initTags(); + return this.tags; + } + + @Override + public void setTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearTags(); + this.tags = null; + return; + } + // Convert tags to lower case and add + this.tags = new HashSet(); + for (String tag : tags) { + this.tags.add(tag.toLowerCase()); + } + } + @Override public EnumSet getApplicationStates() { initApplicationStates(); @@ -194,7 +228,7 @@ public void setApplicationTypes(Set applicationTypes) { } @Override - public void setApplicationStates(EnumSet applicationStates) { + public void setApplicationStates(EnumSet applicationStates) { maybeInitBuilder(); if (applicationStates == null) { builder.clearApplicationStates(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 9716f74..e87259b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -38,6 +38,9 @@ import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.Set; + @Private @Unstable public class ApplicationReportPBImpl extends ApplicationReport { @@ -49,6 +52,7 @@ private ApplicationAttemptId currentApplicationAttemptId; private Token clientToAMToken = null; private Token amRmToken = null; + private Set tags = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -245,6 +249,21 @@ public Token getAMRMToken() { return amRmToken; } + private void initTags() { + if (this.tags != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + this.tags = new HashSet(); + this.tags.addAll(p.getTagsList()); + } + + @Override + public Set getTags() { + initTags(); + return this.tags; + } + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); @@ -356,6 +375,15 @@ public void setApplicationType(String applicationType) { } @Override + public void setTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearTags(); + } + this.tags = tags; + } + + @Override public void setDiagnostics(String diagnostics) { maybeInitBuilder(); if (diagnostics == null) { @@ -450,6 +478,9 @@ private void mergeLocalToBuilder() { builder.getAmRmToken())) { builder.setAmRmToken(convertToProtoFormat(this.amRmToken)); } + if (this.tags != null && !this.tags.isEmpty()) { + builder.addAllTags(this.tags); + } } private void mergeLocalToProto() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 5b48141..a200e23 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -34,6 +34,10 @@ import com.google.protobuf.TextFormat; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + @Private @Unstable public class ApplicationSubmissionContextPBImpl @@ -47,6 +51,7 @@ private Priority priority = null; private ContainerLaunchContext amContainer = null; private Resource resource = null; + private Set tags = null; public ApplicationSubmissionContextPBImpl() { builder = ApplicationSubmissionContextProto.newBuilder(); @@ -100,6 +105,9 @@ private void mergeLocalToBuilder() { builder.getResource())) { builder.setResource(convertToProtoFormat(this.resource)); } + if (this.tags != null && !this.tags.isEmpty()) { + builder.addAllTags(this.tags); + } } private void mergeLocalToProto() { @@ -196,7 +204,22 @@ public String getApplicationType() { } return (p.getApplicationType()); } - + + private void initTags() { + if (this.tags != null) { + return; + } + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + this.tags = new HashSet(); + this.tags.addAll(p.getTagsList()); + } + + @Override + public Set getTags() { + initTags(); + return this.tags; + } + @Override public void setQueue(String queue) { maybeInitBuilder(); @@ -218,6 +241,21 @@ public void setApplicationType(String applicationType) { } @Override + public void setTags(Set tags) { + maybeInitBuilder(); + if (tags == null || tags.isEmpty()) { + builder.clearTags(); + this.tags = null; + return; + } + // Convert tags to lower case and add + this.tags = new HashSet(); + for (String tag : tags) { + this.tags.add(tag.toLowerCase()); + } + } + + @Override public ContainerLaunchContext getAMContainerSpec() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; if (this.amContainer != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 1b2a03e..04fc8f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -26,6 +26,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; @@ -312,7 +313,7 @@ public static ApplicationReport newApplicationReport( String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String appType, Token amRmToken) { + float progress, String appType, Token amRmToken, Set tags) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -334,6 +335,7 @@ public static ApplicationReport newApplicationReport( report.setProgress(progress); report.setApplicationType(appType); report.setAMRMToken(amRmToken); + report.setTags(tags); return report; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index cd2226f..3133a77 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -431,6 +431,7 @@ public GetApplicationsResponse getApplications( request.getApplicationStates(); Set users = request.getUsers(); Set queues = request.getQueues(); + Set tags = request.getTags(); long limit = request.getLimit(); LongRange start = request.getStartRange(); LongRange finish = request.getFinishRange(); @@ -509,6 +510,23 @@ public void remove() { continue; } + if (tags != null && !tags.isEmpty()) { + Set appTags = application.getTags(); + if (appTags == null || appTags.isEmpty()) { + continue; + } + boolean match = false; + for (String tag : tags) { + if (appTags.contains(tag)) { + match = true; + break; + } + } + if (!match) { + continue; + } + } + boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); reports.add(application.createAndGetApplicationReport( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7dbc4cc..348713d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -320,7 +320,8 @@ private RMAppImpl createAndPopulateNewRMApp( submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, - submitTime, submissionContext.getApplicationType()); + submitTime, submissionContext.getApplicationType(), + submissionContext.getTags()); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 1809a4b..f2f5717 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -194,7 +195,13 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * Returns the application type * @return the application type. */ - String getApplicationType(); + String getApplicationType(); + + /** + * Get tags for the application + * @return tags corresponding to the application + */ + Set getTags(); /** * Check whether this application is safe to terminate. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 0bf7c81..726c640 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -98,6 +98,7 @@ private final long submitTime; private final Set updatedNodes = new HashSet(); private final String applicationType; + private final Set tags; // Mutable fields private long startTime; @@ -279,9 +280,9 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, - ApplicationSubmissionContext submissionContext, - YarnScheduler scheduler, - ApplicationMasterService masterService, long submitTime, String applicationType) { + ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, + ApplicationMasterService masterService, long submitTime, + String applicationType, Set tags) { this.applicationId = applicationId; this.name = name; @@ -297,6 +298,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.submitTime = submitTime; this.startTime = System.currentTimeMillis(); this.applicationType = applicationType; + this.tags = tags; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -528,7 +530,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken); + amrmToken, tags); } finally { this.readLock.unlock(); } @@ -1002,6 +1004,11 @@ public String getApplicationType() { } @Override + public Set getTags() { + return tags; + } + + @Override public boolean isAppSafeToTerminate() { RMAppState state = getState(); return state.equals(RMAppState.FINISHING) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index ed147fb..5e606b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -110,6 +110,7 @@ protected void render(Block html) { _("User:", app.getUser()). _("Name:", app.getName()). _("Application Type:", app.getApplicationType()). + _("Tags:", app.getTags()). _("State:", app.getState()). _("FinalStatus:", app.getFinalStatus()). _("Started:", Times.format(app.getStartTime())). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index cec95ac..da5f9c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -92,6 +92,8 @@ .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( appInfo.getApplicationType()))).append("\",\"") .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + appInfo.getTags()))).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( appInfo.getQueue()))).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index e18c047..34e4295 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -261,12 +261,14 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, - @QueryParam("applicationTypes") Set applicationTypes) { + @QueryParam("applicationTypes") Set applicationTypes, + @QueryParam("tags") Set tags) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; + boolean checkAppTags = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -327,6 +329,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, checkAppTypes = true; } + Set appTags = parseQueries(tags, false); + if (!appTags.isEmpty()) { + checkAppTags = true; + } + // stateQuery is deprecated. if (stateQuery != null && !stateQuery.isEmpty()) { statesQuery.add(stateQuery); @@ -354,6 +361,10 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, request.setApplicationTypes(appTypes); } + if (checkAppTags) { + request.setTags(appTags); + } + if (checkAppStates) { request.setApplicationStates(appStates); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 295e1f3..d7ccd05 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -39,6 +39,8 @@ import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import java.util.Set; + @XmlRootElement(name = "app") @XmlAccessorType(XmlAccessType.FIELD) public class AppInfo { @@ -67,6 +69,7 @@ protected String diagnostics; protected long clusterId; protected String applicationType; + protected String tags = ""; // initialize to an empty string // these are only allowed if acls allow protected long startedTime; @@ -117,6 +120,12 @@ public AppInfo(RMApp app, Boolean hasAccess) { if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } + if (app.getTags() != null && !app.getTags().isEmpty()) { + for (String tag : app.getTags()) { + this.tags += tag + ","; + } + this.tags = this.tags.substring(0, tags.length() - 1); + } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { @@ -239,6 +248,10 @@ public long getClusterId() { public String getApplicationType() { return this.applicationType; } + + public String getTags() { + return this.tags; + } public int getRunningContainers() { return this.runningContainers; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index d425dda..73a42ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -41,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -449,6 +450,7 @@ public void handle(Event event) {} {MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName()}; ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), getApplicationId(103)}; + List tags = Arrays.asList("Tag1", "Tag2", "Tag3"); // Submit applications for (int i = 0; i < appIds.length; i++) { @@ -456,7 +458,8 @@ public void handle(Event event) {} when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); SubmitApplicationRequest submitRequest = mockSubmitAppRequest( - appId, appNames[i], queues[i % queues.length]); + appId, appNames[i], queues[i % queues.length], + new HashSet(tags.subList(0, i + 1))); rmService.submitApplication(submitRequest); } @@ -497,6 +500,28 @@ public void handle(Event event) {} userSet.add(UserGroupInformation.getCurrentUser().getShortUserName()); assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + + // Check tags + request = GetApplicationsRequest.newInstance(); + Set tagSet = new HashSet(); + request.setTags(tagSet); + assertEquals("Incorrect number of matching tags", 6, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(0)); + request.setTags(tagSet); + assertEquals("Incorrect number of matching tags", 3, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(1)); + request.setTags(tagSet); + assertEquals("Incorrect number of matching tags", 2, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(2)); + request.setTags(tagSet); + assertEquals("Incorrect number of matching tags", 1, + rmService.getApplications(request).getApplicationList().size()); } @Test(timeout=4000) @@ -567,6 +592,11 @@ public void run() { private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { + return mockSubmitAppRequest(appId, name, queue, null); + } + + private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, + String name, String queue, Set tags) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( @@ -580,6 +610,7 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); + submissionContext.setTags(tags); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -646,7 +677,7 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, when(asContext.getMaxAppAttempts()).thenReturn(1); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis(), "YARN")); + .currentTimeMillis(), "YARN", null)); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId3, 1); RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index aa116bf..672877e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; @@ -140,6 +141,11 @@ public String getApplicationType() { } @Override + public Set getTags() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override public void setQueue(String name) { throw new UnsupportedOperationException("Not supported yet."); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index debcffe..a19034e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -218,6 +219,11 @@ public String getApplicationType() { } @Override + public Set getTags() { + return null; + } + + @Override public boolean isAppSafeToTerminate() { return true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index ba255d3..a904bf1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -225,7 +225,7 @@ protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b601b38..6d06b8e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -615,7 +615,7 @@ public void testUserAsDefaultQueue() throws Exception { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAttemptAddedSchedulerEvent appAddedEvent = @@ -637,7 +637,7 @@ public void testNotUserAsDefaultQueue() throws Exception { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAttemptAddedSchedulerEvent appAddedEvent2 = @@ -1746,7 +1746,7 @@ public void testNotAllowSubmitApplication() throws Exception { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START));