diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index ba02d86..eef1675 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -118,8 +118,9 @@ protected void serviceStop() throws Exception { try { Set appTypes = new HashSet(1); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); + Set appStates = new HashSet(1); return TypeConverter.fromYarnApps( - client.getApplications(appTypes), this.conf); + client.getApplications(appTypes, appStates), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -299,12 +300,27 @@ public ApplicationReport getApplicationReport(ApplicationId appId) } @Override - public List getApplications( - Set applicationTypes) throws YarnException, IOException { + public List getApplications(Set applicationTypes) + throws YarnException, + IOException { return client.getApplications(applicationTypes); } @Override + public List getApplicationWithApplicationStatesFilter( + Set applicationStates) throws YarnException, + IOException { + return client.getApplicationWithApplicationStatesFilter(applicationStates); + } + + @Override + public List getApplications( + Set applicationTypes, Set applicationStates) + throws YarnException, IOException { + return client.getApplications(applicationTypes, applicationStates); + } + + @Override public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException { return client.getYarnClusterMetrics(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 9a73221..a59c822 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -45,13 +45,64 @@ public static GetApplicationsRequest newInstance() { return request; } + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest + newInstance(Set applicationTypes) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationTypes(applicationTypes); + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application states in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstanceWithApplicationStatesFilter( + Set applicationStates) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationStates(applicationStates); + return request; + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving and application types and application types in the cluster from the + * ResourceManager. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ @Public @Stable public static GetApplicationsRequest newInstance( - Set applicationTypes) { + Set applicationTypes, Set applicationStates) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationTypes(applicationTypes); + request.setApplicationStates(applicationStates); return request; } @@ -75,4 +126,25 @@ public static GetApplicationsRequest newInstance( @Unstable public abstract void setApplicationTypes(Set applicationTypes); + + /** + * Get the application states to filter applications on + * + * @return Set of Application states to filter on + */ + @Public + @Stable + public abstract Set getApplicationStates(); + + /** + * Set the application states to filter applications on + * + * @param applicationStates + * A Set of Application states to filter on. + * If not defined, match all running applications + */ + @Private + @Unstable + public abstract void + setApplicationStates(Set applicationStates); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index bd009e0..dbbb015 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -122,6 +122,7 @@ message GetClusterMetricsResponseProto { message GetApplicationsRequestProto { repeated string application_types = 1; + repeated string application_states = 2; } message GetApplicationsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index c3587b2..8addaa3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -171,13 +171,13 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) *

* Get a report (ApplicationReport) of all Applications in the cluster. *

- * + * *

* If the user does not have VIEW_APP access for an application * then the corresponding report will be filtered as described in * {@link #getApplicationReport(ApplicationId)}. *

- * + * * @return a list of reports of all running applications * @throws YarnException * @throws IOException @@ -207,6 +207,49 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) /** *

+ * Get a report (ApplicationReport) of Applications matching the given + * application states in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationStates + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List + getApplicationWithApplicationStatesFilter(Set applicationStates) + throws YarnException, IOException; + + /** + *

+ * Get a report (ApplicationReport) of Applications matching the given + * application types and application states in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationTypes + * @param applicationStates + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplications( + Set applicationTypes, Set applicationStates) + throws YarnException, IOException; + + /** + *

* Get metrics ({@link YarnClusterMetrics}) about the cluster. *

* diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 4a1b83c..006ca31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -211,15 +212,36 @@ public ApplicationReport getApplicationReport(ApplicationId appId) @Override public List getApplications() throws YarnException, IOException { - return getApplications(null); + return getApplications(null, null); } @Override + public List getApplications(Set applicationTypes) + throws YarnException, + IOException { + return getApplications(applicationTypes, null); + } + + @Override + public List getApplicationWithApplicationStatesFilter( + Set applicationStates) + throws YarnException, IOException { + return getApplications(null, applicationStates); + } + @Override public List getApplications( - Set applicationTypes) throws YarnException, IOException { + Set applicationTypes, Set applicationStates) + throws YarnException, IOException { + Set appTypes = new HashSet(); + Set appStates = new HashSet(); + if(applicationTypes != null && !applicationTypes.isEmpty()) { + appTypes.addAll(applicationTypes); + } + if(applicationStates != null && !applicationStates.isEmpty()) { + appStates.addAll(applicationStates); + } GetApplicationsRequest request = - applicationTypes == null ? GetApplicationsRequest.newInstance() - : GetApplicationsRequest.newInstance(applicationTypes); + GetApplicationsRequest.newInstance(appTypes, appStates); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 16e55a6..e9ef89a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -49,6 +49,7 @@ System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; + private static final String APP_STATE_CMD ="appStates"; public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); @@ -66,7 +67,8 @@ public int run(String[] args) throws Exception { opts.addOption(STATUS_CMD, true, "Prints the status of the application."); opts.addOption(LIST_CMD, false, "List applications from the RM. " + "Supports optional use of --appTypes to filter applications " + - "based on application type."); + "based on application type, " + + "and --appStates to filter applications based on application state"); opts.addOption(KILL_CMD, true, "Kills the application."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); Option appTypeOpt = new Option(APP_TYPE_CMD, true, @@ -75,6 +77,12 @@ public int run(String[] args) throws Exception { appTypeOpt.setArgs(Option.UNLIMITED_VALUES); appTypeOpt.setArgName("Comma-separated list of application types"); opts.addOption(appTypeOpt); + Option appStateOpt = new Option(APP_STATE_CMD, true, + "Works with --list to filter applications based on their state"); + appStateOpt.setValueSeparator(','); + appStateOpt.setArgs(Option.UNLIMITED_VALUES); + appStateOpt.setArgName("Comma-separated list of application states"); + opts.addOption(appStateOpt); opts.getOption(KILL_CMD).setArgName("Application ID"); opts.getOption(STATUS_CMD).setArgName("Application ID"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -98,7 +106,18 @@ public int run(String[] args) throws Exception { } } } - listApplications(appTypes); + Set appStates = new HashSet(); + if(cliParser.hasOption(APP_STATE_CMD)) { + String[] types = cliParser.getOptionValues(APP_STATE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + appStates.add(type.trim()); + } + } + } + } + listApplications(appTypes, appStates); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); @@ -127,23 +146,28 @@ void printUsage(Options opts) { /** * Lists the applications matching the given application Types - * present in the Resource Manager + * And application States present in the Resource Manager * * @param appTypes + * @param appStates * @throws YarnException * @throws IOException */ - private void listApplications(Set appTypes) + private void listApplications(Set appTypes, Set appStates) throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); List appsReport = - client.getApplications(appTypes); + client.getApplications(appTypes, appStates); writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name","Application-Type", "User", "Queue", "State", "Final-State","Progress", "Tracking-URL"); for (ApplicationReport appReport : appsReport) { + if (appStates.isEmpty() && appReport.getYarnApplicationState() + != YarnApplicationState.RUNNING) { + continue; + } DecimalFormat formatter = new DecimalFormat("###.##%"); String progress = formatter.format(appReport.getProgress()); writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 8df430a..7eae357 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -55,10 +55,12 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -163,23 +165,51 @@ public void testGetApplications() throws YarnException, IOException { List expectedReports = ((MockYarnClient)client).getReports(); + List reports = client.getApplications(); + Assert.assertEquals(reports.size(), 2); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports + .get(1).getApplicationType().equals("MAPREDUCE")) + || (reports.get(1).getApplicationType().equals("YARN") && reports + .get(0).getApplicationType().equals("MAPREDUCE"))); + for(ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } + Set appTypes = new HashSet(); appTypes.add("YARN"); appTypes.add("NON-YARN"); - List reports = client.getApplications(appTypes); - Assert.assertEquals(reports.size(), 2); + reports = + client.getApplications(appTypes, null); + Assert.assertEquals(reports.size(), 1); Assert - .assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports - .get(1).getApplicationType().equals("NON-YARN")) - || (reports.get(1).getApplicationType().equals("YARN") && reports - .get(0).getApplicationType().equals("NON-YARN"))); + .assertTrue(reports.get(0).getApplicationType().equals("YARN")); for(ApplicationReport report : reports) { Assert.assertTrue(expectedReports.contains(report)); } - reports = client.getApplications(); - Assert.assertEquals(reports, expectedReports); + Set appStates = new HashSet(); + appStates.add("FINISHED"); + appStates.add("FAILED"); + reports = client.getApplications(null, appStates); + Assert.assertEquals(reports.size(), 2); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports + .get(1).getApplicationType().equals("NON-MAPREDUCE")) + || (reports.get(1).getApplicationType().equals("NON-YARN") && reports + .get(0).getApplicationType().equals("NON-MAPREDUCE"))); + for (ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } + + reports = client.getApplications(appTypes, appStates); + Assert.assertEquals(reports.size(), 1); + Assert + .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN"))); + for (ApplicationReport report : reports) { + Assert.assertTrue(expectedReports.contains(report)); + } client.stop(); } @@ -212,10 +242,22 @@ public void start() { @Override public List getApplications( - Set applicationTypes) throws YarnException, IOException { + Set applicationTypes, Set applicationStates) + throws YarnException, IOException { + Set appTypes = new HashSet(); + Set appStates = new HashSet(); + if(applicationTypes != null && !applicationTypes.isEmpty()) { + appTypes.addAll(applicationTypes); + } + if(applicationStates != null && !applicationStates.isEmpty()) { + appStates.addAll(applicationStates); + for(String appState : appStates) { + System.out.println(appState); + } + } GetApplicationsRequest request = - applicationTypes == null ? GetApplicationsRequest.newInstance() - : GetApplicationsRequest.newInstance(applicationTypes); + GetApplicationsRequest.newInstance(appTypes, + appStates); when(rmClient.getApplications(request)) .thenReturn( getApplicationReports(reports, @@ -243,7 +285,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); @@ -262,10 +304,21 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); + + ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); + ApplicationReport newApplicationReport4 = + ApplicationReport.newInstance( + applicationId4, + ApplicationAttemptId.newInstance(applicationId4, 4), + "user4", "queue4", "appname4", "host4", 127, null, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, + "NON-MAPREDUCE", null); + applicationReports.add(newApplicationReport4); return applicationReports; } @@ -275,19 +328,54 @@ private GetApplicationsResponse getApplicationReports( List appReports = new ArrayList(); Set appTypes = request.getApplicationTypes(); - boolean bypassFilter = appTypes.isEmpty(); + Set appStates = request.getApplicationStates(); + boolean bypassFilteronAppTypes = appTypes.isEmpty(); + boolean bypassFilteronAppStates = appStates.isEmpty(); for (ApplicationReport appReport : applicationReports) { - if (!(bypassFilter || appTypes.contains( + if (!(bypassFilteronAppTypes || appTypes.contains( appReport.getApplicationType()))) { continue; } + if (bypassFilteronAppStates) { + if (createApplicationState(appReport.getYarnApplicationState()) + != RMAppState.RUNNING) { + continue; + } + } else { + if (!appStates.contains(createApplicationState(appReport + .getYarnApplicationState()).toString())) { + continue; + } + } appReports.add(appReport); } GetApplicationsResponse response = GetApplicationsResponse.newInstance(appReports); return response; } + + private RMAppState createApplicationState(YarnApplicationState yarnAppState) { + switch(yarnAppState) { + case NEW: + return RMAppState.NEW; + case NEW_SAVING: + return RMAppState.NEW_SAVING; + case SUBMITTED: + return RMAppState.SUBMITTED; + case ACCEPTED: + return RMAppState.ACCEPTED; + case RUNNING: + return RMAppState.RUNNING; + case FINISHED: + return RMAppState.FINISHED; + case KILLED: + return RMAppState.KILLED; + case FAILED: + return RMAppState.FAILED; + } + throw new YarnRuntimeException("Unknown state passed!"); + } } @Test(timeout = 30000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 8be8b68..8508a82 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -134,7 +134,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -152,23 +152,31 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); - Set appType1 = new HashSet(); - appType1.add("YARN"); + ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); + ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( + applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), + "user4", "queue4", "appname4", "host4", 127, null, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", + null); + applicationReports.add(newApplicationReport4); - when(client.getApplications(appType1)).thenReturn( - getApplicationReports(applicationReports, appType1)); - int result = cli.run(new String[] { "-list", "-appTypes", "YARN" }); + Set appType1 = new HashSet(); + Set appState1 = new HashSet(); + when(client.getApplications(appType1, appState1)).thenReturn( + getApplicationReports(applicationReports, appType1, appState1)); + int result = cli.run(new String[] { "-list" }); assertEquals(0, result); - verify(client).getApplications(appType1); + verify(client).getApplications(appType1, appState1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.println("Total Applications:1"); + pw.println("Total Applications:2"); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -176,9 +184,14 @@ public void testGetApplications() throws Exception { pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); + pw.print("queue\t RUNNING\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t RUNNING\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); pw.close(); String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); @@ -187,13 +200,16 @@ public void testGetApplications() throws Exception { sysOutStream.reset(); Set appType2 = new HashSet(); appType2.add("YARN"); - appType2.add("FOO-YARN"); - when(client.getApplications(appType2)).thenReturn( - getApplicationReports(applicationReports, appType2)); - cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN", - ",,,,, YARN,," }); + appType2.add("NON-YARN"); + + Set appState2 = new HashSet(); + when(client.getApplications(appType2, appState2)).thenReturn( + getApplicationReports(applicationReports, appType2, appState2)); + result = + cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN", + " ,, ,," }); assertEquals(0, result); - verify(client).getApplications(appType2); + verify(client).getApplications(appType2, appState2); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total Applications:1"); @@ -204,7 +220,7 @@ public void testGetApplications() throws Exception { pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); + pw.print("queue\t RUNNING\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); pw.close(); @@ -214,14 +230,17 @@ public void testGetApplications() throws Exception { sysOutStream.reset(); Set appType3 = new HashSet(); - appType3.add("YARN"); - appType3.add("NON-YARN"); - when(client.getApplications(appType3)).thenReturn( - getApplicationReports(applicationReports, appType3)); - result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" }); + Set appState3 = new HashSet(); + appState3.add("FINISHED"); + appState3.add("FAILED"); + + when(client.getApplications(appType3, appState3)).thenReturn( + getApplicationReports(applicationReports, appType3, appState3)); + cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , FAILED", + ",,FINISHED" }); assertEquals(0, result); - verify(client).getApplications(appType3); + verify(client).getApplications(appType3, appState3); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total Applications:2"); @@ -230,16 +249,16 @@ public void testGetApplications() throws Exception { pw.print("\t User\t Queue\t State\t "); pw.print("Final-State\t Progress"); pw.println("\t Tracking-URL"); - pw.print(" application_1234_0005\t "); - pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); - pw.print("SUCCEEDED\t 53.79%"); - pw.println("\t N/A"); pw.print(" application_1234_0006\t "); pw.print("appname2\t NON-YARN\t user2\t "); pw.print("queue2\t FINISHED\t "); pw.print("SUCCEEDED\t 63.79%"); pw.println("\t N/A"); + pw.print(" application_1234_0008\t "); + pw.print("appname4\t NON-MAPREDUCE\t user4\t "); + pw.print("queue4\t FAILED\t "); + pw.print("SUCCEEDED\t 83.79%"); + pw.println("\t N/A"); pw.close(); appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); @@ -247,35 +266,32 @@ public void testGetApplications() throws Exception { sysOutStream.reset(); Set appType4 = new HashSet(); - when(client.getApplications(appType4)).thenReturn( - getApplicationReports(applicationReports, appType4)); - result = cli.run(new String[] { "-list" }); - assertEquals(0, result); - verify(client).getApplications(appType4); + appType4.add("YARN"); + appType4.add("NON-YARN"); + Set appState4 = new HashSet(); + appState4.add("FINISHED"); + appState4.add("FAILED"); + + when(client.getApplications(appType4, appState4)).thenReturn( + getApplicationReports(applicationReports, appType4, appState4)); + cli.run(new String[] { "-list", "--appTypes", "YARN,NON-YARN", + "--appStates", "FINISHED ,, , FAILED" }); + assertEquals(0, result); + verify(client).getApplications(appType2, appState2); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); - pw.println("Total Applications:3"); + pw.println("Total Applications:1"); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); pw.print("Final-State\t Progress"); pw.println("\t Tracking-URL"); - pw.print(" application_1234_0005\t "); - pw.print("appname\t YARN\t user\t "); - pw.print("queue\t FINISHED\t "); - pw.print("SUCCEEDED\t 53.79%"); - pw.println("\t N/A"); pw.print(" application_1234_0006\t "); pw.print("appname2\t NON-YARN\t user2\t "); pw.print("queue2\t FINISHED\t "); pw.print("SUCCEEDED\t 63.79%"); pw.println("\t N/A"); - pw.print(" application_1234_0007\t "); - pw.print("appname3\t MAPREDUCE\t user3\t "); - pw.print("queue3\t FINISHED\t "); - pw.print("SUCCEEDED\t 73.79%"); - pw.println("\t N/A"); pw.close(); appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); @@ -284,16 +300,26 @@ public void testGetApplications() throws Exception { private List getApplicationReports( List applicationReports, - Set appTypes) { + Set appTypes, Set appStates) { List appReports = new ArrayList(); - boolean bypassFilter = appTypes.isEmpty(); + boolean bypassFilterOnAppTypes = appTypes.isEmpty(); + boolean bypassFilterOnAppStates = appStates.isEmpty(); for (ApplicationReport appReport : applicationReports) { - if (!(bypassFilter || appTypes.contains( + if (!(bypassFilterOnAppTypes || appTypes.contains( appReport.getApplicationType()))) { continue; } + if (bypassFilterOnAppStates) { + if (!appReport.getYarnApplicationState().toString().equals("RUNNING")) { + continue; + } + } else { + if (!appStates.contains(appReport.getYarnApplicationState().toString())) { + continue; + } + } appReports.add(appReport); } return appReports; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index 48a8d85..194ee15 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -38,6 +38,7 @@ boolean viaProto = false; Set applicationTypes = null; + Set applicationStates = null; public GetApplicationsRequestPBImpl() { builder = GetApplicationsRequestProto.newBuilder(); @@ -67,6 +68,9 @@ private void mergeLocalToBuilder() { if (this.applicationTypes != null) { addLocalApplicationTypesToProto(); } + if (this.applicationStates != null) { + addLocalApplicationStatesToProto(); + } } private void addLocalApplicationTypesToProto() { @@ -77,6 +81,15 @@ private void addLocalApplicationTypesToProto() { builder.addAllApplicationTypes(applicationTypes); } + private void addLocalApplicationStatesToProto() { + maybeInitBuilder(); + builder.clearApplicationStates(); + if (this.applicationStates == null) { + return; + } + builder.addAllApplicationStates(applicationStates); + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = GetApplicationsRequestProto.newBuilder(proto); @@ -94,6 +107,16 @@ private void initApplicationTypes() { this.applicationTypes.addAll(appTypeList); } + private void initApplicationStates() { + if (this.applicationStates != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + List appStateList = p.getApplicationStatesList(); + this.applicationStates = new HashSet(); + this.applicationStates.addAll(appStateList); + } + @Override public Set getApplicationTypes() { initApplicationTypes(); @@ -109,6 +132,21 @@ public void setApplicationTypes(Set applicationTypes) { } @Override + public Set getApplicationStates() { + initApplicationStates(); + return this.applicationStates; + } + + @Override + public void setApplicationStates(Set applicationStates) { + maybeInitBuilder(); + if (applicationStates == null) { + builder.clearApplicationStates(); + } + this.applicationStates = applicationStates; + } + + @Override public int hashCode() { return getProto().hashCode(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 97f0ef8..3548564 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -394,7 +395,6 @@ public GetClusterMetricsResponse getClusterMetrics( @Override public GetApplicationsResponse getApplications( GetApplicationsRequest request) throws YarnException { - UserGroupInformation callerUGI; try { callerUGI = UserGroupInformation.getCurrentUser(); @@ -404,12 +404,23 @@ public GetApplicationsResponse getApplications( } Set applicationTypes = request.getApplicationTypes(); - boolean bypassFilter = applicationTypes.isEmpty(); + Set applicationStates = request.getApplicationStates(); + boolean bypassFilter = + applicationTypes.isEmpty() && applicationStates.isEmpty(); List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { - if (!(bypassFilter || applicationTypes.contains(application - .getApplicationType()))) { - continue; + if (!bypassFilter) { + if (!applicationTypes.isEmpty()) { + if (!applicationTypes.contains(application.getApplicationType())) { + continue; + } + } + + if (!applicationStates.isEmpty()) { + if (!applicationStates.contains(application.getState().toString())) { + continue; + } + } } boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index 8c28355..85aa275 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -22,8 +22,10 @@ import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import junit.framework.Assert; @@ -211,10 +213,16 @@ private void verifyOwnerAccess() throws Exception { // View as owner rmClient.getApplicationReport(appReportRequest); + Set appStates = allAppStates(); + Set appTypes = new HashSet(); + // List apps as owner - Assert.assertEquals("App view by owner should list the apps!!", 1, - rmClient.getApplications( - recordFactory.newRecordInstance(GetApplicationsRequest.class)) + Assert.assertEquals( + "App view by owner should list the apps!!", + 1, + rmClient + .getApplications( + GetApplicationsRequest.newInstance(appTypes, appStates)) .getApplicationList().size()); // Kill app as owner @@ -242,10 +250,12 @@ private void verifySuperUserAccess() throws Exception { // View as the superUser superUserClient.getApplicationReport(appReportRequest); + Set appTypes = new HashSet(); + Set appStates = allAppStates(); // List apps as superUser Assert.assertEquals("App view by super-user should list the apps!!", 2, superUserClient.getApplications( - recordFactory.newRecordInstance(GetApplicationsRequest.class)) + GetApplicationsRequest.newInstance(appTypes, appStates)) .getApplicationList().size()); // Kill app as the superUser @@ -273,10 +283,13 @@ private void verifyFriendAccess() throws Exception { // View as the friend friendClient.getApplicationReport(appReportRequest); + Set appTypes = new HashSet(); + Set appStates = allAppStates(); + // List apps as friend Assert.assertEquals("App view by a friend should list the apps!!", 3, friendClient.getApplications( - recordFactory.newRecordInstance(GetApplicationsRequest.class)) + GetApplicationsRequest.newInstance(appTypes, appStates)) .getApplicationList().size()); // Kill app as the friend @@ -306,11 +319,14 @@ private void verifyEnemyAccess() throws Exception { appReportRequest).getApplicationReport(); verifyEnemyAppReport(appReport); + Set appTypes = new HashSet(); + Set appStates = allAppStates(); + // List apps as enemy - List appReports = enemyRmClient - .getApplications(recordFactory - .newRecordInstance(GetApplicationsRequest.class)) - .getApplicationList(); + List appReports = + enemyRmClient.getApplications( + GetApplicationsRequest.newInstance(appTypes, appStates)) + .getApplicationList(); Assert.assertEquals("App view by enemy should list the apps!!", 4, appReports.size()); for (ApplicationReport report : appReports) { @@ -358,4 +374,12 @@ private void verifyEnemyAppReport(ApplicationReport appReport) { Assert.assertEquals("Enemy should not see app needed resources", -1, usageReport.getNeededResources().getMemory()); } + + private Set allAppStates() { + Set appStates = new HashSet(); + for(RMAppState appState : RMAppState.values()) { + appStates.add(appState.toString()); + } + return appStates; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index ff3c3aa..82c04db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -22,6 +22,7 @@ import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; import java.io.IOException; import java.net.InetSocketAddress; @@ -78,6 +79,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -379,8 +381,12 @@ public void handle(Event event) {} e.getMessage().contains("Cannot add a duplicate!")); } + Set appStates = new HashSet (); + appStates.add("FINISHED"); + appStates.add("NEW"); GetApplicationsRequest getAllAppsRequest = - GetApplicationsRequest.newInstance(new HashSet()); + GetApplicationsRequest.newInstance(new HashSet(), + appStates); GetApplicationsResponse getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest); Assert.assertEquals(5, @@ -389,7 +395,7 @@ public void handle(Event event) {} Set appTypes = new HashSet(); appTypes.add("matchType"); - getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes); + getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes, appStates); getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest); Assert.assertEquals(1, @@ -526,9 +532,11 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName) { ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); when(asContext.getMaxAppAttempts()).thenReturn(1); - return new RMAppImpl(applicationId3, rmContext, config, null, null, + RMAppImpl spyRMApp = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis(), "YARN"); + .currentTimeMillis(), "YARN")); + when(spyRMApp.getState()).thenReturn(RMAppState.FINISHED); + return spyRMApp; } private static YarnScheduler mockYarnScheduler() {