diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 03552e4..01ef949 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -90,7 +90,7 @@ private ApplicationReport getUnknownApplicationReport() { return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null); } NotRunningJob(ApplicationReport applicationReport, JobState jobState) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 82e8bdb..40ed28e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -456,4 +456,34 @@ public ReservationDeleteResponse deleteReservation( throws YarnException, IOException { return client.getClusterNodeLabels(); } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId applicationAttemptId, + boolean includeAppAttemptMetrics) throws YarnException, IOException { + return client.getApplicationAttemptReport(applicationAttemptId, + includeAppAttemptMetrics); + } + + @Override + public List getApplicationAttempts( + ApplicationId applicationId, boolean includeAppAttemptMetrics) + throws YarnException, IOException { + return client.getApplicationAttempts(applicationId, + includeAppAttemptMetrics); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId, + boolean includeApplicationMetrics) throws YarnException, IOException { + return client.getApplicationReport(appId, includeApplicationMetrics); + } + + @Override + public List getApplications(Set applicationTypes, + EnumSet applicationStates, + boolean includeApplicationMetrics) throws YarnException, IOException { + return client.getApplications(applicationTypes, applicationStates, + includeApplicationMetrics); + } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index 7d6b2f3..1b6cb97 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -472,7 +472,8 @@ private ApplicationReport getFinishedApplicationReport() { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, + null, null); } private ApplicationReport getRunningApplicationReport(String host, int port) { @@ -482,7 +483,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null); } private ResourceMgrDelegate getRMDelegate() throws IOException { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index c427975..40762f1 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -196,7 +196,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, - "tmp", null)); + "tmp", null, null, null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptReportRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptReportRequest.java index fb50952..4a553c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptReportRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptReportRequest.java @@ -50,9 +50,22 @@ public static GetApplicationAttemptReportRequest newInstance( GetApplicationAttemptReportRequest request = Records.newRecord(GetApplicationAttemptReportRequest.class); request.setApplicationAttemptId(applicationAttemptId); + request.setIncludeAppAttemptMetrics(false); return request; } + @Public + @Unstable + public static GetApplicationAttemptReportRequest newInstance( + ApplicationAttemptId applicationAttemptId, + boolean includeAppAttemptMetrics) { + GetApplicationAttemptReportRequest request = + Records.newRecord(GetApplicationAttemptReportRequest.class); + request.setApplicationAttemptId(applicationAttemptId); + request.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); + return request; + } + /** * Get the ApplicationAttemptId of an application attempt. * @@ -72,4 +85,32 @@ public static GetApplicationAttemptReportRequest newInstance( @Unstable public abstract void setApplicationAttemptId( ApplicationAttemptId applicationAttemptId); + + /** + * Get the flag which indicates whether + * the ApplicationAttemptReport includes + * ApplicationAttemptMetrics. + * + * @return the flag which indicates whether + * the ApplicationAttemptReport includes + * ApplicationAttemptMetrics. + */ + @Public + @Unstable + public abstract boolean includeAppAttemptMetrics(); + + /** + * Set the flag which indicates whether + * the ApplicationAttemptReport includes + * ApplicationAttemptMetrics. + * + * @param includeAppAttemptMetrics + * the flag which indicates whether + * the ApplicationAttemptReport includes + * ApplicationAttemptMetrics. + */ + @Public + @Unstable + public abstract void setIncludeAppAttemptMetrics( + boolean includeAppAttemptMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptsRequest.java index 4c721a7..d9bbec3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationAttemptsRequest.java @@ -43,6 +43,18 @@ public static GetApplicationAttemptsRequest newInstance( GetApplicationAttemptsRequest request = Records.newRecord(GetApplicationAttemptsRequest.class); request.setApplicationId(applicationId); + request.setIncludeAppAttemptMetrics(false); + return request; + } + + @Public + @Unstable + public static GetApplicationAttemptsRequest newInstance( + ApplicationId applicationId, boolean includeAppAttemptMetrics) { + GetApplicationAttemptsRequest request = + Records.newRecord(GetApplicationAttemptsRequest.class); + request.setApplicationId(applicationId); + request.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); return request; } @@ -64,4 +76,32 @@ public static GetApplicationAttemptsRequest newInstance( @Public @Unstable public abstract void setApplicationId(ApplicationId applicationId); + + /** + * Get the flag which indicates whether + * these ApplicationAttemptReport include + * ApplicationAttemptMetrics. + * + * @return the flag which indicates whether + * these ApplicationAttemptReport include + * ApplicationAttemptMetrics. + */ + @Public + @Unstable + public abstract boolean includeAppAttemptMetrics(); + + /** + * Set the flag which indicates whether + * these ApplicationAttemptReport include + * ApplicationAttemptMetrics. + * + * @param includeAppAttemptMetrics + * the flag which indicates whether + * these ApplicationAttemptReport include + * ApplicationAttemptMetrics. + */ + @Public + @Unstable + public abstract void setIncludeAppAttemptMetrics( + boolean includeAppAttemptMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationReportRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationReportRequest.java index 59f0cd2..556456e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationReportRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationReportRequest.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -46,6 +47,18 @@ public static GetApplicationReportRequest newInstance( GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class); request.setApplicationId(applicationId); + request.setIncludeApplicationMetrics(false); + return request; + } + + @Public + @Stable + public static GetApplicationReportRequest newInstance( + ApplicationId applicationId, boolean includeApplicationMetrics) { + GetApplicationReportRequest request = + Records.newRecord(GetApplicationReportRequest.class); + request.setApplicationId(applicationId); + request.setIncludeApplicationMetrics(includeApplicationMetrics); return request; } @@ -64,4 +77,30 @@ public static GetApplicationReportRequest newInstance( @Public @Stable public abstract void setApplicationId(ApplicationId applicationId); + + /** + * Get the flag which indicates whether this + * ApplicationReport includes ApplicationMetrics. + * + * @return the flag which indicates whether + * this ApplicationReport includes + * ApplicationMetrics. + */ + @Public + @Unstable + public abstract boolean includeApplicationMetrics(); + + /** + * Set the flag which indicates whether this + * ApplicationReport includes ApplicationMetrics. + * + * @param includeApplicationMetrics + * the flag which indicates whether + * the ApplicationReport includes + * ApplicationMetrics. + */ + @Public + @Unstable + public abstract void setIncludeApplicationMetrics( + boolean includeApplicationMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java index 35392e4..cce44a7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Records; @@ -83,6 +84,49 @@ public static GetApplicationsRequest newInstance( LongRange startRange, LongRange finishRange, Long limit) { + return newInstance(scope, users, queues, applicationTypes, applicationTags, + applicationStates, startRange, finishRange, limit, false); + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving application types in the cluster from the + * ResourceManager. + *

+ * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + * + *

Setting any of the parameters to null, would just disable that + * filter

+ * + * @param scope {@link ApplicationsRequestScope} to filter by + * @param users list of users to filter by + * @param queues list of scheduler queues to filter by + * @param applicationTypes types of applications + * @param applicationTags application tags to filter by + * @param applicationStates application states to filter by + * @param startRange range of application start times to filter by + * @param finishRange range of application finish times to filter by + * @param limit number of applications to limit to + * @param includeApplicationMetrics + * indicates whether the reports include {@link ApplicationMetrics} + * @return {@link GetApplicationsRequest} to be used with + * {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)} + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + ApplicationsRequestScope scope, + Set users, + Set queues, + Set applicationTypes, + Set applicationTags, + EnumSet applicationStates, + LongRange startRange, + LongRange finishRange, + Long limit, + boolean includeApplicationMetrics) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); if (scope != null) { @@ -104,9 +148,9 @@ public static GetApplicationsRequest newInstance( if (limit != null) { request.setLimit(limit); } + request.setIncludeApplicationMetrics(includeApplicationMetrics); return request; } - /** *

* The request from clients to get a report of Applications matching the @@ -182,13 +226,34 @@ public static GetApplicationsRequest newInstance( public static GetApplicationsRequest newInstance( Set applicationTypes, EnumSet applicationStates) { + return newInstance(applicationTypes, applicationStates, false); + } + + /** + *

+ * The request from clients to get a report of Applications matching the + * giving and application types and application types in the cluster from the + * ResourceManager. + * If includeApplicationMetrics is set true, the {@link ApplicationMetrics} + * will be included in this report. + *

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ + @Public + @Stable + public static GetApplicationsRequest newInstance( + Set applicationTypes, + EnumSet applicationStates, + boolean includeApplicationMetrics) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationTypes(applicationTypes); request.setApplicationStates(applicationStates); + request.setIncludeApplicationMetrics(includeApplicationMetrics); return request; } - /** * Get the application types to filter applications on * @@ -389,4 +454,30 @@ public abstract void setStartRange(long begin, long end) @Private @Unstable public abstract void setScope(ApplicationsRequestScope scope); + + /** + * Get the flag which indicates whether these + * ApplicationReport include ApplicationMetrics. + * + * @return the flag which indicates whether + * this ApplicationReport includes + * ApplicationMetrics. + */ + @Public + @Unstable + public abstract boolean includeApplicationMetrics(); + + /** + * Set the flag which indicates whether these + * ApplicationReport include ApplicationMetrics. + * + * @param includeApplicationMetrics + * the flag which indicates whether + * the ApplicationReport includes + * ApplicationMetrics. + */ + @Public + @Unstable + public abstract void setIncludeApplicationMetrics( + boolean includeApplicationMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java new file mode 100644 index 0000000..6017ac0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains various metrics for the applicationAttempt + * to be reported by UI and CLI. + */ +@Public +@Unstable +public abstract class ApplicationAttemptMetrics { + + @Private + @Unstable + public static ApplicationAttemptMetrics newInstance( + boolean isPreempted, Resource resourcePreempted, + int numNonAMContainersPreempted, int totalAllocatedContainers, + String localityStatistics, List resourceRequests, + Resource applicationHeadRoom) { + ApplicationAttemptMetrics metrics = + Records.newRecord(ApplicationAttemptMetrics.class); + metrics.setIsPreempted(isPreempted); + metrics.setResourcePreempted(resourcePreempted); + metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted); + metrics.setTotalAllocatedContainers(totalAllocatedContainers); + metrics.setLocalityStatistics(localityStatistics); + metrics.setResourceRequests(resourceRequests); + metrics.setApplicationHeadRoom(applicationHeadRoom); + return metrics; + } + + /** + * Get the total resource preempted for applicationAttempt. + * @return the total resource preempted for applicationAttempt + */ + @Public + @Unstable + public abstract Resource getResourcePreempted(); + + @Private + @Unstable + public abstract void setResourcePreempted(Resource resourcePreempted); + + /** + * Get the number of non-AM containers preempted for applicationAttempt + * @return the number of non-AM containers preempted for applicationAttempt + */ + @Public + @Unstable + public abstract int getNumNonAMContainerPreempted(); + + @Private + @Unstable + public abstract void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted); + + /** + * indicate whether this applicationAttempt has been preempted + * @return the flag to indicate whether this applicationAttempt + * has been preempted + */ + @Public + @Unstable + public abstract boolean isPreempted(); + + @Private + @Unstable + public abstract void setIsPreempted(boolean isPreempted); + + /** + * Get a list of {@link ResourceRequest} which this attempt has made. + * + * @return a list of {@link ResourceRequest} which this attempt has made. + */ + @Public + @Unstable + public abstract List getResourceRequests(); + + @Private + @Unstable + public abstract void setResourceRequests( + List resourceRequests); + + /** + * Get the total number of allocated containers for this attempt + * @return the total number of allocated containers for this attempt + */ + @Public + @Unstable + public abstract int getTotalAllocatedContainers(); + + @Private + @Unstable + public abstract void + setTotalAllocatedContainers(int totalAllocatedContainers); + + /** + * Get the number of NodeLocal/RackLocal/OffSwitch containers for this attempt + * @return a string contains the number of + * NodeLocal/RackLocal/OffSwitch containers + */ + @Public + @Unstable + public abstract String getLocalityStatistics(); + + @Private + @Unstable + public abstract void setLocalityStatistics(String localityStatistics); + + /** + * Get the application headroom. + * @return application headroom + */ + @Public + @Unstable + public abstract Resource getApplicationHeadRoom(); + + @Public + @Unstable + public abstract void setApplicationHeadRoom(Resource applicationHeadRoom); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java index b7f9c1b..0d87f60 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java @@ -36,6 +36,7 @@ *
  • Diagnostic information in case of errors.
  • *
  • {@link YarnApplicationAttemptState} of the application attempt.
  • *
  • {@link ContainerId} of the master Container.
  • + *
  • {@link ApplicationAttemptMetrics} of the application attempt
  • * */ @Public @@ -47,7 +48,8 @@ public static ApplicationAttemptReport newInstance( ApplicationAttemptId applicationAttemptId, String host, int rpcPort, String url, String oUrl, String diagnostics, - YarnApplicationAttemptState state, ContainerId amContainerId) { + YarnApplicationAttemptState state, ContainerId amContainerId, + ApplicationAttemptMetrics applicationAttemptMetrics) { ApplicationAttemptReport report = Records.newRecord(ApplicationAttemptReport.class); report.setApplicationAttemptId(applicationAttemptId); @@ -58,6 +60,8 @@ public static ApplicationAttemptReport newInstance( report.setDiagnostics(diagnostics); report.setYarnApplicationAttemptState(state); report.setAMContainerId(amContainerId); + report.setApplicationAttemptMetrics( + applicationAttemptMetrics); return report; } @@ -171,4 +175,19 @@ public abstract void setApplicationAttemptId( @Private @Unstable public abstract void setAMContainerId(ContainerId amContainerId); + + /** + * Get the ApplicationAttemptMetrics of this attempt. + * + * @return ApplicationAttemptMetrics of this attempt + */ + @Public + @Unstable + public abstract ApplicationAttemptMetrics + getApplicationAttemptMetrics(); + + @Private + @Unstable + public abstract void setApplicationAttemptMetrics( + ApplicationAttemptMetrics applicationAttemptMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java new file mode 100644 index 0000000..acd0e3b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains various metrics for the application + * to be reported by UI and CLI. + */ +@Public +@Unstable +public abstract class ApplicationMetrics { + + @Private + @Unstable + public static ApplicationMetrics newInstance( + Resource resourcePreempted, int numNonAMContainersPreempted, + int numAMContainersPreempted) { + ApplicationMetrics metrics = + Records.newRecord(ApplicationMetrics.class); + metrics.setResourcePreempted(resourcePreempted); + metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted); + metrics.setNumAMContainersPreempted(numAMContainersPreempted); + return metrics; + } + + /** + * Get the total resource preempted of the application. + * @return the total resource preempted of the application + */ + @Public + @Unstable + public abstract Resource getResourcePreempted(); + + @Private + @Unstable + public abstract void setResourcePreempted(Resource resourcePreempted); + + /** + * Get the number of non-AM containers preempted for this application. + * @return the number of non-AM containers preempted for this application + */ + @Public + @Unstable + public abstract int getNumNonAMContainersPreempted(); + + @Private + @Unstable + public abstract void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted); + + /** + * Get the number of AM containers preempted for this application. + * @return the number of non-AM containers preempted for this application + */ + @Public + @Unstable + public abstract int getNumAMContainersPreempted(); + + @Private + @Unstable + public abstract void + setNumAMContainersPreempted(int numAMContainersPreempted); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index ff4fb52..3d2c737 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -43,6 +43,7 @@ *
  • Diagnostic information in case of errors.
  • *
  • Start time of the application.
  • *
  • Client {@link Token} of the application (if security is enabled).
  • + *
  • {@link ApplicationMetrics} of the application
  • * * * @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest) @@ -59,7 +60,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String applicationType, Token amRmToken) { + float progress, String applicationType, Token amRmToken, Set tags, + ApplicationMetrics applicationMetrics) { ApplicationReport report = Records.newRecord(ApplicationReport.class); report.setApplicationId(applicationId); report.setCurrentApplicationAttemptId(applicationAttemptId); @@ -80,6 +82,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setProgress(progress); report.setApplicationType(applicationType); report.setAMRMToken(amRmToken); + report.setApplicationTags(tags); + report.setApplicationMetrics(applicationMetrics); return report; } @@ -361,5 +365,19 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Public @Stable public abstract Token getAMRMToken(); + + /** + * Get the ApplicationMetrics of this application. + * + * @return ApplicationMetrics of this application + */ + @Public + @Unstable + public abstract ApplicationMetrics getApplicationMetrics(); + + @Private + @Unstable + public abstract void setApplicationMetrics( + ApplicationMetrics applicationMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 2edff99..a99c493 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -194,6 +194,13 @@ message ApplicationReportProto { optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; + optional ApplicationMetricsProto application_metrics = 21; +} + +message ApplicationMetricsProto { + optional ResourceProto resource_preempted = 1; + optional int32 num_non_am_containers_preempted = 2; + optional int32 num_am_containers_preempted = 3; } message ApplicationAttemptReportProto { @@ -205,6 +212,17 @@ message ApplicationAttemptReportProto { optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6; optional ContainerIdProto am_container_id = 7; optional string original_tracking_url = 8; + optional ApplicationAttemptMetricsProto application_attempt_metrics = 9; +} + +message ApplicationAttemptMetricsProto { + optional bool is_preempted = 1; + optional ResourceProto resource_preempted = 2; + optional int32 num_non_am_containers_preempted = 3; + repeated ResourceRequestProto resource_requests = 4; + optional int32 total_allocated_containers = 5; + optional string locality_statistics = 6; + optional ResourceProto application_headroom = 7; } enum NodeStateProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 33d1207..7e9f139 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -108,6 +108,7 @@ message GetNewApplicationResponseProto { message GetApplicationReportRequestProto { optional ApplicationIdProto application_id = 1; + optional bool includeApplicationMetrics = 2 [default = false]; } message GetApplicationReportResponseProto { @@ -162,6 +163,7 @@ message GetApplicationsRequestProto { optional int64 finish_end = 9; repeated string applicationTags = 10; optional ApplicationsRequestScopeProto scope = 11 [default = ALL]; + optional bool includeApplicationMetrics = 12 [default = false]; } message GetApplicationsResponseProto { @@ -284,6 +286,7 @@ message GetContainerStatusesResponseProto { message GetApplicationAttemptReportRequestProto { optional ApplicationAttemptIdProto application_attempt_id = 1; + optional bool includeAppAttemptMetrics = 2 [default = false]; } message GetApplicationAttemptReportResponseProto { @@ -292,6 +295,7 @@ message GetApplicationAttemptReportResponseProto { message GetApplicationAttemptsRequestProto { optional ApplicationIdProto application_id = 1; + optional bool includeAppAttemptMetrics = 2 [default = false]; } message GetApplicationAttemptsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index f617731..12a26d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -39,7 +39,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -182,6 +184,43 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException; /** + *

    + * Get a report of the given Application. + * If includeApplicationMetrics is true, {@link ApplicationMetrics} will + * be included in these reports. + *

    + * + *

    + * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

    + * + *

    + * If the user does not have VIEW_APP access then the following + * fields in the report will be set to stubbed values: + *

      + *
    • host - set to "N/A"
    • + *
    • RPC port - set to -1
    • + *
    • client token - set to "N/A"
    • + *
    • diagnostics - set to "N/A"
    • + *
    • tracking URL - set to "N/A"
    • + *
    • original tracking URL - set to "N/A"
    • + *
    • resource usage report - all values are -1
    • + *
    + * + * @param appId + * {@link ApplicationId} of the application that needs a report + * @param includeApplicationMetrics + * whether {@link ApplicationMetrics} of the application will be + * included in this report + * @return application report + * @throws YarnException + * @throws IOException + */ + public abstract ApplicationReport getApplicationReport(ApplicationId appId, + boolean includeApplicationMetrics) throws YarnException, IOException; + + /** * Get the AMRM token of the application. *

    * The AMRM token is required for AM to RM scheduling operations. For @@ -291,6 +330,33 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) /** *

    + * Get a report (ApplicationReport) of Applications matching the given + * application types and application states in the cluster. + * If includeApplicationMetrics is true, {@link ApplicationMetrics} will + * be included in these reports. + *

    + * + *

    + * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

    + * + * @param applicationTypes + * @param applicationStates + * @param includeApplicationMetrics + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplications( + Set applicationTypes, + EnumSet applicationStates, + boolean includeApplicationMetrics) throws YarnException, + IOException; + + /** + *

    * Get metrics ({@link YarnClusterMetrics}) about the cluster. *

    * @@ -423,6 +489,34 @@ public abstract ApplicationAttemptReport getApplicationAttemptReport( /** *

    + * Get a report of the given ApplicationAttempt. + * If includeAppAttemptMetrics is set true, {@link ApplicationAttemptMetrics} + * will be included in the report. + *

    + * + *

    + * In secure mode, YARN verifies access to the application, queue + * etc. before accepting the request. + *

    + * + * @param applicationAttemptId + * {@link ApplicationAttemptId} of the application attempt that needs + * a report + * @param includeAppAttemptMetrics + * whether {@link ApplicationAttemptMetrics} of the attempt will be + * included in this report + * @return application attempt report + * @throws YarnException + * @throws ApplicationAttemptNotFoundException if application attempt + * not found + * @throws IOException + */ + public abstract ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId applicationAttemptId, + boolean includeAppAttemptMetrics) throws YarnException, IOException; + + /** + *

    * Get a report of all (ApplicationAttempts) of Application in the cluster. *

    * @@ -437,6 +531,24 @@ public abstract ApplicationAttemptReport getApplicationAttemptReport( /** *

    + * Get a report of all (ApplicationAttempts) of Application in the cluster. + * If includeAppAttemptMetrics is set true, {@link ApplicationAttemptMetrics} + * will be included in these reports. + *

    + * + * @param applicationId + * @param includeAppAttemptMetrics + * @return a list of reports for all application attempts for specified + * application. + * @throws YarnException + * @throws IOException + */ + public abstract List getApplicationAttempts( + ApplicationId applicationId, boolean includeAppAttemptMetrics) + throws YarnException, IOException; + + /** + *

    * Get a report of the given Container. *

    * diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index d6b36bb..307330d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -422,11 +422,18 @@ boolean enforceAsyncAPITimeout() { @Override public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException { + return getApplicationReport(appId, false); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId, + boolean includeApplicationMetrics) throws YarnException, IOException { GetApplicationReportResponse response = null; try { GetApplicationReportRequest request = Records .newRecord(GetApplicationReportRequest.class); request.setApplicationId(appId); + request.setIncludeApplicationMetrics(includeApplicationMetrics); response = rmClient.getApplicationReport(request); } catch (YarnException e) { if (!historyServiceEnabled) { @@ -478,8 +485,16 @@ public ApplicationReport getApplicationReport(ApplicationId appId) public List getApplications(Set applicationTypes, EnumSet applicationStates) throws YarnException, IOException { + return getApplications(applicationTypes, applicationStates, false); + } + + @Override + public List getApplications(Set applicationTypes, + EnumSet applicationStates, + boolean includeApplicationMetrics) throws YarnException, IOException { GetApplicationsRequest request = - GetApplicationsRequest.newInstance(applicationTypes, applicationStates); + GetApplicationsRequest.newInstance(applicationTypes, applicationStates, + includeApplicationMetrics); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); } @@ -605,10 +620,18 @@ public void setRMClient(ApplicationClientProtocol rmClient) { @Override public ApplicationAttemptReport getApplicationAttemptReport( ApplicationAttemptId appAttemptId) throws YarnException, IOException { + return getApplicationAttemptReport(appAttemptId, false); + } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId appAttemptId, boolean includeAppAttemptMetrics) + throws YarnException, IOException { try { GetApplicationAttemptReportRequest request = Records .newRecord(GetApplicationAttemptReportRequest.class); request.setApplicationAttemptId(appAttemptId); + request.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); GetApplicationAttemptReportResponse response = rmClient .getApplicationAttemptReport(request); return response.getApplicationAttemptReport(); @@ -629,10 +652,18 @@ public ApplicationAttemptReport getApplicationAttemptReport( @Override public List getApplicationAttempts( ApplicationId appId) throws YarnException, IOException { + return getApplicationAttempts(appId, false); + } + + @Override + public List getApplicationAttempts( + ApplicationId appId, boolean includeAppAttemptMetrics) + throws YarnException, IOException { try { GetApplicationAttemptsRequest request = Records .newRecord(GetApplicationAttemptsRequest.class); request.setApplicationId(appId); + request.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); GetApplicationAttemptsResponse response = rmClient .getApplicationAttempts(request); return response.getApplicationAttemptList(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index dd4a949..602e282 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -38,11 +38,15 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; @@ -50,7 +54,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; - +import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @Private @@ -65,6 +69,11 @@ private static final String CONTAINER_PATTERN = "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s" + System.getProperty("line.separator"); + private static final String RESOURCE_REQUESTS_TABLE_PATTERN = + "%16s\t%30s\t%35s\t%20s\t%20s\t%35s" + + System.getProperty("line.separator"); + private static final String LOCALITY_TABLE_PATTERN = + "%48s\t%25s\t%25s\t%25s" + System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; private static final String APP_STATE_CMD = "appStates"; @@ -73,6 +82,7 @@ public static final String APPLICATION = "application"; public static final String APPLICATION_ATTEMPT = "applicationattempt"; public static final String CONTAINER = "container"; + public static final String METRICS_OPTION = "metrics"; private boolean allAppStates; @@ -117,6 +127,8 @@ public int run(String[] args) throws Exception { appStateOpt.setArgs(Option.UNLIMITED_VALUES); appStateOpt.setArgName("States"); opts.addOption(appStateOpt); + opts.addOption(METRICS_OPTION, false, "Works with -status to get " + + "more details for this applicaiton, such as preemption metrics"); opts.getOption(KILL_CMD).setArgName("Application ID"); opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID"); opts.getOption(QUEUE_CMD).setArgName("Queue Name"); @@ -128,6 +140,9 @@ public int run(String[] args) throws Exception { opts.addOption(LIST_CMD, true, "List application attempts for aplication."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); + opts.addOption(METRICS_OPTION, false, "Works with -status to get " + + "more details for this attempt, such as preemption metrics, " + + "resource requests and container locality statistics."); opts.getOption(STATUS_CMD).setArgName("Application Attempt ID"); opts.getOption(LIST_CMD).setArgName("Application ID"); } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) { @@ -152,16 +167,31 @@ public int run(String[] args) throws Exception { } if (cliParser.hasOption(STATUS_CMD)) { - if (args.length != 3) { - printUsage(title, opts); - return exitCode; - } + boolean showMetrics = cliParser.hasOption(METRICS_OPTION); if (args[0].equalsIgnoreCase(APPLICATION)) { - exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); + if (args.length != 3) { + if (!(showMetrics && args.length == 4)) { + printUsage(title, opts); + return exitCode; + } + } + exitCode = + printApplicationReport(cliParser.getOptionValue(STATUS_CMD), + showMetrics); } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (args.length != 3) { + if (!(showMetrics && args.length == 4)) { + printUsage(title, opts); + return exitCode; + } + } exitCode = printApplicationAttemptReport(cliParser - .getOptionValue(STATUS_CMD)); + .getOptionValue(STATUS_CMD), showMetrics); } else if (args[0].equalsIgnoreCase(CONTAINER)) { + if (args.length != 3) { + printUsage(title, opts); + return exitCode; + } exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD)); } return exitCode; @@ -262,12 +292,12 @@ void printUsage(String title, Options opts) { * @return exitCode * @throws YarnException */ - private int printApplicationAttemptReport(String applicationAttemptId) - throws YarnException, IOException { + private int printApplicationAttemptReport(String applicationAttemptId, + boolean showMetrics) throws YarnException, IOException { ApplicationAttemptReport appAttemptReport = null; try { appAttemptReport = client.getApplicationAttemptReport(ConverterUtils - .toApplicationAttemptId(applicationAttemptId)); + .toApplicationAttemptId(applicationAttemptId), showMetrics); } catch (ApplicationNotFoundException e) { sysout.println("Application for AppAttempt with id '" + applicationAttemptId + "' doesn't exist in RM or Timeline Server."); @@ -282,6 +312,8 @@ private int printApplicationAttemptReport(String applicationAttemptId) PrintWriter appAttemptReportStr = new PrintWriter( new OutputStreamWriter(baos, Charset.forName("UTF-8"))); if (appAttemptReport != null) { + ApplicationAttemptMetrics attemptMetrics = + appAttemptReport.getApplicationAttemptMetrics(); appAttemptReportStr.println("Application Attempt Report : "); appAttemptReportStr.print("\tApplicationAttempt-Id : "); appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId()); @@ -298,7 +330,24 @@ private int printApplicationAttemptReport(String applicationAttemptId) appAttemptReportStr.print("\tAM Host : "); appAttemptReportStr.println(appAttemptReport.getHost()); appAttemptReportStr.print("\tDiagnostics : "); - appAttemptReportStr.print(appAttemptReport.getDiagnostics()); + appAttemptReportStr.println(appAttemptReport.getDiagnostics()); + appAttemptReportStr.flush(); + + if (attemptMetrics != null) { + appAttemptReportStr.print( + "\tResource Preempted from Current Attempt : "); + appAttemptReportStr.println(attemptMetrics.getResourcePreempted()); + appAttemptReportStr.print( + "\tNumber of Non-AM Containers Preempted from Current Attempt : "); + appAttemptReportStr.println(attemptMetrics + .getNumNonAMContainerPreempted()); + + // Print ResourceRequests table + printResourceRequestsTable(appAttemptReportStr, attemptMetrics); + + // Print LocalityStatistics table + printLocalityStatisticsTable(appAttemptReportStr, attemptMetrics); + } } else { appAttemptReportStr.print("Application Attempt with id '" + applicationAttemptId + "' doesn't exist in Timeline Server."); @@ -471,15 +520,16 @@ private void moveApplicationAcrossQueues(String applicationId, String queue) * Prints the application report for an application id. * * @param applicationId + * @param showMetrics * @return exitCode * @throws YarnException */ - private int printApplicationReport(String applicationId) + private int printApplicationReport(String applicationId, boolean showMetrics) throws YarnException, IOException { ApplicationReport appReport = null; try { appReport = client.getApplicationReport(ConverterUtils - .toApplicationId(applicationId)); + .toApplicationId(applicationId), showMetrics); } catch (ApplicationNotFoundException e) { sysout.println("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."); @@ -531,7 +581,18 @@ private int printApplicationReport(String applicationId) appReportStr.println("N/A"); } appReportStr.print("\tDiagnostics : "); - appReportStr.print(appReport.getDiagnostics()); + appReportStr.println(appReport.getDiagnostics()); + appReportStr.print("\tApplication Tags : "); + appReportStr.println(appReport.getApplicationTags()); + ApplicationMetrics appMetrics = appReport.getApplicationMetrics(); + if (showMetrics && appMetrics != null) { + appReportStr.print("\tTotal Resource Preempted : "); + appReportStr.println(appMetrics.getResourcePreempted()); + appReportStr.print("\tTotal Number of Non-AM Containers Preempted : "); + appReportStr.println(appMetrics.getNumNonAMContainersPreempted()); + appReportStr.print("\tTotal Number of AM Containers Preempted : "); + appReportStr.print(appMetrics.getNumAMContainersPreempted()); + } } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); @@ -612,4 +673,78 @@ private void listContainers(String appAttemptId) throws YarnException, } writer.flush(); } + + private void printResourceRequestsTable(PrintWriter reportStr, + ApplicationAttemptMetrics attemptMetrics) { + reportStr.println(); + // Print ResourceRequests table + List resouceRequests = + attemptMetrics.getResourceRequests(); + + Resource totalResource = Resource.newInstance(0, 0); + reportStr.println("ResourceRequests : "); + for (ResourceRequest request : resouceRequests) { + if (request.getResourceName().equals(ResourceRequest.ANY)) { + Resources.addTo( + totalResource, + Resources.multiply(request.getCapability(), + request.getNumContainers())); + } + } + reportStr.print("\ttotalResourceRequests : "); + reportStr.println(totalResource); + reportStr.printf(RESOURCE_REQUESTS_TABLE_PATTERN, "Priority", + "ResourceName", "Capability", "NumContainers", "RelaxLocality", + "NodeLabelExpression"); + for (ResourceRequest request : resouceRequests) { + if (request.getNumContainers() == 0) { + continue; + } + reportStr.printf( + RESOURCE_REQUESTS_TABLE_PATTERN, + request.getPriority(), + request.getResourceName(), + request.getCapability(), + request.getNumContainers(), + request.getRelaxLocality(), + request.getNodeLabelExpression() == null ? "N/A" : request + .getNodeLabelExpression()); + } + reportStr.flush(); + } + + private void printLocalityStatisticsTable(PrintWriter reportStr, + ApplicationAttemptMetrics attemptMetrics) { + reportStr.println(); + reportStr.println("Container Locality Statistics Table: "); + reportStr.print("\tTotal Allocated Containers : "); + reportStr.println(attemptMetrics + .getTotalAllocatedContainers()); + reportStr + .println("\tEach table cell represents the number of " + + "NodeLocal/RackLocal/OffSwitch containers satisfied by " + + "NodeLocal/RackLocal/OffSwitch resource requests."); + String[] containersType = { + "Num Node Local Containers (satisfied by)", + "Num Rack Local Containers (satisfied by)", + "Num Off Switch Containers (satisfied by)" + }; + int[][] localityStatistics = + ConverterUtils.covertLocalityStatisticsStringToArray(attemptMetrics + .getLocalityStatistics()); + reportStr.printf(LOCALITY_TABLE_PATTERN, "", + "Node Local Request", + "Rack Local Request", + "Off Switch Request"); + for (int i = 0; i < localityStatistics.length; i++) { + reportStr.printf(LOCALITY_TABLE_PATTERN, + containersType[i], + localityStatistics[i][0] == -1 ? "N/A" : localityStatistics[i][0], + i == 0 ? "" : localityStatistics[i][1] == -1 ? + "N/A" : localityStatistics[i][1], + i <= 1 ? "" : localityStatistics[i][2] == -1 ? + "N/A" : localityStatistics[i][2]); + } + reportStr.flush(); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index f468bc1..6f7652a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -647,7 +647,7 @@ public ApplicationReport createFakeAppReport() { "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, - "fakeApplicationType", null); + "fakeApplicationType", null, null, null); return report; } @@ -700,7 +700,7 @@ public QueueInfo createFakeQueueInfo() { public ApplicationAttemptReport createFakeApplicationAttemptReport() { return ApplicationAttemptReport.newInstance( createFakeApplicationAttemptId(), "localhost", 0, "", "", "", - YarnApplicationAttemptState.RUNNING, createFakeContainerId()); + YarnApplicationAttemptState.RUNNING, createFakeContainerId(), null); } public List diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c41..8ffa1ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -334,7 +334,7 @@ private void createAppReports() { "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", - null); + null, null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -350,7 +350,8 @@ private void createAppReports() { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 1)); + newApplicationReport.getCurrentApplicationAttemptId(), 1), + null); appAttempts.add(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( @@ -362,7 +363,8 @@ private void createAppReports() { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 2)); + newApplicationReport.getCurrentApplicationAttemptId(), 2), + null); appAttempts.add(attempt1); attempts.put(applicationId, appAttempts); @@ -391,7 +393,7 @@ private void createAppReports() { "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, - "NON-YARN", null); + "NON-YARN", null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -401,7 +403,7 @@ private void createAppReports() { "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, - "MAPREDUCE", null); + "MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -411,7 +413,7 @@ private void createAppReports() { "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null); + "NON-MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport4); reports = applicationReports; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index de669f2..60b3f56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -580,7 +580,8 @@ public void setYarnApplicationState(YarnApplicationState state) { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, + null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); List appAttempts = new ArrayList(); @@ -593,7 +594,8 @@ public void setYarnApplicationState(YarnApplicationState state) { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 1)); + newApplicationReport.getCurrentApplicationAttemptId(), 1), + null); appAttempts.add(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( ApplicationAttemptId.newInstance(applicationId, 2), @@ -604,7 +606,8 @@ public void setYarnApplicationState(YarnApplicationState state) { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 2)); + newApplicationReport.getCurrentApplicationAttemptId(), 2), + null); appAttempts.add(attempt1); attempts.put(applicationId, appAttempts); @@ -655,7 +658,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null); + null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -664,7 +667,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -675,7 +678,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null); + "NON-MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport4); return applicationReports; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 4b60c52..5457bf3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -27,7 +28,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -44,8 +44,10 @@ import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -59,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -98,17 +101,23 @@ public void testGetApplicationReport() throws Exception { ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport.newInstance( 2, 0, null, null, null, 123456, 4567); + Set appTags = new HashSet(); + appTags.add("test"); + ApplicationMetrics metrics = + ApplicationMetrics.newInstance(Resource.newInstance(6144, 10), 5, 2); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null); - when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( - newApplicationReport); - int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); + null, appTags, metrics); + when(client.getApplicationReport(any(ApplicationId.class), eq(true))) + .thenReturn(newApplicationReport); + int result = + cli.run(new String[] { "application", "-status", + applicationId.toString(), "-metrics" }); assertEquals(0, result); - verify(client, times(1 + i)).getApplicationReport(applicationId); + verify(client, times(1 + i)).getApplicationReport(applicationId, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Application Report : "); @@ -128,6 +137,10 @@ public void testGetApplicationReport() throws Exception { pw.println("\tAggregate Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tApplication Tags : [test]"); + pw.println("\tTotal Resource Preempted : "); + pw.println("\tTotal Number of Non-AM Containers Preempted : 5"); + pw.println("\tTotal Number of AM Containers Preempted : 2"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -139,21 +152,36 @@ public void testGetApplicationReport() throws Exception { @Test public void testGetApplicationAttemptReport() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); + List resourceRequests = new ArrayList(); + ResourceRequest request = + ResourceRequest.newInstance(Priority.newInstance(20), + ResourceRequest.ANY, Resource.newInstance(1024, 2), 10); + request.setNodeLabelExpression("dummy_label_for_test"); + resourceRequests.add(request); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( applicationId, 1); + String localityStatistics = "NODE_LOCAL:NODE_LOCAL:8," + + "NODE_LOCAL:RACK_LOCAL:0,NODE_LOCAL:OFF_SWITCH:0" + + "|RACK_LOCAL:NODE_LOCAL:2,RACK_LOCAL:RACK_LOCAL:5," + + "RACK_LOCAL:OFF_SWITCH:0|OFF_SWITCH:NODE_LOCAL:1," + + "OFF_SWITCH:RACK_LOCAL:4,OFF_SWITCH:OFF_SWITCH:10"; + ApplicationAttemptMetrics metrics = + ApplicationAttemptMetrics.newInstance(false, + Resource.newInstance(6144, 10), 10, 20, localityStatistics, + resourceRequests, Resource.newInstance(0, 0)); ApplicationAttemptReport attemptReport = ApplicationAttemptReport .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId, 1)); + attemptId, 1), metrics); when( client - .getApplicationAttemptReport(any(ApplicationAttemptId.class))) - .thenReturn(attemptReport); + .getApplicationAttemptReport(any(ApplicationAttemptId.class), + eq(true))).thenReturn(attemptReport); int result = cli.run(new String[] { "applicationattempt", "-status", - attemptId.toString() }); + attemptId.toString(), "-metrics" }); assertEquals(0, result); - verify(client).getApplicationAttemptReport(attemptId); + verify(client).getApplicationAttemptReport(attemptId, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Application Attempt Report : "); @@ -164,8 +192,49 @@ public void testGetApplicationAttemptReport() throws Exception { pw.println("\tRPC Port : 124"); pw.println("\tAM Host : host"); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tResource Preempted from Current Attempt : " + + ""); + pw.println("\tNumber of Non-AM Containers Preempted from " + + "Current Attempt : 10"); + pw.println(); + pw.println("ResourceRequests : "); + pw.println("\ttotalResourceRequests : "); + pw.println(" Priority\t ResourceName" + + "\t Capability\t NumContainers" + + "\t RelaxLocality\t NodeLabelExpression"); + pw.println(" 20\t *" + + "\t \t 10" + + "\t true\t dummy_label_for_test"); + pw.println(); + pw.println("Container Locality Statistics Table: "); + pw.println("\tTotal Allocated Containers : 20"); + pw.println("\tEach table cell represents the number of " + + "NodeLocal/RackLocal/OffSwitch containers satisfied by " + + "NodeLocal/RackLocal/OffSwitch resource requests."); + pw.print(" "); + pw.println("\t Node Local Request" + + "\t Rack Local Request" + + "\t Off Switch Request"); + pw.println(" Num Node Local Containers (satisfied by)" + + "\t 8" + + "\t " + + "\t "); + pw.println(" Num Rack Local Containers (satisfied by)" + + "\t 2" + + "\t 5" + + "\t "); + pw.println(" Num Off Switch Containers (satisfied by)" + + "\t 1" + + "\t 4" + + "\t 10"); + pw.println(); pw.close(); String appReportStr = baos.toString("UTF-8"); + Log.info("ExpectedOutput"); + Log.info("["+appReportStr+"]"); + Log.info("OutputFrom command"); + String actualOutput = sysOutStream.toString(); + Log.info("["+actualOutput+"]"); Assert.assertEquals(appReportStr, sysOutStream.toString()); verify(sysOut, times(1)).println(isA(String.class)); } @@ -181,11 +250,11 @@ public void testGetApplicationAttempts() throws Exception { ApplicationAttemptReport attemptReport = ApplicationAttemptReport .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId, 1)); + attemptId, 1), null); ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport .newInstance(attemptId1, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId1, 1)); + attemptId1, 1), null); List reports = new ArrayList(); reports.add(attemptReport); reports.add(attemptReport1); @@ -328,7 +397,7 @@ public void testGetContainers() throws Exception { public void testGetApplicationReportException() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); - when(client.getApplicationReport(any(ApplicationId.class))).thenThrow( + when(client.getApplicationReport(any(ApplicationId.class), eq(false))).thenThrow( new ApplicationNotFoundException("History file for application" + applicationId + " is not found")); int exitCode = cli.run(new String[] { "application", "-status", @@ -347,7 +416,7 @@ public void testGetApplications() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -357,7 +426,7 @@ public void testGetApplications() throws Exception { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null); + null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -366,7 +435,7 @@ public void testGetApplications() throws Exception { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -375,7 +444,7 @@ public void testGetApplications() throws Exception { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport4); ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9); @@ -384,7 +453,7 @@ public void testGetApplications() throws Exception { "user5", "queue5", "appname5", "host5", 128, null, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", - null); + null, null, null); applicationReports.add(newApplicationReport5); ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10); @@ -393,7 +462,7 @@ public void testGetApplications() throws Exception { "user6", "queue6", "appname6", "host6", 129, null, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", - null); + null, null, null); applicationReports.add(newApplicationReport6); // Test command yarn application -list @@ -817,7 +886,7 @@ public void testKillApplication() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -830,7 +899,7 @@ public void testKillApplication() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -865,7 +934,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application", "-movetoqueue", @@ -880,7 +949,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application", "-movetoqueue", @@ -1334,7 +1403,7 @@ public void testGetApplicationAttemptReportException() throws Exception { ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance( applicationId, 1); - when(client.getApplicationAttemptReport(attemptId1)).thenThrow( + when(client.getApplicationAttemptReport(attemptId1, false)).thenThrow( new ApplicationNotFoundException("History file for application" + applicationId + " is not found")); @@ -1347,7 +1416,7 @@ public void testGetApplicationAttemptReportException() throws Exception { ApplicationAttemptId attemptId2 = ApplicationAttemptId.newInstance( applicationId, 2); - when(client.getApplicationAttemptReport(attemptId2)).thenThrow( + when(client.getApplicationAttemptReport(attemptId2, false)).thenThrow( new ApplicationAttemptNotFoundException( "History file for application attempt" + attemptId2 + " is not found")); @@ -1468,6 +1537,9 @@ private String createApplicationCLIHelpMessage() throws IOException { pw.println(" on application type, and -appStates to"); pw.println(" filter applications based on application"); pw.println(" state."); + pw.println(" -metrics Works with -status to get more details"); + pw.println(" for this applicaiton, such as preemption"); + pw.println(" metrics"); pw.println(" -movetoqueue Moves the application to a different"); pw.println(" queue."); pw.println(" -queue Works with the movetoqueue command to"); @@ -1486,6 +1558,10 @@ private String createApplicationAttemptCLIHelpMessage() throws IOException { pw.println(" -help Displays help for all commands."); pw.println(" -list List application attempts for"); pw.println(" aplication."); + pw.println(" -metrics Works with -status to get more details"); + pw.println(" for this attempt, such as preemption"); + pw.println(" metrics, resource requests and"); + pw.println(" container locality statistics."); pw.println(" -status Prints the status of the application"); pw.println(" attempt."); pw.close(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptReportRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptReportRequestPBImpl.java index d44ad5f..4e0f823 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptReportRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptReportRequestPBImpl.java @@ -137,4 +137,17 @@ private ApplicationAttemptIdPBImpl convertFromProtoFormat( return ((ApplicationAttemptIdPBImpl) t).getProto(); } + @Override + public boolean includeAppAttemptMetrics() { + GetApplicationAttemptReportRequestProtoOrBuilder p = + viaProto ? proto : builder; + return p.getIncludeAppAttemptMetrics(); + } + + @Override + public void setIncludeAppAttemptMetrics(boolean includeAppAttemptMetrics) { + maybeInitBuilder(); + builder.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptsRequestPBImpl.java index 8470053..bd5c21a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationAttemptsRequestPBImpl.java @@ -131,4 +131,16 @@ private ApplicationIdProto convertToProtoFormat(ApplicationId t) { return ((ApplicationIdPBImpl) t).getProto(); } + @Override + public boolean includeAppAttemptMetrics() { + GetApplicationAttemptsRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getIncludeAppAttemptMetrics(); + } + + @Override + public void setIncludeAppAttemptMetrics(boolean includeAppAttemptMetrics) { + maybeInitBuilder(); + builder.setIncludeAppAttemptMetrics(includeAppAttemptMetrics); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationReportRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationReportRequestPBImpl.java index 47f4318..5f4cde2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationReportRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationReportRequestPBImpl.java @@ -127,6 +127,15 @@ private ApplicationIdProto convertToProtoFormat(ApplicationId t) { return ((ApplicationIdPBImpl)t).getProto(); } + @Override + public boolean includeApplicationMetrics() { + GetApplicationReportRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getIncludeApplicationMetrics(); + } - + @Override + public void setIncludeApplicationMetrics(boolean includeApplicationMetrics) { + maybeInitBuilder(); + builder.setIncludeApplicationMetrics(includeApplicationMetrics); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java index ad009d6..518ed87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -389,4 +389,16 @@ public boolean equals(Object other) { public String toString() { return TextFormat.shortDebugString(getProto()); } + + @Override + public boolean includeApplicationMetrics() { + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder; + return p.getIncludeApplicationMetrics(); + } + + @Override + public void setIncludeApplicationMetrics(boolean includeApplicationMetrics) { + maybeInitBuilder(); + builder.setIncludeApplicationMetrics(includeApplicationMetrics); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java new file mode 100644 index 0000000..ad79fca --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java @@ -0,0 +1,305 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; + +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class ApplicationAttemptMetricsPBImpl extends + ApplicationAttemptMetrics { + + ApplicationAttemptMetricsProto proto = + ApplicationAttemptMetricsProto.getDefaultInstance(); + ApplicationAttemptMetricsProto.Builder builder = null; + boolean viaProto = false; + + Resource totalResourcePreempted; + private List resourceRequests; + Resource applicationHeadRoom; + + public ApplicationAttemptMetricsPBImpl() { + builder = ApplicationAttemptMetricsProto.newBuilder(); + } + + public ApplicationAttemptMetricsPBImpl( + ApplicationAttemptMetricsProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ApplicationAttemptMetricsProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.totalResourcePreempted != null + && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals( + builder.getResourcePreempted())) { + builder.setResourcePreempted( + convertToProtoFormat(this.totalResourcePreempted)); + } + if (this.applicationHeadRoom != null + && !((ResourcePBImpl) this.applicationHeadRoom).getProto().equals( + builder.getApplicationHeadroom())) { + builder.setApplicationHeadroom( + convertToProtoFormat(this.applicationHeadRoom)); + } + if (this.resourceRequests != null) { + addResourceRequestsToProto(); + } + } + + private void addResourceRequestsToProto() { + maybeInitBuilder(); + builder.clearResourceRequests(); + if (null == this.resourceRequests) { + return; + } + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = resourceRequests.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ResourceRequestProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllResourceRequests(iterable); + } + + @Override + public synchronized List getResourceRequests() { + initResourceRequests(); + return this.resourceRequests; + } + + private void initResourceRequests() { + if (resourceRequests != null) { + return; + } + ApplicationAttemptMetricsProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getResourceRequestsList(); + resourceRequests = new ArrayList(); + + for (ResourceRequestProto rr : list) { + resourceRequests.add(convertFromProtoFormat(rr)); + } + } + + @Override + public synchronized void setResourceRequests(List resourceRequests) { + if (null == this.resourceRequests) { + builder.clearResourceRequests(); + } + this.resourceRequests = resourceRequests; + } + + private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) { + return new ResourceRequestPBImpl(p); + } + + private ResourceRequestProto convertToProtoFormat(ResourceRequest t) { + return ((ResourceRequestPBImpl)t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationAttemptMetricsProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized Resource getResourcePreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (this.totalResourcePreempted != null) { + return this.totalResourcePreempted; + } + if (!p.hasResourcePreempted()) { + return null; + } + this.totalResourcePreempted = + convertFromProtoFormat(p.getResourcePreempted()); + return this.totalResourcePreempted; + } + + @Override + public synchronized void setResourcePreempted(Resource resourcePreempted) { + maybeInitBuilder(); + if (resourcePreempted == null) + builder.clearResourcePreempted(); + this.totalResourcePreempted = resourcePreempted; + } + + @Override + public synchronized int getNumNonAMContainerPreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return (p.getNumNonAmContainersPreempted()); + } + + @Override + public synchronized void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted); + } + + @Override + public synchronized boolean isPreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return p.getIsPreempted(); + } + + @Override + public synchronized void setIsPreempted(boolean isPreempted) { + maybeInitBuilder(); + builder.setIsPreempted(isPreempted); + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl)t).getProto(); + } + + @Override + public synchronized int getTotalAllocatedContainers() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return p.getTotalAllocatedContainers(); + } + + @Override + public synchronized void setTotalAllocatedContainers(int totalAllocatedContainers) { + maybeInitBuilder(); + builder.setTotalAllocatedContainers(totalAllocatedContainers); + } + + @Override + public synchronized String getLocalityStatistics() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (! p.hasLocalityStatistics()) { + return null; + } + return p.getLocalityStatistics(); + } + + @Override + public synchronized void setLocalityStatistics(String localityStatistics) { + maybeInitBuilder(); + if (localityStatistics == null) { + builder.clearLocalityStatistics(); + return; + } + builder.setLocalityStatistics(localityStatistics); + } + + @Override + public Resource getApplicationHeadRoom() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (this.applicationHeadRoom != null) { + return this.applicationHeadRoom; + } + if (!p.hasApplicationHeadroom()) { + return null; + } + this.applicationHeadRoom = + convertFromProtoFormat(p.getApplicationHeadroom()); + return this.applicationHeadRoom; + } + + @Override + public void setApplicationHeadRoom(Resource applicationHeadRoom) { + maybeInitBuilder(); + if (applicationHeadRoom == null) + builder.clearApplicationHeadroom(); + this.applicationHeadRoom = applicationHeadRoom; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java index c3c0221..3894f34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java @@ -19,10 +19,12 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; @@ -38,6 +40,7 @@ private ApplicationAttemptId ApplicationAttemptId; private ContainerId amContainerId; + private ApplicationAttemptMetrics attemptMetrics; public ApplicationAttemptReportPBImpl() { builder = ApplicationAttemptReportProto.newBuilder(); @@ -244,6 +247,13 @@ private void mergeLocalToBuilder() { builder.getAmContainerId())) { builder.setAmContainerId(convertToProtoFormat(this.amContainerId)); } + + if (this.attemptMetrics != null + && !((ApplicationAttemptMetricsPBImpl) this.attemptMetrics) + .getProto().equals(builder.getApplicationAttemptMetrics())) { + builder.setApplicationAttemptMetrics( + convertToProtoFormat(this.attemptMetrics)); + } } private ContainerIdProto convertToProtoFormat(ContainerId amContainerId) { @@ -286,4 +296,39 @@ public void setAMContainerId(ContainerId amContainerId) { builder.clearAmContainerId(); this.amContainerId = amContainerId; } + + @Override + public ApplicationAttemptMetrics + getApplicationAttemptMetrics() { + if (this.attemptMetrics != null) { + return this.attemptMetrics; + } + + ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationAttemptMetrics()) { + return null; + } + this.attemptMetrics = + convertFromProtoFormat(p.getApplicationAttemptMetrics()); + return this.attemptMetrics; + } + + @Override + public void setApplicationAttemptMetrics( + ApplicationAttemptMetrics applicationAttemptMetrics) { + maybeInitBuilder(); + if (applicationAttemptMetrics == null) + builder.clearApplicationAttemptMetrics(); + this.attemptMetrics = applicationAttemptMetrics; + } + + private ApplicationAttemptMetricsProto + convertToProtoFormat(ApplicationAttemptMetrics t) { + return ((ApplicationAttemptMetricsPBImpl) t).getProto(); + } + + private ApplicationAttemptMetricsPBImpl convertFromProtoFormat( + ApplicationAttemptMetricsProto t) { + return new ApplicationAttemptMetricsPBImpl(t); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java new file mode 100644 index 0000000..6843d97 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; + +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class ApplicationMetricsPBImpl extends ApplicationMetrics { + + ApplicationMetricsProto proto = + ApplicationMetricsProto.getDefaultInstance(); + ApplicationMetricsProto.Builder builder = null; + boolean viaProto = false; + + Resource totalResourcePreempted; + + public ApplicationMetricsPBImpl() { + builder = ApplicationMetricsProto.newBuilder(); + } + + public ApplicationMetricsPBImpl( + ApplicationMetricsProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ApplicationMetricsProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.totalResourcePreempted != null + && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals( + builder.getResourcePreempted())) { + builder.setResourcePreempted( + convertToProtoFormat(this.totalResourcePreempted)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationMetricsProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized Resource getResourcePreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + if (this.totalResourcePreempted != null) { + return this.totalResourcePreempted; + } + if (!p.hasResourcePreempted()) { + return null; + } + this.totalResourcePreempted = + convertFromProtoFormat(p.getResourcePreempted()); + return this.totalResourcePreempted; + } + + @Override + public synchronized void setResourcePreempted(Resource resourcePreempted) { + maybeInitBuilder(); + if (resourcePreempted == null) + builder.clearResourcePreempted(); + this.totalResourcePreempted = resourcePreempted; + } + + @Override + public synchronized int getNumNonAMContainersPreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + return (p.getNumNonAmContainersPreempted()); + } + + @Override + public synchronized void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted); + } + + @Override + public synchronized int getNumAMContainersPreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + return (p.getNumAmContainersPreempted()); + } + + @Override + public synchronized void setNumAMContainersPreempted( + int numAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumAmContainersPreempted(numAMContainersPreempted); + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl)t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index dd3e2bc..daf15cc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; @@ -53,6 +55,7 @@ private Token clientToAMToken = null; private Token amRmToken = null; private Set applicationTags = null; + private ApplicationMetrics applicationMetrics = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -482,6 +485,12 @@ private void mergeLocalToBuilder() { builder.clearApplicationTags(); builder.addAllApplicationTags(this.applicationTags); } + if (this.applicationMetrics != null + && !((ApplicationMetricsPBImpl) this.applicationMetrics) + .getProto().equals(builder.getApplicationMetrics())) { + builder.setApplicationMetrics( + convertToProtoFormat(this.applicationMetrics)); + } } private void mergeLocalToProto() { @@ -548,4 +557,38 @@ private TokenPBImpl convertFromProtoFormat(TokenProto p) { private TokenProto convertToProtoFormat(Token t) { return ((TokenPBImpl)t).getProto(); } + + @Override + public ApplicationMetrics getApplicationMetrics() { + if (this.applicationMetrics != null) { + return this.applicationMetrics; + } + + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationMetrics()) { + return null; + } + this.applicationMetrics = + convertFromProtoFormat(p.getApplicationMetrics()); + return this.applicationMetrics; + } + + @Override + public void setApplicationMetrics( + ApplicationMetrics applicationMetrics) { + maybeInitBuilder(); + if (applicationMetrics == null) + builder.clearApplicationMetrics(); + this.applicationMetrics = applicationMetrics; + } + + private ApplicationMetricsProto convertToProtoFormat( + ApplicationMetrics t) { + return ((ApplicationMetricsPBImpl) t).getProto(); + } + + private ApplicationMetricsPBImpl convertFromProtoFormat( + ApplicationMetricsProto t) { + return new ApplicationMetricsPBImpl(t); +} } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index e9674cf..64a15f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -259,4 +259,42 @@ public static ApplicationId toApplicationId( } return token; } + + /** + * Convert LocalityStatistics from String format to 3X3 array. + * + * @param localityStatistics String + */ + public static int[][] covertLocalityStatisticsStringToArray( + String localityStatistics) { + int[][] coverted = new int[][] { + {-1, -1, -1}, + {-1, -1, -1}, + {-1, -1, -1} + }; + if (localityStatistics != null && !localityStatistics.isEmpty()) { + String[] localityStatisticsForTypes = localityStatistics.split("\\|"); + if (localityStatisticsForTypes.length == 3) { + for (int i = 0; i < 3; i++) { + String[] numOfLocalityStatisticsForTypes = + localityStatisticsForTypes[i].trim().split(","); + if (numOfLocalityStatisticsForTypes.length != 3) { + continue; + } + for (int j = 0; j < 3; j++) { + String[] num = numOfLocalityStatisticsForTypes[j].trim().split(":"); + if (num.length != 3) { + continue; + } + try { + coverted[i][j] = Integer.parseInt(num[2]); + } catch (NumberFormatException ex) { + // Do Nothing + } + } + } + } + } + return coverted; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 9302d4b..debd067 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -47,6 +47,7 @@ public void testApplicationReport() { Assert.assertNull(appReport2.getCurrentApplicationAttemptId()); Assert.assertNotSame(appReport2, appReport3); Assert.assertNull(appReport1.getAMRMToken()); + Assert.assertNull(appReport1.getApplicationMetrics()); } protected static ApplicationReport createApplicationReport( @@ -58,7 +59,8 @@ protected static ApplicationReport createApplicationReport( ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, + null, null); return appReport; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 8b48798..5c7b3c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -194,7 +194,9 @@ public static void setup() throws Exception { generateByNewInstance(Token.class); generateByNewInstance(NMToken.class); generateByNewInstance(ResourceRequest.class); + generateByNewInstance(ApplicationAttemptMetrics.class); generateByNewInstance(ApplicationAttemptReport.class); + generateByNewInstance(ApplicationMetrics.class); generateByNewInstance(ApplicationResourceUsageReport.class); generateByNewInstance(ApplicationReport.class); generateByNewInstance(Container.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index c7cf07b..0b54c97 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport( appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + appHistory.getApplicationType(), null, null, null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) @@ -166,7 +166,7 @@ private ApplicationAttemptReport convertToApplicationAttemptReport( appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(), null, appAttemptHistory.getDiagnosticsInfo(), appAttemptHistory.getYarnApplicationAttemptState(), - appAttemptHistory.getMasterContainerId()); + appAttemptHistory.getMasterContainerId(), null); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 1010f62..7702ce4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -61,7 +64,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; - import com.google.common.annotations.VisibleForTesting; public class ApplicationHistoryManagerOnTimelineStore extends AbstractService @@ -226,6 +228,7 @@ private static ApplicationReportExt convertToApplicationReport( ApplicationResourceUsageReport appResources = null; Map appViewACLs = new HashMap(); + Set appTags = null; Map entityInfo = entity.getOtherInfo(); if (entityInfo != null) { if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) { @@ -245,7 +248,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, 1.0F, type, null), appViewACLs); + null, 1.0F, type, null, appTags, null), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -270,6 +273,17 @@ private static ApplicationReportExt convertToApplicationReport( appResources=ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS)) { + appTags = new HashSet(); + Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS); + if (obj instanceof Collection) { + for(Object o : (Collection)obj) { + if (o != null) { + appTags.add(o.toString()); + } + } + } + } } List events = entity.getEvents(); if (events != null) { @@ -321,7 +335,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, 1.0F, type, null), appViewACLs); + null, 1.0F, type, null, appTags, null), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( @@ -400,7 +414,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport( return ApplicationAttemptReport.newInstance( ConverterUtils.toApplicationAttemptId(entity.getEntityId()), host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo, - state, amContainerId); + state, amContainerId, null); } private static ContainerReport convertToContainerReport( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index df8eecb..df2b15d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,4 +73,6 @@ public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + public static final String APP_TAGS = "YARN_APPLICATION_TAGS"; + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index eeccf0f..8c258b9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -85,7 +86,7 @@ protected void render(Block html) { ApplicationAttemptReport appAttemptReport = null; try { final GetApplicationAttemptReportRequest request = - GetApplicationAttemptReportRequest.newInstance(appAttemptId); + GetApplicationAttemptReportRequest.newInstance(appAttemptId, true); if (callerUGI == null) { appAttemptReport = appBaseProt.getApplicationAttemptReport(request) @@ -183,14 +184,15 @@ public ApplicationAttemptReport run() throws Exception { return; } - // TODO need to render applicationHeadRoom value from - // ApplicationAttemptMetrics after YARN-3284 if (webUiType.equals(YarnWebParams.RM_WEB_UI)) { if (!isApplicationInFinalState(appAttempt.getAppAttemptState())) { DIV pdiv = html._(InfoBlock.class).div(_INFO_WRAP); info("Application Attempt Overview").clear(); info("Application Attempt Metrics")._( - "Application Attempt Headroom : ", 0); + "Application Attempt Headroom : ", + appAttemptReport.getApplicationAttemptMetrics() == null ? "N/A" + : appAttemptReport.getApplicationAttemptMetrics() + .getApplicationHeadRoom()); pdiv._(); } } @@ -238,20 +240,23 @@ public ApplicationAttemptReport run() throws Exception { tbody._()._(); if (webUiType.equals(YarnWebParams.RM_WEB_UI)) { - createContainerLocalityTable(html); // TODO:YARN-3284 + createContainerLocalityTable(html, + appAttemptReport.getApplicationAttemptMetrics()); } } - //TODO: YARN-3284 - //The containerLocality metrics will be exposed from AttemptReport - private void createContainerLocalityTable(Block html) { - int totalAllocatedContainers = 0; //TODO: YARN-3284 - int[][] localityStatistics = new int[0][0];//TODO:YARN-3284 + private void createContainerLocalityTable(Block html, + ApplicationAttemptMetrics attemptMetrics) { + int[][] localityStatistics = + ConverterUtils + .covertLocalityStatisticsStringToArray(attemptMetrics == null ? null + : attemptMetrics.getLocalityStatistics()); DIV div = html.div(_INFO_WRAP); TABLE> table = div.h3( "Total Allocated Containers: " - + totalAllocatedContainers).h3("Each table cell" + + (attemptMetrics != null ? attemptMetrics.getTotalAllocatedContainers():"N/A")) + .h3("Each table cell" + " represents the number of NodeLocal/RackLocal/OffSwitch containers" + " satisfied by NodeLocal/RackLocal/OffSwitch resource requests.").table( "#containerLocality"); @@ -263,15 +268,20 @@ private void createContainerLocalityTable(Block html) { th(_TH, "Off Switch Request"). _(); - String[] containersType = - { "Num Node Local Containers (satisfied by)", "Num Rack Local Containers (satisfied by)", - "Num Off Switch Containers (satisfied by)" }; + String[] containersType = { + "Num Node Local Containers (satisfied by)", + "Num Rack Local Containers (satisfied by)", + "Num Off Switch Containers (satisfied by)" + }; boolean odd = false; for (int i = 0; i < localityStatistics.length; i++) { table.tr((odd = !odd) ? _ODD : _EVEN).td(containersType[i]) - .td(String.valueOf(localityStatistics[i][0])) - .td(i == 0 ? "" : String.valueOf(localityStatistics[i][1])) - .td(i <= 1 ? "" : String.valueOf(localityStatistics[i][2]))._(); + .td(localityStatistics[i][0] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][0])) + .td(i == 0 ? "" : localityStatistics[i][1] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][1])) + .td(i <= 1 ? "" : localityStatistics[i][2] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][2]))._(); } table._(); div._(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 5fc5fa0..e2eaaa1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; + import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.List; @@ -36,8 +37,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -61,7 +64,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; - import com.google.inject.Inject; public class AppBlock extends HtmlBlock { @@ -98,7 +100,7 @@ protected void render(Block html) { ApplicationReport appReport = null; try { final GetApplicationReportRequest request = - GetApplicationReportRequest.newInstance(appID); + GetApplicationReportRequest.newInstance(appID, true); if (callerUGI == null) { appReport = appBaseProt.getApplicationReport(request).getApplicationReport(); @@ -191,7 +193,7 @@ public ApplicationReport run() throws Exception { Collection attempts; try { final GetApplicationAttemptsRequest request = - GetApplicationAttemptsRequest.newInstance(appID); + GetApplicationAttemptsRequest.newInstance(appID, true); if (callerUGI == null) { attempts = appBaseProt.getApplicationAttempts(request) .getApplicationAttemptList(); @@ -213,32 +215,43 @@ public ApplicationReport run() throws Exception { return; } - //TODO:YARN-3284 - //The preemption metrics will be exposed from ApplicationReport - // and ApplicationAttemptReport ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport(); - DIV pdiv = html. - _(InfoBlock.class). - div(_INFO_WRAP); - info("Application Overview").clear(); - info("Application Metrics") - ._("Total Resource Preempted:", - Resources.none()) // TODO: YARN-3284 - ._("Total Number of Non-AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 - ._("Total Number of AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 - ._("Resource Preempted from Current Attempt:", - Resources.none()) // TODO: YARN-3284 - ._("Number of Non-AM Containers Preempted from Current Attempt:", - 0) // TODO: YARN-3284 - ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", usageReport == null - ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0 - : usageReport.getVcoreSeconds())); - pdiv._(); + ApplicationMetrics appMetrics = appReport.getApplicationMetrics(); + ApplicationAttemptMetrics currentAttemptMetrics = null; + for (final ApplicationAttemptReport report : attempts) { + if (report.getApplicationAttemptId().equals( + appReport.getCurrentApplicationAttemptId())) { + currentAttemptMetrics = report.getApplicationAttemptMetrics(); + break; + } + } + if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) { + DIV pdiv = html. + _(InfoBlock.class). + div(_INFO_WRAP); + info("Application Overview").clear(); + info("Application Metrics") + ._("Total Resource Preempted:", + appMetrics != null ? appMetrics.getResourcePreempted() : "N/A") + ._("Total Number of Non-AM Containers Preempted:", + appMetrics != null ? appMetrics.getNumNonAMContainersPreempted() + : "N/A") + ._("Total Number of AM Containers Preempted:", + appMetrics != null ? appMetrics.getNumAMContainersPreempted() : "N/A") + ._("Resource Preempted from Current Attempt:", + currentAttemptMetrics != null ? + currentAttemptMetrics.getResourcePreempted() : "N/A") + ._("Number of Non-AM Containers Preempted from Current Attempt:", + currentAttemptMetrics != null ? + currentAttemptMetrics.getNumNonAMContainerPreempted() : "N/A") + ._("Aggregate Resource Allocation:", + String.format("%d MB-seconds, %d vcore-seconds", usageReport == null + ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0 + : usageReport.getVcoreSeconds())); + pdiv._(); + } html._(InfoBlock.class); // Application Attempt Table @@ -321,13 +334,13 @@ public ContainerReport run() throws Exception { tbody._()._(); if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) { - createResourceRequestsTable(html, null); // TODO:YARN-3284 + createResourceRequestsTable(html, currentAttemptMetrics == null ? null + : currentAttemptMetrics.getResourceRequests()); } } - //TODO:YARN-3284 - //The resource requests metrics will be exposed from attemptReport - private void createResourceRequestsTable(Block html, List resouceRequests) { + private void createResourceRequestsTable(Block html, + List resouceRequests) { TBODY> tbody = html.table("#ResourceRequests").thead().tr() .th(".priority", "Priority") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 21d70b4..a42f609 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -333,7 +333,7 @@ public GetApplicationReportResponse getApplicationReport( ApplicationAccessType.VIEW_APP, application); ApplicationReport report = application.createAndGetApplicationReport(callerUGI.getUserName(), - allowAccess); + allowAccess, request.includeApplicationMetrics()); GetApplicationReportResponse response = recordFactory .newRecordInstance(GetApplicationReportResponse.class); @@ -374,7 +374,7 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport( "' doesn't exist in RM."); } ApplicationAttemptReport attemptReport = appAttempt - .createApplicationAttemptReport(); + .createApplicationAttemptReport(request.includeAppAttemptMetrics()); response = GetApplicationAttemptReportResponse.newInstance(attemptReport); }else{ throw new YarnException("User " + callerUGI.getShortUserName() @@ -413,7 +413,8 @@ public GetApplicationAttemptsResponse getApplicationAttempts( .entrySet().iterator(); while (iter.hasNext()) { listAttempts.add(iter.next().getValue() - .createApplicationAttemptReport()); + .createApplicationAttemptReport( + request.includeAppAttemptMetrics())); } response = GetApplicationAttemptsResponse.newInstance(listAttempts); } else { @@ -801,7 +802,8 @@ public void remove() { } reports.add(application.createAndGetApplicationReport( - callerUGI.getUserName(), allowAccess)); + callerUGI.getUserName(), allowAccess, + request.includeApplicationMetrics())); } GetApplicationsResponse response = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 2373b3b..7c43aa4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; +import java.util.Set; + import org.apache.hadoop.yarn.api.records.ApplicationId; public class ApplicationCreatedEvent extends @@ -29,6 +31,7 @@ private String user; private String queue; private long submittedTime; + private Set appTags; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -36,7 +39,8 @@ public ApplicationCreatedEvent(ApplicationId appId, String user, String queue, long submittedTime, - long createdTime) { + long createdTime, + Set appTags) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -44,6 +48,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.user = user; this.queue = queue; this.submittedTime = submittedTime; + this.appTags = appTags; } @Override @@ -75,4 +80,7 @@ public long getSubmittedTime() { return submittedTime; } + public Set getAppTags() { + return appTags; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b849b00..8b932a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,7 @@ public void appCreated(RMApp app, long createdTime) { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime)); + createdTime, app.getApplicationTags())); } } @@ -251,6 +251,7 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); + entityInfo.put(ApplicationMetricsConstants.APP_TAGS, event.getAppTags()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( @@ -283,7 +284,7 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); - + tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index fbcaab9..5c36347 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -138,7 +139,28 @@ */ ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess); - + + /** + * To get the status of an application in the RM, this method can be used. + * If full access is not allowed then the following fields in the report + * will be stubbed: + *
      + *
    • host - set to "N/A"
    • + *
    • RPC port - set to -1
    • + *
    • client token - set to "N/A"
    • + *
    • diagnostics - set to "N/A"
    • + *
    • tracking URL - set to "N/A"
    • + *
    • original tracking URL - set to "N/A"
    • + *
    • resource usage report - all values are -1
    • + *
    + * + * @param clientUserName the user name of the client requesting the report + * @param allowAccess whether to allow full access to the report + * @param includeApplicationMetrics whether includes {@link ApplicationMetrics} + * @return the {@link ApplicationReport} detailing the status of the application. + */ + ApplicationReport createAndGetApplicationReport(String clientUserName, + boolean allowAccess, boolean includeApplicationMetrics); /** * To receive the collection of all {@link RMNode}s whose updates have been * received by the RMApp. Updates can be node becoming lost or becoming diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 2d1737a..f608729 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -543,10 +544,16 @@ public int pullRMNodeUpdates(Collection updatedNodes) { this.writeLock.unlock(); } } - + @Override public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) { + return createAndGetApplicationReport(clientUserName, allowAccess, false); + } + + @Override + public ApplicationReport createAndGetApplicationReport(String clientUserName, + boolean allowAccess, boolean includeApplicationMetrics) { this.readLock.lock(); try { @@ -558,6 +565,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, int rpcPort = -1; ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; + ApplicationMetrics appMetrics = null; FinalApplicationStatus finishState = getFinalApplicationStatus(); String diags = UNAVAILABLE; float progress = 0.0f; @@ -604,6 +612,10 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, RMAppMetrics rmAppMetrics = getRMAppMetrics(); appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); + + if (includeApplicationMetrics) { + appMetrics = rmAppMetrics.convertToApplicationMetrics(); + } } if (currentApplicationAttemptId == null) { @@ -612,13 +624,12 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, DUMMY_APPLICATION_ATTEMPT_NUMBER); } - return BuilderUtils.newApplicationReport(this.applicationId, - currentApplicationAttemptId, this.user, this.queue, - this.name, host, rpcPort, clientToAMToken, - createApplicationState(), diags, - trackingUrl, this.startTime, this.finishTime, finishState, - appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken, applicationTags); + return ApplicationReport.newInstance(this.applicationId, + currentApplicationAttemptId, this.user, this.queue, this.name, host, + rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, + this.startTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, appMetrics); } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 5091470..d4c8f51 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.Resource; public class RMAppMetrics { @@ -56,4 +57,15 @@ public long getMemorySeconds() { public long getVcoreSeconds() { return vcoreSeconds; } + + /** + * Convert RMAppMetrics object to ApplicationMetrics object + * @return applicationMetrics + */ + public ApplicationMetrics convertToApplicationMetrics() { + return ApplicationMetrics + .newInstance(this.getResourcePreempted(), + this.getNumNonAMContainersPreempted(), + this.getNumAMContainersPreempted()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index b85174e..3f9413f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -214,6 +215,16 @@ ApplicationAttemptReport createApplicationAttemptReport(); /** + * Create the Application attempt report from the {@link RMAppAttempt} + * + * @param includeAppAttemptMetrics indicates whether + * includes the {@link ApplicationAttemptMetrics} + * @return {@link ApplicationAttemptReport} + */ + ApplicationAttemptReport createApplicationAttemptReport( + boolean includeAppAttemptMetrics); + + /** * Return the flag which indicates whether the attempt failure should be * counted to attempt retry count. *

    diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1be1727..c29d4fe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -36,7 +36,6 @@ import javax.crypto.SecretKey; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -1902,6 +1901,12 @@ private static String sanitizeTrackingUrl(String url) { @Override public ApplicationAttemptReport createApplicationAttemptReport() { + return createApplicationAttemptReport(false); + } + + @Override + public ApplicationAttemptReport createApplicationAttemptReport( + boolean includeAppAttemptMetrics) { this.readLock.lock(); ApplicationAttemptReport attemptReport = null; try { @@ -1909,10 +1914,13 @@ public ApplicationAttemptReport createApplicationAttemptReport() { // am container. ContainerId amId = masterContainer == null ? null : masterContainer.getId(); - attemptReport = ApplicationAttemptReport.newInstance(this - .getAppAttemptId(), this.getHost(), this.getRpcPort(), this - .getTrackingUrl(), this.getOriginalTrackingUrl(), this.getDiagnostics(), - YarnApplicationAttemptState .valueOf(this.getState().toString()), amId); + attemptReport = + ApplicationAttemptReport.newInstance(this.getAppAttemptId(), + this.getHost(), this.getRpcPort(), this.getTrackingUrl(), + this.getOriginalTrackingUrl(), this.getDiagnostics(), + YarnApplicationAttemptState.valueOf(this.getState().toString()), + amId, includeAppAttemptMetrics ? attemptMetrics + .convertToApplicationAttemptMetrics() : null); } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 0a3638b..9edd00e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -28,10 +29,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.util.resource.Resources; @@ -148,6 +152,47 @@ public int getTotalAllocatedContainers() { return this.totalAllocatedContainers; } + /** + * Convert RMAppAttemptMetrics object to ApplicationAttemptMetrics object + * @return applicationAttemptMetrics + */ + public ApplicationAttemptMetrics convertToApplicationAttemptMetrics() { + return ApplicationAttemptMetrics.newInstance( + this.getIsPreempted(), + this.getResourcePreempted(), + this.getNumNonAMContainersPreempted(), + this.getTotalAllocatedContainers(), + this.ConvertLocalityStatisticsToString(), + this.getResourceRequests(), + this.getApplicationAttemptHeadroom()); + } + + private String ConvertLocalityStatisticsToString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < NodeType.values().length; i++) { + for (int j = 0; j < NodeType.values().length; j++) { + sb.append(NodeType.values()[i] + ":" + NodeType.values()[j] + ":" + + localityStatistics[i][j]); + if (j != NodeType.values().length - 1) { + sb.append(","); + } + } + if (i != NodeType.values().length - 1) { + sb.append("|"); + } + } + return sb.toString(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private List getResourceRequests() { + if (rmContext.getScheduler() instanceof AbstractYarnScheduler) { + return ((AbstractYarnScheduler) rmContext.getScheduler()) + .getPendingResourceRequestsForAttempt(attemptId); + } + return null; + } + public Resource getApplicationAttemptHeadroom() { return applicationHeadroom; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 430763c..43794af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1229,9 +1229,10 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, Resource.newInstance(1024, 1), 1)){ @Override public ApplicationReport createAndGetApplicationReport( - String clientUserName, boolean allowAccess) { + String clientUserName, boolean allowAccess, + boolean includeApplicationMetrics) { ApplicationReport report = super.createAndGetApplicationReport( - clientUserName, allowAccess); + clientUserName, allowAccess, includeApplicationMetrics); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); usageReport.setMemorySeconds(memorySeconds); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f8d92aa..55d306f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -190,6 +190,13 @@ public ReservationId getReservationId() { public ResourceRequest getAMResourceRequest() { return this.amReq; } + + @Override + public ApplicationReport createAndGetApplicationReport( + String clientUserName, boolean allowAccess, + boolean includeApplicationMetrics) { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { @@ -289,7 +296,7 @@ public ApplicationReport createAndGetApplicationReport( getName(), null, 0, null, null, getDiagnostics().toString(), getTrackingUrl(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), - type, null); + type, null, null, null); return report; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7ed3835..701730b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import java.util.EnumSet; +import java.util.HashSet; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -350,6 +351,7 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + when(app.getApplicationTags()).thenReturn(new HashSet()); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); return app; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index ec990f9..8290879 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -271,4 +271,10 @@ public ReservationId getReservationId() { public ResourceRequest getAMResourceRequest() { return this.amReq; } + + @Override + public ApplicationReport createAndGetApplicationReport(String clientUserName, + boolean allowAccess, boolean includeApplicationMetrics) { + throw new UnsupportedOperationException("Not supported yet."); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 481a53b..2f3063d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -253,7 +253,8 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), - app.getProgress(), app.getApplicationType(), (Token) null); + app.getProgress(), app.getApplicationType(), (Token) null, null, + null); appReports.add(appReport); } GetApplicationsResponse response = mock(GetApplicationsResponse.class);