diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 03552e4..01ef949 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -90,7 +90,7 @@ private ApplicationReport getUnknownApplicationReport() { return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null); } NotRunningJob(ApplicationReport applicationReport, JobState jobState) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index 7d6b2f3..1b6cb97 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -472,7 +472,8 @@ private ApplicationReport getFinishedApplicationReport() { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, + null, null); } private ApplicationReport getRunningApplicationReport(String host, int port) { @@ -482,7 +483,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null); } private ResourceMgrDelegate getRMDelegate() throws IOException { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index c427975..40762f1 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -196,7 +196,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, - "tmp", null)); + "tmp", null, null, null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java new file mode 100644 index 0000000..576a3f8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains various metrics for the applicationAttempt + * to be reported by UI and CLI. + */ +@Public +@Unstable +public abstract class ApplicationAttemptMetrics { + + @Private + @Unstable + public static ApplicationAttemptMetrics newInstance( + boolean isPreempted, Resource resourcePreempted, + int numNonAMContainersPreempted, int totalAllocatedContainers, + String localityStatistics, Resource applicationHeadRoom) { + ApplicationAttemptMetrics metrics = + Records.newRecord(ApplicationAttemptMetrics.class); + metrics.setIsPreempted(isPreempted); + metrics.setResourcePreempted(resourcePreempted); + metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted); + metrics.setTotalAllocatedContainers(totalAllocatedContainers); + metrics.setLocalityStatistics(localityStatistics); + return metrics; + } + + /** + * Get the total resource preempted for applicationAttempt. + * @return the total resource preempted for applicationAttempt + */ + @Public + @Unstable + public abstract Resource getResourcePreempted(); + + @Private + @Unstable + public abstract void setResourcePreempted(Resource resourcePreempted); + + /** + * Get the number of non-AM containers preempted for applicationAttempt + * @return the number of non-AM containers preempted for applicationAttempt + */ + @Public + @Unstable + public abstract int getNumNonAMContainerPreempted(); + + @Private + @Unstable + public abstract void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted); + + /** + * indicate whether this applicationAttempt has been preempted + * @return the flag to indicate whether this applicationAttempt + * has been preempted + */ + @Public + @Unstable + public abstract boolean isPreempted(); + + @Private + @Unstable + public abstract void setIsPreempted(boolean isPreempted); + + /** + * Get the total number of allocated containers for this attempt + * @return the total number of allocated containers for this attempt + */ + @Public + @Unstable + public abstract int getTotalAllocatedContainers(); + + @Private + @Unstable + public abstract void + setTotalAllocatedContainers(int totalAllocatedContainers); + + /** + * Get the number of NodeLocal/RackLocal/OffSwitch containers for this attempt + * @return a string contains the number of + * NodeLocal/RackLocal/OffSwitch containers + */ + @Public + @Unstable + public abstract String getLocalityStatistics(); + + @Private + @Unstable + public abstract void setLocalityStatistics(String localityStatistics); + + /** + * Get the application headroom. + * @return application headroom + */ + @Public + @Unstable + public abstract Resource getApplicationHeadRoom(); + + @Public + @Unstable + public abstract void setApplicationHeadRoom(Resource applicationHeadRoom); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java index 53c18ae..e677a5d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java @@ -39,6 +39,8 @@ *
  • Diagnostic information in case of errors.
  • *
  • {@link YarnApplicationAttemptState} of the application attempt.
  • *
  • {@link ContainerId} of the master Container.
  • + *
  • {@link ApplicationAttemptMetrics} of + * the application attempt
  • * *

    * @@ -52,7 +54,8 @@ public static ApplicationAttemptReport newInstance( ApplicationAttemptId applicationAttemptId, String host, int rpcPort, String url, String oUrl, String diagnostics, - YarnApplicationAttemptState state, ContainerId amContainerId) { + YarnApplicationAttemptState state, ContainerId amContainerId, + ApplicationAttemptMetrics applicationAttemptMetrics) { ApplicationAttemptReport report = Records.newRecord(ApplicationAttemptReport.class); report.setApplicationAttemptId(applicationAttemptId); @@ -63,6 +66,8 @@ public static ApplicationAttemptReport newInstance( report.setDiagnostics(diagnostics); report.setYarnApplicationAttemptState(state); report.setAMContainerId(amContainerId); + report.setApplicationAttemptMetrics( + applicationAttemptMetrics); return report; } @@ -176,4 +181,19 @@ public abstract void setApplicationAttemptId( @Private @Unstable public abstract void setAMContainerId(ContainerId amContainerId); + + /** + * Get the ApplicationAttemptMetrics of this attempt. + * + * @return ApplicationAttemptMetrics of this attempt + */ + @Public + @Unstable + public abstract ApplicationAttemptMetrics + getApplicationAttemptMetrics(); + + @Private + @Unstable + public abstract void setApplicationAttemptMetrics( + ApplicationAttemptMetrics applicationAttemptMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java new file mode 100644 index 0000000..acd0e3b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains various metrics for the application + * to be reported by UI and CLI. + */ +@Public +@Unstable +public abstract class ApplicationMetrics { + + @Private + @Unstable + public static ApplicationMetrics newInstance( + Resource resourcePreempted, int numNonAMContainersPreempted, + int numAMContainersPreempted) { + ApplicationMetrics metrics = + Records.newRecord(ApplicationMetrics.class); + metrics.setResourcePreempted(resourcePreempted); + metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted); + metrics.setNumAMContainersPreempted(numAMContainersPreempted); + return metrics; + } + + /** + * Get the total resource preempted of the application. + * @return the total resource preempted of the application + */ + @Public + @Unstable + public abstract Resource getResourcePreempted(); + + @Private + @Unstable + public abstract void setResourcePreempted(Resource resourcePreempted); + + /** + * Get the number of non-AM containers preempted for this application. + * @return the number of non-AM containers preempted for this application + */ + @Public + @Unstable + public abstract int getNumNonAMContainersPreempted(); + + @Private + @Unstable + public abstract void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted); + + /** + * Get the number of AM containers preempted for this application. + * @return the number of non-AM containers preempted for this application + */ + @Public + @Unstable + public abstract int getNumAMContainersPreempted(); + + @Private + @Unstable + public abstract void + setNumAMContainersPreempted(int numAMContainersPreempted); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 412c22b..56034be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -43,6 +43,7 @@ *
  • Diagnostic information in case of errors.
  • *
  • Start time of the application.
  • *
  • Client {@link Token} of the application (if security is enabled).
  • + *
  • {@link ApplicationMetrics} of the application
  • * *

    * @@ -60,7 +61,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String applicationType, Token amRmToken) { + float progress, String applicationType, Token amRmToken, Set tags, + ApplicationMetrics applicationMetrics) { ApplicationReport report = Records.newRecord(ApplicationReport.class); report.setApplicationId(applicationId); report.setCurrentApplicationAttemptId(applicationAttemptId); @@ -81,6 +83,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setProgress(progress); report.setApplicationType(applicationType); report.setAMRMToken(amRmToken); + report.setApplicationTags(tags); + report.setApplicationMetrics(applicationMetrics); return report; } @@ -362,5 +366,19 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Public @Stable public abstract Token getAMRMToken(); + + /** + * Get the ApplicationMetrics of this application. + * + * @return ApplicationMetrics of this application + */ + @Public + @Unstable + public abstract ApplicationMetrics getApplicationMetrics(); + + @Private + @Unstable + public abstract void setApplicationMetrics( + ApplicationMetrics applicationMetrics); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 2edff99..a99c493 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -194,6 +194,13 @@ message ApplicationReportProto { optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; + optional ApplicationMetricsProto application_metrics = 21; +} + +message ApplicationMetricsProto { + optional ResourceProto resource_preempted = 1; + optional int32 num_non_am_containers_preempted = 2; + optional int32 num_am_containers_preempted = 3; } message ApplicationAttemptReportProto { @@ -205,6 +212,17 @@ message ApplicationAttemptReportProto { optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6; optional ContainerIdProto am_container_id = 7; optional string original_tracking_url = 8; + optional ApplicationAttemptMetricsProto application_attempt_metrics = 9; +} + +message ApplicationAttemptMetricsProto { + optional bool is_preempted = 1; + optional ResourceProto resource_preempted = 2; + optional int32 num_non_am_containers_preempted = 3; + repeated ResourceRequestProto resource_requests = 4; + optional int32 total_allocated_containers = 5; + optional string locality_statistics = 6; + optional ResourceProto application_headroom = 7; } enum NodeStateProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index dd4a949..40300d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -23,6 +23,7 @@ import java.io.PrintWriter; import java.nio.charset.Charset; import java.text.DecimalFormat; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -38,11 +39,15 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; @@ -50,8 +55,12 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.codehaus.jettison.json.JSONException; import com.google.common.annotations.VisibleForTesting; +import com.sun.jersey.api.client.ClientHandlerException; @Private @Unstable @@ -65,6 +74,11 @@ private static final String CONTAINER_PATTERN = "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s" + System.getProperty("line.separator"); + private static final String RESOURCE_REQUESTS_TABLE_PATTERN = + "%16s\t%30s\t%35s\t%20s\t%20s\t%35s" + + System.getProperty("line.separator"); + private static final String LOCALITY_TABLE_PATTERN = + "%48s\t%25s\t%25s\t%25s" + System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; private static final String APP_STATE_CMD = "appStates"; @@ -282,6 +296,8 @@ private int printApplicationAttemptReport(String applicationAttemptId) PrintWriter appAttemptReportStr = new PrintWriter( new OutputStreamWriter(baos, Charset.forName("UTF-8"))); if (appAttemptReport != null) { + ApplicationAttemptMetrics attemptMetrics = + appAttemptReport.getApplicationAttemptMetrics(); appAttemptReportStr.println("Application Attempt Report : "); appAttemptReportStr.print("\tApplicationAttempt-Id : "); appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId()); @@ -298,7 +314,25 @@ private int printApplicationAttemptReport(String applicationAttemptId) appAttemptReportStr.print("\tAM Host : "); appAttemptReportStr.println(appAttemptReport.getHost()); appAttemptReportStr.print("\tDiagnostics : "); - appAttemptReportStr.print(appAttemptReport.getDiagnostics()); + appAttemptReportStr.println(appAttemptReport.getDiagnostics()); + appAttemptReportStr.print( + "\tResource Preempted from Current Attempt : "); + appAttemptReportStr.println(attemptMetrics == null ? "N/A" + : attemptMetrics.getResourcePreempted()); + appAttemptReportStr.print( + "\tNumber of Non-AM Containers Preempted from Current Attempt : "); + appAttemptReportStr.println(attemptMetrics == null ? "N/A" + : attemptMetrics.getNumNonAMContainerPreempted()); + appAttemptReportStr.flush(); + + // Print ResourceRequests table + printResourceRequestsTable(appAttemptReportStr, appAttemptReport + .getApplicationAttemptId().getApplicationId().toString()); + + if (attemptMetrics != null) { + // Print LocalityStatistics table + printLocalityStatisticsTable(appAttemptReportStr, attemptMetrics); + } } else { appAttemptReportStr.print("Application Attempt with id '" + applicationAttemptId + "' doesn't exist in Timeline Server."); @@ -531,7 +565,19 @@ private int printApplicationReport(String applicationId) appReportStr.println("N/A"); } appReportStr.print("\tDiagnostics : "); - appReportStr.print(appReport.getDiagnostics()); + appReportStr.println(appReport.getDiagnostics()); + appReportStr.print("\tApplication Tags : "); + appReportStr.println(appReport.getApplicationTags()); + ApplicationMetrics appMetrics = appReport.getApplicationMetrics(); + appReportStr.print("\tTotal Resource Preempted : "); + appReportStr.println(appMetrics != null ? appMetrics + .getResourcePreempted() : "N/A"); + appReportStr.print("\tTotal Number of Non-AM Containers Preempted : "); + appReportStr.println(appMetrics != null ? appMetrics + .getNumNonAMContainersPreempted() : "N/A"); + appReportStr.print("\tTotal Number of AM Containers Preempted : "); + appReportStr.print(appMetrics != null ? appMetrics + .getNumAMContainersPreempted() : "N/A"); } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); @@ -612,4 +658,89 @@ private void listContainers(String appAttemptId) throws YarnException, } writer.flush(); } + + private void printResourceRequestsTable(PrintWriter reportStr, String appID) { + reportStr.println(); + // Print ResourceRequests table + List resouceRequests = getResourceRequests(appID); + + Resource totalResource = Resource.newInstance(0, 0); + reportStr.println("ResourceRequests : "); + for (ResourceRequest request : resouceRequests) { + if (request.getResourceName().equals(ResourceRequest.ANY)) { + Resources.addTo( + totalResource, + Resources.multiply(request.getCapability(), + request.getNumContainers())); + } + } + reportStr.print("\ttotalResourceRequests : "); + reportStr.println(totalResource); + reportStr.printf(RESOURCE_REQUESTS_TABLE_PATTERN, "Priority", + "ResourceName", "Capability", "NumContainers", "RelaxLocality", + "NodeLabelExpression"); + for (ResourceRequest request : resouceRequests) { + if (request.getNumContainers() == 0) { + continue; + } + reportStr.printf( + RESOURCE_REQUESTS_TABLE_PATTERN, + request.getPriority(), + request.getResourceName(), + request.getCapability(), + request.getNumContainers(), + request.getRelaxLocality(), + request.getNodeLabelExpression() == null ? "N/A" : request + .getNodeLabelExpression()); + } + reportStr.flush(); + } + + private void printLocalityStatisticsTable(PrintWriter reportStr, + ApplicationAttemptMetrics attemptMetrics) { + reportStr.println(); + reportStr.println("Container Locality Statistics Table: "); + reportStr.print("\tTotal Allocated Containers : "); + reportStr.println(attemptMetrics + .getTotalAllocatedContainers()); + reportStr + .println("\tEach table cell represents the number of " + + "NodeLocal/RackLocal/OffSwitch containers satisfied by " + + "NodeLocal/RackLocal/OffSwitch resource requests."); + String[] containersType = { + "Num Node Local Containers (satisfied by)", + "Num Rack Local Containers (satisfied by)", + "Num Off Switch Containers (satisfied by)" + }; + int[][] localityStatistics = + ConverterUtils.covertLocalityStatisticsStringToArray(attemptMetrics + .getLocalityStatistics()); + reportStr.printf(LOCALITY_TABLE_PATTERN, "", + "Node Local Request", + "Rack Local Request", + "Off Switch Request"); + for (int i = 0; i < localityStatistics.length; i++) { + reportStr.printf(LOCALITY_TABLE_PATTERN, + containersType[i], + localityStatistics[i][0] == -1 ? "N/A" : localityStatistics[i][0], + i == 0 ? "" : localityStatistics[i][1] == -1 ? + "N/A" : localityStatistics[i][1], + i <= 1 ? "" : localityStatistics[i][2] == -1 ? + "N/A" : localityStatistics[i][2]); + } + reportStr.flush(); + } + + @VisibleForTesting + public List getResourceRequests(String appId) { + List resouceRequests = new ArrayList(); + try { + resouceRequests = WebAppUtils.getResourceRequests(getConf(), appId); + } catch (JSONException ex) { + // DO NOTHING + } catch (ClientHandlerException ex) { + // DO NOTHING + } + return resouceRequests; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index f468bc1..6f7652a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -647,7 +647,7 @@ public ApplicationReport createFakeAppReport() { "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, - "fakeApplicationType", null); + "fakeApplicationType", null, null, null); return report; } @@ -700,7 +700,7 @@ public QueueInfo createFakeQueueInfo() { public ApplicationAttemptReport createFakeApplicationAttemptReport() { return ApplicationAttemptReport.newInstance( createFakeApplicationAttemptId(), "localhost", 0, "", "", "", - YarnApplicationAttemptState.RUNNING, createFakeContainerId()); + YarnApplicationAttemptState.RUNNING, createFakeContainerId(), null); } public List diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c41..8ffa1ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -334,7 +334,7 @@ private void createAppReports() { "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", - null); + null, null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -350,7 +350,8 @@ private void createAppReports() { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 1)); + newApplicationReport.getCurrentApplicationAttemptId(), 1), + null); appAttempts.add(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( @@ -362,7 +363,8 @@ private void createAppReports() { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 2)); + newApplicationReport.getCurrentApplicationAttemptId(), 2), + null); appAttempts.add(attempt1); attempts.put(applicationId, appAttempts); @@ -391,7 +393,7 @@ private void createAppReports() { "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, - "NON-YARN", null); + "NON-YARN", null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -401,7 +403,7 @@ private void createAppReports() { "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, - "MAPREDUCE", null); + "MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -411,7 +413,7 @@ private void createAppReports() { "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null); + "NON-MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport4); reports = applicationReports; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index de669f2..60b3f56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -580,7 +580,8 @@ public void setYarnApplicationState(YarnApplicationState state) { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, + null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); List appAttempts = new ArrayList(); @@ -593,7 +594,8 @@ public void setYarnApplicationState(YarnApplicationState state) { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 1)); + newApplicationReport.getCurrentApplicationAttemptId(), 1), + null); appAttempts.add(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( ApplicationAttemptId.newInstance(applicationId, 2), @@ -604,7 +606,8 @@ public void setYarnApplicationState(YarnApplicationState state) { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 2)); + newApplicationReport.getCurrentApplicationAttemptId(), 2), + null); appAttempts.add(attempt1); attempts.put(applicationId, appAttempts); @@ -655,7 +658,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null); + null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -664,7 +667,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -675,7 +678,7 @@ public void setYarnApplicationState(YarnApplicationState state) { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null); + "NON-MAPREDUCE", null, null, null); applicationReports.add(newApplicationReport4); return applicationReports; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 4b60c52..2e64059 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doReturn; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -44,8 +45,10 @@ import org.apache.commons.cli.Options; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -98,12 +102,16 @@ public void testGetApplicationReport() throws Exception { ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport.newInstance( 2, 0, null, null, null, 123456, 4567); + Set appTags = new HashSet(); + appTags.add("test"); + ApplicationMetrics metrics = + ApplicationMetrics.newInstance(Resource.newInstance(6144, 10), 5, 2); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null); + null, appTags, metrics); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -128,6 +136,10 @@ public void testGetApplicationReport() throws Exception { pw.println("\tAggregate Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tApplication Tags : [test]"); + pw.println("\tTotal Resource Preempted : "); + pw.println("\tTotal Number of Non-AM Containers Preempted : 5"); + pw.println("\tTotal Number of AM Containers Preempted : 2"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -139,18 +151,36 @@ public void testGetApplicationReport() throws Exception { @Test public void testGetApplicationAttemptReport() throws Exception { ApplicationCLI cli = createAndGetAppCLI(); + ApplicationCLI spyCLI = spy(cli); + List resourceRequests = new ArrayList(); + ResourceRequest request = + ResourceRequest.newInstance(Priority.newInstance(20), + ResourceRequest.ANY, Resource.newInstance(1024, 2), 10); + request.setNodeLabelExpression("dummy_label_for_test"); + resourceRequests.add(request); + doReturn(resourceRequests).when(spyCLI).getResourceRequests( + any(String.class)); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( applicationId, 1); + String localityStatistics = "NODE_LOCAL:NODE_LOCAL:8," + + "NODE_LOCAL:RACK_LOCAL:0,NODE_LOCAL:OFF_SWITCH:0" + + "|RACK_LOCAL:NODE_LOCAL:2,RACK_LOCAL:RACK_LOCAL:5," + + "RACK_LOCAL:OFF_SWITCH:0|OFF_SWITCH:NODE_LOCAL:1," + + "OFF_SWITCH:RACK_LOCAL:4,OFF_SWITCH:OFF_SWITCH:10"; + ApplicationAttemptMetrics metrics = + ApplicationAttemptMetrics.newInstance(false, + Resource.newInstance(6144, 10), 10, 20, localityStatistics, + Resource.newInstance(0, 0)); ApplicationAttemptReport attemptReport = ApplicationAttemptReport .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId, 1)); + attemptId, 1), metrics); when( client .getApplicationAttemptReport(any(ApplicationAttemptId.class))) .thenReturn(attemptReport); - int result = cli.run(new String[] { "applicationattempt", "-status", + int result = spyCLI.run(new String[] { "applicationattempt", "-status", attemptId.toString() }); assertEquals(0, result); verify(client).getApplicationAttemptReport(attemptId); @@ -164,8 +194,49 @@ public void testGetApplicationAttemptReport() throws Exception { pw.println("\tRPC Port : 124"); pw.println("\tAM Host : host"); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tResource Preempted from Current Attempt : " + + ""); + pw.println("\tNumber of Non-AM Containers Preempted from " + + "Current Attempt : 10"); + pw.println(); + pw.println("ResourceRequests : "); + pw.println("\ttotalResourceRequests : "); + pw.println(" Priority\t ResourceName" + + "\t Capability\t NumContainers" + + "\t RelaxLocality\t NodeLabelExpression"); + pw.println(" 20\t *" + + "\t \t 10" + + "\t true\t dummy_label_for_test"); + pw.println(); + pw.println("Container Locality Statistics Table: "); + pw.println("\tTotal Allocated Containers : 20"); + pw.println("\tEach table cell represents the number of " + + "NodeLocal/RackLocal/OffSwitch containers satisfied by " + + "NodeLocal/RackLocal/OffSwitch resource requests."); + pw.print(" "); + pw.println("\t Node Local Request" + + "\t Rack Local Request" + + "\t Off Switch Request"); + pw.println(" Num Node Local Containers (satisfied by)" + + "\t 8" + + "\t " + + "\t "); + pw.println(" Num Rack Local Containers (satisfied by)" + + "\t 2" + + "\t 5" + + "\t "); + pw.println(" Num Off Switch Containers (satisfied by)" + + "\t 1" + + "\t 4" + + "\t 10"); + pw.println(); pw.close(); String appReportStr = baos.toString("UTF-8"); + Log.info("ExpectedOutput"); + Log.info("["+appReportStr+"]"); + Log.info("OutputFrom command"); + String actualOutput = sysOutStream.toString(); + Log.info("["+actualOutput+"]"); Assert.assertEquals(appReportStr, sysOutStream.toString()); verify(sysOut, times(1)).println(isA(String.class)); } @@ -181,11 +252,11 @@ public void testGetApplicationAttempts() throws Exception { ApplicationAttemptReport attemptReport = ApplicationAttemptReport .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId, 1)); + attemptId, 1), null); ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport .newInstance(attemptId1, "host", 124, "url", "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId1, 1)); + attemptId1, 1), null); List reports = new ArrayList(); reports.add(attemptReport); reports.add(attemptReport1); @@ -347,7 +418,7 @@ public void testGetApplications() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -357,7 +428,7 @@ public void testGetApplications() throws Exception { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null); + null, null, null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -366,7 +437,7 @@ public void testGetApplications() throws Exception { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -375,7 +446,7 @@ public void testGetApplications() throws Exception { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", - null); + null, null, null); applicationReports.add(newApplicationReport4); ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9); @@ -384,7 +455,7 @@ public void testGetApplications() throws Exception { "user5", "queue5", "appname5", "host5", 128, null, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", - null); + null, null, null); applicationReports.add(newApplicationReport5); ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10); @@ -393,7 +464,7 @@ public void testGetApplications() throws Exception { "user6", "queue6", "appname6", "host6", 129, null, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", - null); + null, null, null); applicationReports.add(newApplicationReport6); // Test command yarn application -list @@ -817,7 +888,7 @@ public void testKillApplication() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -830,7 +901,7 @@ public void testKillApplication() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -865,7 +936,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application", "-movetoqueue", @@ -880,7 +951,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application", "-movetoqueue", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java new file mode 100644 index 0000000..a00adf8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class ApplicationAttemptMetricsPBImpl extends + ApplicationAttemptMetrics { + + ApplicationAttemptMetricsProto proto = + ApplicationAttemptMetricsProto.getDefaultInstance(); + ApplicationAttemptMetricsProto.Builder builder = null; + boolean viaProto = false; + + Resource totalResourcePreempted; + Resource applicationHeadRoom; + + public ApplicationAttemptMetricsPBImpl() { + builder = ApplicationAttemptMetricsProto.newBuilder(); + } + + public ApplicationAttemptMetricsPBImpl( + ApplicationAttemptMetricsProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ApplicationAttemptMetricsProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.totalResourcePreempted != null + && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals( + builder.getResourcePreempted())) { + builder.setResourcePreempted( + convertToProtoFormat(this.totalResourcePreempted)); + } + if (this.applicationHeadRoom != null + && !((ResourcePBImpl) this.applicationHeadRoom).getProto().equals( + builder.getApplicationHeadroom())) { + builder.setApplicationHeadroom( + convertToProtoFormat(this.applicationHeadRoom)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationAttemptMetricsProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized Resource getResourcePreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (this.totalResourcePreempted != null) { + return this.totalResourcePreempted; + } + if (!p.hasResourcePreempted()) { + return null; + } + this.totalResourcePreempted = + convertFromProtoFormat(p.getResourcePreempted()); + return this.totalResourcePreempted; + } + + @Override + public synchronized void setResourcePreempted(Resource resourcePreempted) { + maybeInitBuilder(); + if (resourcePreempted == null) + builder.clearResourcePreempted(); + this.totalResourcePreempted = resourcePreempted; + } + + @Override + public synchronized int getNumNonAMContainerPreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return (p.getNumNonAmContainersPreempted()); + } + + @Override + public synchronized void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted); + } + + @Override + public synchronized boolean isPreempted() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return p.getIsPreempted(); + } + + @Override + public synchronized void setIsPreempted(boolean isPreempted) { + maybeInitBuilder(); + builder.setIsPreempted(isPreempted); + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl)t).getProto(); + } + + @Override + public synchronized int getTotalAllocatedContainers() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + return p.getTotalAllocatedContainers(); + } + + @Override + public synchronized void setTotalAllocatedContainers(int totalAllocatedContainers) { + maybeInitBuilder(); + builder.setTotalAllocatedContainers(totalAllocatedContainers); + } + + @Override + public synchronized String getLocalityStatistics() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (! p.hasLocalityStatistics()) { + return null; + } + return p.getLocalityStatistics(); + } + + @Override + public synchronized void setLocalityStatistics(String localityStatistics) { + maybeInitBuilder(); + if (localityStatistics == null) { + builder.clearLocalityStatistics(); + return; + } + builder.setLocalityStatistics(localityStatistics); + } + + @Override + public Resource getApplicationHeadRoom() { + ApplicationAttemptMetricsProtoOrBuilder p = + viaProto ? proto : builder; + if (this.applicationHeadRoom != null) { + return this.applicationHeadRoom; + } + if (!p.hasApplicationHeadroom()) { + return null; + } + this.applicationHeadRoom = + convertFromProtoFormat(p.getApplicationHeadroom()); + return this.applicationHeadRoom; + } + + @Override + public void setApplicationHeadRoom(Resource applicationHeadRoom) { + maybeInitBuilder(); + if (applicationHeadRoom == null) + builder.clearApplicationHeadroom(); + this.applicationHeadRoom = applicationHeadRoom; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java index c3c0221..3894f34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java @@ -19,10 +19,12 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; @@ -38,6 +40,7 @@ private ApplicationAttemptId ApplicationAttemptId; private ContainerId amContainerId; + private ApplicationAttemptMetrics attemptMetrics; public ApplicationAttemptReportPBImpl() { builder = ApplicationAttemptReportProto.newBuilder(); @@ -244,6 +247,13 @@ private void mergeLocalToBuilder() { builder.getAmContainerId())) { builder.setAmContainerId(convertToProtoFormat(this.amContainerId)); } + + if (this.attemptMetrics != null + && !((ApplicationAttemptMetricsPBImpl) this.attemptMetrics) + .getProto().equals(builder.getApplicationAttemptMetrics())) { + builder.setApplicationAttemptMetrics( + convertToProtoFormat(this.attemptMetrics)); + } } private ContainerIdProto convertToProtoFormat(ContainerId amContainerId) { @@ -286,4 +296,39 @@ public void setAMContainerId(ContainerId amContainerId) { builder.clearAmContainerId(); this.amContainerId = amContainerId; } + + @Override + public ApplicationAttemptMetrics + getApplicationAttemptMetrics() { + if (this.attemptMetrics != null) { + return this.attemptMetrics; + } + + ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationAttemptMetrics()) { + return null; + } + this.attemptMetrics = + convertFromProtoFormat(p.getApplicationAttemptMetrics()); + return this.attemptMetrics; + } + + @Override + public void setApplicationAttemptMetrics( + ApplicationAttemptMetrics applicationAttemptMetrics) { + maybeInitBuilder(); + if (applicationAttemptMetrics == null) + builder.clearApplicationAttemptMetrics(); + this.attemptMetrics = applicationAttemptMetrics; + } + + private ApplicationAttemptMetricsProto + convertToProtoFormat(ApplicationAttemptMetrics t) { + return ((ApplicationAttemptMetricsPBImpl) t).getProto(); + } + + private ApplicationAttemptMetricsPBImpl convertFromProtoFormat( + ApplicationAttemptMetricsProto t) { + return new ApplicationAttemptMetricsPBImpl(t); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java new file mode 100644 index 0000000..6843d97 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; + +import com.google.protobuf.TextFormat; + +@Private +@Unstable +public class ApplicationMetricsPBImpl extends ApplicationMetrics { + + ApplicationMetricsProto proto = + ApplicationMetricsProto.getDefaultInstance(); + ApplicationMetricsProto.Builder builder = null; + boolean viaProto = false; + + Resource totalResourcePreempted; + + public ApplicationMetricsPBImpl() { + builder = ApplicationMetricsProto.newBuilder(); + } + + public ApplicationMetricsPBImpl( + ApplicationMetricsProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ApplicationMetricsProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.totalResourcePreempted != null + && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals( + builder.getResourcePreempted())) { + builder.setResourcePreempted( + convertToProtoFormat(this.totalResourcePreempted)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ApplicationMetricsProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized Resource getResourcePreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + if (this.totalResourcePreempted != null) { + return this.totalResourcePreempted; + } + if (!p.hasResourcePreempted()) { + return null; + } + this.totalResourcePreempted = + convertFromProtoFormat(p.getResourcePreempted()); + return this.totalResourcePreempted; + } + + @Override + public synchronized void setResourcePreempted(Resource resourcePreempted) { + maybeInitBuilder(); + if (resourcePreempted == null) + builder.clearResourcePreempted(); + this.totalResourcePreempted = resourcePreempted; + } + + @Override + public synchronized int getNumNonAMContainersPreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + return (p.getNumNonAmContainersPreempted()); + } + + @Override + public synchronized void setNumNonAMContainersPreempted( + int numNonAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted); + } + + @Override + public synchronized int getNumAMContainersPreempted() { + ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder; + return (p.getNumAmContainersPreempted()); + } + + @Override + public synchronized void setNumAMContainersPreempted( + int numAMContainersPreempted) { + maybeInitBuilder(); + builder.setNumAmContainersPreempted(numAMContainersPreempted); + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ((ResourcePBImpl)t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index dd3e2bc..daf15cc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto; @@ -53,6 +55,7 @@ private Token clientToAMToken = null; private Token amRmToken = null; private Set applicationTags = null; + private ApplicationMetrics applicationMetrics = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -482,6 +485,12 @@ private void mergeLocalToBuilder() { builder.clearApplicationTags(); builder.addAllApplicationTags(this.applicationTags); } + if (this.applicationMetrics != null + && !((ApplicationMetricsPBImpl) this.applicationMetrics) + .getProto().equals(builder.getApplicationMetrics())) { + builder.setApplicationMetrics( + convertToProtoFormat(this.applicationMetrics)); + } } private void mergeLocalToProto() { @@ -548,4 +557,38 @@ private TokenPBImpl convertFromProtoFormat(TokenProto p) { private TokenProto convertToProtoFormat(Token t) { return ((TokenPBImpl)t).getProto(); } + + @Override + public ApplicationMetrics getApplicationMetrics() { + if (this.applicationMetrics != null) { + return this.applicationMetrics; + } + + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationMetrics()) { + return null; + } + this.applicationMetrics = + convertFromProtoFormat(p.getApplicationMetrics()); + return this.applicationMetrics; + } + + @Override + public void setApplicationMetrics( + ApplicationMetrics applicationMetrics) { + maybeInitBuilder(); + if (applicationMetrics == null) + builder.clearApplicationMetrics(); + this.applicationMetrics = applicationMetrics; + } + + private ApplicationMetricsProto convertToProtoFormat( + ApplicationMetrics t) { + return ((ApplicationMetricsPBImpl) t).getProto(); + } + + private ApplicationMetricsPBImpl convertFromProtoFormat( + ApplicationMetricsProto t) { + return new ApplicationMetricsPBImpl(t); +} } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index e9674cf..64a15f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -259,4 +259,42 @@ public static ApplicationId toApplicationId( } return token; } + + /** + * Convert LocalityStatistics from String format to 3X3 array. + * + * @param localityStatistics String + */ + public static int[][] covertLocalityStatisticsStringToArray( + String localityStatistics) { + int[][] coverted = new int[][] { + {-1, -1, -1}, + {-1, -1, -1}, + {-1, -1, -1} + }; + if (localityStatistics != null && !localityStatistics.isEmpty()) { + String[] localityStatisticsForTypes = localityStatistics.split("\\|"); + if (localityStatisticsForTypes.length == 3) { + for (int i = 0; i < 3; i++) { + String[] numOfLocalityStatisticsForTypes = + localityStatisticsForTypes[i].trim().split(","); + if (numOfLocalityStatisticsForTypes.length != 3) { + continue; + } + for (int j = 0; j < 3; j++) { + String[] num = numOfLocalityStatisticsForTypes[j].trim().split(":"); + if (num.length != 3) { + continue; + } + try { + coverted[i][j] = Integer.parseInt(num[2]); + } catch (NumberFormatException ex) { + // Do Nothing + } + } + } + } + } + return coverted; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 3aeb33e..1c86b1f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -24,7 +24,12 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; + +import javax.ws.rs.core.MediaType; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -32,10 +37,22 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.RMHAUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; @Private @Evolving @@ -347,4 +364,97 @@ static String getPassword(Configuration conf, String alias) { } return password; } + + /** + * Get a list of ResourceRequests for the application by calling + * RM Web Service. + * @param conf Configuration + * @param appId ApplicationId + * @return resourceRequests List + * @throws ClientHandlerException + * @throws UniformInterfaceException + * @throws JSONException + */ + public static List getResourceRequests(Configuration conf, + String appId) throws ClientHandlerException, + UniformInterfaceException, JSONException { + List resouceRequests = new ArrayList(); + Client webServiceClient = Client.create(); + String webAppAddress = + WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.RM_BIND_HOST, + WebAppUtils.getRMWebAppURLWithScheme(conf)); + WebResource webResource = webServiceClient.resource(webAppAddress); + + ClientResponse response = + webResource.path("ws").path("v1").path("cluster").path("apps") + .path(appId).accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + JSONObject json = response.getEntity(JSONObject.class).getJSONObject("app"); + JSONArray requests = json.getJSONArray("resourceRequests"); + for (int i = 0; i < requests.length(); i++) { + resouceRequests.add(parseFromJSONObject(requests + .getJSONObject(i))); + } + return resouceRequests; + } + + private static ResourceRequest parseFromJSONObject(JSONObject resourceRequest) + throws JSONException { + Map out = new HashMap(); + parseJSONObject(resourceRequest, out); + Priority priority = null; + if (out.containsKey("priority")) { + priority = Priority.newInstance(Integer.parseInt(out.get("priority"))); + } + String hostName = null; + if (out.containsKey("resourceName")) { + hostName = out.get("resourceName"); + } + int numContainers = -1; + if (out.containsKey("numContainers")) { + numContainers = Integer.parseInt(out.get("numContainers")); + } + boolean relaxLocality = true; + if (out.containsKey("relaxLocality")) { + relaxLocality = Boolean.parseBoolean(out.get("relaxLocality")); + } + int memory = 0; + if (out.containsKey("memory")) { + memory = Integer.parseInt(out.get("memory")); + } + int virtualCores = 0; + if (out.containsKey("virtualCores")) { + virtualCores = Integer.parseInt(out.get("virtualCores")); + } + Resource capability = Resource.newInstance(memory, virtualCores); + String nodeLabelExpression = null; + if (out.containsKey("nodeLabelExpression")) { + nodeLabelExpression = out.get(out.get("nodeLabelExpression")); + } + ResourceRequest request = + ResourceRequest.newInstance(priority, hostName, capability, + numContainers, relaxLocality, nodeLabelExpression); + return request; + } + + @SuppressWarnings("unchecked") + private static Map parseJSONObject(JSONObject json, + Map out) throws JSONException { + Iterator keys = json.keys(); + while (keys.hasNext()) { + String key = keys.next(); + String val = null; + try { + JSONObject value = json.getJSONObject(key); + parseJSONObject(value, out); + } catch (Exception e) { + val = json.getString(key); + } + + if (val != null) { + out.put(key.trim(), val.trim()); + } + } + return out; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 9302d4b..debd067 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -47,6 +47,7 @@ public void testApplicationReport() { Assert.assertNull(appReport2.getCurrentApplicationAttemptId()); Assert.assertNotSame(appReport2, appReport3); Assert.assertNull(appReport1.getAMRMToken()); + Assert.assertNull(appReport1.getApplicationMetrics()); } protected static ApplicationReport createApplicationReport( @@ -58,7 +59,8 @@ protected static ApplicationReport createApplicationReport( ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, + null, null); return appReport; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 8b48798..5c7b3c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -194,7 +194,9 @@ public static void setup() throws Exception { generateByNewInstance(Token.class); generateByNewInstance(NMToken.class); generateByNewInstance(ResourceRequest.class); + generateByNewInstance(ApplicationAttemptMetrics.class); generateByNewInstance(ApplicationAttemptReport.class); + generateByNewInstance(ApplicationMetrics.class); generateByNewInstance(ApplicationResourceUsageReport.class); generateByNewInstance(ApplicationReport.class); generateByNewInstance(Container.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index c7cf07b..0b54c97 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport( appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + appHistory.getApplicationType(), null, null, null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) @@ -166,7 +166,7 @@ private ApplicationAttemptReport convertToApplicationAttemptReport( appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(), null, appAttemptHistory.getDiagnosticsInfo(), appAttemptHistory.getYarnApplicationAttemptState(), - appAttemptHistory.getMasterContainerId()); + appAttemptHistory.getMasterContainerId(), null); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 1010f62..9ea318f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -31,8 +34,10 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; @@ -55,12 +60,14 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.MetricsConstantsUtils; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.codehaus.jackson.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -226,6 +233,8 @@ private static ApplicationReportExt convertToApplicationReport( ApplicationResourceUsageReport appResources = null; Map appViewACLs = new HashMap(); + Set appTags = null; + ApplicationMetrics appMetrics = null; Map entityInfo = entity.getOtherInfo(); if (entityInfo != null) { if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) { @@ -245,7 +254,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, 1.0F, type, null), appViewACLs); + null, 1.0F, type, null, appTags, appMetrics), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -270,6 +279,17 @@ private static ApplicationReportExt convertToApplicationReport( appResources=ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS)) { + appTags = new HashSet(); + Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS); + if (obj instanceof Collection) { + for(Object o : (Collection)obj) { + if (o != null) { + appTags.add(o.toString()); + } + } + } + } } List events = entity.getEvents(); if (events != null) { @@ -314,6 +334,13 @@ private static ApplicationReportExt convertToApplicationReport( YarnApplicationState.valueOf(eventInfo.get( ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); } + if (eventInfo.containsKey(ApplicationMetricsConstants.APP_METRICS)) { + Object obj = eventInfo.get(ApplicationMetricsConstants.APP_METRICS); + if (obj instanceof JsonNode) { + appMetrics = MetricsConstantsUtils + .convertJSONtoApplicationMetrics((JsonNode)obj); + } + } } } } @@ -321,7 +348,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, 1.0F, type, null), appViewACLs); + null, 1.0F, type, null, appTags, appMetrics), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( @@ -333,6 +360,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport( String originalTrackingUrl = null; String diagnosticsInfo = null; YarnApplicationAttemptState state = null; + ApplicationAttemptMetrics appAttemptMetrics = null; List events = entity.getEvents(); if (events != null) { for (TimelineEvent event : events) { @@ -394,13 +422,23 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport( AppAttemptMetricsConstants.STATE_EVENT_INFO) .toString()); } + if (eventInfo + .containsKey(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS)) { + Object ob = + eventInfo.get(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS); + if (ob instanceof JsonNode) { + appAttemptMetrics = + MetricsConstantsUtils + .convertJSONtoApplicationAttemptMetrics((JsonNode) ob); + } + } } } } return ApplicationAttemptReport.newInstance( ConverterUtils.toApplicationAttemptId(entity.getEntityId()), host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo, - state, amContainerId); + state, amContainerId, appAttemptMetrics); } private static ContainerReport convertToContainerReport( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java index a7809cf..5ce1aef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java @@ -61,4 +61,7 @@ public static final String STATE_EVENT_INFO = "YARN_APPLICATION_ATTEMPT_STATE"; + public static final String APP_ATTEMPT_METRICS = + "YARN_APPLICATION_ATTEMPT_METRICS"; + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index df8eecb..0055de3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,4 +73,8 @@ public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + public static final String APP_TAGS = "YARN_APPLICATION_TAGS"; + + public static final String APP_METRICS = "YARN_APPLICATION_METRICS"; + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java new file mode 100644 index 0000000..b544117 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.metrics; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; +import org.apache.hadoop.yarn.api.records.Resource; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.node.ObjectNode; + +public class MetricsConstantsUtils { + + /** + * Convert ApplicationMetrics object to JSON + * @param appMetrics ApplicationMetrics + * @return node JsonNode + */ + public static JsonNode appMetricsToJSON(ApplicationMetrics appMetrics) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode node = mapper.createObjectNode(); + if (appMetrics != null) { + if (appMetrics.getResourcePreempted() != null) { + node.put("MEMORY", appMetrics.getResourcePreempted().getMemory()); + node.put("VIRTUAL_CORES", appMetrics.getResourcePreempted() + .getVirtualCores()); + } + node.put("NUM_NON_AMCONTAINERS_PREEMPTED", + appMetrics.getNumNonAMContainersPreempted()); + node.put("NUM_AMCONTAINERS_PREEMPTED", + appMetrics.getNumAMContainersPreempted()); + } + return node; + } + + /** + * Convert JSON to ApplicationMetrics Object + * @param node JSON + * @return applicationMetrics ApplicationMetrics + */ + public static ApplicationMetrics + convertJSONtoApplicationMetrics(JsonNode node) { + if (node == null) { + return null; + } + int memory = 0; + if (node.has("MEMORY")) { + memory = node.path("MEMORY").getIntValue(); + } + int vcores = 0; + if (node.has("VIRTUAL_CORES")) { + vcores = node.path("VIRTUAL_CORES").getIntValue(); + } + int numNonAMContainersPreempted = 0; + if (node.has("NUM_NON_AMCONTAINERS_PREEMPTED")) { + numNonAMContainersPreempted = + node.path("NUM_NON_AMCONTAINERS_PREEMPTED").getIntValue(); + } + int numAMContainersPreempted = 0; + if (node.has("NUM_AMCONTAINERS_PREEMPTED")) { + numAMContainersPreempted = + node.path("NUM_AMCONTAINERS_PREEMPTED").getIntValue(); + } + return ApplicationMetrics.newInstance(Resource.newInstance(memory, vcores), + numNonAMContainersPreempted, numAMContainersPreempted); + } + + public static JsonNode appAttemptMetricsToJSON( + ApplicationAttemptMetrics attemptMetrics) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode node = mapper.createObjectNode(); + if (attemptMetrics != null) { + if (attemptMetrics.getResourcePreempted() != null) { + node.put("MEMORY", attemptMetrics.getResourcePreempted().getMemory()); + node.put("VIRTUAL_CORES", attemptMetrics.getResourcePreempted() + .getVirtualCores()); + } + node.put("NUM_NON_AMCONTAINER_PREEMPTED", + attemptMetrics.getNumNonAMContainerPreempted()); + node.put("IS_PREEMPTED", attemptMetrics.isPreempted()); + node.put("TOTAL_ALLOCATED_CONTAINERS", + attemptMetrics.getTotalAllocatedContainers()); + node.put("LOCALITY_STATISTICS", attemptMetrics.getLocalityStatistics()); + if (attemptMetrics.getApplicationHeadRoom() != null) { + node.put("HEADROOM_MEMORY", attemptMetrics.getApplicationHeadRoom() + .getMemory()); + node.put("HEADROOM_VIRTUAL_CORES", attemptMetrics + .getApplicationHeadRoom().getVirtualCores()); + } + } + return node; + } + + public static ApplicationAttemptMetrics + convertJSONtoApplicationAttemptMetrics(JsonNode node) { + if (node == null) { + return null; + } + int memory = 0; + if (node.has("MEMORY")) { + memory = node.path("MEMORY").getIntValue(); + } + int vcores = 0; + if (node.has("VIRTUAL_CORES")) { + vcores = node.path("VIRTUAL_CORES").getIntValue(); + } + int numNonAMContainerPreempted = 0; + if (node.has("NUM_NON_AMCONTAINER_PREEMPTED")) { + numNonAMContainerPreempted = + node.path("NUM_NON_AMCONTAINER_PREEMPTED").getIntValue(); + } + boolean isPreempted = false; + if (node.has("IS_PREEMPTED")) { + isPreempted = node.path("IS_PREEMPTED").getBooleanValue(); + } + int totalAllocatedContainers = 0; + if (node.has("TOTAL_ALLOCATED_CONTAINERS")) { + totalAllocatedContainers = + node.path("TOTAL_ALLOCATED_CONTAINERS").getIntValue(); + } + String localityStatistics = null; + if (node.has("LOCALITY_STATISTICS")) { + localityStatistics = node.path("LOCALITY_STATISTICS").getTextValue(); + } + int headroomMemory = 0; + if (node.has("HEADROOM_MEMORY")) { + memory = node.path("HEADROOM_MEMORY").getIntValue(); + } + int headroomVcores = 0; + if (node.has("HEADROOM_VIRTUAL_CORES")) { + vcores = node.path("HEADROOM_VIRTUAL_CORES").getIntValue(); + } + return ApplicationAttemptMetrics.newInstance(isPreempted, + Resource.newInstance(memory, vcores), numNonAMContainerPreempted, + totalAllocatedContainers, localityStatistics, + Resource.newInstance(headroomMemory, headroomVcores)); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 4a82c93..433bc58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -225,20 +226,23 @@ public ApplicationAttemptReport run() throws Exception { tbody._()._(); if (webUiType.equals(YarnWebParams.RM_WEB_UI)) { - createContainerLocalityTable(html); // TODO:YARN-3284 + createContainerLocalityTable(html, + appAttemptReport.getApplicationAttemptMetrics()); } } - //TODO: YARN-3284 - //The containerLocality metrics will be exposed from AttemptReport - private void createContainerLocalityTable(Block html) { - int totalAllocatedContainers = 0; //TODO: YARN-3284 - int[][] localityStatistics = new int[0][0];//TODO:YARN-3284 + private void createContainerLocalityTable(Block html, + ApplicationAttemptMetrics attemptMetrics) { + int[][] localityStatistics = + ConverterUtils + .covertLocalityStatisticsStringToArray(attemptMetrics == null ? null + : attemptMetrics.getLocalityStatistics()); DIV div = html.div(_INFO_WRAP); TABLE> table = div.h3( "Total Allocated Containers: " - + totalAllocatedContainers).h3("Each table cell" + + (attemptMetrics != null ? attemptMetrics.getTotalAllocatedContainers():"N/A")) + .h3("Each table cell" + " represents the number of NodeLocal/RackLocal/OffSwitch containers" + " satisfied by NodeLocal/RackLocal/OffSwitch resource requests.").table( "#containerLocality"); @@ -250,15 +254,20 @@ private void createContainerLocalityTable(Block html) { th(_TH, "Off Switch Request"). _(); - String[] containersType = - { "Num Node Local Containers (satisfied by)", "Num Rack Local Containers (satisfied by)", - "Num Off Switch Containers (satisfied by)" }; + String[] containersType = { + "Num Node Local Containers (satisfied by)", + "Num Rack Local Containers (satisfied by)", + "Num Off Switch Containers (satisfied by)" + }; boolean odd = false; for (int i = 0; i < localityStatistics.length; i++) { table.tr((odd = !odd) ? _ODD : _EVEN).td(containersType[i]) - .td(String.valueOf(localityStatistics[i][0])) - .td(i == 0 ? "" : String.valueOf(localityStatistics[i][1])) - .td(i <= 1 ? "" : String.valueOf(localityStatistics[i][2]))._(); + .td(localityStatistics[i][0] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][0])) + .td(i == 0 ? "" : localityStatistics[i][1] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][1])) + .td(i <= 1 ? "" : localityStatistics[i][2] == -1 ? + "N/A" : String.valueOf(localityStatistics[i][2]))._(); } table._(); div._(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 5fc5fa0..0970ca8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -22,7 +22,9 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; + import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -36,8 +38,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -59,10 +63,12 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; - +import org.codehaus.jettison.json.JSONException; import com.google.inject.Inject; +import com.sun.jersey.api.client.ClientHandlerException; public class AppBlock extends HtmlBlock { @@ -213,26 +219,35 @@ public ApplicationReport run() throws Exception { return; } - //TODO:YARN-3284 - //The preemption metrics will be exposed from ApplicationReport - // and ApplicationAttemptReport ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport(); + ApplicationMetrics appMetrics = appReport.getApplicationMetrics(); + ApplicationAttemptMetrics currentAttemptMetrics = null; + for (final ApplicationAttemptReport report : attempts) { + if (report.getApplicationAttemptId().equals( + appReport.getCurrentApplicationAttemptId())) { + currentAttemptMetrics = report.getApplicationAttemptMetrics(); + break; + } + } DIV pdiv = html. _(InfoBlock.class). div(_INFO_WRAP); info("Application Overview").clear(); info("Application Metrics") ._("Total Resource Preempted:", - Resources.none()) // TODO: YARN-3284 + appMetrics != null ? appMetrics.getResourcePreempted() : "N/A") ._("Total Number of Non-AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 + appMetrics != null ? appMetrics.getNumNonAMContainersPreempted() + : "N/A") ._("Total Number of AM Containers Preempted:", - String.valueOf(0)) // TODO: YARN-3284 + appMetrics != null ? appMetrics.getNumAMContainersPreempted() : "N/A") ._("Resource Preempted from Current Attempt:", - Resources.none()) // TODO: YARN-3284 + currentAttemptMetrics != null ? + currentAttemptMetrics.getResourcePreempted() : "N/A") ._("Number of Non-AM Containers Preempted from Current Attempt:", - 0) // TODO: YARN-3284 + currentAttemptMetrics != null ? + currentAttemptMetrics.getNumNonAMContainerPreempted() : "N/A") ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", usageReport == null ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0 @@ -321,13 +336,21 @@ public ContainerReport run() throws Exception { tbody._()._(); if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) { - createResourceRequestsTable(html, null); // TODO:YARN-3284 + // Call RMWebService to get ResourceRequests table for the application + List resouceRequests = new ArrayList(); + try { + resouceRequests = WebAppUtils.getResourceRequests(conf, appID.toString()); + } catch (JSONException ex) { + LOG.warn(ex.getMessage()); + } catch (ClientHandlerException ex) { + LOG.warn(ex.getMessage()); + } + createResourceRequestsTable(html, resouceRequests); } } - //TODO:YARN-3284 - //The resource requests metrics will be exposed from attemptReport - private void createResourceRequestsTable(Block html, List resouceRequests) { + private void createResourceRequestsTable(Block html, + List resouceRequests) { TBODY> tbody = html.table("#ResourceRequests").thead().tr() .th(".priority", "Priority") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java index 12e646d..8fcd553 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java @@ -28,7 +28,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; - import org.iq80.leveldb.DBException; import org.iq80.leveldb.DBIterator; import org.junit.Test; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 3d28bb7..7de9546 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -32,6 +32,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -255,7 +256,12 @@ public static YarnApplicationAttemptState createApplicationAttemptState( Resources.createResource(-1, -1), Resources.createResource(-1, -1), Resources.createResource(-1, -1), 0, 0); - + /** + * Statically defined dummy ApplicationMetrics. Used as + * a return value when a valid report cannot be found. + */ + public static final ApplicationMetrics DUMMY_APPLICATION_METRICS = + ApplicationMetrics.newInstance(Resources.createResource(-1, -1), -1, -1); /** * Find all configs whose name starts with diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java index 71d9363..34fcdb0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -31,6 +32,7 @@ private String diagnosticsInfo; private FinalApplicationStatus appStatus; private YarnApplicationAttemptState state; + private ApplicationAttemptMetrics appAttemptMetrics; public AppAttemptFinishedEvent( ApplicationAttemptId appAttemptId, @@ -39,7 +41,8 @@ public AppAttemptFinishedEvent( String diagnosticsInfo, FinalApplicationStatus appStatus, YarnApplicationAttemptState state, - long finishedTime) { + long finishedTime, + ApplicationAttemptMetrics appAttemptMetrics) { super(SystemMetricsEventType.APP_ATTEMPT_FINISHED, finishedTime); this.appAttemptId = appAttemptId; // This is the tracking URL after the application attempt is finished @@ -48,6 +51,7 @@ public AppAttemptFinishedEvent( this.diagnosticsInfo = diagnosticsInfo; this.appStatus = appStatus; this.state = state; + this.appAttemptMetrics = appAttemptMetrics; } @Override @@ -79,4 +83,7 @@ public YarnApplicationAttemptState getYarnApplicationAttemptState() { return state; } + public ApplicationAttemptMetrics getAppAttemptMetrics() { + return appAttemptMetrics; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 2373b3b..7c43aa4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; +import java.util.Set; + import org.apache.hadoop.yarn.api.records.ApplicationId; public class ApplicationCreatedEvent extends @@ -29,6 +31,7 @@ private String user; private String queue; private long submittedTime; + private Set appTags; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -36,7 +39,8 @@ public ApplicationCreatedEvent(ApplicationId appId, String user, String queue, long submittedTime, - long createdTime) { + long createdTime, + Set appTags) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -44,6 +48,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.user = user; this.queue = queue; this.submittedTime = submittedTime; + this.appTags = appTags; } @Override @@ -75,4 +80,7 @@ public long getSubmittedTime() { return submittedTime; } + public Set getAppTags() { + return appTags; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b849b00..be95e27 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.MetricsConstantsUtils; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -106,7 +107,7 @@ public void appCreated(RMApp app, long createdTime) { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime)); + createdTime, app.getApplicationTags())); } } @@ -168,7 +169,8 @@ public void appAttemptFinished(RMAppAttempt appAttempt, // based on app state if it doesn't exist app.getFinalApplicationStatus(), RMServerUtils.createApplicationAttemptState(appAttemtpState), - finishedTime)); + finishedTime, appAttempt.getRMAppAttemptMetrics() + .convertToApplicationAttemptMetrics())); } } @@ -251,6 +253,7 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); + entityInfo.put(ApplicationMetricsConstants.APP_TAGS, event.getAppTags()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( @@ -283,7 +286,9 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); - + entity.addOtherInfo(ApplicationMetricsConstants.APP_METRICS, + MetricsConstantsUtils.appMetricsToJSON(appMetrics + .convertToApplicationMetrics())); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); @@ -359,6 +364,9 @@ private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) { event.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationAttemptState().toString()); + eventInfo.put(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS, + MetricsConstantsUtils.appAttemptMetricsToJSON(event + .getAppAttemptMetrics())); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 2d1737a..ec3199d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -558,6 +559,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, int rpcPort = -1; ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; + ApplicationMetrics appMetrics = RMServerUtils.DUMMY_APPLICATION_METRICS; FinalApplicationStatus finishState = getFinalApplicationStatus(); String diags = UNAVAILABLE; float progress = 0.0f; @@ -604,6 +606,8 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, RMAppMetrics rmAppMetrics = getRMAppMetrics(); appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds()); appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds()); + + appMetrics = rmAppMetrics.convertToApplicationMetrics(); } if (currentApplicationAttemptId == null) { @@ -612,13 +616,12 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, DUMMY_APPLICATION_ATTEMPT_NUMBER); } - return BuilderUtils.newApplicationReport(this.applicationId, - currentApplicationAttemptId, this.user, this.queue, - this.name, host, rpcPort, clientToAMToken, - createApplicationState(), diags, - trackingUrl, this.startTime, this.finishTime, finishState, - appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken, applicationTags); + return ApplicationReport.newInstance(this.applicationId, + currentApplicationAttemptId, this.user, this.queue, this.name, host, + rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, + this.startTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, appMetrics); } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 5091470..d4c8f51 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; +import org.apache.hadoop.yarn.api.records.ApplicationMetrics; import org.apache.hadoop.yarn.api.records.Resource; public class RMAppMetrics { @@ -56,4 +57,15 @@ public long getMemorySeconds() { public long getVcoreSeconds() { return vcoreSeconds; } + + /** + * Convert RMAppMetrics object to ApplicationMetrics object + * @return applicationMetrics + */ + public ApplicationMetrics convertToApplicationMetrics() { + return ApplicationMetrics + .newInstance(this.getResourcePreempted(), + this.getNumNonAMContainersPreempted(), + this.getNumAMContainersPreempted()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1be1727..7bb4b95 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -36,7 +36,6 @@ import javax.crypto.SecretKey; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -1912,7 +1911,8 @@ public ApplicationAttemptReport createApplicationAttemptReport() { attemptReport = ApplicationAttemptReport.newInstance(this .getAppAttemptId(), this.getHost(), this.getRpcPort(), this .getTrackingUrl(), this.getOriginalTrackingUrl(), this.getDiagnostics(), - YarnApplicationAttemptState .valueOf(this.getState().toString()), amId); + YarnApplicationAttemptState .valueOf(this.getState().toString()), amId, + attemptMetrics.convertToApplicationAttemptMetrics()); } finally { this.readLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index bc22073..47da21a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -145,4 +146,35 @@ public void incNumAllocatedContainers(NodeType containerType, public int getTotalAllocatedContainers() { return this.totalAllocatedContainers; } + + /** + * Convert RMAppAttemptMetrics object to ApplicationAttemptMetrics object + * @return applicationAttemptMetrics + */ + //TODO: YARN-3273 to find a way to expose application headroom + public ApplicationAttemptMetrics convertToApplicationAttemptMetrics() { + return ApplicationAttemptMetrics.newInstance( + this.getIsPreempted(), + this.getResourcePreempted(), + this.getNumNonAMContainersPreempted(), + this.getTotalAllocatedContainers(), + this.ConvertLocalityStatisticsToString(), Resource.newInstance(0, 0)); + } + + private String ConvertLocalityStatisticsToString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < NodeType.values().length; i++) { + for (int j = 0; j < NodeType.values().length; j++) { + sb.append(NodeType.values()[i] + ":" + NodeType.values()[j] + ":" + + localityStatistics[i][j]); + if (j != NodeType.values().length - 1) { + sb.append(","); + } + } + if (i != NodeType.values().length - 1) { + sb.append("|"); + } + } + return sb.toString(); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f8d92aa..4477538 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -289,7 +289,7 @@ public ApplicationReport createAndGetApplicationReport( getName(), null, 0, null, null, getDiagnostics().toString(), getTrackingUrl(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), - type, null); + type, null, null, null); return report; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7ed3835..21ce5e7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import java.util.EnumSet; +import java.util.HashSet; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; @@ -350,6 +352,7 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + when(app.getApplicationTags()).thenReturn(new HashSet()); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); return app; @@ -369,6 +372,8 @@ private static RMAppAttempt createRMAppAttempt( when(appAttempt.getTrackingUrl()).thenReturn("test tracking url"); when(appAttempt.getOriginalTrackingUrl()).thenReturn( "test original tracking url"); + RMAppAttemptMetrics metrics = mock(RMAppAttemptMetrics.class); + when(appAttempt.getRMAppAttemptMetrics()).thenReturn(metrics); return appAttempt; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 6e2b56f..12f7423 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 481a53b..2f3063d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -253,7 +253,8 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), - app.getProgress(), app.getApplicationType(), (Token) null); + app.getProgress(), app.getApplicationType(), (Token) null, null, + null); appReports.add(appReport); } GetApplicationsResponse response = mock(GetApplicationsResponse.class);