diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
index 03552e4..01ef949 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
@@ -90,7 +90,7 @@ private ApplicationReport getUnknownApplicationReport() {
return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null);
}
NotRunningJob(ApplicationReport applicationReport, JobState jobState) {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index 7d6b2f3..1b6cb97 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -472,7 +472,8 @@ private ApplicationReport getFinishedApplicationReport() {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
+ null, null);
}
private ApplicationReport getRunningApplicationReport(String host, int port) {
@@ -482,7 +483,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
"url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, null);
}
private ResourceMgrDelegate getRMDelegate() throws IOException {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index c427975..40762f1 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -196,7 +196,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation)
ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp",
"tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp",
0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f,
- "tmp", null));
+ "tmp", null, null, null));
yarnRunner.killJob(jobId);
verify(clientDelegate).killJob(jobId);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java
new file mode 100644
index 0000000..f45dc7e
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptMetrics.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Contains various metrics for the applicationAttempt
+ * to be reported by UI and CLI.
+ */
+@Public
+@Unstable
+public abstract class ApplicationAttemptMetrics {
+
+ @Private
+ @Unstable
+ public static ApplicationAttemptMetrics newInstance(
+ boolean isPreempted, Resource resourcePreempted,
+ int numNonAMContainersPreempted, int totalAllocatedContainers,
+ String localityStatistics) {
+ ApplicationAttemptMetrics metrics =
+ Records.newRecord(ApplicationAttemptMetrics.class);
+ metrics.setIsPreempted(isPreempted);
+ metrics.setResourcePreempted(resourcePreempted);
+ metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted);
+ metrics.setTotalAllocatedContainers(totalAllocatedContainers);
+ metrics.setLocalityStatistics(localityStatistics);
+ return metrics;
+ }
+
+ /**
+ * Get the total resource preempted for applicationAttempt.
+ * @return the total resource preempted for applicationAttempt
+ */
+ @Public
+ @Unstable
+ public abstract Resource getResourcePreempted();
+
+ @Private
+ @Unstable
+ public abstract void setResourcePreempted(Resource resourcePreempted);
+
+ /**
+ * Get the number of non-AM containers preempted for applicationAttempt
+ * @return the number of non-AM containers preempted for applicationAttempt
+ */
+ @Public
+ @Unstable
+ public abstract int getNumNonAMContainerPreempted();
+
+ @Private
+ @Unstable
+ public abstract void setNumNonAMContainersPreempted(
+ int numNonAMContainersPreempted);
+
+ /**
+ * indicate whether this applicationAttempt has been preempted
+ * @return the flag to indicate whether this applicationAttempt
+ * has been preempted
+ */
+ @Public
+ @Unstable
+ public abstract boolean isPreempted();
+
+ @Private
+ @Unstable
+ public abstract void setIsPreempted(boolean isPreempted);
+
+ /**
+ * Get the total number of allocated containers for this attempt
+ * @return the total number of allocated containers for this attempt
+ */
+ @Public
+ @Unstable
+ public abstract int getTotalAllocatedContainers();
+
+ @Private
+ @Unstable
+ public abstract void
+ setTotalAllocatedContainers(int totalAllocatedContainers);
+
+ /**
+ * Get the number of NodeLocal/RackLocal/OffSwitch containers for this attempt
+ * @return a string contains the number of
+ * NodeLocal/RackLocal/OffSwitch containers
+ */
+ @Public
+ @Unstable
+ public abstract String getLocalityStatistics();
+
+ @Private
+ @Unstable
+ public abstract void setLocalityStatistics(String localityStatistics);
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
index 53c18ae..e677a5d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
@@ -39,6 +39,8 @@
*
Diagnostic information in case of errors.
* {@link YarnApplicationAttemptState} of the application attempt.
* {@link ContainerId} of the master Container.
+ * {@link ApplicationAttemptMetrics} of
+ * the application attempt
*
*
*
@@ -52,7 +54,8 @@
public static ApplicationAttemptReport newInstance(
ApplicationAttemptId applicationAttemptId, String host, int rpcPort,
String url, String oUrl, String diagnostics,
- YarnApplicationAttemptState state, ContainerId amContainerId) {
+ YarnApplicationAttemptState state, ContainerId amContainerId,
+ ApplicationAttemptMetrics applicationAttemptMetrics) {
ApplicationAttemptReport report =
Records.newRecord(ApplicationAttemptReport.class);
report.setApplicationAttemptId(applicationAttemptId);
@@ -63,6 +66,8 @@ public static ApplicationAttemptReport newInstance(
report.setDiagnostics(diagnostics);
report.setYarnApplicationAttemptState(state);
report.setAMContainerId(amContainerId);
+ report.setApplicationAttemptMetrics(
+ applicationAttemptMetrics);
return report;
}
@@ -176,4 +181,19 @@ public abstract void setApplicationAttemptId(
@Private
@Unstable
public abstract void setAMContainerId(ContainerId amContainerId);
+
+ /**
+ * Get the ApplicationAttemptMetrics of this attempt.
+ *
+ * @return ApplicationAttemptMetrics of this attempt
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationAttemptMetrics
+ getApplicationAttemptMetrics();
+
+ @Private
+ @Unstable
+ public abstract void setApplicationAttemptMetrics(
+ ApplicationAttemptMetrics applicationAttemptMetrics);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java
new file mode 100644
index 0000000..acd0e3b
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMetrics.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Contains various metrics for the application
+ * to be reported by UI and CLI.
+ */
+@Public
+@Unstable
+public abstract class ApplicationMetrics {
+
+ @Private
+ @Unstable
+ public static ApplicationMetrics newInstance(
+ Resource resourcePreempted, int numNonAMContainersPreempted,
+ int numAMContainersPreempted) {
+ ApplicationMetrics metrics =
+ Records.newRecord(ApplicationMetrics.class);
+ metrics.setResourcePreempted(resourcePreempted);
+ metrics.setNumNonAMContainersPreempted(numNonAMContainersPreempted);
+ metrics.setNumAMContainersPreempted(numAMContainersPreempted);
+ return metrics;
+ }
+
+ /**
+ * Get the total resource preempted of the application.
+ * @return the total resource preempted of the application
+ */
+ @Public
+ @Unstable
+ public abstract Resource getResourcePreempted();
+
+ @Private
+ @Unstable
+ public abstract void setResourcePreempted(Resource resourcePreempted);
+
+ /**
+ * Get the number of non-AM containers preempted for this application.
+ * @return the number of non-AM containers preempted for this application
+ */
+ @Public
+ @Unstable
+ public abstract int getNumNonAMContainersPreempted();
+
+ @Private
+ @Unstable
+ public abstract void setNumNonAMContainersPreempted(
+ int numNonAMContainersPreempted);
+
+ /**
+ * Get the number of AM containers preempted for this application.
+ * @return the number of non-AM containers preempted for this application
+ */
+ @Public
+ @Unstable
+ public abstract int getNumAMContainersPreempted();
+
+ @Private
+ @Unstable
+ public abstract void
+ setNumAMContainersPreempted(int numAMContainersPreempted);
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index 412c22b..56034be 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -43,6 +43,7 @@
* Diagnostic information in case of errors.
* Start time of the application.
* Client {@link Token} of the application (if security is enabled).
+ * {@link ApplicationMetrics} of the application
*
*
*
@@ -60,7 +61,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
YarnApplicationState state, String diagnostics, String url,
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
- float progress, String applicationType, Token amRmToken) {
+ float progress, String applicationType, Token amRmToken, Set tags,
+ ApplicationMetrics applicationMetrics) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -81,6 +83,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
report.setProgress(progress);
report.setApplicationType(applicationType);
report.setAMRMToken(amRmToken);
+ report.setApplicationTags(tags);
+ report.setApplicationMetrics(applicationMetrics);
return report;
}
@@ -362,5 +366,19 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
@Public
@Stable
public abstract Token getAMRMToken();
+
+ /**
+ * Get the ApplicationMetrics of this application.
+ *
+ * @return ApplicationMetrics of this application
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationMetrics getApplicationMetrics();
+
+ @Private
+ @Unstable
+ public abstract void setApplicationMetrics(
+ ApplicationMetrics applicationMetrics);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 2edff99..b9908f2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -194,6 +194,13 @@ message ApplicationReportProto {
optional string applicationType = 18;
optional hadoop.common.TokenProto am_rm_token = 19;
repeated string applicationTags = 20;
+ optional ApplicationMetricsProto application_metrics = 21;
+}
+
+message ApplicationMetricsProto {
+ optional ResourceProto resource_preempted = 1;
+ optional int32 num_non_am_containers_preempted = 2;
+ optional int32 num_am_containers_preempted = 3;
}
message ApplicationAttemptReportProto {
@@ -205,6 +212,16 @@ message ApplicationAttemptReportProto {
optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6;
optional ContainerIdProto am_container_id = 7;
optional string original_tracking_url = 8;
+ optional ApplicationAttemptMetricsProto application_attempt_metrics = 9;
+}
+
+message ApplicationAttemptMetricsProto {
+ optional bool is_preempted = 1;
+ optional ResourceProto resource_preempted = 2;
+ optional int32 num_non_am_containers_preempted = 3;
+ repeated ResourceRequestProto resource_requests = 4;
+ optional int32 total_allocated_containers = 5;
+ optional string locality_statistics = 6;
}
enum NodeStateProto {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index dd4a949..40300d2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -23,6 +23,7 @@
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -38,11 +39,15 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
@@ -50,8 +55,12 @@
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.codehaus.jettison.json.JSONException;
import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.ClientHandlerException;
@Private
@Unstable
@@ -65,6 +74,11 @@
private static final String CONTAINER_PATTERN =
"%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
+ private static final String RESOURCE_REQUESTS_TABLE_PATTERN =
+ "%16s\t%30s\t%35s\t%20s\t%20s\t%35s"
+ + System.getProperty("line.separator");
+ private static final String LOCALITY_TABLE_PATTERN =
+ "%48s\t%25s\t%25s\t%25s" + System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
private static final String APP_STATE_CMD = "appStates";
@@ -282,6 +296,8 @@ private int printApplicationAttemptReport(String applicationAttemptId)
PrintWriter appAttemptReportStr = new PrintWriter(
new OutputStreamWriter(baos, Charset.forName("UTF-8")));
if (appAttemptReport != null) {
+ ApplicationAttemptMetrics attemptMetrics =
+ appAttemptReport.getApplicationAttemptMetrics();
appAttemptReportStr.println("Application Attempt Report : ");
appAttemptReportStr.print("\tApplicationAttempt-Id : ");
appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId());
@@ -298,7 +314,25 @@ private int printApplicationAttemptReport(String applicationAttemptId)
appAttemptReportStr.print("\tAM Host : ");
appAttemptReportStr.println(appAttemptReport.getHost());
appAttemptReportStr.print("\tDiagnostics : ");
- appAttemptReportStr.print(appAttemptReport.getDiagnostics());
+ appAttemptReportStr.println(appAttemptReport.getDiagnostics());
+ appAttemptReportStr.print(
+ "\tResource Preempted from Current Attempt : ");
+ appAttemptReportStr.println(attemptMetrics == null ? "N/A"
+ : attemptMetrics.getResourcePreempted());
+ appAttemptReportStr.print(
+ "\tNumber of Non-AM Containers Preempted from Current Attempt : ");
+ appAttemptReportStr.println(attemptMetrics == null ? "N/A"
+ : attemptMetrics.getNumNonAMContainerPreempted());
+ appAttemptReportStr.flush();
+
+ // Print ResourceRequests table
+ printResourceRequestsTable(appAttemptReportStr, appAttemptReport
+ .getApplicationAttemptId().getApplicationId().toString());
+
+ if (attemptMetrics != null) {
+ // Print LocalityStatistics table
+ printLocalityStatisticsTable(appAttemptReportStr, attemptMetrics);
+ }
} else {
appAttemptReportStr.print("Application Attempt with id '"
+ applicationAttemptId + "' doesn't exist in Timeline Server.");
@@ -531,7 +565,19 @@ private int printApplicationReport(String applicationId)
appReportStr.println("N/A");
}
appReportStr.print("\tDiagnostics : ");
- appReportStr.print(appReport.getDiagnostics());
+ appReportStr.println(appReport.getDiagnostics());
+ appReportStr.print("\tApplication Tags : ");
+ appReportStr.println(appReport.getApplicationTags());
+ ApplicationMetrics appMetrics = appReport.getApplicationMetrics();
+ appReportStr.print("\tTotal Resource Preempted : ");
+ appReportStr.println(appMetrics != null ? appMetrics
+ .getResourcePreempted() : "N/A");
+ appReportStr.print("\tTotal Number of Non-AM Containers Preempted : ");
+ appReportStr.println(appMetrics != null ? appMetrics
+ .getNumNonAMContainersPreempted() : "N/A");
+ appReportStr.print("\tTotal Number of AM Containers Preempted : ");
+ appReportStr.print(appMetrics != null ? appMetrics
+ .getNumAMContainersPreempted() : "N/A");
} else {
appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM.");
@@ -612,4 +658,89 @@ private void listContainers(String appAttemptId) throws YarnException,
}
writer.flush();
}
+
+ private void printResourceRequestsTable(PrintWriter reportStr, String appID) {
+ reportStr.println();
+ // Print ResourceRequests table
+ List resouceRequests = getResourceRequests(appID);
+
+ Resource totalResource = Resource.newInstance(0, 0);
+ reportStr.println("ResourceRequests : ");
+ for (ResourceRequest request : resouceRequests) {
+ if (request.getResourceName().equals(ResourceRequest.ANY)) {
+ Resources.addTo(
+ totalResource,
+ Resources.multiply(request.getCapability(),
+ request.getNumContainers()));
+ }
+ }
+ reportStr.print("\ttotalResourceRequests : ");
+ reportStr.println(totalResource);
+ reportStr.printf(RESOURCE_REQUESTS_TABLE_PATTERN, "Priority",
+ "ResourceName", "Capability", "NumContainers", "RelaxLocality",
+ "NodeLabelExpression");
+ for (ResourceRequest request : resouceRequests) {
+ if (request.getNumContainers() == 0) {
+ continue;
+ }
+ reportStr.printf(
+ RESOURCE_REQUESTS_TABLE_PATTERN,
+ request.getPriority(),
+ request.getResourceName(),
+ request.getCapability(),
+ request.getNumContainers(),
+ request.getRelaxLocality(),
+ request.getNodeLabelExpression() == null ? "N/A" : request
+ .getNodeLabelExpression());
+ }
+ reportStr.flush();
+ }
+
+ private void printLocalityStatisticsTable(PrintWriter reportStr,
+ ApplicationAttemptMetrics attemptMetrics) {
+ reportStr.println();
+ reportStr.println("Container Locality Statistics Table: ");
+ reportStr.print("\tTotal Allocated Containers : ");
+ reportStr.println(attemptMetrics
+ .getTotalAllocatedContainers());
+ reportStr
+ .println("\tEach table cell represents the number of "
+ + "NodeLocal/RackLocal/OffSwitch containers satisfied by "
+ + "NodeLocal/RackLocal/OffSwitch resource requests.");
+ String[] containersType = {
+ "Num Node Local Containers (satisfied by)",
+ "Num Rack Local Containers (satisfied by)",
+ "Num Off Switch Containers (satisfied by)"
+ };
+ int[][] localityStatistics =
+ ConverterUtils.covertLocalityStatisticsStringToArray(attemptMetrics
+ .getLocalityStatistics());
+ reportStr.printf(LOCALITY_TABLE_PATTERN, "",
+ "Node Local Request",
+ "Rack Local Request",
+ "Off Switch Request");
+ for (int i = 0; i < localityStatistics.length; i++) {
+ reportStr.printf(LOCALITY_TABLE_PATTERN,
+ containersType[i],
+ localityStatistics[i][0] == -1 ? "N/A" : localityStatistics[i][0],
+ i == 0 ? "" : localityStatistics[i][1] == -1 ?
+ "N/A" : localityStatistics[i][1],
+ i <= 1 ? "" : localityStatistics[i][2] == -1 ?
+ "N/A" : localityStatistics[i][2]);
+ }
+ reportStr.flush();
+ }
+
+ @VisibleForTesting
+ public List getResourceRequests(String appId) {
+ List resouceRequests = new ArrayList();
+ try {
+ resouceRequests = WebAppUtils.getResourceRequests(getConf(), appId);
+ } catch (JSONException ex) {
+ // DO NOTHING
+ } catch (ClientHandlerException ex) {
+ // DO NOTHING
+ }
+ return resouceRequests;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index f468bc1..6f7652a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -647,7 +647,7 @@ public ApplicationReport createFakeAppReport() {
"fakeQueue", "fakeApplicationName", "localhost", 0, null,
YarnApplicationState.FINISHED, "fake an application report", "",
1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
- "fakeApplicationType", null);
+ "fakeApplicationType", null, null, null);
return report;
}
@@ -700,7 +700,7 @@ public QueueInfo createFakeQueueInfo() {
public ApplicationAttemptReport createFakeApplicationAttemptReport() {
return ApplicationAttemptReport.newInstance(
createFakeApplicationAttemptId(), "localhost", 0, "", "", "",
- YarnApplicationAttemptState.RUNNING, createFakeContainerId());
+ YarnApplicationAttemptState.RUNNING, createFakeContainerId(), null);
}
public List
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index c3e3c41..8ffa1ef 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -334,7 +334,7 @@ private void createAppReports() {
"queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
- null);
+ null, null, null);
List applicationReports =
new ArrayList();
applicationReports.add(newApplicationReport);
@@ -350,7 +350,8 @@ private void createAppReports() {
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newContainerId(
- newApplicationReport.getCurrentApplicationAttemptId(), 1));
+ newApplicationReport.getCurrentApplicationAttemptId(), 1),
+ null);
appAttempts.add(attempt);
ApplicationAttemptReport attempt1 =
ApplicationAttemptReport.newInstance(
@@ -362,7 +363,8 @@ private void createAppReports() {
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newContainerId(
- newApplicationReport.getCurrentApplicationAttemptId(), 2));
+ newApplicationReport.getCurrentApplicationAttemptId(), 2),
+ null);
appAttempts.add(attempt1);
attempts.put(applicationId, appAttempts);
@@ -391,7 +393,7 @@ private void createAppReports() {
"queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f,
- "NON-YARN", null);
+ "NON-YARN", null, null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -401,7 +403,7 @@ private void createAppReports() {
"queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f,
- "MAPREDUCE", null);
+ "MAPREDUCE", null, null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -411,7 +413,7 @@ private void createAppReports() {
"queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null, null);
applicationReports.add(newApplicationReport4);
reports = applicationReports;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index de669f2..60b3f56 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -580,7 +580,8 @@ public void setYarnApplicationState(YarnApplicationState state) {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null, null);
List applicationReports = new ArrayList();
applicationReports.add(newApplicationReport);
List appAttempts = new ArrayList();
@@ -593,7 +594,8 @@ public void setYarnApplicationState(YarnApplicationState state) {
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newContainerId(
- newApplicationReport.getCurrentApplicationAttemptId(), 1));
+ newApplicationReport.getCurrentApplicationAttemptId(), 1),
+ null);
appAttempts.add(attempt);
ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance(
ApplicationAttemptId.newInstance(applicationId, 2),
@@ -604,7 +606,8 @@ public void setYarnApplicationState(YarnApplicationState state) {
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newContainerId(
- newApplicationReport.getCurrentApplicationAttemptId(), 2));
+ newApplicationReport.getCurrentApplicationAttemptId(), 2),
+ null);
appAttempts.add(attempt1);
attempts.put(applicationId, appAttempts);
@@ -655,7 +658,7 @@ public void setYarnApplicationState(YarnApplicationState state) {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -664,7 +667,7 @@ public void setYarnApplicationState(YarnApplicationState state) {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -675,7 +678,7 @@ public void setYarnApplicationState(YarnApplicationState state) {
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null, null);
applicationReports.add(newApplicationReport4);
return applicationReports;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 4b60c52..3e9c121 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -27,6 +27,7 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.doReturn;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -44,8 +45,10 @@
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -59,6 +62,7 @@
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
@@ -98,12 +102,16 @@ public void testGetApplicationReport() throws Exception {
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport.newInstance(
2, 0, null, null, null, 123456, 4567);
+ Set appTags = new HashSet();
+ appTags.add("test");
+ ApplicationMetrics metrics =
+ ApplicationMetrics.newInstance(Resource.newInstance(6144, 10), 5, 2);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
- null);
+ null, appTags, metrics);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@@ -128,6 +136,10 @@ public void testGetApplicationReport() throws Exception {
pw.println("\tAggregate Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tDiagnostics : diagnostics");
+ pw.println("\tApplication Tags : [test]");
+ pw.println("\tTotal Resource Preempted : ");
+ pw.println("\tTotal Number of Non-AM Containers Preempted : 5");
+ pw.println("\tTotal Number of AM Containers Preempted : 2");
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString());
@@ -139,18 +151,35 @@ public void testGetApplicationReport() throws Exception {
@Test
public void testGetApplicationAttemptReport() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationCLI spyCLI = spy(cli);
+ List resourceRequests = new ArrayList();
+ ResourceRequest request =
+ ResourceRequest.newInstance(Priority.newInstance(20),
+ ResourceRequest.ANY, Resource.newInstance(1024, 2), 10);
+ request.setNodeLabelExpression("dummy_label_for_test");
+ resourceRequests.add(request);
+ doReturn(resourceRequests).when(spyCLI).getResourceRequests(
+ any(String.class));
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
applicationId, 1);
+ String localityStatistics = "NODE_LOCAL:NODE_LOCAL:8,"
+ + "NODE_LOCAL:RACK_LOCAL:0,NODE_LOCAL:OFF_SWITCH:0"
+ + "|RACK_LOCAL:NODE_LOCAL:2,RACK_LOCAL:RACK_LOCAL:5,"
+ + "RACK_LOCAL:OFF_SWITCH:0|OFF_SWITCH:NODE_LOCAL:1,"
+ + "OFF_SWITCH:RACK_LOCAL:4,OFF_SWITCH:OFF_SWITCH:10";
+ ApplicationAttemptMetrics metrics =
+ ApplicationAttemptMetrics.newInstance(false,
+ Resource.newInstance(6144, 10), 10, 20, localityStatistics);
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
.newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(
- attemptId, 1));
+ attemptId, 1), metrics);
when(
client
.getApplicationAttemptReport(any(ApplicationAttemptId.class)))
.thenReturn(attemptReport);
- int result = cli.run(new String[] { "applicationattempt", "-status",
+ int result = spyCLI.run(new String[] { "applicationattempt", "-status",
attemptId.toString() });
assertEquals(0, result);
verify(client).getApplicationAttemptReport(attemptId);
@@ -164,8 +193,49 @@ public void testGetApplicationAttemptReport() throws Exception {
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
pw.println("\tDiagnostics : diagnostics");
+ pw.println("\tResource Preempted from Current Attempt : "
+ + "");
+ pw.println("\tNumber of Non-AM Containers Preempted from "
+ + "Current Attempt : 10");
+ pw.println();
+ pw.println("ResourceRequests : ");
+ pw.println("\ttotalResourceRequests : ");
+ pw.println(" Priority\t ResourceName"
+ + "\t Capability\t NumContainers"
+ + "\t RelaxLocality\t NodeLabelExpression");
+ pw.println(" 20\t *"
+ + "\t \t 10"
+ + "\t true\t dummy_label_for_test");
+ pw.println();
+ pw.println("Container Locality Statistics Table: ");
+ pw.println("\tTotal Allocated Containers : 20");
+ pw.println("\tEach table cell represents the number of "
+ + "NodeLocal/RackLocal/OffSwitch containers satisfied by "
+ + "NodeLocal/RackLocal/OffSwitch resource requests.");
+ pw.print(" ");
+ pw.println("\t Node Local Request"
+ + "\t Rack Local Request"
+ + "\t Off Switch Request");
+ pw.println(" Num Node Local Containers (satisfied by)"
+ + "\t 8"
+ + "\t "
+ + "\t ");
+ pw.println(" Num Rack Local Containers (satisfied by)"
+ + "\t 2"
+ + "\t 5"
+ + "\t ");
+ pw.println(" Num Off Switch Containers (satisfied by)"
+ + "\t 1"
+ + "\t 4"
+ + "\t 10");
+ pw.println();
pw.close();
String appReportStr = baos.toString("UTF-8");
+ Log.info("ExpectedOutput");
+ Log.info("["+appReportStr+"]");
+ Log.info("OutputFrom command");
+ String actualOutput = sysOutStream.toString();
+ Log.info("["+actualOutput+"]");
Assert.assertEquals(appReportStr, sysOutStream.toString());
verify(sysOut, times(1)).println(isA(String.class));
}
@@ -181,11 +251,11 @@ public void testGetApplicationAttempts() throws Exception {
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
.newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(
- attemptId, 1));
+ attemptId, 1), null);
ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport
.newInstance(attemptId1, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(
- attemptId1, 1));
+ attemptId1, 1), null);
List reports = new ArrayList();
reports.add(attemptReport);
reports.add(attemptReport1);
@@ -347,7 +417,7 @@ public void testGetApplications() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null);
List applicationReports = new ArrayList();
applicationReports.add(newApplicationReport);
@@ -357,7 +427,7 @@ public void testGetApplications() throws Exception {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -366,7 +436,7 @@ public void testGetApplications() throws Exception {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -375,7 +445,7 @@ public void testGetApplications() throws Exception {
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport4);
ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -384,7 +454,7 @@ public void testGetApplications() throws Exception {
"user5", "queue5", "appname5", "host5", 128, null,
YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport5);
ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -393,7 +463,7 @@ public void testGetApplications() throws Exception {
"user6", "queue6", "appname6", "host6", 129, null,
YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
- null);
+ null, null, null);
applicationReports.add(newApplicationReport6);
// Test command yarn application -list
@@ -817,7 +887,7 @@ public void testKillApplication() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2);
int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -830,7 +900,7 @@ public void testKillApplication() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -865,7 +935,7 @@ public void testMoveApplicationAcrossQueues() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2);
int result = cli.run(new String[] { "application", "-movetoqueue",
@@ -880,7 +950,7 @@ public void testMoveApplicationAcrossQueues() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, null, null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
result = cli.run(new String[] { "application", "-movetoqueue",
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java
new file mode 100644
index 0000000..712ac86
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptMetricsPBImpl.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class ApplicationAttemptMetricsPBImpl extends
+ ApplicationAttemptMetrics {
+
+ ApplicationAttemptMetricsProto proto =
+ ApplicationAttemptMetricsProto.getDefaultInstance();
+ ApplicationAttemptMetricsProto.Builder builder = null;
+ boolean viaProto = false;
+
+ Resource totalResourcePreempted;
+
+ public ApplicationAttemptMetricsPBImpl() {
+ builder = ApplicationAttemptMetricsProto.newBuilder();
+ }
+
+ public ApplicationAttemptMetricsPBImpl(
+ ApplicationAttemptMetricsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized ApplicationAttemptMetricsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.totalResourcePreempted != null
+ && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals(
+ builder.getResourcePreempted())) {
+ builder.setResourcePreempted(
+ convertToProtoFormat(this.totalResourcePreempted));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationAttemptMetricsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized Resource getResourcePreempted() {
+ ApplicationAttemptMetricsProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.totalResourcePreempted != null) {
+ return this.totalResourcePreempted;
+ }
+ if (!p.hasResourcePreempted()) {
+ return null;
+ }
+ this.totalResourcePreempted =
+ convertFromProtoFormat(p.getResourcePreempted());
+ return this.totalResourcePreempted;
+ }
+
+ @Override
+ public synchronized void setResourcePreempted(Resource resourcePreempted) {
+ maybeInitBuilder();
+ if (resourcePreempted == null)
+ builder.clearResourcePreempted();
+ this.totalResourcePreempted = resourcePreempted;
+ }
+
+ @Override
+ public synchronized int getNumNonAMContainerPreempted() {
+ ApplicationAttemptMetricsProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return (p.getNumNonAmContainersPreempted());
+ }
+
+ @Override
+ public synchronized void setNumNonAMContainersPreempted(
+ int numNonAMContainersPreempted) {
+ maybeInitBuilder();
+ builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted);
+ }
+
+ @Override
+ public synchronized boolean isPreempted() {
+ ApplicationAttemptMetricsProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getIsPreempted();
+ }
+
+ @Override
+ public synchronized void setIsPreempted(boolean isPreempted) {
+ maybeInitBuilder();
+ builder.setIsPreempted(isPreempted);
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl)t).getProto();
+ }
+
+ @Override
+ public synchronized int getTotalAllocatedContainers() {
+ ApplicationAttemptMetricsProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getTotalAllocatedContainers();
+ }
+
+ @Override
+ public synchronized void setTotalAllocatedContainers(int totalAllocatedContainers) {
+ maybeInitBuilder();
+ builder.setTotalAllocatedContainers(totalAllocatedContainers);
+ }
+
+ @Override
+ public synchronized String getLocalityStatistics() {
+ ApplicationAttemptMetricsProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (! p.hasLocalityStatistics()) {
+ return null;
+ }
+ return p.getLocalityStatistics();
+ }
+
+ @Override
+ public synchronized void setLocalityStatistics(String localityStatistics) {
+ maybeInitBuilder();
+ if (localityStatistics == null) {
+ builder.clearLocalityStatistics();
+ return;
+ }
+ builder.setLocalityStatistics(localityStatistics);
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
index c3c0221..3894f34 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
@@ -19,10 +19,12 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptMetricsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
@@ -38,6 +40,7 @@
private ApplicationAttemptId ApplicationAttemptId;
private ContainerId amContainerId;
+ private ApplicationAttemptMetrics attemptMetrics;
public ApplicationAttemptReportPBImpl() {
builder = ApplicationAttemptReportProto.newBuilder();
@@ -244,6 +247,13 @@ private void mergeLocalToBuilder() {
builder.getAmContainerId())) {
builder.setAmContainerId(convertToProtoFormat(this.amContainerId));
}
+
+ if (this.attemptMetrics != null
+ && !((ApplicationAttemptMetricsPBImpl) this.attemptMetrics)
+ .getProto().equals(builder.getApplicationAttemptMetrics())) {
+ builder.setApplicationAttemptMetrics(
+ convertToProtoFormat(this.attemptMetrics));
+ }
}
private ContainerIdProto convertToProtoFormat(ContainerId amContainerId) {
@@ -286,4 +296,39 @@ public void setAMContainerId(ContainerId amContainerId) {
builder.clearAmContainerId();
this.amContainerId = amContainerId;
}
+
+ @Override
+ public ApplicationAttemptMetrics
+ getApplicationAttemptMetrics() {
+ if (this.attemptMetrics != null) {
+ return this.attemptMetrics;
+ }
+
+ ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationAttemptMetrics()) {
+ return null;
+ }
+ this.attemptMetrics =
+ convertFromProtoFormat(p.getApplicationAttemptMetrics());
+ return this.attemptMetrics;
+ }
+
+ @Override
+ public void setApplicationAttemptMetrics(
+ ApplicationAttemptMetrics applicationAttemptMetrics) {
+ maybeInitBuilder();
+ if (applicationAttemptMetrics == null)
+ builder.clearApplicationAttemptMetrics();
+ this.attemptMetrics = applicationAttemptMetrics;
+ }
+
+ private ApplicationAttemptMetricsProto
+ convertToProtoFormat(ApplicationAttemptMetrics t) {
+ return ((ApplicationAttemptMetricsPBImpl) t).getProto();
+ }
+
+ private ApplicationAttemptMetricsPBImpl convertFromProtoFormat(
+ ApplicationAttemptMetricsProto t) {
+ return new ApplicationAttemptMetricsPBImpl(t);
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java
new file mode 100644
index 0000000..6843d97
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMetricsPBImpl.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class ApplicationMetricsPBImpl extends ApplicationMetrics {
+
+ ApplicationMetricsProto proto =
+ ApplicationMetricsProto.getDefaultInstance();
+ ApplicationMetricsProto.Builder builder = null;
+ boolean viaProto = false;
+
+ Resource totalResourcePreempted;
+
+ public ApplicationMetricsPBImpl() {
+ builder = ApplicationMetricsProto.newBuilder();
+ }
+
+ public ApplicationMetricsPBImpl(
+ ApplicationMetricsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized ApplicationMetricsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.totalResourcePreempted != null
+ && !((ResourcePBImpl) this.totalResourcePreempted).getProto().equals(
+ builder.getResourcePreempted())) {
+ builder.setResourcePreempted(
+ convertToProtoFormat(this.totalResourcePreempted));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationMetricsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized Resource getResourcePreempted() {
+ ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.totalResourcePreempted != null) {
+ return this.totalResourcePreempted;
+ }
+ if (!p.hasResourcePreempted()) {
+ return null;
+ }
+ this.totalResourcePreempted =
+ convertFromProtoFormat(p.getResourcePreempted());
+ return this.totalResourcePreempted;
+ }
+
+ @Override
+ public synchronized void setResourcePreempted(Resource resourcePreempted) {
+ maybeInitBuilder();
+ if (resourcePreempted == null)
+ builder.clearResourcePreempted();
+ this.totalResourcePreempted = resourcePreempted;
+ }
+
+ @Override
+ public synchronized int getNumNonAMContainersPreempted() {
+ ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNumNonAmContainersPreempted());
+ }
+
+ @Override
+ public synchronized void setNumNonAMContainersPreempted(
+ int numNonAMContainersPreempted) {
+ maybeInitBuilder();
+ builder.setNumNonAmContainersPreempted(numNonAMContainersPreempted);
+ }
+
+ @Override
+ public synchronized int getNumAMContainersPreempted() {
+ ApplicationMetricsProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNumAmContainersPreempted());
+ }
+
+ @Override
+ public synchronized void setNumAMContainersPreempted(
+ int numAMContainersPreempted) {
+ maybeInitBuilder();
+ builder.setNumAmContainersPreempted(numAMContainersPreempted);
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl)t).getProto();
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
index dd3e2bc..daf15cc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -30,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMetricsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
@@ -53,6 +55,7 @@
private Token clientToAMToken = null;
private Token amRmToken = null;
private Set applicationTags = null;
+ private ApplicationMetrics applicationMetrics = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@@ -482,6 +485,12 @@ private void mergeLocalToBuilder() {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
+ if (this.applicationMetrics != null
+ && !((ApplicationMetricsPBImpl) this.applicationMetrics)
+ .getProto().equals(builder.getApplicationMetrics())) {
+ builder.setApplicationMetrics(
+ convertToProtoFormat(this.applicationMetrics));
+ }
}
private void mergeLocalToProto() {
@@ -548,4 +557,38 @@ private TokenPBImpl convertFromProtoFormat(TokenProto p) {
private TokenProto convertToProtoFormat(Token t) {
return ((TokenPBImpl)t).getProto();
}
+
+ @Override
+ public ApplicationMetrics getApplicationMetrics() {
+ if (this.applicationMetrics != null) {
+ return this.applicationMetrics;
+ }
+
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationMetrics()) {
+ return null;
+ }
+ this.applicationMetrics =
+ convertFromProtoFormat(p.getApplicationMetrics());
+ return this.applicationMetrics;
+ }
+
+ @Override
+ public void setApplicationMetrics(
+ ApplicationMetrics applicationMetrics) {
+ maybeInitBuilder();
+ if (applicationMetrics == null)
+ builder.clearApplicationMetrics();
+ this.applicationMetrics = applicationMetrics;
+ }
+
+ private ApplicationMetricsProto convertToProtoFormat(
+ ApplicationMetrics t) {
+ return ((ApplicationMetricsPBImpl) t).getProto();
+ }
+
+ private ApplicationMetricsPBImpl convertFromProtoFormat(
+ ApplicationMetricsProto t) {
+ return new ApplicationMetricsPBImpl(t);
+}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index e9674cf..64a15f6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -259,4 +259,42 @@ public static ApplicationId toApplicationId(
}
return token;
}
+
+ /**
+ * Convert LocalityStatistics from String format to 3X3 array.
+ *
+ * @param localityStatistics String
+ */
+ public static int[][] covertLocalityStatisticsStringToArray(
+ String localityStatistics) {
+ int[][] coverted = new int[][] {
+ {-1, -1, -1},
+ {-1, -1, -1},
+ {-1, -1, -1}
+ };
+ if (localityStatistics != null && !localityStatistics.isEmpty()) {
+ String[] localityStatisticsForTypes = localityStatistics.split("\\|");
+ if (localityStatisticsForTypes.length == 3) {
+ for (int i = 0; i < 3; i++) {
+ String[] numOfLocalityStatisticsForTypes =
+ localityStatisticsForTypes[i].trim().split(",");
+ if (numOfLocalityStatisticsForTypes.length != 3) {
+ continue;
+ }
+ for (int j = 0; j < 3; j++) {
+ String[] num = numOfLocalityStatisticsForTypes[j].trim().split(":");
+ if (num.length != 3) {
+ continue;
+ }
+ try {
+ coverted[i][j] = Integer.parseInt(num[2]);
+ } catch (NumberFormatException ex) {
+ // Do Nothing
+ }
+ }
+ }
+ }
+ }
+ return coverted;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 3aeb33e..390bc8f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -24,7 +24,12 @@
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+
+import javax.ws.rs.core.MediaType;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -32,10 +37,22 @@
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
@Private
@Evolving
@@ -347,4 +364,96 @@ static String getPassword(Configuration conf, String alias) {
}
return password;
}
+
+ /**
+ * Get a list of ResourceRequests for the application by calling
+ * RM Web Service.
+ * @param conf Configuration
+ * @param appId ApplicationId
+ * @return resourceRequests List
+ * @throws ClientHandlerException
+ * @throws UniformInterfaceException
+ * @throws JSONException
+ */
+ public static List getResourceRequests(Configuration conf,
+ String appId) throws ClientHandlerException,
+ UniformInterfaceException, JSONException {
+ List resouceRequests = new ArrayList();
+ Client webServiceClient = Client.create();
+ String webAppAddress =
+ WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.RM_BIND_HOST,
+ WebAppUtils.getRMWebAppURLWithScheme(conf));
+ WebResource webResource = webServiceClient.resource(webAppAddress);
+
+ ClientResponse response =
+ webResource.path("ws").path("v1").path("cluster").path("apps")
+ .path(appId).accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ JSONObject json = response.getEntity(JSONObject.class).getJSONObject("app");
+ JSONArray requests = json.getJSONArray("resourceRequests");
+ for (int i = 0; i < requests.length(); i++) {
+ resouceRequests.add(parseFromJSONObject(requests
+ .getJSONObject(i)));
+ }
+ return resouceRequests;
+ }
+
+ private static ResourceRequest parseFromJSONObject(JSONObject resourceRequest)
+ throws JSONException {
+ Map out = new HashMap();
+ parseJSONObject(resourceRequest, out);
+ Priority priority = null;
+ if (out.containsKey("priority")) {
+ priority = Priority.newInstance(Integer.parseInt(out.get("priority")));
+ }
+ String hostName = null;
+ if (out.containsKey("resourceName")) {
+ hostName = out.get("resourceName");
+ }
+ int numContainers = -1;
+ if (out.containsKey("numContainers")) {
+ numContainers = Integer.parseInt(out.get("numContainers"));
+ }
+ boolean relaxLocality = true;
+ if (out.containsKey("relaxLocality")) {
+ relaxLocality = Boolean.parseBoolean(out.get("relaxLocality"));
+ }
+ int memory = 0;
+ if (out.containsKey("memory")) {
+ memory = Integer.parseInt(out.get("memory"));
+ }
+ int virtualCores = 0;
+ if (out.containsKey("virtualCores")) {
+ virtualCores = Integer.parseInt(out.get("virtualCores"));
+ }
+ Resource capability = Resource.newInstance(memory, virtualCores);
+ String nodeLabelExpression = null;
+ if (out.containsKey("nodeLabelExpression")) {
+ nodeLabelExpression = out.get(out.get("nodeLabelExpression"));
+ }
+ ResourceRequest request =
+ ResourceRequest.newInstance(priority, hostName, capability,
+ numContainers, relaxLocality, nodeLabelExpression);
+ return request;
+ }
+
+ private static Map parseJSONObject(JSONObject json,
+ Map out) throws JSONException {
+ Iterator keys = json.keys();
+ while (keys.hasNext()) {
+ String key = keys.next();
+ String val = null;
+ try {
+ JSONObject value = json.getJSONObject(key);
+ parseJSONObject(value, out);
+ } catch (Exception e) {
+ val = json.getString(key);
+ }
+
+ if (val != null) {
+ out.put(key.trim(), val.trim());
+ }
+ }
+ return out;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
index 9302d4b..debd067 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
@@ -47,6 +47,7 @@ public void testApplicationReport() {
Assert.assertNull(appReport2.getCurrentApplicationAttemptId());
Assert.assertNotSame(appReport2, appReport3);
Assert.assertNull(appReport1.getAMRMToken());
+ Assert.assertNull(appReport1.getApplicationMetrics());
}
protected static ApplicationReport createApplicationReport(
@@ -58,7 +59,8 @@ protected static ApplicationReport createApplicationReport(
ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
+ null, null);
return appReport;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 8b48798..5c7b3c5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -194,7 +194,9 @@ public static void setup() throws Exception {
generateByNewInstance(Token.class);
generateByNewInstance(NMToken.class);
generateByNewInstance(ResourceRequest.class);
+ generateByNewInstance(ApplicationAttemptMetrics.class);
generateByNewInstance(ApplicationAttemptReport.class);
+ generateByNewInstance(ApplicationMetrics.class);
generateByNewInstance(ApplicationResourceUsageReport.class);
generateByNewInstance(ApplicationReport.class);
generateByNewInstance(Container.class);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..0b54c97 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport(
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
appHistory.getFinalApplicationStatus(), null, "", 100,
- appHistory.getApplicationType(), null);
+ appHistory.getApplicationType(), null, null, null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)
@@ -166,7 +166,7 @@ private ApplicationAttemptReport convertToApplicationAttemptReport(
appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(), null,
appAttemptHistory.getDiagnosticsInfo(),
appAttemptHistory.getYarnApplicationAttemptState(),
- appAttemptHistory.getMasterContainerId());
+ appAttemptHistory.getMasterContainerId(), null);
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 1010f62..9ea318f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -19,11 +19,14 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
@@ -31,8 +34,10 @@
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -55,12 +60,14 @@
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.MetricsConstantsUtils;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.codehaus.jackson.JsonNode;
import com.google.common.annotations.VisibleForTesting;
@@ -226,6 +233,8 @@ private static ApplicationReportExt convertToApplicationReport(
ApplicationResourceUsageReport appResources = null;
Map appViewACLs =
new HashMap();
+ Set appTags = null;
+ ApplicationMetrics appMetrics = null;
Map entityInfo = entity.getOtherInfo();
if (entityInfo != null) {
if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) {
@@ -245,7 +254,7 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
- null, 1.0F, type, null), appViewACLs);
+ null, 1.0F, type, null, appTags, appMetrics), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@@ -270,6 +279,17 @@ private static ApplicationReportExt convertToApplicationReport(
appResources=ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
}
+ if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS)) {
+ appTags = new HashSet();
+ Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS);
+ if (obj instanceof Collection>) {
+ for(Object o : (Collection>)obj) {
+ if (o != null) {
+ appTags.add(o.toString());
+ }
+ }
+ }
+ }
}
List events = entity.getEvents();
if (events != null) {
@@ -314,6 +334,13 @@ private static ApplicationReportExt convertToApplicationReport(
YarnApplicationState.valueOf(eventInfo.get(
ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
}
+ if (eventInfo.containsKey(ApplicationMetricsConstants.APP_METRICS)) {
+ Object obj = eventInfo.get(ApplicationMetricsConstants.APP_METRICS);
+ if (obj instanceof JsonNode) {
+ appMetrics = MetricsConstantsUtils
+ .convertJSONtoApplicationMetrics((JsonNode)obj);
+ }
+ }
}
}
}
@@ -321,7 +348,7 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
- null, 1.0F, type, null), appViewACLs);
+ null, 1.0F, type, null, appTags, appMetrics), appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(
@@ -333,6 +360,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport(
String originalTrackingUrl = null;
String diagnosticsInfo = null;
YarnApplicationAttemptState state = null;
+ ApplicationAttemptMetrics appAttemptMetrics = null;
List events = entity.getEvents();
if (events != null) {
for (TimelineEvent event : events) {
@@ -394,13 +422,23 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport(
AppAttemptMetricsConstants.STATE_EVENT_INFO)
.toString());
}
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS)) {
+ Object ob =
+ eventInfo.get(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS);
+ if (ob instanceof JsonNode) {
+ appAttemptMetrics =
+ MetricsConstantsUtils
+ .convertJSONtoApplicationAttemptMetrics((JsonNode) ob);
+ }
+ }
}
}
}
return ApplicationAttemptReport.newInstance(
ConverterUtils.toApplicationAttemptId(entity.getEntityId()),
host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
- state, amContainerId);
+ state, amContainerId, appAttemptMetrics);
}
private static ContainerReport convertToContainerReport(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
index a7809cf..5ce1aef 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
@@ -61,4 +61,7 @@
public static final String STATE_EVENT_INFO =
"YARN_APPLICATION_ATTEMPT_STATE";
+ public static final String APP_ATTEMPT_METRICS =
+ "YARN_APPLICATION_ATTEMPT_METRICS";
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index df8eecb..0055de3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -73,4 +73,8 @@
public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
"YARN_APPLICATION_LATEST_APP_ATTEMPT";
+ public static final String APP_TAGS = "YARN_APPLICATION_TAGS";
+
+ public static final String APP_METRICS = "YARN_APPLICATION_METRICS";
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java
new file mode 100644
index 0000000..cbd4aca
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/MetricsConstantsUtils.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.metrics;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ObjectNode;
+
+public class MetricsConstantsUtils {
+
+ /**
+ * Convert ApplicationMetrics object to JSON
+ * @param appMetrics ApplicationMetrics
+ * @return node JsonNode
+ */
+ public static JsonNode appMetricsToJSON(ApplicationMetrics appMetrics) {
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode node = mapper.createObjectNode();
+ if (appMetrics != null) {
+ node.put("MEMORY", appMetrics.getResourcePreempted().getMemory());
+ node.put("VIRTUAL_CORES", appMetrics.getResourcePreempted()
+ .getVirtualCores());
+ node.put("NUM_NON_AMCONTAINERS_PREEMPTED",
+ appMetrics.getNumNonAMContainersPreempted());
+ node.put("NUM_AMCONTAINERS_PREEMPTED",
+ appMetrics.getNumAMContainersPreempted());
+ }
+ return node;
+ }
+
+ /**
+ * Convert JSON to ApplicationMetrics Object
+ * @param node JSON
+ * @return applicationMetrics ApplicationMetrics
+ */
+ public static ApplicationMetrics
+ convertJSONtoApplicationMetrics(JsonNode node) {
+ if (node == null) {
+ return null;
+ }
+ int memory = 0;
+ if (node.has("MEMORY")) {
+ memory = node.path("MEMORY").getIntValue();
+ }
+ int vcores = 0;
+ if (node.has("VIRTUAL_CORES")) {
+ vcores = node.path("VIRTUAL_CORES").getIntValue();
+ }
+ int numNonAMContainersPreempted = 0;
+ if (node.has("NUM_NON_AMCONTAINERS_PREEMPTED")) {
+ numNonAMContainersPreempted =
+ node.path("NUM_NON_AMCONTAINERS_PREEMPTED").getIntValue();
+ }
+ int numAMContainersPreempted = 0;
+ if (node.has("NUM_AMCONTAINERS_PREEMPTED")) {
+ numAMContainersPreempted =
+ node.path("NUM_AMCONTAINERS_PREEMPTED").getIntValue();
+ }
+ return ApplicationMetrics.newInstance(Resource.newInstance(memory, vcores),
+ numNonAMContainersPreempted, numAMContainersPreempted);
+ }
+
+ public static JsonNode appAttemptMetricsToJSON(
+ ApplicationAttemptMetrics attemptMetrics) {
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode node = mapper.createObjectNode();
+ if (attemptMetrics != null) {
+ node.put("MEMORY", attemptMetrics.getResourcePreempted().getMemory());
+ node.put("VIRTUAL_CORES", attemptMetrics.getResourcePreempted()
+ .getVirtualCores());
+ node.put("NUM_NON_AMCONTAINER_PREEMPTED",
+ attemptMetrics.getNumNonAMContainerPreempted());
+ node.put("IS_PREEMPTED", attemptMetrics.isPreempted());
+ node.put("TOTAL_ALLOCATED_CONTAINERS",
+ attemptMetrics.getTotalAllocatedContainers());
+ node.put("LOCALITY_STATISTICS", attemptMetrics.getLocalityStatistics());
+ }
+ return node;
+ }
+
+ public static ApplicationAttemptMetrics
+ convertJSONtoApplicationAttemptMetrics(JsonNode node) {
+ if (node == null) {
+ return null;
+ }
+ int memory = 0;
+ if (node.has("MEMORY")) {
+ memory = node.path("MEMORY").getIntValue();
+ }
+ int vcores = 0;
+ if (node.has("VIRTUAL_CORES")) {
+ vcores = node.path("VIRTUAL_CORES").getIntValue();
+ }
+ int numNonAMContainerPreempted = 0;
+ if (node.has("NUM_NON_AMCONTAINER_PREEMPTED")) {
+ numNonAMContainerPreempted =
+ node.path("NUM_NON_AMCONTAINER_PREEMPTED").getIntValue();
+ }
+ boolean isPreempted = false;
+ if (node.has("IS_PREEMPTED")) {
+ isPreempted = node.path("IS_PREEMPTED").getBooleanValue();
+ }
+ int totalAllocatedContainers = 0;
+ if (node.has("TOTAL_ALLOCATED_CONTAINERS")) {
+ totalAllocatedContainers =
+ node.path("TOTAL_ALLOCATED_CONTAINERS").getIntValue();
+ }
+ String localityStatistics = null;
+ if (node.has("LOCALITY_STATISTICS")) {
+ localityStatistics = node.path("LOCALITY_STATISTICS").getTextValue();
+ }
+ return ApplicationAttemptMetrics.newInstance(isPreempted,
+ Resource.newInstance(memory, vcores), numNonAMContainerPreempted,
+ totalAllocatedContainers, localityStatistics);
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index 4a82c93..433bc58 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -36,6 +36,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
@@ -225,20 +226,23 @@ public ApplicationAttemptReport run() throws Exception {
tbody._()._();
if (webUiType.equals(YarnWebParams.RM_WEB_UI)) {
- createContainerLocalityTable(html); // TODO:YARN-3284
+ createContainerLocalityTable(html,
+ appAttemptReport.getApplicationAttemptMetrics());
}
}
- //TODO: YARN-3284
- //The containerLocality metrics will be exposed from AttemptReport
- private void createContainerLocalityTable(Block html) {
- int totalAllocatedContainers = 0; //TODO: YARN-3284
- int[][] localityStatistics = new int[0][0];//TODO:YARN-3284
+ private void createContainerLocalityTable(Block html,
+ ApplicationAttemptMetrics attemptMetrics) {
+ int[][] localityStatistics =
+ ConverterUtils
+ .covertLocalityStatisticsStringToArray(attemptMetrics == null ? null
+ : attemptMetrics.getLocalityStatistics());
DIV div = html.div(_INFO_WRAP);
TABLE> table =
div.h3(
"Total Allocated Containers: "
- + totalAllocatedContainers).h3("Each table cell"
+ + (attemptMetrics != null ? attemptMetrics.getTotalAllocatedContainers():"N/A"))
+ .h3("Each table cell"
+ " represents the number of NodeLocal/RackLocal/OffSwitch containers"
+ " satisfied by NodeLocal/RackLocal/OffSwitch resource requests.").table(
"#containerLocality");
@@ -250,15 +254,20 @@ private void createContainerLocalityTable(Block html) {
th(_TH, "Off Switch Request").
_();
- String[] containersType =
- { "Num Node Local Containers (satisfied by)", "Num Rack Local Containers (satisfied by)",
- "Num Off Switch Containers (satisfied by)" };
+ String[] containersType = {
+ "Num Node Local Containers (satisfied by)",
+ "Num Rack Local Containers (satisfied by)",
+ "Num Off Switch Containers (satisfied by)"
+ };
boolean odd = false;
for (int i = 0; i < localityStatistics.length; i++) {
table.tr((odd = !odd) ? _ODD : _EVEN).td(containersType[i])
- .td(String.valueOf(localityStatistics[i][0]))
- .td(i == 0 ? "" : String.valueOf(localityStatistics[i][1]))
- .td(i <= 1 ? "" : String.valueOf(localityStatistics[i][2]))._();
+ .td(localityStatistics[i][0] == -1 ?
+ "N/A" : String.valueOf(localityStatistics[i][0]))
+ .td(i == 0 ? "" : localityStatistics[i][1] == -1 ?
+ "N/A" : String.valueOf(localityStatistics[i][1]))
+ .td(i <= 1 ? "" : localityStatistics[i][2] == -1 ?
+ "N/A" : String.valueOf(localityStatistics[i][2]))._();
}
table._();
div._();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 5fc5fa0..0970ca8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -22,7 +22,9 @@
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -36,8 +38,10 @@
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -59,10 +63,12 @@
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-
+import org.codehaus.jettison.json.JSONException;
import com.google.inject.Inject;
+import com.sun.jersey.api.client.ClientHandlerException;
public class AppBlock extends HtmlBlock {
@@ -213,26 +219,35 @@ public ApplicationReport run() throws Exception {
return;
}
- //TODO:YARN-3284
- //The preemption metrics will be exposed from ApplicationReport
- // and ApplicationAttemptReport
ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
+ ApplicationMetrics appMetrics = appReport.getApplicationMetrics();
+ ApplicationAttemptMetrics currentAttemptMetrics = null;
+ for (final ApplicationAttemptReport report : attempts) {
+ if (report.getApplicationAttemptId().equals(
+ appReport.getCurrentApplicationAttemptId())) {
+ currentAttemptMetrics = report.getApplicationAttemptMetrics();
+ break;
+ }
+ }
DIV
pdiv = html.
_(InfoBlock.class).
div(_INFO_WRAP);
info("Application Overview").clear();
info("Application Metrics")
._("Total Resource Preempted:",
- Resources.none()) // TODO: YARN-3284
+ appMetrics != null ? appMetrics.getResourcePreempted() : "N/A")
._("Total Number of Non-AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
+ appMetrics != null ? appMetrics.getNumNonAMContainersPreempted()
+ : "N/A")
._("Total Number of AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
+ appMetrics != null ? appMetrics.getNumAMContainersPreempted() : "N/A")
._("Resource Preempted from Current Attempt:",
- Resources.none()) // TODO: YARN-3284
+ currentAttemptMetrics != null ?
+ currentAttemptMetrics.getResourcePreempted() : "N/A")
._("Number of Non-AM Containers Preempted from Current Attempt:",
- 0) // TODO: YARN-3284
+ currentAttemptMetrics != null ?
+ currentAttemptMetrics.getNumNonAMContainerPreempted() : "N/A")
._("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
@@ -321,13 +336,21 @@ public ContainerReport run() throws Exception {
tbody._()._();
if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) {
- createResourceRequestsTable(html, null); // TODO:YARN-3284
+ // Call RMWebService to get ResourceRequests table for the application
+ List resouceRequests = new ArrayList();
+ try {
+ resouceRequests = WebAppUtils.getResourceRequests(conf, appID.toString());
+ } catch (JSONException ex) {
+ LOG.warn(ex.getMessage());
+ } catch (ClientHandlerException ex) {
+ LOG.warn(ex.getMessage());
+ }
+ createResourceRequestsTable(html, resouceRequests);
}
}
- //TODO:YARN-3284
- //The resource requests metrics will be exposed from attemptReport
- private void createResourceRequestsTable(Block html, List resouceRequests) {
+ private void createResourceRequestsTable(Block html,
+ List resouceRequests) {
TBODY> tbody =
html.table("#ResourceRequests").thead().tr()
.th(".priority", "Priority")
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java
index 12e646d..8fcd553 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestLeveldbIterator.java
@@ -28,7 +28,6 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
-
import org.iq80.leveldb.DBException;
import org.iq80.leveldb.DBIterator;
import org.junit.Test;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 3d28bb7..7de9546 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -255,7 +256,12 @@ public static YarnApplicationAttemptState createApplicationAttemptState(
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
Resources.createResource(-1, -1), 0, 0);
-
+ /**
+ * Statically defined dummy ApplicationMetrics. Used as
+ * a return value when a valid report cannot be found.
+ */
+ public static final ApplicationMetrics DUMMY_APPLICATION_METRICS =
+ ApplicationMetrics.newInstance(Resources.createResource(-1, -1), -1, -1);
/**
* Find all configs whose name starts with
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
index 71d9363..34fcdb0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@@ -31,6 +32,7 @@
private String diagnosticsInfo;
private FinalApplicationStatus appStatus;
private YarnApplicationAttemptState state;
+ private ApplicationAttemptMetrics appAttemptMetrics;
public AppAttemptFinishedEvent(
ApplicationAttemptId appAttemptId,
@@ -39,7 +41,8 @@ public AppAttemptFinishedEvent(
String diagnosticsInfo,
FinalApplicationStatus appStatus,
YarnApplicationAttemptState state,
- long finishedTime) {
+ long finishedTime,
+ ApplicationAttemptMetrics appAttemptMetrics) {
super(SystemMetricsEventType.APP_ATTEMPT_FINISHED, finishedTime);
this.appAttemptId = appAttemptId;
// This is the tracking URL after the application attempt is finished
@@ -48,6 +51,7 @@ public AppAttemptFinishedEvent(
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.state = state;
+ this.appAttemptMetrics = appAttemptMetrics;
}
@Override
@@ -79,4 +83,7 @@ public YarnApplicationAttemptState getYarnApplicationAttemptState() {
return state;
}
+ public ApplicationAttemptMetrics getAppAttemptMetrics() {
+ return appAttemptMetrics;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
index 2373b3b..7c43aa4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+import java.util.Set;
+
import org.apache.hadoop.yarn.api.records.ApplicationId;
public class ApplicationCreatedEvent extends
@@ -29,6 +31,7 @@
private String user;
private String queue;
private long submittedTime;
+ private Set appTags;
public ApplicationCreatedEvent(ApplicationId appId,
String name,
@@ -36,7 +39,8 @@ public ApplicationCreatedEvent(ApplicationId appId,
String user,
String queue,
long submittedTime,
- long createdTime) {
+ long createdTime,
+ Set appTags) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@@ -44,6 +48,7 @@ public ApplicationCreatedEvent(ApplicationId appId,
this.user = user;
this.queue = queue;
this.submittedTime = submittedTime;
+ this.appTags = appTags;
}
@Override
@@ -75,4 +80,7 @@ public long getSubmittedTime() {
return submittedTime;
}
+ public Set getAppTags() {
+ return appTags;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index b849b00..be95e27 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -43,6 +43,7 @@
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.MetricsConstantsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -106,7 +107,7 @@ public void appCreated(RMApp app, long createdTime) {
app.getUser(),
app.getQueue(),
app.getSubmitTime(),
- createdTime));
+ createdTime, app.getApplicationTags()));
}
}
@@ -168,7 +169,8 @@ public void appAttemptFinished(RMAppAttempt appAttempt,
// based on app state if it doesn't exist
app.getFinalApplicationStatus(),
RMServerUtils.createApplicationAttemptState(appAttemtpState),
- finishedTime));
+ finishedTime, appAttempt.getRMAppAttemptMetrics()
+ .convertToApplicationAttemptMetrics()));
}
}
@@ -251,6 +253,7 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
event.getQueue());
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS, event.getAppTags());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(
@@ -283,7 +286,9 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
appMetrics.getVcoreSeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
appMetrics.getMemorySeconds());
-
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_METRICS,
+ MetricsConstantsUtils.appMetricsToJSON(appMetrics
+ .convertToApplicationMetrics()));
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);
@@ -359,6 +364,9 @@ private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
event.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
event.getYarnApplicationAttemptState().toString());
+ eventInfo.put(AppAttemptMetricsConstants.APP_ATTEMPT_METRICS,
+ MetricsConstantsUtils.appAttemptMetricsToJSON(event
+ .getAppAttemptMetrics()));
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 2d1737a..ec3199d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -558,6 +559,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
int rpcPort = -1;
ApplicationResourceUsageReport appUsageReport =
RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
+ ApplicationMetrics appMetrics = RMServerUtils.DUMMY_APPLICATION_METRICS;
FinalApplicationStatus finishState = getFinalApplicationStatus();
String diags = UNAVAILABLE;
float progress = 0.0f;
@@ -604,6 +606,8 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
RMAppMetrics rmAppMetrics = getRMAppMetrics();
appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
+
+ appMetrics = rmAppMetrics.convertToApplicationMetrics();
}
if (currentApplicationAttemptId == null) {
@@ -612,13 +616,12 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
DUMMY_APPLICATION_ATTEMPT_NUMBER);
}
- return BuilderUtils.newApplicationReport(this.applicationId,
- currentApplicationAttemptId, this.user, this.queue,
- this.name, host, rpcPort, clientToAMToken,
- createApplicationState(), diags,
- trackingUrl, this.startTime, this.finishTime, finishState,
- appUsageReport, origTrackingUrl, progress, this.applicationType,
- amrmToken, applicationTags);
+ return ApplicationReport.newInstance(this.applicationId,
+ currentApplicationAttemptId, this.user, this.queue, this.name, host,
+ rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl,
+ this.startTime, this.finishTime, finishState, appUsageReport,
+ origTrackingUrl, progress, this.applicationType, amrmToken,
+ applicationTags, appMetrics);
} finally {
this.readLock.unlock();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 5091470..d4c8f51 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+import org.apache.hadoop.yarn.api.records.ApplicationMetrics;
import org.apache.hadoop.yarn.api.records.Resource;
public class RMAppMetrics {
@@ -56,4 +57,15 @@ public long getMemorySeconds() {
public long getVcoreSeconds() {
return vcoreSeconds;
}
+
+ /**
+ * Convert RMAppMetrics object to ApplicationMetrics object
+ * @return applicationMetrics
+ */
+ public ApplicationMetrics convertToApplicationMetrics() {
+ return ApplicationMetrics
+ .newInstance(this.getResourcePreempted(),
+ this.getNumNonAMContainersPreempted(),
+ this.getNumAMContainersPreempted());
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 1be1727..7bb4b95 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -36,7 +36,6 @@
import javax.crypto.SecretKey;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -1912,7 +1911,8 @@ public ApplicationAttemptReport createApplicationAttemptReport() {
attemptReport = ApplicationAttemptReport.newInstance(this
.getAppAttemptId(), this.getHost(), this.getRpcPort(), this
.getTrackingUrl(), this.getOriginalTrackingUrl(), this.getDiagnostics(),
- YarnApplicationAttemptState .valueOf(this.getState().toString()), amId);
+ YarnApplicationAttemptState .valueOf(this.getState().toString()), amId,
+ attemptMetrics.convertToApplicationAttemptMetrics());
} finally {
this.readLock.unlock();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index bc22073..5e4f162 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -28,6 +28,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptMetrics;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -145,4 +146,34 @@ public void incNumAllocatedContainers(NodeType containerType,
public int getTotalAllocatedContainers() {
return this.totalAllocatedContainers;
}
+
+ /**
+ * Convert RMAppAttemptMetrics object to ApplicationAttemptMetrics object
+ * @return applicationAttemptMetrics
+ */
+ public ApplicationAttemptMetrics convertToApplicationAttemptMetrics() {
+ return ApplicationAttemptMetrics.newInstance(
+ this.getIsPreempted(),
+ this.getResourcePreempted(),
+ this.getNumNonAMContainersPreempted(),
+ this.getTotalAllocatedContainers(),
+ this.ConvertLocalityStatisticsToString());
+ }
+
+ private String ConvertLocalityStatisticsToString() {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < NodeType.values().length; i++) {
+ for (int j = 0; j < NodeType.values().length; j++) {
+ sb.append(NodeType.values()[i] + ":" + NodeType.values()[j] + ":"
+ + localityStatistics[i][j]);
+ if (j != NodeType.values().length - 1) {
+ sb.append(",");
+ }
+ }
+ if (i != NodeType.values().length - 1) {
+ sb.append("|");
+ }
+ }
+ return sb.toString();
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index f8d92aa..4477538 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -289,7 +289,7 @@ public ApplicationReport createAndGetApplicationReport(
getName(), null, 0, null, null, getDiagnostics().toString(),
getTrackingUrl(), getStartTime(), getFinishTime(),
getFinalApplicationStatus(), usageReport , null, getProgress(),
- type, null);
+ type, null, null, null);
return report;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 7ed3835..21ce5e7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -22,6 +22,7 @@
import static org.mockito.Mockito.when;
import java.util.EnumSet;
+import java.util.HashSet;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -45,6 +46,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
@@ -350,6 +352,7 @@ private static RMApp createRMApp(ApplicationId appId) {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus()).thenReturn(
FinalApplicationStatus.UNDEFINED);
+ when(app.getApplicationTags()).thenReturn(new HashSet());
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
return app;
@@ -369,6 +372,8 @@ private static RMAppAttempt createRMAppAttempt(
when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
when(appAttempt.getOriginalTrackingUrl()).thenReturn(
"test original tracking url");
+ RMAppAttemptMetrics metrics = mock(RMAppAttemptMetrics.class);
+ when(appAttempt.getRMAppAttemptMetrics()).thenReturn(metrics);
return appAttempt;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 6e2b56f..12f7423 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -63,7 +63,6 @@
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 481a53b..2f3063d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -253,7 +253,8 @@ public static ClientRMService mockClientRMService(RMContext rmContext) {
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
(ApplicationResourceUsageReport) null, app.getTrackingUrl(),
- app.getProgress(), app.getApplicationType(), (Token) null);
+ app.getProgress(), app.getApplicationType(), (Token) null, null,
+ null);
appReports.add(appReport);
}
GetApplicationsResponse response = mock(GetApplicationsResponse.class);