diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 5a945daf864..3d37dee9588 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -82,6 +82,10 @@ public static final String APPS_APPID_APPATTEMPTS_APPATTEMPTID_CONTAINERS = "/apps/{appid}/appattempts/{appattemptid}/containers"; + /** Path for {@code RMWebServiceProtocol#getAppLogAggregationStatus}. */ + public static final String APPS_APPID_LOGAGGREGATIONSTATUS = + "/apps/{appid}/logaggregationstatus"; + /** Path for {@code RMWebServiceProtocol#getNodeToLabels}. */ public static final String GET_NODE_TO_LABELS = "/get-node-to-labels"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index c5d52854717..194044360ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -108,6 +108,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; @@ -126,12 +127,14 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; @@ -151,6 +154,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppLogAggregationStatusInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; @@ -166,6 +170,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLogAggregationStatusInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewReservation; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -1464,6 +1469,38 @@ public Response updateAppQueue(AppQueue targetQueue, return Response.status(Status.OK).entity(ret).build(); } + @GET + @Path(RMWSConsts.APPS_APPID_LOGAGGREGATIONSTATUS) + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppLogAggregationStatusInfo getAppLogAggregationStatus( + @Context HttpServletRequest hsr, @PathParam("appid") String appId) + throws AuthorizationException { + init(); + AppLogAggregationStatusInfo ret = new AppLogAggregationStatusInfo(); + RMApp app = getRMAppForAppId(appId); + if (app != null) { + Map logAggregationReports = + app.getLogAggregationReportsForApp(); + if (logAggregationReports != null && !logAggregationReports.isEmpty()) { + for (Entry entry : + logAggregationReports.entrySet()) { + NodeId nodeId = entry.getKey(); + LogAggregationReport report = entry.getValue(); + if (nodeId != null && report != null) { + LogAggregationStatus status = report.getLogAggregationStatus(); + String diagnosticMessage = report.getDiagnosticMessage(); + String failureMessages = ((RMAppImpl) app) + .getLogAggregationFailureMessagesForNM(nodeId); + ret.addLogAggregationStatusInfo(new NodeLogAggregationStatusInfo( + nodeId.toString(), status, diagnosticMessage, failureMessages + )); + } + } + } + } + return ret; + } + protected Response moveApp(RMApp app, UserGroupInformation callerUGI, String targetQueue) throws IOException, InterruptedException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppLogAggregationStatusInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppLogAggregationStatusInfo.java new file mode 100644 index 00000000000..72fb48fdae2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppLogAggregationStatusInfo.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement(name = "app") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppLogAggregationStatusInfo { + + private ArrayList nodeLogAggregationStatus = + new ArrayList<>(); + + public AppLogAggregationStatusInfo() { + } + + public ArrayList + getLogAggregationStatusInfos() { + return nodeLogAggregationStatus; + } + + public void setLogAggregationStatusInfos( + ArrayList nodeLogAggregationStatusInfos) { + this.nodeLogAggregationStatus = nodeLogAggregationStatusInfos; + } + + public void addLogAggregationStatusInfo(NodeLogAggregationStatusInfo info) { + nodeLogAggregationStatus.add(info); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLogAggregationStatusInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLogAggregationStatusInfo.java new file mode 100644 index 00000000000..0ac298aa081 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLogAggregationStatusInfo.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.yarn.api.records.LogAggregationStatus; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "logaggregationstatus") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodeLogAggregationStatusInfo { + + private String nodeId; + private LogAggregationStatus logAggregationStatus; + private String diagnostics; + private String failureMessage; + + public NodeLogAggregationStatusInfo() { + } + + public NodeLogAggregationStatusInfo(String nodeId, + LogAggregationStatus logAggregationStatus, + String diagnostics, + String failureMessage) { + this.nodeId = nodeId; + this.logAggregationStatus = logAggregationStatus; + this.diagnostics = diagnostics; + this.failureMessage = failureMessage; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public LogAggregationStatus getLogAggregationStatus() { + return logAggregationStatus; + } + + public void setLogAggregationStatus( + LogAggregationStatus logAggregationStatus) { + this.logAggregationStatus = logAggregationStatus; + } + + public String getDiagnostics() { + return diagnostics; + } + + public void setDiagnostics(String diagnostics) { + this.diagnostics = diagnostics; + } + + public String getFailureMessage() { + return failureMessage; + } + + public void setFailureMessage(String failureMessage) { + this.failureMessage = failureMessage; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 09e472717ca..2cd98e98e6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -3256,6 +3256,96 @@ Response Body: 8 +Cluster Application Log Aggregation Status API +----------------------------- + +With the application log aggregation status API, you can get the log aggregation statuses of all the nodes running the application. + +### URI + + * http://rm-http-address:port/ws/v1/cluster/apps/{appid}/logaggregationstatus + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + + None + +### Elements of *nodeLogAggregationStatus* object + +When you make a request for the log aggregation status of an app, the information returned has the following fields + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| nodeId | string | The node id of the node the container ran on | +| logAggregationStatus | string | Status of log aggregation - valid values are the members of the LogAggregationStatus enum: DISABLED, NOT\_START, RUNNING, RUNNING\_WITH\_FAILURE, SUCCEEDED, FAILED, TIME\_OUT | +| diagnostics | string | The diagnositic information of the log aggregation | +| failureMessage | string | The failure messages of the log aggregation | + +### Response Examples + +**JSON responses** + +HTTP Request + + GET http://rm-http-address:port/ws/v1/cluster/apps/application_1399397633663_0003/logaggregationstatus + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) + +Response Body: + + { + "nodeLogAggregationStatus": { + "nodeId": "host1.domain.com:8041", + "logAggregationStatus": "NOT_START", + "diagnostics": "", + "failureMessage": "" + }, + "nodeLogAggregationStatus": { + "nodeId": "host2.domain.com:8041", + "logAggregationStatus": "SUCCEEDED", + "diagnostics": "Log uploaded successfully for Application: application_1399397633663_0003 in NodeManager: host2.domain.com:8041 at Fri Feb 02 14:18:12 +0000 2018", + "failureMessage": "" + } + } + +**XML responses** + +HTTP Request + + GET http://rm-http-address:port/ws/v1/cluster/apps/application_1399397633663_0003/logaggregationstatus + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 98 + Server: Jetty(6.1.26) + +Response Body: + + + + host1.domain.com:8041 + NOT_START + + + + + host2.domain.com:8041 + SUCCEEDED + Log uploaded successfully for Application: application_1399397633663_0003 in NodeManager: host2.domain.com:8041 at Fri Feb 02 14:18:12 +0000 2018 + + + + Cluster Delegation Tokens API -----------------------------