From 09698f5cc1b20429c16c5b7ad48853811a6f6f27 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Thu, 2 Jan 2020 16:07:35 +0100 Subject: [PATCH] YARN-10026. Pull out common code pieces from ATS v1.5 and v2 --- .../webapp/AHSWebServices.java | 90 +---------- .../yarn/server/webapp/AppInfoProvider.java | 47 ++++++ .../yarn/server/webapp/BasicAppInfo.java | 47 ++++++ .../hadoop/yarn/server/webapp/LogServlet.java | 139 +++++++++++++++++ .../yarn/server/webapp/LogWebService.java | 146 +++--------------- .../yarn/server/webapp/WebServices.java | 31 +++- .../yarn/server/webapp/TestLogWebService.java | 12 +- 7 files changed, 293 insertions(+), 219 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index e30d6c6c76df176d349e002f5ddf41deb2710ab1..fc23c8c88ec3676e68fe1b3cdd36c87218e047fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import org.apache.hadoop.yarn.server.webapp.LogServlet; import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; @@ -61,7 +62,6 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; -import org.apache.hadoop.yarn.webapp.NotFoundException; import com.google.common.base.Joiner; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -75,12 +75,15 @@ private static final Logger LOG = LoggerFactory .getLogger(AHSWebServices.class); + // TODO NM_DOWNLOAD_URI_STR should be removed from here private static final String NM_DOWNLOAD_URI_STR = "/ws/v1/node/containers"; + // TODO JOINER should be removed private static final Joiner JOINER = Joiner.on(""); - private static final Joiner DOT_JOINER = Joiner.on(". "); + private final Configuration conf; private final LogAggregationFileControllerFactory factory; + private final LogServlet logServlet; @Inject public AHSWebServices(ApplicationBaseProtocol appBaseProt, @@ -88,6 +91,7 @@ public AHSWebServices(ApplicationBaseProtocol appBaseProt, super(appBaseProt); this.conf = conf; this.factory = new LogAggregationFileControllerFactory(conf); + this.logServlet = new LogServlet(conf, this); } @GET @@ -242,87 +246,9 @@ public Response getContainerLogsInfo( @QueryParam(YarnWebServiceParams.NM_ID) String nmId, @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) @DefaultValue("false") boolean redirected_from_node) { - ContainerId containerId = null; initForReadableEndpoints(res); - try { - containerId = ContainerId.fromString(containerIdStr); - } catch (IllegalArgumentException e) { - throw new BadRequestException("invalid container id, " + containerIdStr); - } - - ApplicationId appId = containerId.getApplicationAttemptId() - .getApplicationId(); - AppInfo appInfo; - try { - appInfo = super.getApp(req, res, appId.toString()); - } catch (Exception ex) { - // directly find logs from HDFS. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); - } - // if the application finishes, directly find logs - // from HDFS. - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); - } - if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) { - String appOwner = appInfo.getUser(); - String nodeHttpAddress = null; - if (nmId != null && !nmId.isEmpty()) { - try { - nodeHttpAddress = getNMWebAddressFromRM(conf, nmId); - } catch (Exception ex) { - LOG.debug("{}", ex); - } - } - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { - ContainerInfo containerInfo; - try { - containerInfo = super.getContainer( - req, res, appId.toString(), - containerId.getApplicationAttemptId().toString(), - containerId.toString()); - } catch (Exception ex) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); - } - nodeHttpAddress = containerInfo.getNodeHttpAddress(); - // make sure nodeHttpAddress is not null and not empty. Otherwise, - // we would only get log meta for aggregated logs instead of - // re-directing the request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() - || redirected_from_node) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - // If this is the redirect request from NM, we should not - // re-direct the request back. Simply output the aggregated log meta. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); - } - } - String uri = "/" + containerId.toString() + "/logs"; - String resURI = JOINER.join( - LogWebServiceUtils.getAbsoluteNMWebAddress(conf, nodeHttpAddress), - NM_DOWNLOAD_URI_STR, uri); - String query = req.getQueryString(); - if (query != null && !query.isEmpty()) { - resURI += "?" + query; - } - ResponseBuilder response = Response.status( - HttpServletResponse.SC_TEMPORARY_REDIRECT); - response.header("Location", resURI); - return response.build(); - } else { - throw new NotFoundException( - "The application is not at Running or Finished State."); - } + return logServlet.logRequest(req, containerIdStr, nmId, + redirected_from_node, null); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..5d319bce3ccf88aa8fd8ed77baa75d306e6332f9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webapp; + +import org.apache.hadoop.classification.InterfaceAudience; + +import javax.servlet.http.HttpServletRequest; + + +@InterfaceAudience.LimitedPrivate({"YARN"}) +public interface AppInfoProvider { + + /** + * + * @param req + * @param appId + * @param appAttemptId + * @param containerId + * @return + */ + String getNodeHttpAddress(HttpServletRequest req, + String appId, String appAttemptId, String containerId, String clusterId); + + /** + * + * @param req + * @param appId + * @return + */ + BasicAppInfo getApp(HttpServletRequest req, String appId, String clusterId); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..f33a915af0b50e8d17fc32d061ef9a2711ff7609 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp; + +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; + +/** + * Utility class that wraps the application information + * required by the {@link LogServlet}. + */ +class BasicAppInfo { + private final YarnApplicationState appState; + private final String user; + + BasicAppInfo(YarnApplicationState appState, String user) { + this.appState = appState; + this.user = user; + } + + static BasicAppInfo fromAppInfo(AppInfo report) { + return new BasicAppInfo(report.getAppState(), report.getUser()); + } + + YarnApplicationState getAppState() { + return this.appState; + } + + String getUser() { + return this.user; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java new file mode 100644 index 0000000000000000000000000000000000000000..9936b4bff4952669f6833e5003a2e01753876a04 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.webapp; + +import com.google.common.base.Joiner; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; + +/** + * + */ +public class LogServlet extends Configured { + private static final Logger LOG = LoggerFactory + .getLogger(LogServlet.class); + + private static final Joiner JOINER = Joiner.on(""); + private static final String NM_DOWNLOAD_URI_STR = "/ws/v1/node/containers"; + + private final LogAggregationFileControllerFactory factory; + private final AppInfoProvider appInfoProvider; + + public LogServlet(Configuration conf, AppInfoProvider appInfoProvider) { + super(conf); + this.factory = new LogAggregationFileControllerFactory(conf); + this.appInfoProvider = appInfoProvider; + } + + public Response logRequest(HttpServletRequest req, String containerIdStr, + String nmId, boolean redirected_from_node, String clusterId) { + ContainerId containerId = null; + try { + containerId = ContainerId.fromString(containerIdStr); + } catch (IllegalArgumentException e) { + throw new BadRequestException("invalid container id, " + containerIdStr); + } + + ApplicationId appId = containerId.getApplicationAttemptId() + .getApplicationId(); + BasicAppInfo appInfo; + try { + appInfo = appInfoProvider.getApp(req, appId.toString(), clusterId); + } catch (Exception ex) { + // directly find logs from HDFS. + return LogWebServiceUtils + .getContainerLogMeta(factory, appId, null, null, containerIdStr, + false); + } + // if the application finishes, directly find logs + // from HDFS. + if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { + return LogWebServiceUtils + .getContainerLogMeta(factory, appId, null, null, containerIdStr, + false); + } + if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) { + String appOwner = appInfo.getUser(); + String nodeHttpAddress = null; + if (nmId != null && !nmId.isEmpty()) { + try { + nodeHttpAddress = + LogWebServiceUtils.getNMWebAddressFromRM(getConf(), nmId); + } catch (Exception ex) { + LOG.debug("{}", ex); + } + } + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + try { + nodeHttpAddress = appInfoProvider.getNodeHttpAddress( + req, appId.toString(), + containerId.getApplicationAttemptId().toString(), + containerId.toString(), clusterId); + } catch (Exception ex) { + // return log meta for the aggregated logs if exists. + // It will also return empty log meta for the local logs. + return LogWebServiceUtils + .getContainerLogMeta(factory, appId, appOwner, null, + containerIdStr, true); + } + // make sure nodeHttpAddress is not null and not empty. Otherwise, + // we would only get log meta for aggregated logs instead of + // re-directing the request + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirected_from_node) { + // return log meta for the aggregated logs if exists. + // It will also return empty log meta for the local logs. + // If this is the redirect request from NM, we should not + // re-direct the request back. Simply output the aggregated log meta. + return LogWebServiceUtils + .getContainerLogMeta(factory, appId, appOwner, null, + containerIdStr, true); + } + } + String uri = "/" + containerId.toString() + "/logs"; + String resURI = JOINER.join( + LogWebServiceUtils.getAbsoluteNMWebAddress(getConf(), + nodeHttpAddress), + NM_DOWNLOAD_URI_STR, uri); + String query = req.getQueryString(); + if (query != null && !query.isEmpty()) { + resURI += "?" + query; + } + Response.ResponseBuilder response = Response.status( + HttpServletResponse.SC_TEMPORARY_REDIRECT); + response.header("Location", resURI); + return response.build(); + } else { + throw new NotFoundException( + "The application is not at Running or Finished State."); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java index b51375128d54187d66ab3650589bebbb310166dc..f7ab44af16bb17459115a5d2b8b5025e2a961e6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; -import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -71,7 +70,7 @@ /** * Support only ATSv2 client only. */ -@Singleton @Path("/ws/v2/applicationlog") public class LogWebService { +@Singleton @Path("/ws/v2/applicationlog") public class LogWebService implements AppInfoProvider { private static final Logger LOG = LoggerFactory.getLogger(LogWebService.class); private static final String RESOURCE_URI_STR_V2 = "/ws/v2/timeline/"; @@ -81,6 +80,8 @@ private static LogAggregationFileControllerFactory factory; private static String base; private static String defaultClusterid; + + private final LogServlet logServlet; private volatile Client webTimelineClient; static { @@ -99,6 +100,10 @@ private static void init() { + " for URI: " + base); } + public LogWebService() { + this.logServlet = new LogServlet(yarnConf, this); + } + private Client createTimelineWebClient() { ClientConfig cfg = new DefaultClientConfig(); cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class); @@ -145,91 +150,13 @@ public Response getContainerLogsInfo(@Context HttpServletRequest req, @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) @DefaultValue("false") boolean redirectedFromNode, @QueryParam(YarnWebServiceParams.CLUSTER_ID) String clusterId) { - ContainerId containerId = null; initForReadableEndpoints(res); - try { - containerId = ContainerId.fromString(containerIdStr); - } catch (IllegalArgumentException e) { - throw new BadRequestException("invalid container id, " + containerIdStr); - } - - ApplicationId appId = - containerId.getApplicationAttemptId().getApplicationId(); - AppInfo appInfo; - try { - appInfo = getApp(req, appId.toString(), clusterId); - } catch (Exception ex) { - // directly find logs from HDFS. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); - } - // if the application finishes, directly find logs - // from HDFS. - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, null, null, containerIdStr, - false); - } - if (LogWebServiceUtils.isRunningState(appInfo.getAppState())) { - String appOwner = appInfo.getUser(); - String nodeHttpAddress = null; - if (nmId != null && !nmId.isEmpty()) { - try { - nodeHttpAddress = - LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId); - } catch (Exception ex) { - LOG.debug("{}", ex); - } - } - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { - ContainerInfo containerInfo; - try { - containerInfo = - getContainer(req, appId.toString(), containerId.toString(), - clusterId); - } catch (Exception ex) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); - } - nodeHttpAddress = containerInfo.getNodeHttpAddress(); - // make sure nodeHttpAddress is not null and not empty. Otherwise, - // we would only get log meta for aggregated logs instead of - // re-directing the request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() - || redirectedFromNode) { - // return log meta for the aggregated logs if exists. - // It will also return empty log meta for the local logs. - // If this is the redirect request from NM, we should not - // re-direct the request back. Simply output the aggregated log meta. - return LogWebServiceUtils - .getContainerLogMeta(factory, appId, appOwner, null, - containerIdStr, true); - } - } - String uri = "/" + containerId.toString() + "/logs"; - String resURI = JOINER.join( - LogWebServiceUtils.getAbsoluteNMWebAddress(yarnConf, nodeHttpAddress), - NM_DOWNLOAD_URI_STR, uri); - String query = req.getQueryString(); - if (query != null && !query.isEmpty()) { - resURI += "?" + query; - } - Response.ResponseBuilder response = - Response.status(HttpServletResponse.SC_TEMPORARY_REDIRECT); - response.header("Location", resURI); - return response.build(); - } else { - throw new NotFoundException( - "The application is not at Running or Finished State."); - } + return logServlet.logRequest(req, containerIdStr, nmId, redirectedFromNode, clusterId); } - protected ContainerInfo getContainer(HttpServletRequest req, String appId, - String containerId, String clusterId) { + @Override + public String getNodeHttpAddress(HttpServletRequest req, String appId, + String appAttemptId, String containerId, String clusterId) { UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req); String cId = clusterId != null ? clusterId : defaultClusterid; MultivaluedMap params = new MultivaluedMapImpl(); @@ -255,14 +182,12 @@ protected ContainerInfo getContainer(HttpServletRequest req, String appId, if (conEntity == null) { return null; } - String nodeHttpAddress = (String) conEntity.getInfo() + return (String) conEntity.getInfo() .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO); - - ContainerInfo info = new ContainerInfo(nodeHttpAddress); - return info; } - protected AppInfo getApp(HttpServletRequest req, String appId, + @Override + public BasicAppInfo getApp(HttpServletRequest req, String appId, String clusterId) { UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req); @@ -296,8 +221,7 @@ protected AppInfo getApp(HttpServletRequest req, String appId, String state = (String) appEntity.getInfo() .get(ApplicationMetricsConstants.STATE_EVENT_INFO); YarnApplicationState appState = YarnApplicationState.valueOf(state); - AppInfo info = new AppInfo(appState, appOwner); - return info; + return new BasicAppInfo(appState, appOwner); } /** @@ -358,7 +282,7 @@ public Response getLogs(@Context HttpServletRequest req, ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId(); - AppInfo appInfo; + BasicAppInfo appInfo; try { appInfo = getApp(req, appId.toString(), clusterId); } catch (Exception ex) { @@ -386,18 +310,16 @@ public Response getLogs(@Context HttpServletRequest req, } } if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { - ContainerInfo containerInfo; try { - containerInfo = - getContainer(req, appId.toString(), containerId.toString(), - clusterId); + nodeHttpAddress = + getNodeHttpAddress(req, appId.toString(), null, //appAttemptId + containerId.toString(), clusterId); } catch (Exception ex) { // output the aggregated logs return LogWebServiceUtils .sendStreamOutputResponse(factory, appId, appOwner, null, containerIdStr, filename, format, length, true); } - nodeHttpAddress = containerInfo.getNodeHttpAddress(); // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get aggregated logs instead of re-directing the // request. @@ -429,36 +351,6 @@ public Response getLogs(@Context HttpServletRequest req, } } - protected static class AppInfo { - private YarnApplicationState appState; - private String user; - - AppInfo(YarnApplicationState appState, String user) { - this.appState = appState; - this.user = user; - } - - public YarnApplicationState getAppState() { - return this.appState; - } - - public String getUser() { - return this.user; - } - } - - protected static class ContainerInfo { - private String nodeHttpAddress; - - ContainerInfo(String nodeHttpAddress) { - this.nodeHttpAddress = nodeHttpAddress; - } - - public String getNodeHttpAddress() { - return nodeHttpAddress; - } - } - @VisibleForTesting protected TimelineEntity getEntity(String path, MultivaluedMap params) throws IOException { ClientResponse resp = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 5bb5448492a83a4d4c04f80d437a446cc2aaa048..bced43f40e65999e7b9552d0a0bbdb8515e49bd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; -public class WebServices { +public class WebServices implements AppInfoProvider { protected ApplicationBaseProtocol appBaseProt; @@ -214,8 +214,17 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, return allApps; } - public AppInfo getApp(HttpServletRequest req, HttpServletResponse res, - String appId) { + public AppInfo getApp(HttpServletRequest req, + HttpServletResponse res, String appId) { + return getApp(req, appId); + } + + @Override + public BasicAppInfo getApp(HttpServletRequest req, String appId, String clusterId) { + return BasicAppInfo.fromAppInfo(getApp(req, appId)); + } + + public AppInfo getApp(HttpServletRequest req, String appId) { UserGroupInformation callerUGI = getUser(req); final ApplicationId id = parseApplicationId(appId); ApplicationReport app = null; @@ -351,8 +360,16 @@ public ContainersInfo getContainers(HttpServletRequest req, return containersInfo; } + @Override + public String getNodeHttpAddress(HttpServletRequest req, + String appId, String appAttemptId, + String containerId, String clusterId) { + ContainerInfo containerInfo = getContainer(req, appId, appAttemptId, containerId); + return containerInfo.getNodeHttpAddress(); + } + public ContainerInfo getContainer(HttpServletRequest req, - HttpServletResponse res, String appId, String appAttemptId, + String appId, String appAttemptId, String containerId) { UserGroupInformation callerUGI = getUser(req); ApplicationId aid = parseApplicationId(appId); @@ -387,6 +404,12 @@ public ContainerReport run() throws Exception { return new ContainerInfo(container); } + public ContainerInfo getContainer(HttpServletRequest req, + HttpServletResponse res, String appId, String appAttemptId, + String containerId) { + return getContainer(req, appId, appAttemptId, containerId); + } + protected void initForReadableEndpoints(HttpServletResponse response) { // clear content type response.setContentType(null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java index 9ceb629791c9550d1c810684511d53d200f99e31..9290939572113e913c3c0ce2420db709da64ba12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/TestLogWebService.java @@ -62,18 +62,18 @@ } - @Test public void testGetApp() { - - LogWebService.AppInfo app = + @Test + public void testGetApp() { + BasicAppInfo app = logWebService.getApp(request, appId.toString(), null); Assert.assertEquals("RUNNING", app.getAppState().toString()); Assert.assertEquals(user, app.getUser()); } @Test public void testGetContainer() { - LogWebService.ContainerInfo container = logWebService - .getContainer(request, appId.toString(), cId.toString(), null); - Assert.assertEquals(nodeHttpAddress, container.getNodeHttpAddress()); + String address = logWebService + .getNodeHttpAddress(request, appId.toString(), null, cId.toString(), null); + Assert.assertEquals(this.nodeHttpAddress, address); } class LogWebServiceTest extends LogWebService { -- 2.21.0