From bedf47b6c88d850df11da5abab7227786f446541 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Thu, 9 Jan 2020 17:46:24 +0100 Subject: [PATCH] YARN-10028. Integrate the new abstract log servlet to the JobHistory server --- .../mapreduce/v2/hs/HistoryClientService.java | 6 ++- .../mapreduce/v2/hs/webapp/HsWebServices.java | 53 +++++++++++++++++-- .../v2/hs/webapp/TestHsWebServicesAcls.java | 2 +- .../org/apache/hadoop/yarn/webapp/WebApp.java | 1 + .../apache/hadoop/yarn/webapp/WebApps.java | 14 ++++- .../yarn/server/webapp/AppInfoProvider.java | 2 + .../yarn/server/webapp/BasicAppInfo.java | 4 ++ 7 files changed, 75 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index b0bf41b5e42..90d0c5d6f5c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -79,6 +79,8 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -150,9 +152,10 @@ protected void serviceStart() throws Exception { } @VisibleForTesting - protected void initializeWebApp(Configuration conf) { + protected void initializeWebApp(Configuration conf) throws IOException { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); + ApplicationClientProtocol appClientProtocol = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") @@ -163,6 +166,7 @@ protected void initializeWebApp(Configuration conf) { JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .withCSRFProtection(JHAdminConfig.MR_HISTORY_CSRF_PREFIX) .withXFSProtection(JHAdminConfig.MR_HISTORY_XFS_PREFIX) + .withAppClientProtocol(appClientProtocol) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index dabb760d1ce..d5e3334cfa1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -22,6 +22,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -30,9 +31,12 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.UriInfo; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.mapreduce.JobACL; @@ -62,7 +66,11 @@ import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; +import org.apache.hadoop.yarn.server.webapp.LogServlet; +import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; @@ -71,19 +79,21 @@ import com.google.inject.Inject; @Path("/ws/v1/history") -public class HsWebServices { +public class HsWebServices extends WebServices { private final HistoryContext ctx; private WebApp webapp; + private final LogServlet logServlet; private @Context HttpServletResponse response; - @Context - UriInfo uriInfo; + @Context UriInfo uriInfo; @Inject public HsWebServices(final HistoryContext ctx, final Configuration conf, - final WebApp webapp) { + final WebApp webapp, ApplicationClientProtocol appBaseProto) { + super(appBaseProto); this.ctx = ctx; this.webapp = webapp; + this.logServlet = new LogServlet(conf, this); } private boolean hasAccess(Job job, HttpServletRequest request) { @@ -409,4 +419,39 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + @GET + @Path("/containers/{containerid}/logs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getLogs(@Context HttpServletRequest hsr, + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirectedFromNode) { + init(); + return logServlet.getContainerLogsInfo(hsr, containerIdStr, nmId, + redirectedFromNode, null); + } + + @GET + @Path("/containerlogs/{containerid}/{filename}") + @Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getLogs(@Context HttpServletRequest req, + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) + String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) + String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) + String size, + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { + init(); + return logServlet.getLogFile(req, containerIdStr, filename, format, size, + nmId, redirected_from_node, null); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java index 867c6615c7a..960993ed7f7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java @@ -84,7 +84,7 @@ public void setup() throws IOException { this.ctx = buildHistoryContext(this.conf); WebApp webApp = mock(HsWebApp.class); when(webApp.name()).thenReturn("hsmockwebapp"); - this.hsWebServices= new HsWebServices(ctx, conf, webApp); + this.hsWebServices = new HsWebServices(ctx, conf, webApp, null); this.hsWebServices.setResponse(mock(HttpServletResponse.class)); Job job = ctx.getAllJobs().values().iterator().next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index fad6fe29462..a02e492d8f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.webapp.view.RobotsTextPage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index ced5cedd1ef..bd74fd7057b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -42,7 +42,11 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.http.RestCsrfPreventionFilter; import org.apache.hadoop.security.http.XFrameOptionsFilter; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.eclipse.jetty.webapp.WebAppContext; import org.slf4j.Logger; @@ -104,6 +108,7 @@ private String xfsConfigPrefix; private final HashSet servlets = new HashSet(); private final HashMap attributes = new HashMap(); + private ApplicationClientProtocol appClientProtocol; Builder(String name, Class api, T application, String wsName) { this.name = name; @@ -232,6 +237,11 @@ return this; } + public Builder withAppClientProtocol(ApplicationClientProtocol appClientProtocol) { + this.appClientProtocol = appClientProtocol; + return this; + } + public WebApp build(WebApp webapp) { if (webapp == null) { webapp = new WebApp() { @@ -401,7 +411,6 @@ public void setup() { webapp.setConf(conf); webapp.setHttpServer(server); - } catch (ClassNotFoundException e) { throw new WebAppException("Error starting http server", e); } catch (IOException e) { @@ -413,6 +422,9 @@ protected void configure() { if (api != null) { bind(api).toInstance(application); } + if (appClientProtocol != null) { + bind(ApplicationClientProtocol.class).toInstance(appClientProtocol); + } } }); LOG.info("Registered webapp guice modules"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java index 945135ce7e2..1919e28abb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import javax.servlet.http.HttpServletRequest; @@ -26,6 +27,7 @@ * for providing various application related information. */ @InterfaceAudience.LimitedPrivate({"YARN"}) +@InterfaceStability.Unstable public interface AppInfoProvider { /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java index 0181fd182f3..3594b5d032c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.server.webapp; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; @@ -24,6 +26,8 @@ * Utility class that wraps application information * required by the {@link LogServlet} class. */ +@InterfaceAudience.LimitedPrivate({"YARN"}) +@InterfaceStability.Unstable class BasicAppInfo { private final YarnApplicationState appState; private final String user; -- 2.21.0