From 081e19a4f322a8a845b92d7be5f0290a95aab581 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Wed, 8 Jan 2020 18:45:26 +0100 Subject: [PATCH] YARN-10028. Integrate the new abstract log servlet to the JobHistory server --- .../mapreduce/v2/hs/webapp/HsWebApp.java | 13 +++++ .../mapreduce/v2/hs/webapp/HsWebServices.java | 53 +++++++++++++++++-- .../v2/hs/webapp/TestHsWebServicesAcls.java | 2 +- .../yarn/server/webapp/AppInfoProvider.java | 2 + .../yarn/server/webapp/BasicAppInfo.java | 4 ++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java index d1309102671..ec7465f9fc1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java @@ -28,15 +28,27 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.webapp.AMParams; import org.apache.hadoop.mapreduce.v2.hs.HistoryContext; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; +import java.io.IOException; + public class HsWebApp extends WebApp implements AMParams { private HistoryContext history; + private ApplicationBaseProtocol appBaseProt; public HsWebApp(HistoryContext history) { this.history = history; + try { + this.appBaseProt = + ClientRMProxy.createRMProxy(conf(), ApplicationBaseProtocol.class); + } catch (IOException e) { + throw new YarnRuntimeException(e); + } } @Override @@ -46,6 +58,7 @@ public void setup() { bind(GenericExceptionHandler.class); bind(AppContext.class).toInstance(history); bind(HistoryContext.class).toInstance(history); + bind(ApplicationBaseProtocol.class).toInstance(appBaseProt); route("/", HsController.class); route("/app", HsController.class); route(pajoin("/job", JOB_ID), HsController.class, "job"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index dabb760d1ce..d242193c118 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -22,6 +22,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -30,9 +31,12 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.UriInfo; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.mapreduce.JobACL; @@ -62,7 +66,11 @@ import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; +import org.apache.hadoop.yarn.server.webapp.LogServlet; +import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.WebApp; @@ -71,19 +79,21 @@ import com.google.inject.Inject; @Path("/ws/v1/history") -public class HsWebServices { +public class HsWebServices extends WebServices { private final HistoryContext ctx; private WebApp webapp; + private final LogServlet logServlet; private @Context HttpServletResponse response; - @Context - UriInfo uriInfo; + @Context UriInfo uriInfo; @Inject public HsWebServices(final HistoryContext ctx, final Configuration conf, - final WebApp webapp) { + final WebApp webapp, ApplicationBaseProtocol appBaseProto) { + super(appBaseProto); this.ctx = ctx; this.webapp = webapp; + this.logServlet = new LogServlet(conf, this); } private boolean hasAccess(Job job, HttpServletRequest request) { @@ -409,4 +419,39 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + @GET + @Path("/containers/{containerid}/logs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getLogs(@Context HttpServletRequest hsr, + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirectedFromNode) { + init(); + return logServlet.getContainerLogsInfo(hsr, containerIdStr, nmId, + redirectedFromNode, null); + } + + @GET + @Path("/containerlogs/{containerid}/{filename}") + @Produces({ MediaType.TEXT_PLAIN + "; " + JettyUtils.UTF_8 }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getLogs(@Context HttpServletRequest req, + @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) + String filename, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) + String format, + @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) + String size, + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { + init(); + return logServlet.getLogFile(req, containerIdStr, filename, format, size, + nmId, redirected_from_node, null); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java index 867c6615c7a..960993ed7f7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java @@ -84,7 +84,7 @@ public void setup() throws IOException { this.ctx = buildHistoryContext(this.conf); WebApp webApp = mock(HsWebApp.class); when(webApp.name()).thenReturn("hsmockwebapp"); - this.hsWebServices= new HsWebServices(ctx, conf, webApp); + this.hsWebServices = new HsWebServices(ctx, conf, webApp, null); this.hsWebServices.setResponse(mock(HttpServletResponse.class)); Job job = ctx.getAllJobs().values().iterator().next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java index 945135ce7e2..1919e28abb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppInfoProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import javax.servlet.http.HttpServletRequest; @@ -26,6 +27,7 @@ * for providing various application related information. */ @InterfaceAudience.LimitedPrivate({"YARN"}) +@InterfaceStability.Unstable public interface AppInfoProvider { /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java index 0181fd182f3..3594b5d032c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/BasicAppInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.server.webapp; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; @@ -24,6 +26,8 @@ * Utility class that wraps application information * required by the {@link LogServlet} class. */ +@InterfaceAudience.LimitedPrivate({"YARN"}) +@InterfaceStability.Unstable class BasicAppInfo { private final YarnApplicationState appState; private final String user; -- 2.21.0