diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 4ba8fe0b377..0f1ebaa77be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -423,6 +423,23 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @param req HttpServletRequest + * @return Path names grouped by file controller name + */ + @GET + @Path("/remote-log-dir") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response getRemoteLogDirPath(@Context HttpServletRequest req, + @QueryParam(YarnWebServiceParams.REMOTE_USER) String user) + throws IOException { + init(); + return logServlet.getRemoteLogDirPath(user); + } + @GET @Path("/aggregatedlogs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java index 22aa3acd9a3..2bb6aa52830 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesLogs.java @@ -48,9 +48,13 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController; import org.apache.hadoop.yarn.server.webapp.LogServlet; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; @@ -68,6 +72,7 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -76,6 +81,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -576,6 +583,48 @@ public void testGetContainerLogFileForRunningContainer() throws Exception { + ContainerLogAggregationType.AGGREGATED, "Hello-" + CONTAINER_2_2_3); } + @Test + public void testRemoteLogDir() { + String testSuffix = "test-logs"; + WebResource r = resource(); + String[] fileFormats = {"IFile", "TFile"}; + String remoteUser = "TEST"; + String logDirBefore = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR); + String suffixBefore = conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX); + + conf.setStrings(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, + fileFormats); + conf.setClass(String.format( + YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT, "IFile"), + LogAggregationIndexedFileController.class, + LogAggregationFileController.class); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, testSuffix); + + ClientResponse response = r.path("ws").path("v1") + .path("history").path("remote-log-dir") + .queryParam(YarnWebServiceParams.REMOTE_USER, + remoteUser) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + RemoteLogPaths res = response. + getEntity(new GenericType(){}); + + for (RemoteLogPathEntry entry: res.getPaths()) { + String path = String.format("%s/%s/bucket-%s-%s", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, remoteUser, + testSuffix, entry.getFileController().toLowerCase()); + assertTrue(Arrays.asList(fileFormats).contains( + entry.getFileController())); + assertEquals(entry.getPath(), path); + } + + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, logDirBefore); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffixBefore); + } + @Test public void testNonExistingAppId() { ApplicationId nonExistingApp = ApplicationId.newInstance(99, 99); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java index c653691ff82..4b4c3e93fb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java @@ -205,4 +205,15 @@ private boolean validateAggregatedFileControllerName(String name) { getConfiguredLogAggregationFileControllerList() { return this.controllers; } + + public Map + getGroupedLogAggregationFileControllers() { + HashMap fileControllers = + new HashMap<>(); + for (LogAggregationFileController controller : controllers) { + fileControllers.put(controller.fileControllerName, controller); + } + + return fileControllers; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 33de8df0111..a69eb2530b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -24,13 +24,18 @@ import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry; +import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -45,10 +50,12 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; +import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; /** * Extracts aggregated logs and related information. @@ -174,6 +181,39 @@ private void validateUserInput(ApplicationId applicationId, } } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @return {@link Response} object containing remote log dir path names + */ + public Response getRemoteLogDirPath(String user) throws IOException { + String remoteUser = user; + if (remoteUser == null) { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + remoteUser = ugi.getUserName(); + } + Map fileControllers = + getOrCreateFactory().getGroupedLogAggregationFileControllers(); + RemoteLogPaths result = new RemoteLogPaths(); + List paths = new ArrayList<>(); + + for (Map.Entry controllerEntry + : fileControllers.entrySet()) { + LogAggregationFileController controller = controllerEntry.getValue(); + String path = LogAggregationUtils.getRemoteLogSuffixedDir( + controller.getRemoteRootLogDir(), + remoteUser, controller.getRemoteRootLogDirSuffix()).toString(); + + paths.add(new RemoteLogPathEntry(controllerEntry.getKey(), path)); + } + + result.setPaths(paths); + Response.ResponseBuilder response = Response.ok().entity(result); + response.header("X-Content-Type-Options", "nosniff"); + return response.build(); + } + public Response getLogsInfo(HttpServletRequest hsr, String appIdStr, String appAttemptIdStr, String containerIdStr, String nmId, boolean redirectedFromNode, boolean manualRedirection) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 0d9e9f68c10..3aade3faafc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -39,4 +39,5 @@ String REDIRECTED_FROM_NODE = "redirected_from_node"; String CLUSTER_ID = "clusterid"; String MANUAL_REDIRECTION = "manual_redirection"; + String REMOTE_USER = "user"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java new file mode 100644 index 00000000000..f82d253f035 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPathEntry.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp.dao; + +/** + * A remote log path for a log aggregation file controller. + *
+ *   /%USER/
+ * 
+ */ +public class RemoteLogPathEntry { + private String fileController; + private String path; + + public RemoteLogPathEntry() {} + + public RemoteLogPathEntry(String fileController, String path) { + this.fileController = fileController; + this.path = path; + } + + public String getFileController() { + return fileController; + } + + public void setFileController(String fileController) { + this.fileController = fileController; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java new file mode 100644 index 00000000000..cf41a2c2da5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/RemoteLogPaths.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +/** + * Container of a list of {@link RemoteLogPathEntry}. + */ +@XmlRootElement(name = "remoteLogDirPathResult") +@XmlAccessorType(XmlAccessType.FIELD) +public class RemoteLogPaths { + + @XmlElement(name = "paths") + private List paths; + public RemoteLogPaths() {} + + public RemoteLogPaths(List paths) { + this.paths = paths; + } + + public List getPaths() { + return paths; + } + + public void setPaths(List paths) { + this.paths = paths; + } +} \ No newline at end of file