diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
index 4ba8fe0b377..15b6b666f3e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
@@ -423,6 +423,22 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters(
return new JobTaskAttemptCounterInfo(ta);
}
+ /**
+ * Returns the user qualified path name of the remote log directory for
+ * each pre-configured log aggregation file controller.
+ *
+ * @param req HttpServletRequest
+ * @return Path names grouped by file controller name
+ */
+ @GET
+ @Path("/remote-log-dir")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response getRemoteLogDirPath(@Context HttpServletRequest req)
+ throws IOException {
+ init();
+ return logServlet.getRemoteLogDirPath();
+ }
+
@GET
@Path("/aggregatedlogs")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java
new file mode 100644
index 00000000000..a211633d837
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helps building the log aggregation remote directory path from pre-configured
+ * values. The path consists of 3 parts, from which the USER part is not
+ * configured.
+ *
+ * /%USER/
+ *
+ */
+public class RemoteLogPathFactory {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(RemoteLogPathFactory.class);
+
+ private final Configuration conf;
+ private Map remoteRootLogDir;
+ private Map remoteRootLogDirSuffix;
+
+ @XmlRootElement(name = "remoteLogDirPathResult")
+ @XmlAccessorType(XmlAccessType.FIELD)
+ public static class RemoteLogPathResult {
+
+ @XmlElement(name = "paths")
+ protected List paths;
+ public RemoteLogPathResult() {}
+
+ public RemoteLogPathResult(List paths) {
+ this.paths = paths;
+ }
+
+ public List getPaths() {
+ return paths;
+ }
+
+ public void setPaths(List paths) {
+ this.paths = paths;
+ }
+ }
+
+ public static class RemoteLogPath {
+ protected String fileController;
+ protected String path;
+
+ public RemoteLogPath() {}
+
+ public RemoteLogPath(String fileController, String path) {
+ this.fileController = fileController;
+ this.path = path;
+ }
+
+ public String getFileController() {
+ return fileController;
+ }
+
+ public void setFileController(String fileController) {
+ this.fileController = fileController;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+ }
+
+ public RemoteLogPathFactory(Configuration conf) {
+ this.conf = conf;
+ this.remoteRootLogDir = new HashMap<>();
+ this.remoteRootLogDirSuffix = new HashMap<>();
+ }
+
+ /**
+ * Initializes the root and suffix part of the remote log directory path.
+ */
+ public void initialize() {
+ Collection fileControllers = conf.getStringCollection(
+ YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS);
+
+ for (String fileControllerName : fileControllers) {
+ loadRootLogDirPath(fileControllerName);
+ loadRootLogDirSuffixPath(fileControllerName);
+ }
+ }
+
+ /**
+ * Returns the root part of the remote log directory path for a specific
+ * file controller.
+ *
+ * @param fileControllerName Name of file controller (eg. IFile)
+ * @return Root part
+ */
+ public Path getRootLogDir(String fileControllerName) {
+ return remoteRootLogDir.get(fileControllerName);
+ }
+ /**
+ * Returns the suffix part of the remote log directory path for a specific
+ * file controller.
+ *
+ * @param fileControllerName Name of file controller (eg. IFile)
+ * @return Suffix part
+ */
+ public Path getRootLogDirSuffix(String fileControllerName) {
+ return remoteRootLogDirSuffix.get(fileControllerName);
+ }
+
+ /**
+ * Returns the user qualified remote log directory path for a specific
+ * file controller.
+ *
+ * @param fileControllerName Name of file controller (eg. IFile)
+ * @param user Name of user
+ * @return Full path
+ */
+ public Path getUserLogDirPath(String fileControllerName, String user) {
+ Path userPathWithoutSuffix = LogAggregationUtils.getRemoteLogUserDir(
+ getRootLogDir(fileControllerName), user);
+
+ return new Path(userPathWithoutSuffix, getRootLogDirSuffix(
+ fileControllerName));
+ }
+
+ /**
+ * Returns all user qualified remote log directory path for each file
+ * controller.
+ *
+ * @param user Name of user
+ * @return User qualified paths grouped by all pre-configured file controller
+ * name
+ */
+ public RemoteLogPathResult getAllUserLogDirPath(String user) {
+ List paths = new ArrayList<>();
+ for(String fileControllerName : remoteRootLogDir.keySet()) {
+ Path path = getUserLogDirPath(
+ fileControllerName, user);
+ paths.add(new RemoteLogPath(fileControllerName, path.toString()));
+ }
+
+ return new RemoteLogPathResult(paths);
+ }
+
+ private void loadRootLogDirPath(String fileControllerName) {
+ String remoteDirStr = String.format(
+ YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT,
+ fileControllerName);
+ String remoteDir = conf.get(remoteDirStr);
+ if (remoteDir == null || remoteDir.isEmpty()) {
+ remoteDir = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR);
+ }
+ remoteRootLogDir.put(fileControllerName, new Path(remoteDir));
+ }
+
+ private void loadRootLogDirSuffixPath(String fileControllerName) {
+ String suffix = String.format(
+ YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT,
+ fileControllerName);
+ String suffixDir= conf.get(suffix);
+ if (suffixDir == null
+ || remoteRootLogDirSuffix.isEmpty()) {
+ suffixDir = conf.get(
+ YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX)
+ + "-" + fileControllerName.toLowerCase();
+ }
+ remoteRootLogDirSuffix.put(fileControllerName, new Path(suffixDir));
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java
new file mode 100644
index 00000000000..e6d84898602
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests the path assembler logic of {@code RemoteLogPathFactory}
+ */
+public class TestRemoteLogPathFactory extends Configured {
+
+ private static final String USER = "USER";
+ private static final String TFILE = "TFile";
+ private static final String IFILE = "IFile";
+
+ private static final String TFILE_NM_REMOTE_APP_LOG_DIR =
+ String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, TFILE);
+ private static final String IFILE_NM_REMOTE_APP_LOG_DIR =
+ String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, IFILE);
+ private static final String TFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX =
+ String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, TFILE);
+ private static final String IFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX =
+ String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, IFILE);
+
+ @Before
+ public void setup() throws IOException {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR);
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX);
+ setConf(conf);
+ }
+
+ /**
+ * Tests whether the {@code RemoteLogPathFactory} correctly assembles
+ * the paths if the configuration is not set (using the default values).
+ */
+ @Test
+ public void testGetDefaultPaths() {
+ String defaultTFilePath = String.format("%s/%s/logs-tfile",
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER);
+ String defaultIFilePath = String.format("%s/%s/logs-ifile",
+ YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER);
+
+ Map expectedPaths = new HashMap<>();
+ expectedPaths.put(TFILE, defaultTFilePath);
+ expectedPaths.put(IFILE, defaultIFilePath);
+
+ assertPaths(expectedPaths);
+ }
+
+ /**
+ * Tests whether the {@code RemoteLogPathFactory} correctly assembles
+ * the paths if only the NodeManager common remote-app-log-dir entry is
+ * set shared between all the FileControllers.
+ */
+ @Test
+ public void testCommonCustomPaths() {
+ String commonRoot = "/tmp/test-logs";
+ String commonSuffix = "logs-test";
+ getConf().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, commonRoot);
+ getConf().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, commonSuffix);
+
+ String commonTFilePath = String.format(
+ "%s/%s/logs-test-tfile", commonRoot, USER);
+ String commonIFilePath = String.format(
+ "%s/%s/logs-test-ifile", commonRoot, USER);
+ Map expectedPaths = new HashMap<>();
+ expectedPaths.put(TFILE, commonTFilePath);
+ expectedPaths.put(IFILE, commonIFilePath);
+
+ assertPaths(expectedPaths);
+ }
+
+ /**
+ * Tests whether the {@code RemoteLogPathFactory} correctly assembles
+ * the paths if the configuration is set for each FileController type.
+ */
+ @Test
+ public void testCustomPaths() {
+ String rootTFile = "/tmp/tfile-logs";
+ String rootIFile = "/tmp/ifile-logs";
+ String suffixTFile = "logs-tfile";
+ String suffixIFile = "logs-ifile";
+
+ getConf().set(TFILE_NM_REMOTE_APP_LOG_DIR, rootTFile);
+ getConf().set(IFILE_NM_REMOTE_APP_LOG_DIR, rootIFile);
+ getConf().set(TFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX, suffixTFile);
+ getConf().set(IFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX, suffixIFile);
+
+ String commonTFilePath = String.format("%s/%s/logs-tfile", rootTFile, USER);
+ String commonIFilePath = String.format("%s/%s/logs-ifile", rootIFile, USER);
+ Map expectedPaths = new HashMap<>();
+ expectedPaths.put(TFILE, commonTFilePath);
+ expectedPaths.put(IFILE, commonIFilePath);
+
+ assertPaths(expectedPaths);
+ }
+
+ private void assertPaths(Map expectedPathNames) {
+ RemoteLogPathFactory pathFactory = new RemoteLogPathFactory(this.getConf());
+ RemoteLogPathFactory.RemoteLogPathResult result =
+ pathFactory.getAllUserLogDirPath(USER);
+
+ for (RemoteLogPathFactory.RemoteLogPath path : result.getPaths()) {
+ assertEquals(expectedPathNames.get(
+ path.getFileController()), path.getPath());
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java
index 991e9842d0e..6d1339823d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java
@@ -24,11 +24,13 @@
import com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
+import org.apache.hadoop.yarn.logaggregation.RemoteLogPathFactory;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.util.Apps;
@@ -45,6 +47,7 @@
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.StreamingOutput;
+import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
@@ -166,6 +169,24 @@ private void validateUserInput(ApplicationId applicationId,
}
}
+ /**
+ * Returns the user qualified path name of the remote log directory for
+ * each pre-configured log aggregation file controller.
+ *
+ * @return {@link Response} object containing remote log dir path names
+ */
+ public Response getRemoteLogDirPath() throws IOException {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String user = ugi.getUserName();
+ RemoteLogPathFactory pathHelper = new RemoteLogPathFactory(getConf());
+ pathHelper.initialize();
+
+ Response.ResponseBuilder response = Response.ok().entity(
+ pathHelper.getAllUserLogDirPath(user));
+ response.header("X-Content-Type-Options", "nosniff");
+ return response.build();
+ }
+
public Response getLogsInfo(HttpServletRequest hsr, String appIdStr,
String appAttemptIdStr, String containerIdStr, String nmId,
boolean redirectedFromNode, boolean manualRedirection) {