diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 4ba8fe0b377..15b6b666f3e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -423,6 +423,22 @@ public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters( return new JobTaskAttemptCounterInfo(ta); } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @param req HttpServletRequest + * @return Path names grouped by file controller name + */ + @GET + @Path("/remote-log-dir") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response getRemoteLogDirPath(@Context HttpServletRequest req) + throws IOException { + init(); + return logServlet.getRemoteLogDirPath(); + } + @GET @Path("/aggregatedlogs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java new file mode 100644 index 00000000000..a211633d837 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/RemoteLogPathFactory.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Helps building the log aggregation remote directory path from pre-configured + * values. The path consists of 3 parts, from which the USER part is not + * configured. + *
+ * /%USER/
+ * 
+ */ +public class RemoteLogPathFactory { + + private static final Logger LOG = LoggerFactory + .getLogger(RemoteLogPathFactory.class); + + private final Configuration conf; + private Map remoteRootLogDir; + private Map remoteRootLogDirSuffix; + + @XmlRootElement(name = "remoteLogDirPathResult") + @XmlAccessorType(XmlAccessType.FIELD) + public static class RemoteLogPathResult { + + @XmlElement(name = "paths") + protected List paths; + public RemoteLogPathResult() {} + + public RemoteLogPathResult(List paths) { + this.paths = paths; + } + + public List getPaths() { + return paths; + } + + public void setPaths(List paths) { + this.paths = paths; + } + } + + public static class RemoteLogPath { + protected String fileController; + protected String path; + + public RemoteLogPath() {} + + public RemoteLogPath(String fileController, String path) { + this.fileController = fileController; + this.path = path; + } + + public String getFileController() { + return fileController; + } + + public void setFileController(String fileController) { + this.fileController = fileController; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + } + + public RemoteLogPathFactory(Configuration conf) { + this.conf = conf; + this.remoteRootLogDir = new HashMap<>(); + this.remoteRootLogDirSuffix = new HashMap<>(); + } + + /** + * Initializes the root and suffix part of the remote log directory path. + */ + public void initialize() { + Collection fileControllers = conf.getStringCollection( + YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS); + + for (String fileControllerName : fileControllers) { + loadRootLogDirPath(fileControllerName); + loadRootLogDirSuffixPath(fileControllerName); + } + } + + /** + * Returns the root part of the remote log directory path for a specific + * file controller. + * + * @param fileControllerName Name of file controller (eg. IFile) + * @return Root part + */ + public Path getRootLogDir(String fileControllerName) { + return remoteRootLogDir.get(fileControllerName); + } + /** + * Returns the suffix part of the remote log directory path for a specific + * file controller. + * + * @param fileControllerName Name of file controller (eg. IFile) + * @return Suffix part + */ + public Path getRootLogDirSuffix(String fileControllerName) { + return remoteRootLogDirSuffix.get(fileControllerName); + } + + /** + * Returns the user qualified remote log directory path for a specific + * file controller. + * + * @param fileControllerName Name of file controller (eg. IFile) + * @param user Name of user + * @return Full path + */ + public Path getUserLogDirPath(String fileControllerName, String user) { + Path userPathWithoutSuffix = LogAggregationUtils.getRemoteLogUserDir( + getRootLogDir(fileControllerName), user); + + return new Path(userPathWithoutSuffix, getRootLogDirSuffix( + fileControllerName)); + } + + /** + * Returns all user qualified remote log directory path for each file + * controller. + * + * @param user Name of user + * @return User qualified paths grouped by all pre-configured file controller + * name + */ + public RemoteLogPathResult getAllUserLogDirPath(String user) { + List paths = new ArrayList<>(); + for(String fileControllerName : remoteRootLogDir.keySet()) { + Path path = getUserLogDirPath( + fileControllerName, user); + paths.add(new RemoteLogPath(fileControllerName, path.toString())); + } + + return new RemoteLogPathResult(paths); + } + + private void loadRootLogDirPath(String fileControllerName) { + String remoteDirStr = String.format( + YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, + fileControllerName); + String remoteDir = conf.get(remoteDirStr); + if (remoteDir == null || remoteDir.isEmpty()) { + remoteDir = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); + } + remoteRootLogDir.put(fileControllerName, new Path(remoteDir)); + } + + private void loadRootLogDirSuffixPath(String fileControllerName) { + String suffix = String.format( + YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, + fileControllerName); + String suffixDir= conf.get(suffix); + if (suffixDir == null + || remoteRootLogDirSuffix.isEmpty()) { + suffixDir = conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX) + + "-" + fileControllerName.toLowerCase(); + } + remoteRootLogDirSuffix.put(fileControllerName, new Path(suffixDir)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java new file mode 100644 index 00000000000..e6d84898602 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestRemoteLogPathFactory.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Tests the path assembler logic of {@code RemoteLogPathFactory} + */ +public class TestRemoteLogPathFactory extends Configured { + + private static final String USER = "USER"; + private static final String TFILE = "TFile"; + private static final String IFILE = "IFile"; + + private static final String TFILE_NM_REMOTE_APP_LOG_DIR = + String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, TFILE); + private static final String IFILE_NM_REMOTE_APP_LOG_DIR = + String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, IFILE); + private static final String TFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX = + String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, TFILE); + private static final String IFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX = + String.format(YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, IFILE); + + @Before + public void setup() throws IOException { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); + setConf(conf); + } + + /** + * Tests whether the {@code RemoteLogPathFactory} correctly assembles + * the paths if the configuration is not set (using the default values). + */ + @Test + public void testGetDefaultPaths() { + String defaultTFilePath = String.format("%s/%s/logs-tfile", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER); + String defaultIFilePath = String.format("%s/%s/logs-ifile", + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER); + + Map expectedPaths = new HashMap<>(); + expectedPaths.put(TFILE, defaultTFilePath); + expectedPaths.put(IFILE, defaultIFilePath); + + assertPaths(expectedPaths); + } + + /** + * Tests whether the {@code RemoteLogPathFactory} correctly assembles + * the paths if only the NodeManager common remote-app-log-dir entry is + * set shared between all the FileControllers. + */ + @Test + public void testCommonCustomPaths() { + String commonRoot = "/tmp/test-logs"; + String commonSuffix = "logs-test"; + getConf().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, commonRoot); + getConf().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, commonSuffix); + + String commonTFilePath = String.format( + "%s/%s/logs-test-tfile", commonRoot, USER); + String commonIFilePath = String.format( + "%s/%s/logs-test-ifile", commonRoot, USER); + Map expectedPaths = new HashMap<>(); + expectedPaths.put(TFILE, commonTFilePath); + expectedPaths.put(IFILE, commonIFilePath); + + assertPaths(expectedPaths); + } + + /** + * Tests whether the {@code RemoteLogPathFactory} correctly assembles + * the paths if the configuration is set for each FileController type. + */ + @Test + public void testCustomPaths() { + String rootTFile = "/tmp/tfile-logs"; + String rootIFile = "/tmp/ifile-logs"; + String suffixTFile = "logs-tfile"; + String suffixIFile = "logs-ifile"; + + getConf().set(TFILE_NM_REMOTE_APP_LOG_DIR, rootTFile); + getConf().set(IFILE_NM_REMOTE_APP_LOG_DIR, rootIFile); + getConf().set(TFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX, suffixTFile); + getConf().set(IFILE_NM_REMOTE_APP_LOG_DIR_SUFFIX, suffixIFile); + + String commonTFilePath = String.format("%s/%s/logs-tfile", rootTFile, USER); + String commonIFilePath = String.format("%s/%s/logs-ifile", rootIFile, USER); + Map expectedPaths = new HashMap<>(); + expectedPaths.put(TFILE, commonTFilePath); + expectedPaths.put(IFILE, commonIFilePath); + + assertPaths(expectedPaths); + } + + private void assertPaths(Map expectedPathNames) { + RemoteLogPathFactory pathFactory = new RemoteLogPathFactory(this.getConf()); + RemoteLogPathFactory.RemoteLogPathResult result = + pathFactory.getAllUserLogDirPath(USER); + + for (RemoteLogPathFactory.RemoteLogPath path : result.getPaths()) { + assertEquals(expectedPathNames.get( + path.getFileController()), path.getPath()); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 991e9842d0e..6d1339823d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -24,11 +24,13 @@ import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; +import org.apache.hadoop.yarn.logaggregation.RemoteLogPathFactory; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.util.Apps; @@ -45,6 +47,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; +import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -166,6 +169,24 @@ private void validateUserInput(ApplicationId applicationId, } } + /** + * Returns the user qualified path name of the remote log directory for + * each pre-configured log aggregation file controller. + * + * @return {@link Response} object containing remote log dir path names + */ + public Response getRemoteLogDirPath() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String user = ugi.getUserName(); + RemoteLogPathFactory pathHelper = new RemoteLogPathFactory(getConf()); + pathHelper.initialize(); + + Response.ResponseBuilder response = Response.ok().entity( + pathHelper.getAllUserLogDirPath(user)); + response.header("X-Content-Type-Options", "nosniff"); + return response.build(); + } + public Response getLogsInfo(HttpServletRequest hsr, String appIdStr, String appAttemptIdStr, String containerIdStr, String nmId, boolean redirectedFromNode, boolean manualRedirection) {