diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index 5fdd957..571e714 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -39,8 +39,8 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -236,9 +236,9 @@ private void printLogs(Block html, ContainerId containerId, logFile.getName(), "?start=0"), "here"). _(" for full log")._(); } - // TODO: Use secure IO Utils to avoid symlink attacks. // TODO Fix findBugs close warning along with IOUtils change - logByteStream = new FileInputStream(logFile); + logByteStream = + SecureIOUtils.openForRead(logFile, application.getUser(), null); IOUtils.skipFully(logByteStream, start); InputStreamReader reader = new InputStreamReader(logByteStream); @@ -260,8 +260,19 @@ private void printLogs(Block html, ContainerId containerId, reader.close(); } catch (IOException e) { - html.h1("Exception reading log-file. Log file was likely aggregated. " - + StringUtils.stringifyException(e)); + LOG.error( + "Exception reading log file " + logFile.getAbsolutePath(), e); + if (e.getMessage().contains( + "did not match " + "expected owner '" + application.getUser() + + "'")) { + html.h1("Exception reading log file. User '" + + application.getUser() + + "' doesn't have permissions to read " + "log file :" + + logFile.getName()); + } else { + html.h1("Exception Reading log file. It might be because log " + + "file was aggregated : " + logFile.getName()); + } } finally { if (logByteStream != null) { try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java index 4594939..3145313 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -18,27 +18,48 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Assert; import org.junit.Test; +import com.google.inject.Injector; +import com.google.inject.Module; + public class TestContainerLogsPage { @Test(timeout=30000) @@ -69,4 +90,99 @@ public void testContainerLogDirs() throws IOException { container1, dirsHandler); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); } + + @Test(timeout = 10000) + public void testContainerLogPageAccess() throws IOException { + // SecureIOUtils require Native IO to be enabled. This test will run + // only if it is enabled. + assumeTrue(NativeIO.isAvailable()); + String user = "randomUser" + System.currentTimeMillis(); + File absLogDir = null, appDir = null, containerDir = null, syslog = null; + try { + // target log directory + absLogDir = + new File("target", TestNMWebServer.class.getSimpleName() + "LogDir") + .getAbsoluteFile(); + absLogDir.mkdir(); + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOG_DIRS, absLogDir.toURI().toString()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + UserGroupInformation.setConfiguration(conf); + + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + // Add an application and the corresponding containers + RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(conf); + long clusterTimeStamp = 1234; + ApplicationId appId = + BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); + Application app = mock(Application.class); + when(app.getAppId()).thenReturn(appId); + + // Making sure that application returns a random user. This is required + // for SecureIOUtils' file owner check. + when(app.getUser()).thenReturn(user); + + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 1); + ContainerId container1 = + BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); + + // Testing secure read access for log files + + // Creating application and container directory and syslog file. + appDir = new File(absLogDir, appId.toString()); + appDir.mkdir(); + containerDir = new File(appDir, container1.toString()); + containerDir.mkdir(); + syslog = new File(containerDir, "syslog"); + syslog.createNewFile(); + BufferedOutputStream out = + new BufferedOutputStream(new FileOutputStream(syslog)); + out.write("Log file Content".getBytes()); + out.close(); + + ApplicationACLsManager aclsManager = mock(ApplicationACLsManager.class); + + Context context = mock(Context.class); + ConcurrentMap appMap = + new ConcurrentHashMap(); + appMap.put(appId, app); + when(context.getApplications()).thenReturn(appMap); + when(context.getContainers()).thenReturn( + new ConcurrentHashMap()); + + ContainersLogsBlock cLogsBlock = + new ContainersLogsBlock(conf, context, aclsManager, dirsHandler); + + Map params = new HashMap(); + params.put(YarnWebParams.CONTAINER_ID, container1.toString()); + params.put(YarnWebParams.CONTAINER_LOG_TYPE, "syslog"); + + Injector injector = + WebAppTests.testPage(ContainerLogsPage.class, + ContainersLogsBlock.class, cLogsBlock, params, (Module[])null); + PrintWriter spyPw = WebAppTests.getPrintWriter(injector); + verify(spyPw).write( + "Exception reading log file. User '" + user + "' " + + "doesn't have permissions to read log file :syslog"); + } finally { + if (syslog != null) { + syslog.delete(); + } + if (containerDir != null) { + containerDir.delete(); + } + if (appDir != null) { + appDir.delete(); + } + if (absLogDir != null) { + absLogDir.delete(); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/krb5.conf hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/krb5.conf new file mode 100644 index 0000000..121ac6d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/krb5.conf @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[libdefaults] + default_realm = APACHE.ORG + udp_preference_limit = 1 + extra_addresses = 127.0.0.1 +[realms] + APACHE.ORG = { + admin_server = localhost:88 + kdc = localhost:88 + } +[domain_realm] + localhost = APACHE.ORG