diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/krb5.conf hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/krb5.conf new file mode 100644 index 0000000..121ac6d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/krb5.conf @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[libdefaults] + default_realm = APACHE.ORG + udp_preference_limit = 1 + extra_addresses = 127.0.0.1 +[realms] + APACHE.ORG = { + admin_server = localhost:88 + kdc = localhost:88 + } +[domain_realm] + localhost = APACHE.ORG diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index c519f17..dae31fe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -50,8 +50,10 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.file.tfile.TFile; +import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -137,12 +139,15 @@ public String toString() { private final List rootLogDirs; private final ContainerId containerId; + private final String user; // TODO Maybe add a version string here. Instead of changing the version of // the entire k-v format - public LogValue(List rootLogDirs, ContainerId containerId) { + public LogValue(List rootLogDirs, ContainerId containerId, + String user) { this.rootLogDirs = new ArrayList(rootLogDirs); this.containerId = containerId; + this.user = user; // Ensure logs are processed in lexical order Collections.sort(this.rootLogDirs); @@ -177,14 +182,16 @@ public void write(DataOutputStream out) throws IOException { // Write the log itself FileInputStream in = null; try { - in = new FileInputStream(logFile); + in = SecureIOUtils.openForRead(logFile, user, null); byte[] buf = new byte[65535]; int len = 0; while ((len = in.read(buf)) != -1) { out.write(buf, 0, len); } } finally { - in.close(); + if (in != null) { + in.close(); + } } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index de755a7..7c08bdd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -25,6 +25,7 @@ import java.io.OutputStreamWriter; import java.io.StringWriter; import java.io.Writer; +import java.util.Arrays; import java.util.Collections; import junit.framework.Assert; @@ -32,11 +33,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; @@ -44,6 +48,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -97,7 +102,7 @@ public void testReadAcontainerLogs1() throws Exception { LogKey logKey = new LogKey(testContainerId); LogValue logValue = new LogValue(Collections.singletonList(srcFileRoot.toString()), - testContainerId); + testContainerId, ugi.getShortUserName()); logWriter.append(logKey, logValue); logWriter.closeWriter(); @@ -131,6 +136,59 @@ public void testReadAcontainerLogs1() throws Exception { Assert.assertEquals(expectedLength, s.length()); } + @Test(timeout=10000) + public void testContainerLogsFileAccess() throws IOException { + // This test will run only if NativeIO is enabled as SecureIOUtils + // require it to be enabled. + Assume.assumeTrue(NativeIO.isAvailable()); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + UserGroupInformation.setConfiguration(conf); + File workDir = new File(testWorkDir, "testContainerLogsFileAccess1"); + Path remoteAppLogFile = + new Path(workDir.getAbsolutePath(), "aggregatedLogFile"); + Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles"); + ContainerId testContainerId = BuilderUtils.newContainerId(1, 1, 1, 1); + Path t = + new Path(srcFileRoot, testContainerId.getApplicationAttemptId() + .getApplicationId().toString()); + Path srcFilePath = new Path(t, testContainerId.toString()); + + int numChars = 80000; + + writeSrcFile(srcFilePath, "stdout", numChars); + + UserGroupInformation ugi = + UserGroupInformation.getCurrentUser(); + LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi); + + LogKey logKey = new LogKey(testContainerId); + LogValue logValue = + new LogValue(Collections.singletonList(srcFileRoot.toString()), + testContainerId, "randomUser"); + + boolean exceptionOccurred = false; + String stdoutFile = + StringUtils.join( + Path.SEPARATOR, + Arrays.asList(new String[] { + srcFileRoot.toUri().toString(), + testContainerId.getApplicationAttemptId().getApplicationId() + .toString(), testContainerId.toString(), "stdout" })); + String message = + "Owner '" + ugi.getShortUserName() + "' for path " + + stdoutFile + " did not match expected owner 'randomUser'"; + try { + logWriter.append(logKey, logValue); + } catch (IOException e) { + // expected + Assert.assertTrue(e.getMessage().contains(message)); + exceptionOccurred = true; + } + Assert.assertTrue("IOException was expected with message :" + message, + exceptionOccurred); + } private void writeSrcFile(Path srcFilePath, String fileName, long length) throws IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index f9a0558..6ef7944 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -123,7 +123,9 @@ private void uploadLogsForContainer(ContainerId containerId) { + ". Current good log dirs are " + StringUtils.join(",", dirsHandler.getLogDirs())); LogKey logKey = new LogKey(containerId); - LogValue logValue = new LogValue(dirsHandler.getLogDirs(), containerId); + LogValue logValue = + new LogValue(dirsHandler.getLogDirs(), containerId, + userUgi.getShortUserName()); try { this.writer.append(logKey, logValue); } catch (IOException e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index 5fdd957..9a49ed3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -39,8 +39,8 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SecureIOUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; @@ -236,9 +236,11 @@ private void printLogs(Block html, ContainerId containerId, logFile.getName(), "?start=0"), "here"). _(" for full log")._(); } - // TODO: Use secure IO Utils to avoid symlink attacks. // TODO Fix findBugs close warning along with IOUtils change - logByteStream = new FileInputStream(logFile); + System.out.println("Opeing file :=" + logFile.getAbsolutePath() + + " with user :" + application.getUser()); + logByteStream = + SecureIOUtils.openForRead(logFile, application.getUser(), null); IOUtils.skipFully(logByteStream, start); InputStreamReader reader = new InputStreamReader(logByteStream); @@ -260,8 +262,19 @@ private void printLogs(Block html, ContainerId containerId, reader.close(); } catch (IOException e) { - html.h1("Exception reading log-file. Log file was likely aggregated. " - + StringUtils.stringifyException(e)); + LOG.error( + "Exception reading log file " + logFile.getAbsolutePath(), e); + if (e.getMessage().contains( + "did not match expected owner '" + application.getUser() + + "'")) { + html.h1("Exception reading log file. Application submitted by '" + + application.getUser() + + "' doesn't own requested log file :" + + logFile.getName()); + } else { + html.h1("Exception Reading log file. It might be because log " + + "file was aggregated : " + logFile.getName()); + } } finally { if (logByteStream != null) { try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 3c97f43..9b1e223 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.*; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; @@ -124,6 +125,7 @@ public void tearDown() throws IOException, InterruptedException { @SuppressWarnings("unchecked") public void testLocalFileDeletionAfterUpload() throws Exception { this.delSrvc = new DeletionService(createContainerExecutor()); + delSrvc = spy(delSrvc); this.delSrvc.init(conf); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -167,7 +169,8 @@ public void testLocalFileDeletionAfterUpload() throws Exception { // ensure filesystems were closed verify(logAggregationService).closeFileSystems( any(UserGroupInformation.class)); - + verify(delSrvc).delete(eq(user), eq((Path) null), + eq(new Path(app1LogDir.getAbsolutePath()))); delSrvc.stop(); String containerIdStr = ConverterUtils.toString(container11); @@ -586,7 +589,7 @@ private void verifyContainerLogs( ContainerId[] expectedContainerIds) throws IOException { AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(this.conf, - logAggregationService.getRemoteNodeLogFileForApp(appId, this.user)); + logAggregationService.getRemoteNodeLogFileForApp(appId, this.user)); Assert.assertEquals(this.user, reader.getApplicationOwner()); verifyAcls(reader.getApplicationAcls()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java index 4594939..986c6af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -18,27 +18,48 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; +import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Assert; import org.junit.Test; +import com.google.inject.Injector; +import com.google.inject.Module; + public class TestContainerLogsPage { @Test(timeout=30000) @@ -69,4 +90,99 @@ public void testContainerLogDirs() throws IOException { container1, dirsHandler); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); } + + @Test(timeout = 10000) + public void testContainerLogPageAccess() throws IOException { + // SecureIOUtils require Native IO to be enabled. This test will run + // only if it is enabled. + assumeTrue(NativeIO.isAvailable()); + String user = "randomUser" + System.currentTimeMillis(); + File absLogDir = null, appDir = null, containerDir = null, syslog = null; + try { + // target log directory + absLogDir = + new File("target", TestContainerLogsPage.class.getSimpleName() + + "LogDir").getAbsoluteFile(); + absLogDir.mkdir(); + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOG_DIRS, absLogDir.toURI().toString()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + UserGroupInformation.setConfiguration(conf); + + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + // Add an application and the corresponding containers + RecordFactory recordFactory = + RecordFactoryProvider.getRecordFactory(conf); + long clusterTimeStamp = 1234; + ApplicationId appId = + BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); + Application app = mock(Application.class); + when(app.getAppId()).thenReturn(appId); + + // Making sure that application returns a random user. This is required + // for SecureIOUtils' file owner check. + when(app.getUser()).thenReturn(user); + + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 1); + ContainerId container1 = + BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); + + // Testing secure read access for log files + + // Creating application and container directory and syslog file. + appDir = new File(absLogDir, appId.toString()); + appDir.mkdir(); + containerDir = new File(appDir, container1.toString()); + containerDir.mkdir(); + syslog = new File(containerDir, "syslog"); + syslog.createNewFile(); + BufferedOutputStream out = + new BufferedOutputStream(new FileOutputStream(syslog)); + out.write("Log file Content".getBytes()); + out.close(); + + ApplicationACLsManager aclsManager = mock(ApplicationACLsManager.class); + + Context context = mock(Context.class); + ConcurrentMap appMap = + new ConcurrentHashMap(); + appMap.put(appId, app); + when(context.getApplications()).thenReturn(appMap); + when(context.getContainers()).thenReturn( + new ConcurrentHashMap()); + + ContainersLogsBlock cLogsBlock = + new ContainersLogsBlock(conf, context, aclsManager, dirsHandler); + + Map params = new HashMap(); + params.put(YarnWebParams.CONTAINER_ID, container1.toString()); + params.put(YarnWebParams.CONTAINER_LOG_TYPE, "syslog"); + + Injector injector = + WebAppTests.testPage(ContainerLogsPage.class, + ContainersLogsBlock.class, cLogsBlock, params, (Module[])null); + PrintWriter spyPw = WebAppTests.getPrintWriter(injector); + verify(spyPw).write( + "Exception reading log file. Application submitted by '" + user + + "' doesn't own requested log file :syslog"); + } finally { + if (syslog != null) { + syslog.delete(); + } + if (containerDir != null) { + containerDir.delete(); + } + if (appDir != null) { + appDir.delete(); + } + if (absLogDir != null) { + absLogDir.delete(); + } + } + } }