diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 2e9e92d..28f0276 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,26 @@ package org.apache.hadoop.yarn.client.cli; +import java.io.FileNotFoundException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.EnumSet; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.util.Progressable; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; @@ -40,7 +60,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -58,6 +77,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; + import org.junit.Before; import org.junit.Test; @@ -68,15 +88,11 @@ ByteArrayOutputStream sysErrStream; private PrintStream sysErr; + private static int DEFAULT_PORT = 1234; + @Before public void setUp() { - sysOutStream = new ByteArrayOutputStream(); - sysOut = new PrintStream(sysOutStream); - System.setOut(sysOut); - - sysErrStream = new ByteArrayOutputStream(); - sysErr = new PrintStream(sysErrStream); - System.setErr(sysErr); + refreshSysOutputs(); } @Test(timeout = 5000l) @@ -107,7 +123,7 @@ public void testInvalidApplicationId() throws Exception { LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); - int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); + int exitCode = cli.run(new String[]{"-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); } @@ -154,6 +170,66 @@ public void testHelpMessage() throws Exception { String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); } + + @Test(timeout = 5000l) + public void testErrorMessageWhenAccessControlException() throws Exception { + URI uri = new URI("dummy://dummy-host:" + DEFAULT_PORT); + ApplicationId appId = ApplicationId.newInstance(0, 0); + Configuration conf = new YarnConfiguration(); + conf.set("fs.AbstractFileSystem.dummy.impl", + AccessControlExceptionFileSystem.class.getName()); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, uri.toASCIIString()); + + LogCLIHelpers cliHelper = new LogCLIHelpers(); + cliHelper.setConf(conf); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI dumper = new LogsCLIForTest(mockYarnClient); + dumper.setConf(conf); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + + // LogCLIHelper-level tests + int exitCode = cliHelper.dumpAContainersLogs(appId.toString(), + "container_0_0", "nonexistentnode:1234", "nobody"); + assertTrue("Should return an error code", exitCode == -1); + pw.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE + " : " + + "/tmp/logs/nobody/logs/application_0_0000"); + pw.close(); + String dumpAContainerLogsErrStr = baos.toString("UTF-8"); + Assert.assertEquals(dumpAContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + exitCode = cliHelper.dumpAllContainersLogs(appId, "nobody", null); + assertTrue("Should return an error code", exitCode == -1); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE + " : " + + "/tmp/logs/nobody/logs/application_0_0000"); + pw.close(); + String dumpAllContainerLogsErrStr = baos.toString("UTF-8"); + Assert.assertEquals(dumpAllContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + // LogsCLI-level tests + exitCode = dumper.run( new String[] { + "-applicationId", "application_0_0", "-containerId", "container_0_0", + "-nodeAddress", "nonexistentnode:1234", "-appOwner", "nobody", + } ); + assertTrue("Should return an error code", exitCode != 0); + Assert.assertEquals(dumpAContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + exitCode = dumper.run( new String[] { + "-applicationId", "application_0_0", "-appOwner", "nobody", + } ); + assertTrue("Should return an error code", exitCode != 0); + Assert.assertEquals(dumpAllContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + } + @Test (timeout = 15000) public void testFetchApplictionLogs() throws Exception { @@ -284,6 +360,23 @@ private YarnClient createMockYarnClientUnknownApp() throws YarnException, return mockClient; } + private void refreshSysOutputs() { + if (sysOut != null) { + sysOut.close(); + } + if (sysErr != null) { + sysErr.close(); + } + + sysOutStream = new ByteArrayOutputStream(); + sysOut = new PrintStream(sysOutStream); + System.setOut(sysOut); + + sysErrStream = new ByteArrayOutputStream(); + sysErr = new PrintStream(sysErrStream); + System.setErr(sysErr); + } + private static class LogsCLIForTest extends LogsCLI { private YarnClient yarnClient; @@ -297,4 +390,125 @@ protected YarnClient createYarnClient() { return yarnClient; } } + + public static class AccessControlExceptionFileSystem + extends AbstractFileSystem { + + public AccessControlExceptionFileSystem(URI uri) throws URISyntaxException { + super(uri, "dummy", true, DEFAULT_PORT); + } + + public AccessControlExceptionFileSystem(URI uri, Configuration conf) throws + URISyntaxException { + this(uri); + } + + @Override + public int getUriDefaultPort() { + return DEFAULT_PORT; + } + + @Override + public FSDataOutputStream createInternal(Path f, EnumSet flag, + FsPermission absolutePermission, int bufferSize, short replication, + long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt, + boolean createParent) throws IOException { + throw new AccessControlException(); + } + + @Override + public boolean delete(Path f, boolean recursive) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException { + throw new AccessControlException(); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path f, long start, long len) + throws IOException { + throw new AccessControlException(); + } + + @Override + public FileChecksum getFileChecksum(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public FsStatus getFsStatus() throws IOException { + throw new AccessControlException(); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + throw new AccessControlException(); + } + + @Override + public FileStatus[] listStatus(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public void mkdir(Path dir, FsPermission permission, boolean createParent) + throws IOException { + throw new AccessControlException(); + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + throw new AccessControlException(); + } + + @Override + public void renameInternal(Path src, Path dst) throws IOException { + throw new AccessControlException(); + } + + @Override + public void setOwner(Path f, String username, String groupname) + throws IOException { + throw new AccessControlException(); + } + + @Override + public void setPermission(Path f, FsPermission permission) + throws IOException { + throw new AccessControlException(); + } + + @Override + public boolean setReplication(Path f, short replication) + throws IOException { + throw new AccessControlException(); + } + + @Override + public void setTimes(Path f, long mtime, long atime) throws IOException { + throw new AccessControlException(); + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) throws IOException { + throw new AccessControlException(); + } + + @Override + public void checkPath(Path p) { + // always success + return; + } + + @Override + public RemoteIterator listStatusIterator(final Path f) + throws AccessControlException { + throw new AccessControlException(); + } + + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index de06d48..700e329 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; @@ -43,6 +44,8 @@ public class LogCLIHelpers implements Configurable { private Configuration conf; + public static final String PERMISSION_DENIED_MESSAGE + = "Permission denied."; @Private @VisibleForTesting @@ -63,6 +66,10 @@ public int dumpAContainersLogs(String appId, String containerId, nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()) .listStatus(remoteAppLogDir); + } catch (AccessControlException ace) { + System.out.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE + " : " + + remoteAppLogDir); + return -1; } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; @@ -96,8 +103,10 @@ public int dumpAContainersLogs(String appId, String containerId, } @Private + @VisibleForTesting public int dumpAContainerLogs(String containerIdStr, - AggregatedLogFormat.LogReader reader, PrintStream out) throws IOException { + AggregatedLogFormat.LogReader reader, PrintStream out) + throws IOException { DataInputStream valueStream; LogKey key = new LogKey(); valueStream = reader.next(key); @@ -129,7 +138,8 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String user = appOwner; - String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); + String logDirSuffix = + LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); // TODO Change this to get a list of files from the LAS. Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, appId, user, logDirSuffix); @@ -139,6 +149,10 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()).listStatus(remoteAppLogDir); + } catch (AccessControlException ace) { + System.out.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE + " : " + + remoteAppLogDir); + return -1; } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; @@ -157,8 +171,8 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, valueStream = reader.next(key); while (valueStream != null) { - String containerString = - "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName(); + String containerString = "\n\nContainer: " + key + " on " + + thisNodeFile.getPath().getName(); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); while (true) { @@ -179,7 +193,7 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, } } } - if (! foundAnyLogs) { + if (!foundAnyLogs) { emptyLogDir(remoteAppLogDir.toString()); return -1; } @@ -202,8 +216,9 @@ private static void containerLogNotFound(String containerId) { } private static void logDirNotExist(String remoteAppLogDir) { - System.out.println(remoteAppLogDir + "does not exist."); - System.out.println("Log aggregation has not completed or is not enabled."); + System.out.println(remoteAppLogDir + " does not exist."); + System.out.println("App is not submitted or app log aggregation has not " + + "completed or is not enabled."); } private static void emptyLogDir(String remoteAppLogDir) {