diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index eb6169c..3d15df4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -144,15 +145,22 @@ public int run(String[] args) throws Exception { Path remoteRootLogDir = new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - AggregatedLogFormat.LogReader reader = - new AggregatedLogFormat.LogReader(getConf(), - LogAggregationUtils.getRemoteNodeLogFileForApp( - remoteRootLogDir, - appId, - appOwner, - ConverterUtils.toNodeId(nodeAddress), - LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()))); - resultCode = logCliHelper.dumpAContainerLogs(containerIdStr, reader, System.out); + Path remoteNodeLogFileForApp = + LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir, + appId, appOwner, ConverterUtils.toNodeId(nodeAddress), + LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf())); + try { + AggregatedLogFormat.LogReader reader = + new AggregatedLogFormat.LogReader(getConf(), + remoteNodeLogFileForApp); + resultCode = logCliHelper.dumpAContainerLogs(containerIdStr, + reader, System.out); + } catch (AccessControlException ace) { + System.out.println(LogCLIHelpers.LOGS_NOT_AVAILABLE_MSG_PREFIX + + remoteNodeLogFileForApp.toString()); + System.out.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE); + return -1; + } } return resultCode; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index f02f335..9c5a939 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,26 @@ package org.apache.hadoop.yarn.client.cli; +import java.io.FileNotFoundException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.EnumSet; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.util.Progressable; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; @@ -29,8 +49,6 @@ import java.io.PrintStream; import java.io.PrintWriter; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -41,9 +59,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; + +import org.junit.Assert; import org.junit.Before; import org.junit.Test; + public class TestLogsCLI { ByteArrayOutputStream sysOutStream; private PrintStream sysOut; @@ -51,15 +72,11 @@ ByteArrayOutputStream sysErrStream; private PrintStream sysErr; + private static int DEFAULT_PORT = 1234; + @Before public void setUp() { - sysOutStream = new ByteArrayOutputStream(); - sysOut = new PrintStream(sysOutStream); - System.setOut(sysOut); - - sysErrStream = new ByteArrayOutputStream(); - sysErr = new PrintStream(sysErrStream); - System.setErr(sysErr); + refreshSysOutputs(); } @Test(timeout = 5000l) @@ -90,7 +107,7 @@ public void testInvalidApplicationId() throws Exception { LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); - int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); + int exitCode = cli.run(new String[]{"-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); } @@ -137,6 +154,68 @@ public void testHelpMessage() throws Exception { String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); } + + @Test(timeout = 5000l) + public void testErrorMessageWhenAccessControlException() throws Exception { + URI uri = new URI("dummy://dummy-host:" + DEFAULT_PORT); + ApplicationId appId = ApplicationId.newInstance(0, 0); + Configuration conf = new YarnConfiguration(); + conf.set("fs.AbstractFileSystem.dummy.impl", + AccessControlExceptionFileSystem.class.getName()); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, uri.toASCIIString()); + + LogCLIHelpers cliHelper = new LogCLIHelpers(); + cliHelper.setConf(conf); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI dumper = new LogsCLIForTest(mockYarnClient); + dumper.setConf(conf); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + + // LogCLIHelper-level tests + int exitCode = cliHelper.dumpAContainersLogs(appId.toString(), + "container_0_0", "nonexistentnode:1234", "nobody"); + assertTrue("Should return an error code", exitCode == -1); + pw.println(LogCLIHelpers.LOGS_NOT_AVAILABLE_MSG_PREFIX + + "/tmp/logs/nobody/logs/application_0_0000/nonexistentnode_1234"); + pw.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE); + pw.close(); + String dumpAContainerLogsErrStr = baos.toString("UTF-8"); + Assert.assertEquals(dumpAContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + exitCode = cliHelper.dumpAllContainersLogs(appId, "nobody", null); + assertTrue("Should return an error code", exitCode == -1); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println(LogCLIHelpers.LOGS_NOT_AVAILABLE_MSG_PREFIX + + "/tmp/logs/nobody/logs/application_0_0000"); + pw.println(LogCLIHelpers.PERMISSION_DENIED_MESSAGE); + pw.close(); + String dumpAllContainerLogsErrStr = baos.toString("UTF-8"); + Assert.assertEquals(dumpAllContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + // LogsCLI-level tests + exitCode = dumper.run( new String[] { + "-applicationId", "application_0_0", "-containerId", "container_0_0", + "-nodeAddress", "nonexistentnode:1234", "-appOwner", "nobody", + } ); + assertTrue("Should return an error code", exitCode != 0); + Assert.assertEquals(dumpAContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + + exitCode = dumper.run( new String[] { + "-applicationId", "application_0_0", "-appOwner", "nobody", + } ); + assertTrue("Should return an error code", exitCode != 0); + Assert.assertEquals(dumpAllContainerLogsErrStr, sysOutStream.toString()); + refreshSysOutputs(); + } + private YarnClient createMockYarnClient(YarnApplicationState appState) throws YarnException, IOException { @@ -156,6 +235,23 @@ private YarnClient createMockYarnClientUnknownApp() throws YarnException, return mockClient; } + private void refreshSysOutputs() { + if (sysOut != null) { + sysOut.close(); + } + if (sysErr != null) { + sysErr.close(); + } + + sysOutStream = new ByteArrayOutputStream(); + sysOut = new PrintStream(sysOutStream); + System.setOut(sysOut); + + sysErrStream = new ByteArrayOutputStream(); + sysErr = new PrintStream(sysErrStream); + System.setErr(sysErr); + } + private static class LogsCLIForTest extends LogsCLI { private YarnClient yarnClient; @@ -169,4 +265,125 @@ protected YarnClient createYarnClient() { return yarnClient; } } + + public static class AccessControlExceptionFileSystem + extends AbstractFileSystem { + + public AccessControlExceptionFileSystem(URI uri) throws URISyntaxException { + super(uri, "dummy", true, DEFAULT_PORT); + } + + public AccessControlExceptionFileSystem(URI uri, Configuration conf) throws + URISyntaxException { + this(uri); + } + + @Override + public int getUriDefaultPort() { + return DEFAULT_PORT; + } + + @Override + public FSDataOutputStream createInternal(Path f, EnumSet flag, + FsPermission absolutePermission, int bufferSize, short replication, + long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt, + boolean createParent) throws IOException { + throw new AccessControlException(); + } + + @Override + public boolean delete(Path f, boolean recursive) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException { + throw new AccessControlException(); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path f, long start, long len) + throws IOException { + throw new AccessControlException(); + } + + @Override + public FileChecksum getFileChecksum(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public FsStatus getFsStatus() throws IOException { + throw new AccessControlException(); + } + + @Override + public FsServerDefaults getServerDefaults() throws IOException { + throw new AccessControlException(); + } + + @Override + public FileStatus[] listStatus(Path f) throws IOException { + throw new AccessControlException(); + } + + @Override + public void mkdir(Path dir, FsPermission permission, boolean createParent) + throws IOException { + throw new AccessControlException(); + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + throw new AccessControlException(); + } + + @Override + public void renameInternal(Path src, Path dst) throws IOException { + throw new AccessControlException(); + } + + @Override + public void setOwner(Path f, String username, String groupname) + throws IOException { + throw new AccessControlException(); + } + + @Override + public void setPermission(Path f, FsPermission permission) + throws IOException { + throw new AccessControlException(); + } + + @Override + public boolean setReplication(Path f, short replication) + throws IOException { + throw new AccessControlException(); + } + + @Override + public void setTimes(Path f, long mtime, long atime) throws IOException { + throw new AccessControlException(); + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) throws IOException { + throw new AccessControlException(); + } + + @Override + public void checkPath(Path p) { + // always success + return; + } + + @Override + public RemoteIterator listStatusIterator(final Path f) + throws AccessControlException { + throw new AccessControlException(); + } + + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 908a0fd..1481065 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; @@ -43,6 +44,10 @@ public class LogCLIHelpers implements Configurable { private Configuration conf; + public static final String PERMISSION_DENIED_MESSAGE + = "Permission denied."; + public static final String LOGS_NOT_AVAILABLE_MSG_PREFIX + = "Logs not available at "; @Private @VisibleForTesting @@ -58,8 +63,12 @@ public int dumpAContainersLogs(String appId, String containerId, AggregatedLogFormat.LogReader reader; try { reader = new AggregatedLogFormat.LogReader(getConf(), logPath); - } catch (FileNotFoundException fnfe) { - System.out.println("Logs not available at " + logPath.toString()); + } catch (AccessControlException ace) { + System.out.println(LOGS_NOT_AVAILABLE_MSG_PREFIX + logPath.toString()); + System.out.println(PERMISSION_DENIED_MESSAGE); + return -1; + } catch (FileNotFoundException fnfe) { + System.out.println(LOGS_NOT_AVAILABLE_MSG_PREFIX + logPath.toString()); System.out .println("Log aggregation has not completed or is not enabled."); return -1; @@ -68,6 +77,7 @@ public int dumpAContainersLogs(String appId, String containerId, } @Private + @VisibleForTesting public int dumpAContainerLogs(String containerIdStr, AggregatedLogFormat.LogReader reader, PrintStream out) throws IOException { DataInputStream valueStream; @@ -109,9 +119,16 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, remoteRootLogDir, appId, user, logDirSuffix); RemoteIterator nodeFiles; try { - nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir); + nodeFiles = FileContext.getFileContext(getConf()) + .listStatus(remoteAppLogDir); + } catch (AccessControlException ace) { + System.out.println(LOGS_NOT_AVAILABLE_MSG_PREFIX + + remoteAppLogDir.toString()); + System.out.println(PERMISSION_DENIED_MESSAGE); + return -1; } catch (FileNotFoundException fnf) { - System.out.println("Logs not available at " + remoteAppLogDir.toString()); + System.out.println(LOGS_NOT_AVAILABLE_MSG_PREFIX + + remoteAppLogDir.toString()); System.out .println("Log aggregation has not completed or is not enabled."); return -1; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index 2b83e69..ca978ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.webapp.log; +import org.apache.hadoop.security.AccessControlException; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_LOG_TYPE; @@ -101,6 +102,10 @@ protected void render(Block html) { + ". Aggregation may not be complete, " + "Check back later or try the nodemanager at " + nodeId)._(); return; + } catch (AccessControlException ace) { + html.h1() + ._("Logs not available for " + logEntity + ". Permission Denied."); + return; } catch (IOException e) { html.h1()._("Error getting logs for " + logEntity)._(); LOG.error("Error getting logs for " + logEntity, e);