diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..e49b035 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -43,10 +43,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ContainerReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -208,13 +205,11 @@ public int run(String[] args) throws Exception { LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); - if (appOwner == null || appOwner.isEmpty()) { - appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); - } - YarnApplicationState appState = YarnApplicationState.NEW; + ApplicationReport appReport = null; try { - appState = getApplicationState(appId); + appReport = getApplicationReport(appId); + appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) { @@ -226,6 +221,16 @@ public int run(String[] args) throws Exception { + " Attempting to fetch logs directly from the filesystem."); } + if (appOwner == null || appOwner.isEmpty()) { + appOwner = guessAppOwner(appReport, appId); + if (appOwner == null) { + System.err.println("Can not find the appOwner. " + + "Please specify the correct appOwner"); + System.out.println("Could not locate application logs for " + appId); + return -1; + } + } + // To get am logs if (getAMContainerLogs) { // if we do not specify the value for CONTAINER_LOG_FILES option, @@ -254,9 +259,9 @@ public int run(String[] args) throws Exception { return printAMContainerLogs(getConf(), appIdStr, amContainersList, logFiles, logCliHelper, appOwner, true); } else { - System.out - .println( - "Can not get AMContainers logs for the application:" + appId); + System.out.println("Can not get AMContainers logs for " + + "the application:" + appId + " with the appOwner: " + + appOwner); System.out.println("This application:" + appId + " is finished." + " Please enable the application history service. Or Using " + "yarn logs -applicationId -containerId " @@ -316,7 +321,8 @@ public int run(String[] args) throws Exception { return resultCode; } catch (IOException | YarnException ex) { System.err.println("Unable to get logs for this container:" - + containerIdStr + "for the application:" + appId); + + containerIdStr + "for the application:" + appId + + " with the appOwner: " + appOwner); if (!getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { System.out.println("Please enable the application history service. Or "); @@ -330,6 +336,10 @@ public int run(String[] args) throws Exception { if (nodeAddress == null) { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); + if (resultCode == -1) { + System.out.println("Can not find the logs for the application: " + + appId + " with the appOwner: " + appOwner); + } } else { System.out.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -339,13 +349,12 @@ public int run(String[] args) throws Exception { return resultCode; } - private YarnApplicationState getApplicationState(ApplicationId appId) + private ApplicationReport getApplicationReport(ApplicationId appId) throws IOException, YarnException { YarnClient yarnClient = createYarnClient(); try { - ApplicationReport appReport = yarnClient.getApplicationReport(appId); - return appReport.getYarnApplicationState(); + return yarnClient.getApplicationReport(appId); } finally { yarnClient.close(); } @@ -689,4 +698,19 @@ public boolean isAppFinished() { return isAppFinished; } } + + private String guessAppOwner(ApplicationReport appReport, + ApplicationId appId) throws IOException { + String appOwner = null; + if (appReport != null) { + //always use the app owner from the app report if possible + appOwner = appReport.getUser(); + } + else { + appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); + appOwner = LogCLIHelpers.getOwnerForAppIdOrNull( + appId, appOwner, getConf()); + } + return appOwner; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index aec7cae..349a4ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -69,7 +70,7 @@ public class TestLogsCLI { ByteArrayOutputStream sysOutStream; private PrintStream sysOut; - + ByteArrayOutputStream sysErrStream; private PrintStream sysErr; @@ -78,7 +79,7 @@ public void setUp() { sysOutStream = new ByteArrayOutputStream(); sysOut = new PrintStream(sysOutStream); System.setOut(sysOut); - + sysErrStream = new ByteArrayOutputStream(); sysErr = new PrintStream(sysErrStream); System.setErr(sysErr); @@ -90,16 +91,18 @@ public void testFailResultCodes() throws Exception { conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); LogCLIHelpers cliHelper = new LogCLIHelpers(); cliHelper.setConf(conf); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); - + // verify dumping a non-existent application's logs returns a failure code int exitCode = dumper.run( new String[] { "-applicationId", "application_0_0" } ); assertTrue("Should return an error code", exitCode != 0); - - // verify dumping a non-existent container log is a failure code + + // verify dumping a non-existent container log is a failure code exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234", "nobody"); assertTrue("Should return an error code", exitCode != 0); @@ -108,10 +111,12 @@ public void testFailResultCodes() throws Exception { @Test(timeout = 5000l) public void testInvalidApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); - + int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); @@ -136,7 +141,9 @@ public void testUnknownApplicationId() throws Exception { @Test(timeout = 5000l) public void testHelpMessage() throws Exception { Configuration conf = new YarnConfiguration(); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); @@ -175,9 +182,13 @@ public void testHelpMessage() throws Exception { String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); } - + @Test (timeout = 15000) public void testFetchApplictionLogs() throws Exception { + final String testUser = "test"; + UserGroupInformation testUgi = UserGroupInformation + .createRemoteUser(testUser); + String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); @@ -197,6 +208,12 @@ public void testFetchApplictionLogs() throws Exception { ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3); NodeId nodeId = NodeId.newInstance("localhost", 1234); + ApplicationId appId2 = ApplicationIdPBImpl.newInstance(0, 2); + ApplicationAttemptId appAttemptId2 = + ApplicationAttemptIdPBImpl.newInstance(appId2, 1); + ContainerId containerId1ForApp2 = ContainerIdPBImpl + .newContainerId(appAttemptId2, 1); + // create local logs String rootLogDir = "target/LocalLogs"; Path rootLogDirPath = new Path(rootLogDir); @@ -210,6 +227,14 @@ public void testFetchApplictionLogs() throws Exception { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); + + // create local app dir for app2 + Path appLogsDir2 = new Path(rootLogDirPath, appId2.toString()); + if (fs.exists(appLogsDir2)) { + fs.delete(appLogsDir2, true); + } + assertTrue(fs.mkdirs(appLogsDir2)); + List rootLogDirs = Arrays.asList(rootLogDir); List logTypes = new ArrayList(); @@ -218,6 +243,10 @@ public void testFetchApplictionLogs() throws Exception { createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes); createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes); + // create container logs for app2 + createContainerLogInLocalDir(appLogsDir2, containerId1ForApp2, + fs, logTypes); + // create two logs for container3 in localLogDir logTypes.add("stdout"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes); @@ -230,6 +259,15 @@ public void testFetchApplictionLogs() throws Exception { } assertTrue(fs.mkdirs(path)); + // create the remote app dir for app2 + Path path2 = + new Path(remoteLogRootDir + testUser + + "/logs/" + appId2); + if (fs.exists(path2)) { + fs.delete(path2, true); + } + assertTrue(fs.mkdirs(path2)); + // upload container logs into remote directory // the first two logs is empty. When we try to read first two logs, // we will meet EOF exception, but it will not impact other logs. @@ -245,8 +283,13 @@ public void testFetchApplictionLogs() throws Exception { uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId3, path, fs); + // upload container logs for app2 into remote dir + uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, + nodeId, containerId1ForApp2, path2, fs); + YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient( + YarnApplicationState.FINISHED, ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); @@ -317,6 +360,27 @@ public void testFetchApplictionLogs() throws Exception { "Hello container_0_0001_01_000003 in syslog!")); sysOutStream.reset(); + // Verify that we can not get the application logs + // if an invalid user is specified + exitCode = cli.run(new String[] { + "-applicationId", appId.toString(), + "-appOwner", "invalid"}); + assertTrue(exitCode == -1); + sysOutStream.reset(); + + // Verify that we do not specify appOwner, and can not + // get appReport from RM, we still can figure out the appOwner + // and can get app logs successfully. + YarnClient mockYarnClient2 = createMockYarnClientUnknownApp(); + cli = new LogsCLIForTest(mockYarnClient2); + cli.setConf(configuration); + exitCode = cli.run(new String[] { + "-applicationId", appId2.toString()}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains("Hello " + containerId1ForApp2 + + " in syslog!")); + sysOutStream.reset(); + fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); } @@ -347,7 +411,8 @@ public void testFetchApplictionLogsHar() throws Exception { assertTrue(fs.exists(harPath)); YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient(YarnApplicationState.FINISHED, + ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); int exitCode = cli.run(new String[]{"-applicationId", @@ -429,10 +494,12 @@ private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ug writer.close(); } - private YarnClient createMockYarnClient(YarnApplicationState appState) + private YarnClient createMockYarnClient(YarnApplicationState appState, + String user) throws YarnException, IOException { YarnClient mockClient = mock(YarnClient.class); ApplicationReport mockAppReport = mock(ApplicationReport.class); + doReturn(user).when(mockAppReport).getUser(); doReturn(appState).when(mockAppReport).getYarnApplicationState(); doReturn(mockAppReport).when(mockClient).getApplicationReport( any(ApplicationId.class)); @@ -448,9 +515,9 @@ private YarnClient createMockYarnClientUnknownApp() throws YarnException, } private static class LogsCLIForTest extends LogsCLI { - + private YarnClient yarnClient; - + public LogsCLIForTest(YarnClient yarnClient) { super(); this.yarnClient = yarnClient; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..5385d24 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -56,6 +56,46 @@ public int dumpAContainersLogs(String appId, String containerId, @Private @VisibleForTesting + /** + * Return the owner for a given AppId + * @param remoteRootLogDir + * @param appId + * @param bestGuess + * @param conf + * @return the owner or null + * @throws IOException + */ + public static String getOwnerForAppIdOrNull( + ApplicationId appId, String bestGuess, + Configuration conf) throws IOException { + Path remoteRootLogDir = new Path(conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); + Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, + appId, bestGuess, suffix); + FileContext fc = + FileContext.getFileContext(remoteRootLogDir.toUri(), conf); + if (fc.util().exists(fullPath)) { + return bestGuess; + } + Path toMatch = LogAggregationUtils. + getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching == null || matching.length != 1) { + return null; + } + //fetch the user from the full path /app-logs/user[/suffix]/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + return parent.getName(); + } + + @Private + @VisibleForTesting public int dumpAContainersLogsForALogType(String appId, String containerId, String nodeId, String jobOwner, List logType) throws IOException { Path remoteRootLogDir = new Path(getConf().get(