From 4d216ec69a2dcf00875380b190e791d960c34fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Hud=C3=A1ky?= Date: Thu, 25 Jun 2020 02:49:09 +0200 Subject: [PATCH] YARN-10106.004 --- .../hadoop/yarn/client/cli/LogsCLI.java | 42 ++++-- .../hadoop/yarn/client/cli/TestLogsCLI.java | 126 +++++++++++++++--- 2 files changed, 143 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 4d67ce8178d..fb312979636 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -65,6 +65,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -95,6 +96,7 @@ private static final String CONTAINER_ID_OPTION = "containerId"; private static final String APPLICATION_ID_OPTION = "applicationId"; + private static final String APPLICATION_ATTEMPT_ID_OPTION = "applicationAttemptId"; private static final String CLUSTER_ID_OPTION = "clusterId"; private static final String NODE_ADDRESS_OPTION = "nodeAddress"; private static final String APP_OWNER_OPTION = "appOwner"; @@ -160,6 +162,7 @@ private int runCommand(String[] args) throws Exception { } CommandLineParser parser = new GnuParser(); String appIdStr = null; + String appAttemptIdStr = null; String clusterIdStr = null; String containerIdStr = null; String nodeAddress = null; @@ -180,6 +183,7 @@ private int runCommand(String[] args) throws Exception { try { CommandLine commandLine = parser.parse(opts, args, false); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); + appAttemptIdStr = commandLine.getOptionValue(APPLICATION_ATTEMPT_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); @@ -240,8 +244,8 @@ private int runCommand(String[] args) throws Exception { return -1; } - if (appIdStr == null && containerIdStr == null) { - System.err.println("Both applicationId and containerId are missing, " + if (appIdStr == null && appAttemptIdStr == null && containerIdStr == null) { + System.err.println("None of applicationId, appAttemptId and containerId is available, " + " one of them must be specified."); printHelpMessage(printOpts); return -1; @@ -252,7 +256,23 @@ private int runCommand(String[] args) throws Exception { try { appId = ApplicationId.fromString(appIdStr); } catch (Exception e) { - System.err.println("Invalid ApplicationId specified"); + System.err.println("Invalid Application ID specified"); + return -1; + } + } + + ApplicationAttemptId appAttemptId = null; + if (appAttemptIdStr != null) { + try { + appAttemptId = ApplicationAttemptId.fromString(appAttemptIdStr); + if (appId == null) { + appId = appAttemptId.getApplicationId(); + } else if (!appId.equals(appAttemptId.getApplicationId())) { + System.err.println("Specified application attempt is not belonging to the specified application."); + return -1; + } + } catch (Exception e) { + System.err.println("Invalid application attempt ID specified"); return -1; } } @@ -260,16 +280,19 @@ private int runCommand(String[] args) throws Exception { if (containerIdStr != null) { try { ContainerId containerId = ContainerId.fromString(containerIdStr); + if (appAttemptId != null && !appAttemptId.equals(containerId.getApplicationAttemptId())) { + System.err.println("Specified container is not belonging to the specified application attempt."); + return -1; + } if (appId == null) { appId = containerId.getApplicationAttemptId().getApplicationId(); } else if (!containerId.getApplicationAttemptId().getApplicationId() - .equals(appId)) { - System.err.println("The Application:" + appId - + " does not have the container:" + containerId); + .equals(appId)) { + System.err.println("Specified container is not belonging to the specified application."); return -1; } } catch (Exception e) { - System.err.println("Invalid ContainerId specified"); + System.err.println("Invalid Container ID specified"); return -1; } } @@ -344,7 +367,7 @@ private int runCommand(String[] args) throws Exception { } - ContainerLogsRequest request = new ContainerLogsRequest(appId, null, + ContainerLogsRequest request = new ContainerLogsRequest(appId, appAttemptId, Apps.isApplicationFinalState(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null); @@ -913,6 +936,9 @@ private Options createCommandOpts() { Option appIdOpt = new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); opts.addOption(appIdOpt); + opts.addOption(APPLICATION_ATTEMPT_ID_OPTION, true, "ApplicationAttemptId. " + + "Lists all logs belonging to the specified application attempt Id. " + + "If specified, the applicationId can be omitted"); opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. " + "By default, it will print all available logs." + " Work with -log_files to get only specific logs. If specified, the" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 24256a01471..9438aa41609 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -238,6 +238,22 @@ public void testUnknownApplicationId() throws Exception { "Unable to get ApplicationState")); } + @Test(timeout = 5000l) + public void testUnknownApplicationAttemptId() throws Exception { + YarnClient mockYarnClient = createMockYarnClientUnknownApp(); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(conf); + ApplicationId appId = ApplicationId.newInstance(0, 1); + + int exitCode = cli.run(new String[] { "-applicationAttemptId", + ApplicationAttemptId.newInstance(appId, 1).toString() }); + + // Error since no logs present for the app. + assertTrue(exitCode != 0); + assertTrue(sysErrStream.toString().contains( + "Unable to get ApplicationState.")); + } + @Test (timeout = 10000) public void testHelpMessage() throws Exception { YarnClient mockYarnClient = createMockYarnClient( @@ -372,12 +388,14 @@ public void testFetchFinishedApplictionLogs() throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ApplicationId appId = ApplicationId.newInstance(0, 1); - ApplicationAttemptId appAttemptId = + ApplicationAttemptId appAttemptId1 = ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0); - ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); - ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2); - ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3); + ApplicationAttemptId appAttemptId2 = + ApplicationAttemptId.newInstance(appId, 2); + ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); + ContainerId containerId1 = ContainerId.newContainerId(appAttemptId1, 1); + ContainerId containerId2 = ContainerId.newContainerId(appAttemptId1, 2); + ContainerId containerId3 = ContainerId.newContainerId(appAttemptId2, 3); final NodeId nodeId = NodeId.newInstance("localhost", 1234); // create local logs @@ -464,6 +482,43 @@ public ContainerReport getContainerReport(String containerIdStr) createEmptyLog("empty"))); sysOutStream.reset(); + // Check fetching data for application attempt with applicationId defined + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-applicationAttemptId", appAttemptId1.toString()}); + LOG.info(sysOutStream.toString()); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + assertTrue(sysOutStream.toString().contains( + createEmptyLog("empty"))); + sysOutStream.reset(); + + // Check fetching data for application attempt without application defined + exitCode = cli.run(new String[] {"-applicationAttemptId", appAttemptId1.toString()}); + LOG.info(sysOutStream.toString()); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId1, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId2, "syslog"))); + assertTrue(sysOutStream.toString().contains( + logMessage(containerId3, "syslog"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout"))); + assertFalse(sysOutStream.toString().contains( + logMessage(containerId3, "stdout1234"))); + assertTrue(sysOutStream.toString().contains( + createEmptyLog("empty"))); + sysOutStream.reset(); + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files_pattern", ".*"}); assertTrue(exitCode == 0); @@ -742,7 +797,7 @@ public ContainerReport getContainerReport(String containerIdStr) exitCode = cli.run(new String[] { "-containerId", "invalid_container" }); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Invalid ContainerId specified")); + "Invalid Container ID specified")); sysErrStream.reset(); fs.delete(new Path(remoteLogRootDir), true); @@ -979,6 +1034,8 @@ public void testFetchRunningApplicationLogs() throws Exception { any(ContainerLogsRequest.class)); cli2.setConf(new YarnConfiguration()); ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); + System.out.println(containerId100.toString()); + System.out.println(appId.toString()); exitCode = cli2.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId100.toString(), "-nodeAddress", "NM:1234"}); assertTrue(exitCode == 0); @@ -1143,7 +1200,13 @@ public void testLogsCLIWithInvalidArgs() throws Exception { String localDir = "target/SaveLogs"; Path localPath = new Path(localDir); FileSystem fs = FileSystem.get(conf); - ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + ApplicationId appId2 = ApplicationId.newInstance(0, 2); + ApplicationAttemptId appAttemptId1 = + ApplicationAttemptId.newInstance(appId1, 1); + ApplicationAttemptId appAttemptId2 = + ApplicationAttemptId.newInstance(appId2, 1); + ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); @@ -1151,24 +1214,53 @@ public void testLogsCLIWithInvalidArgs() throws Exception { cli.setConf(conf); // Specify an invalid applicationId - int exitCode = cli.run(new String[] {"-applicationId", - "123"}); + int exitCode = cli.run(new String[] {"-applicationId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Invalid ApplicationId specified")); + "Invalid Application ID specified")); + sysErrStream.reset(); + + // Specify an invalid appAttemptId + exitCode = cli.run(new String[] {"-applicationAttemptId", "123"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Invalid application attempt ID specified")); sysErrStream.reset(); // Specify an invalid containerId - exitCode = cli.run(new String[] {"-containerId", - "123"}); + exitCode = cli.run(new String[] {"-containerId", "123"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Invalid Container ID specified")); + sysErrStream.reset(); + + // Non-matching applicationId and applicationAttemptId + exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), + "-applicationAttemptId", appAttemptId1.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Specified application attempt is not belonging to the specified application")); + sysErrStream.reset(); + + // Non-matching applicationId and containerId + exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), + "-containerId", containerId0.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( - "Invalid ContainerId specified")); + "Specified container is not belonging to the specified application")); + sysErrStream.reset(); + + // Non-matching applicationAttemptId and containerId + exitCode = cli.run(new String[] {"-applicationAttemptId", appAttemptId2.toString(), + "-containerId", containerId0.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Specified container is not belonging to the specified application attempt")); sysErrStream.reset(); // Specify show_container_log_info and show_application_log_info // at the same time - exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-show_container_log_info", "-show_application_log_info"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " @@ -1178,7 +1270,7 @@ public void testLogsCLIWithInvalidArgs() throws Exception { // Specify log_files and log_files_pattern // at the same time - exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-log_files", "*", "-log_files_pattern", ".*"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " @@ -1194,7 +1286,7 @@ public void testLogsCLIWithInvalidArgs() throws Exception { fs.createNewFile(tmpFilePath); } exitCode = cli.run(new String[] {"-applicationId", - appId.toString(), + appId1.toString(), "-out" , tmpFilePath.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( @@ -1452,7 +1544,7 @@ public void testPrintContainerLogMetadata() throws Exception { "-show_container_log_info", "-nodeAddress", "localhost", "-containerId", "container_1234"}); assertTrue(sysErrStream.toString().contains( - "Invalid ContainerId specified")); + "Invalid Container ID specified")); sysErrStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), -- 2.24.1 (Apple Git-126)