diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index edee8ee..0a651de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -43,13 +43,11 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ContainerReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; @@ -208,13 +206,11 @@ public int run(String[] args) throws Exception { LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); - if (appOwner == null || appOwner.isEmpty()) { - appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); - } - YarnApplicationState appState = YarnApplicationState.NEW; + ApplicationReport appReport = null; try { - appState = getApplicationState(appId); + appReport = getApplicationReport(appId); + appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) { @@ -226,6 +222,22 @@ public int run(String[] args) throws Exception { + " Attempting to fetch logs directly from the filesystem."); } + if (appReport != null) { + //always use the app owner from the app report if possible + appOwner = appReport.getUser(); + } + else { + if (appOwner == null || appOwner.isEmpty()) { + appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); + } + appOwner = LogCLIHelpers.getOwnerForAppIdOrNull( + appId, appOwner, getConf()); + if (appOwner == null) { + System.out.println("Could not locate application logs for " + appId); + return -1; + } + } + // To get am logs if (getAMContainerLogs) { // if we do not specify the value for CONTAINER_LOG_FILES option, @@ -339,13 +351,12 @@ public int run(String[] args) throws Exception { return resultCode; } - private YarnApplicationState getApplicationState(ApplicationId appId) + private ApplicationReport getApplicationReport(ApplicationId appId) throws IOException, YarnException { YarnClient yarnClient = createYarnClient(); try { - ApplicationReport appReport = yarnClient.getApplicationReport(appId); - return appReport.getYarnApplicationState(); + return yarnClient.getApplicationReport(appId); } finally { yarnClient.close(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index fb4d3cd..328eada 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -56,6 +56,46 @@ public int dumpAContainersLogs(String appId, String containerId, @Private @VisibleForTesting + /** + * Return the owner for a given AppId + * @param remoteRootLogDir + * @param appId + * @param bestGuess + * @param conf + * @return the owner or null + * @throws IOException + */ + public static String getOwnerForAppIdOrNull( + ApplicationId appId, String bestGuess, + Configuration conf) throws IOException { + Path remoteRootLogDir = new Path(conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); + Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, + appId, bestGuess, suffix); + FileContext fc = + FileContext.getFileContext(remoteRootLogDir.toUri(), conf); + if (fc.util().exists(fullPath)) { + return bestGuess; + } + Path toMatch = LogAggregationUtils. + getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching == null || matching.length != 1) { + return null; + } + //fetch the user from the full path /app-logs/user[/suffix]/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + return parent.getName(); + } + + @Private + @VisibleForTesting public int dumpAContainersLogsForALogType(String appId, String containerId, String nodeId, String jobOwner, List logType) throws IOException { Path remoteRootLogDir = new Path(getConf().get(