diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsService.java new file mode 100644 index 00000000000..fa6972e8e71 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsService.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.IssueType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Utility methods to launch the diagnostic scripts. + */ +@InterfaceAudience.Private +public final class DiagnosticsService { + private static final Logger LOG = LoggerFactory + .getLogger(DiagnosticsService.class); + private static final String LIST_ISSUES_ARGUMENT = "list_issues"; + private static final String COLON = ":"; + private static final String COMMA = ","; + private static final String EXECUTION_ERROR_MESSAGE = "Error occurred " + + "during the execution of the diagnostic script with the argument '{}'."; + private static final String OPTION_ERROR_MESSAGE = "Error while parsing " + + "diagnostic option: {}. Skipping this option"; + + private static String scriptLocation = "/tmp/diagnostics_collector.sh"; + + + private DiagnosticsService() { + // hidden constructor + } + + public static CommonIssues listCommonIssues() throws Exception { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); + } + + CommonIssues result = new CommonIssues(); + + ProcessBuilder pb = new ProcessBuilder(scriptLocation, + LIST_ISSUES_ARGUMENT); + Process process = pb.start(); + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), + StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + result.add(parseIssueType(line)); + } + process.waitFor(); + } catch (Exception e) { + LOG.error(EXECUTION_ERROR_MESSAGE, LIST_ISSUES_ARGUMENT, e); + throw e; + } + + return result; + } + + public static String collectIssueData(String issueId, String params) { + if (Shell.WINDOWS) { + throw new UnsupportedOperationException("Not implemented for Windows."); + } + throw new NotImplementedException("Code is not implemented"); + } + + @VisibleForTesting + protected static IssueType parseIssueType(String line) { + String[] issueParams = line.split(COLON); + IssueType parsedIssueType = null; + + try { + if (issueParams.length < 2 || issueParams.length > 3) { + LOG.warn(OPTION_ERROR_MESSAGE, "incorrect number of parameters"); + } else { + int id = Integer.parseInt(issueParams[0]); + String name = issueParams[1]; + parsedIssueType = new IssueType(id, name); + if (issueParams.length == 3) { + List parameterList = + Arrays.asList(issueParams[2].split(COMMA)); + parsedIssueType.setParameters(parameterList); + } + } + } catch (NumberFormatException e) { + LOG.warn(OPTION_ERROR_MESSAGE, "id is not a number"); + } + + return parsedIssueType; + } + + @VisibleForTesting + protected static void setScriptLocation(String scriptLocationParam) { + scriptLocation = scriptLocationParam; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 82ceed37c2d..1642e8e9017 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -42,6 +42,13 @@ /** Path for {@code RMWebServiceProtocol#getClusterMetricsInfo}. */ public static final String METRICS = "/metrics"; + /** Path for {@code RMWebServices#getCommonIssueList}. */ + public static final String COMMON_ISSUE_LIST = "/common-issues/list"; + + /** Path for {@code RMWebServices#getCommonIssueData}. */ + public static final String COMMON_ISSUE_COLLECT = + "/common-issues/collect"; + /** Path for {@code RMWebServiceProtocol#getSchedulerInfo}. */ public static final String SCHEDULER = "/scheduler"; @@ -213,6 +220,8 @@ // ----------------QueryParams for RMWebServiceProtocol---------------- + public static final String ISSUEID = "issueId"; + public static final String ISSUEPARAMS = "params"; public static final String TIME = "time"; public static final String STATES = "states"; public static final String NODEID = "nodeId"; @@ -279,4 +288,4 @@ private RMWSConsts() { public enum AppActivitiesRequiredAction { REFRESH, GET } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index f2736e3773c..f096f4b9ac3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -110,6 +111,25 @@ */ ClusterMetricsInfo getClusterMetricsInfo(); + /** + * This method retrieves the common diagnosable issue list, and it is + * reachable by using {@link RMWSConsts#COMMON_ISSUE_LIST}. + * + * @return the list of available diagnostic cases + */ + CommonIssues getCommonIssueList(); + + /** + * This method retrieves the diagnostic information for the selected issue, + * and it is reachable by using {@link RMWSConsts#COMMON_ISSUE_COLLECT}. + * + * @param issueId the selected issue's ID. It is a FormParam. + * @param params the necessary parameters for diagnosing the issue. + * It is a FormParam. + * @return the associated diagnostic information to the selected issue + */ + Response getCommonIssueData(String issueId, String params); + /** * This method retrieves the current scheduler status, and it is reachable by * using {@link RMWSConsts#SCHEDULER}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 7c4e5df5bc7..f6fda01b023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -132,6 +132,7 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.AdminService; +import org.apache.hadoop.yarn.server.resourcemanager.DiagnosticsService; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -171,6 +172,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; @@ -212,6 +214,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -385,6 +388,42 @@ public ClusterMetricsInfo getClusterMetricsInfo() { return new ClusterMetricsInfo(this.rm); } + @GET + @Path(RMWSConsts.COMMON_ISSUE_LIST) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public CommonIssues getCommonIssueList() { + initForReadableEndpoints(); + try { + return DiagnosticsService.listCommonIssues(); + } catch (Exception e) { + throw new WebAppException("Error collecting the common " + + "issue types. Error message: " + e.getMessage() + ". " + + "For more information please check the ResourceManager logs."); + } + } + + @GET + @Path(RMWSConsts.COMMON_ISSUE_COLLECT) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public Response getCommonIssueData( + @QueryParam(RMWSConsts.ISSUEID) String issueId, + @QueryParam(RMWSConsts.ISSUEPARAMS) String params) { + initForReadableEndpoints(); + try { + return Response.status(Status.OK) + .entity(DiagnosticsService.collectIssueData(issueId, params)) + .build(); + } catch (Exception e) { + throw new WebAppException("Error collecting the selected " + + "issue data. Error message: " + e.getMessage() + ". " + + "For more information please check the ResourceManager logs."); + } + } + @GET @Path(RMWSConsts.SCHEDULER) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CommonIssues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CommonIssues.java new file mode 100644 index 00000000000..67ac8546dbd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CommonIssues.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@XmlRootElement(name = "commonIssues") +@XmlAccessorType(XmlAccessType.FIELD) +public class CommonIssues { + private List issue = new ArrayList<>(); + @XmlTransient + private Set idSet = new HashSet<>(); + + public CommonIssues() {} + + public void add(IssueType type) { + if (type != null && !idSet.contains(type.getId())) { + idSet.add(type.getId()); + issue.add(type); + } + } + + public List getIssueList() { + return issue; + } + + public void addAll(List issueTypes) { + issue.addAll(issueTypes); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/IssueType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/IssueType.java new file mode 100644 index 00000000000..38b4a56af8e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/IssueType.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@XmlRootElement(name = "issue") +@XmlAccessorType(XmlAccessType.FIELD) +public class IssueType { + private int id; + private String name; + private List parameters = new ArrayList<>(); + + public IssueType() {} + + public IssueType(int id, String name) { + this.id = id; + this.name = name; + this.parameters = Collections.emptyList(); + } + + public IssueType(int id, String name, List parameters) { + this.id = id; + this.name = name; + this.parameters = new ArrayList<>(parameters); + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getParameters() { + return parameters; + } + + public void setParameters(List parameters) { + this.parameters = parameters; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/diagnostics/diagnostics_collector.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/diagnostics/diagnostics_collector.sh new file mode 100755 index 00000000000..01983036755 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/diagnostics/diagnostics_collector.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ $1 == "list_issues" ]] +then + echo "1:Application Failed:appId" + echo "2:Application Hanging:appId" + echo "3:Scheduler Related Issue" + echo "4:RM failure to start:nodeId" + echo "5:NM failure to start:nodeId" +elif [[ $1 == "collect" ]] +then + echo "collect" +fi diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsServiceTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsServiceTest.java new file mode 100644 index 00000000000..fda352f45c5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/DiagnosticsServiceTest.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + + +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.IssueType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +public class DiagnosticsServiceTest { + private static final String ISSUE_ID_1 = "1"; + private static final String ISSUE_ID_2 = "2"; + private static final String ISSUE_ID_3 = "3"; + private static final String ISSUE_ID_4 = "4"; + private static final String ISSUE_NAME = "Application Failed"; + private static final String ISSUE_PARAMETER_1 = "appId"; + private static final String ISSUE_PARAMETER_2 = "containerId"; + private static final String COLON = ":"; + private static final String COMMA = ","; + + + @Before + public void setUp() { + DiagnosticsService.setScriptLocation("src/test/resources/diagnostics" + + "/diagnostics_collector.sh"); + } + + @Test + public void testListCommonIssuesValidCaseWithOptionsToBeSkipped() + throws Exception { + handleWindowsRuntime(); + + // The test script contains two invalid options: one with an ambigious id + // and one with too many parameters. These should be skipped silently. + CommonIssues commonIssues = DiagnosticsService.listCommonIssues(); + + Assert.assertEquals(4, commonIssues.getIssueList().size()); + Assert.assertEquals(Integer.parseInt(ISSUE_ID_1), + commonIssues.getIssueList().get(0).getId()); + Assert.assertEquals(ISSUE_NAME, + commonIssues.getIssueList().get(0).getName()); + Assert.assertTrue(commonIssues.getIssueList().get(0) + .getParameters().isEmpty()); + + Assert.assertEquals(Integer.parseInt(ISSUE_ID_2), + commonIssues.getIssueList().get(1).getId()); + Assert.assertEquals(ISSUE_NAME, + commonIssues.getIssueList().get(1).getName()); + Assert.assertEquals(1, + commonIssues.getIssueList().get(1).getParameters().size()); + Assert.assertEquals(ISSUE_PARAMETER_1, + commonIssues.getIssueList().get(1).getParameters().get(0)); + + Assert.assertEquals(Integer.parseInt(ISSUE_ID_3), + commonIssues.getIssueList().get(2).getId()); + Assert.assertEquals(ISSUE_NAME, + commonIssues.getIssueList().get(2).getName()); + Assert.assertEquals(1, + commonIssues.getIssueList().get(2).getParameters().size()); + Assert.assertEquals(ISSUE_PARAMETER_2, + commonIssues.getIssueList().get(2).getParameters().get(0)); + + Assert.assertEquals(Integer.parseInt(ISSUE_ID_4), + commonIssues.getIssueList().get(3).getId()); + Assert.assertEquals(ISSUE_NAME, + commonIssues.getIssueList().get(3).getName()); + Assert.assertEquals(2, + commonIssues.getIssueList().get(3).getParameters().size()); + Assert.assertTrue(commonIssues.getIssueList().get(3).getParameters() + .contains(ISSUE_PARAMETER_1)); + Assert.assertTrue(commonIssues.getIssueList().get(3).getParameters() + .contains(ISSUE_PARAMETER_2)); + + } + + @Test(expected = IOException.class) + public void testListCommonIssuesScriptMissing() throws Exception { + handleWindowsRuntime(); + DiagnosticsService.setScriptLocation("/src/invalidLocation/script.sh"); + DiagnosticsService.listCommonIssues(); + } + + @Test + public void testParseIssueTypeValidCases() { + // valid case: id, name, no parameters + String line = ISSUE_ID_1 + COLON + ISSUE_NAME; + + IssueType issueType = DiagnosticsService.parseIssueType(line); + Assert.assertEquals(Integer.parseInt(ISSUE_ID_1), issueType.getId()); + Assert.assertEquals(ISSUE_NAME, issueType.getName()); + Assert.assertTrue(issueType.getParameters().isEmpty()); + + // valid case: id, name, one parameter + line = ISSUE_ID_1 + COLON + ISSUE_NAME + COLON + ISSUE_PARAMETER_1; + + issueType = DiagnosticsService.parseIssueType(line); + Assert.assertEquals(Integer.parseInt(ISSUE_ID_1), issueType.getId()); + Assert.assertEquals(ISSUE_NAME, issueType.getName()); + Assert.assertEquals(1, issueType.getParameters().size()); + Assert.assertEquals(ISSUE_PARAMETER_1, issueType.getParameters().get(0)); + + // valid case: id, name, one parameter + line = ISSUE_ID_1 + COLON + ISSUE_NAME + COLON + ISSUE_PARAMETER_1 + + COMMA + ISSUE_PARAMETER_2; + + issueType = DiagnosticsService.parseIssueType(line); + Assert.assertEquals(Integer.parseInt(ISSUE_ID_1), issueType.getId()); + Assert.assertEquals(ISSUE_NAME, issueType.getName()); + Assert.assertEquals(2, issueType.getParameters().size()); + Assert.assertEquals(ISSUE_PARAMETER_1, issueType.getParameters().get(0)); + Assert.assertEquals(ISSUE_PARAMETER_2, issueType.getParameters().get(1)); + } + + @Test + public void testParseIssueTypeInvalidCases() { + // invalid case: too few values + String line = ISSUE_NAME; + + IssueType issueType = DiagnosticsService.parseIssueType(line); + Assert.assertNull(issueType); + + // invalid case: correct number of values, id is not a number + line = ISSUE_NAME + COLON + ISSUE_NAME; + + issueType = DiagnosticsService.parseIssueType(line); + Assert.assertNull(issueType); + + // invalid case: too many values + line = ISSUE_ID_1 + COLON + ISSUE_NAME + COLON + ISSUE_NAME + + COLON + ISSUE_NAME; + + issueType = DiagnosticsService.parseIssueType(line); + Assert.assertNull(issueType); + } + + private void handleWindowsRuntime() { + if (Shell.WINDOWS) { + try { + DiagnosticsService.listCommonIssues(); + fail("On Windows listCommonIssues should throw " + + "UnsupportedOperationException"); + } catch (Exception e) { + // Exception is expected + } + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/diagnostics/diagnostics_collector.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/diagnostics/diagnostics_collector.sh new file mode 100755 index 00000000000..7cf115e0cc4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/diagnostics/diagnostics_collector.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ $1 == "list_issues" ]] +then + echo "1:Application Failed" + echo "2:Application Failed:appId" + echo "3:Application Failed:containerId" + echo "4:Application Failed:appId,containerId" + echo "4:NM failure to start:nodeId" # invalid type: id is not unique + echo "5:NM failure to start:nodeId:containerId" # invalid type: too many values +elif [[ $1 == "collect" ]] +then + echo "collect method" +fi diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 00a8beb6684..0ab57a9a510 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -128,6 +129,22 @@ public ClusterMetricsInfo getClusterMetricsInfo() { getConf()); } + @Override + public CommonIssues getCommonIssueList() { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + CommonIssues.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.COMMON_ISSUE_LIST, + null, null, getConf()); + } + + @Override + public Response getCommonIssueData(String issueId, String params) { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + Response.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.COMMON_ISSUE_LIST, + null, null, getConf()); + } + @Override public SchedulerTypeInfo getSchedulerInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, @@ -575,4 +592,4 @@ public Response signalToContainer(String containerId, String command, + containerId + "/" + RMWSConsts.SIGNAL + "/" + command, null, null, getConf()); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index ab97b1a7fd9..ff2f72beddf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -1131,6 +1132,16 @@ public ClusterUserInfo getClusterUserInfo(HttpServletRequest hsr) { throw new NotImplementedException("Code is not implemented"); } + @Override + public CommonIssues getCommonIssueList() { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public Response getCommonIssueData(String issueId, String params) { + throw new NotImplementedException("Code is not implemented"); + } + @Override public SchedulerTypeInfo getSchedulerInfo() { throw new NotImplementedException("Code is not implemented"); @@ -1366,4 +1377,4 @@ public void shutdown() { threadpool.shutdown(); } } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index bde46484d6b..fc1faef6525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -356,6 +357,30 @@ public ClusterMetricsInfo getClusterMetricsInfo() { return pipeline.getRootInterceptor().getClusterMetricsInfo(); } + @GET + @Path(RMWSConsts.COMMON_ISSUE_LIST) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public CommonIssues getCommonIssueList() { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(null); + return pipeline.getRootInterceptor().getCommonIssueList(); + } + + @GET + @Path(RMWSConsts.COMMON_ISSUE_COLLECT) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public Response getCommonIssueData( + @QueryParam(RMWSConsts.ISSUEID) String issueId, + @QueryParam(RMWSConsts.ISSUEPARAMS) String params) { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(null); + return pipeline.getRootInterceptor().getCommonIssueData(issueId, params); + } + @GET @Path(RMWSConsts.SCHEDULER) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index 67c9d671fb1..59909de289c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -96,6 +97,16 @@ public ClusterMetricsInfo getClusterMetricsInfo() { return new ClusterMetricsInfo(); } + @Override + public CommonIssues getCommonIssueList() { + return new CommonIssues(); + } + + @Override + public Response getCommonIssueData(String issueId, String params) { + return Response.ok().build(); + } + @Override public SchedulerTypeInfo getSchedulerInfo() { return new SchedulerTypeInfo(); @@ -373,4 +384,4 @@ public Response signalToContainer(String containerId, String command, HttpServletRequest req) { return Response.status(Status.OK).build(); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 142a6511b93..0e7f0ea0831 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CommonIssues; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -121,6 +122,16 @@ public ClusterMetricsInfo getClusterMetricsInfo() { return getNextInterceptor().getClusterMetricsInfo(); } + @Override + public CommonIssues getCommonIssueList() { + return getNextInterceptor().getCommonIssueList(); + } + + @Override + public Response getCommonIssueData(String issueId, String params) { + return getNextInterceptor().getCommonIssueData(issueId, params); + } + @Override public SchedulerTypeInfo getSchedulerInfo() { return getNextInterceptor().getSchedulerInfo();