From 520d095227641f42310fab354a67855f765870f1 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sun, 26 May 2019 19:09:02 +0530 Subject: [PATCH] YARN-9581 --- .../org/apache/hadoop/yarn/client/cli/LogsCLI.java | 42 +++++++++++----- .../hadoop/yarn/client/cli/SchedConfCLI.java | 58 ++++++++++++---------- .../hadoop/yarn/webapp/util/WebAppUtils.java | 14 ++++-- .../yarn/webapp/util/YarnWebServiceUtils.java | 37 ++++++++------ .../hadoop/yarn/conf/TestYarnConfiguration.java | 6 +++ 5 files changed, 100 insertions(+), 57 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 96007f4..bdc6ccd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -79,6 +79,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; @@ -157,6 +158,9 @@ public HttpURLConnection getHttpURLConnection(URL url) if (yarnClient != null) { yarnClient.close(); } + if (webServiceClient != null) { + webServiceClient.destroy(); + } } } @@ -422,20 +426,32 @@ private void printHelpMessage(Options options) { protected List getAMContainerInfoForRMWebService( Configuration conf, String appId) throws ClientHandlerException, UniformInterfaceException, JSONException { - String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); - - WebResource webResource = webServiceClient.resource(webAppAddress); - - ClientResponse response = - webResource.path("ws").path("v1").path("cluster").path("apps") - .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - JSONObject json = - response.getEntity(JSONObject.class).getJSONObject("appAttempts"); - JSONArray requests = json.getJSONArray("appAttempt"); List amContainersList = new ArrayList(); - for (int i = 0; i < requests.length(); i++) { - amContainersList.add(requests.getJSONObject(i)); + int rmIds = HAUtil.isHAEnabled(conf) ? 2 : 1; + for (int i=0; i rmIds = new ArrayList<>(HAUtil.getRMHAIds(conf)); if (rmIds != null && !rmIds.isEmpty()) { - yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(0)); + yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(haIdIndex)); } } } @@ -120,13 +120,19 @@ public static String getRMWebAppURLWithoutScheme(Configuration conf, } } + public static String getRMWebAppURLWithScheme(Configuration conf, + int haIdIndex) { + return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme( + conf, HAUtil.isHAEnabled(conf), haIdIndex); + } + public static String getRMWebAppURLWithScheme(Configuration conf) { return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme( - conf, HAUtil.isHAEnabled(conf)); + conf, HAUtil.isHAEnabled(conf), 0); } public static String getRMWebAppURLWithoutScheme(Configuration conf) { - return getRMWebAppURLWithoutScheme(conf, false); + return getRMWebAppURLWithoutScheme(conf, false, 0); } public static String getRouterWebAppURLWithScheme(Configuration conf) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java index e7bca2c..3e0e27d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java @@ -27,6 +27,7 @@ import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.HAUtil; import org.codehaus.jettison.json.JSONObject; import java.io.StringWriter; @@ -53,23 +54,29 @@ private YarnWebServiceUtils() {} public static JSONObject getNodeInfoFromRMWebService(Configuration conf, String nodeId) throws ClientHandlerException, UniformInterfaceException { - Client webServiceClient = Client.create(); - String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); - - WebResource webResource = webServiceClient.resource(webAppAddress); - - ClientResponse response = null; - try { - response = webResource.path("ws").path("v1").path("cluster") - .path("nodes").path(nodeId).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - return response.getEntity(JSONObject.class); - } finally { - if (response != null) { - response.close(); + int rmIds = HAUtil.isHAEnabled(conf) ? 2 : 1; + for (int i = 0; i < rmIds; i++) { + Client webServiceClient = Client.create(); + ClientResponse response = null; + try { + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, i); + WebResource webResource = webServiceClient.resource(webAppAddress); + response = webResource.path("ws").path("v1").path("cluster") + .path("nodes").path(nodeId).accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + return response.getEntity(JSONObject.class); + } catch (Exception e) { + if (i + 1 == rmIds) { + throw e; + } + } finally { + if (response != null) { + response.close(); + } + webServiceClient.destroy(); } - webServiceClient.destroy(); } + return new JSONObject(); } @SuppressWarnings("rawtypes") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java index a053fdb..212e09c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java @@ -56,6 +56,12 @@ public void testDefaultRMWebUrl() throws Exception { conf2.set("yarn.resourcemanager.hostname.rm2", "40.40.40.40"); String rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2); Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2); + + rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 0); + Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2); + + rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 1); + Assert.assertEquals("http://40.40.40.40:8088", rmWebUrlinHA2); } @Test -- 2.7.4 (Apple Git-66)