From 4f718d215c7f2abdf88a4a704b4e678ddcf5f458 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 29 May 2019 00:48:07 +0530 Subject: [PATCH] YARN-9581 --- .../org/apache/hadoop/yarn/client/cli/LogsCLI.java | 48 ++++++++++++++++------ .../hadoop/yarn/client/cli/SchedConfCLI.java | 22 ++++++++-- .../hadoop/yarn/webapp/util/WebAppUtils.java | 14 +++++-- .../yarn/webapp/util/YarnWebServiceUtils.java | 20 +++++++-- .../hadoop/yarn/conf/TestYarnConfiguration.java | 6 +++ 5 files changed, 86 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 96007f4..f469808 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -79,6 +79,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; @@ -157,6 +158,9 @@ public HttpURLConnection getHttpURLConnection(URL url) if (yarnClient != null) { yarnClient.close(); } + if (webServiceClient != null) { + webServiceClient.destroy(); + } } } @@ -422,22 +426,40 @@ private void printHelpMessage(Options options) { protected List getAMContainerInfoForRMWebService( Configuration conf, String appId) throws ClientHandlerException, UniformInterfaceException, JSONException { - String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); - - WebResource webResource = webServiceClient.resource(webAppAddress); + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 0); + try { + return getAMContainerInfoFromRM(appId, webAppAddress); + } catch (Exception e) { + if (HAUtil.isHAEnabled(conf)) { + webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 1); + return getAMContainerInfoFromRM(appId, webAppAddress); + } + throw e; + } + } - ClientResponse response = - webResource.path("ws").path("v1").path("cluster").path("apps") - .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - JSONObject json = - response.getEntity(JSONObject.class).getJSONObject("appAttempts"); - JSONArray requests = json.getJSONArray("appAttempt"); + private List getAMContainerInfoFromRM( + String appId, String webAppAddress) throws ClientHandlerException, + UniformInterfaceException, JSONException { List amContainersList = new ArrayList(); - for (int i = 0; i < requests.length(); i++) { - amContainersList.add(requests.getJSONObject(i)); + ClientResponse response = null; + try { + WebResource webResource = webServiceClient.resource(webAppAddress); + response = webResource.path("ws").path("v1").path("cluster") + .path("apps").path(appId).path("appattempts") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + JSONObject json = response.getEntity(JSONObject.class) + .getJSONObject("appAttempts"); + JSONArray requests = json.getJSONArray("appAttempt"); + for (int j = 0; j < requests.length(); j++) { + amContainersList.add(requests.getJSONObject(j)); + } + return amContainersList; + } finally { + if (response != null) { + response.close(); + } } - return amContainersList; } private List getAMContainerInfoForAHSWebService( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index a5f3b80..1439fd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -28,8 +28,10 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -131,12 +133,25 @@ public int run(String[] args) throws Exception { return -1; } + Configuration conf = getConf(); + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 0); + try { + return updateSchedulerConfOnRMNode(webAppAddress, updateInfo); + } catch (Exception e) { + if (HAUtil.isHAEnabled(conf)) { + webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 1); + return updateSchedulerConfOnRMNode(webAppAddress, updateInfo); + } + throw e; + } + } + + private int updateSchedulerConfOnRMNode(String webAppAddress, + SchedConfUpdateInfo updateInfo) throws Exception { Client webServiceClient = Client.create(); - WebResource webResource = webServiceClient - .resource(WebAppUtils.getRMWebAppURLWithScheme(getConf())); ClientResponse response = null; - try { + WebResource webResource = webServiceClient.resource(webAppAddress); response = webResource.path("ws").path("v1").path("cluster") .path("scheduler-conf").accept(MediaType.APPLICATION_JSON) @@ -163,6 +178,7 @@ public int run(String[] args) throws Exception { } } + @VisibleForTesting void addQueues(String args, SchedConfUpdateInfo updateInfo) { if (args == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index e62bf10..a926e0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -91,7 +91,7 @@ public static void setNMWebAppHostNameAndPort(Configuration conf, } public static String getRMWebAppURLWithoutScheme(Configuration conf, - boolean isHAEnabled) { + boolean isHAEnabled, int haIdIndex) { YarnConfiguration yarnConfig = new YarnConfiguration(conf); // set RM_ID if we have not configure it. if (isHAEnabled) { @@ -99,7 +99,7 @@ public static String getRMWebAppURLWithoutScheme(Configuration conf, if (rmId == null || rmId.isEmpty()) { List rmIds = new ArrayList<>(HAUtil.getRMHAIds(conf)); if (rmIds != null && !rmIds.isEmpty()) { - yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(0)); + yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(haIdIndex)); } } } @@ -120,13 +120,19 @@ public static String getRMWebAppURLWithoutScheme(Configuration conf, } } + public static String getRMWebAppURLWithScheme(Configuration conf, + int haIdIndex) { + return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme( + conf, HAUtil.isHAEnabled(conf), haIdIndex); + } + public static String getRMWebAppURLWithScheme(Configuration conf) { return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme( - conf, HAUtil.isHAEnabled(conf)); + conf, HAUtil.isHAEnabled(conf), 0); } public static String getRMWebAppURLWithoutScheme(Configuration conf) { - return getRMWebAppURLWithoutScheme(conf, false); + return getRMWebAppURLWithoutScheme(conf, false, 0); } public static String getRouterWebAppURLWithScheme(Configuration conf) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java index e7bca2c..d98559c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java @@ -27,6 +27,7 @@ import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.HAUtil; import org.codehaus.jettison.json.JSONObject; import java.io.StringWriter; @@ -53,13 +54,24 @@ private YarnWebServiceUtils() {} public static JSONObject getNodeInfoFromRMWebService(Configuration conf, String nodeId) throws ClientHandlerException, UniformInterfaceException { - Client webServiceClient = Client.create(); - String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); - - WebResource webResource = webServiceClient.resource(webAppAddress); + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 0); + try { + return getNodeInfoFromRM(webAppAddress, nodeId); + } catch (Exception e) { + if (HAUtil.isHAEnabled(conf)) { + webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf, 1); + return getNodeInfoFromRM(webAppAddress, nodeId); + } + throw e; + } + } + private static JSONObject getNodeInfoFromRM(String webAppAddress, + String nodeId) throws ClientHandlerException, UniformInterfaceException { + Client webServiceClient = Client.create(); ClientResponse response = null; try { + WebResource webResource = webServiceClient.resource(webAppAddress); response = webResource.path("ws").path("v1").path("cluster") .path("nodes").path(nodeId).accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java index a053fdb..212e09c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java @@ -56,6 +56,12 @@ public void testDefaultRMWebUrl() throws Exception { conf2.set("yarn.resourcemanager.hostname.rm2", "40.40.40.40"); String rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2); Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2); + + rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 0); + Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2); + + rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 1); + Assert.assertEquals("http://40.40.40.40:8088", rmWebUrlinHA2); } @Test -- 2.7.4 (Apple Git-66)