diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index cf0c5e97d8d..5fa3ea9a0c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3956,6 +3956,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + /** + * Connection and Read timeout from the Router to RM. + */ + public static final String ROUTER_WEBAPP_CONNECT_TIMEOUT = + ROUTER_WEBAPP_PREFIX + "connect-timeout"; + public static final long DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT = + TimeUnit.SECONDS.toMillis(30); + public static final String ROUTER_WEBAPP_READ_TIMEOUT = + ROUTER_WEBAPP_PREFIX + "read-timeout"; + public static final long DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT = + TimeUnit.SECONDS.toMillis(30); + //////////////////////////////// // CSI Volume configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 6f781fa7c0d..9fda8094c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -183,6 +183,10 @@ public void initializeMemberVariables() { .add(YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY); configurationPrefixToSkipCompare .add(YarnConfiguration.ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED); + configurationPrefixToSkipCompare + .add(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT); // Set by container-executor.cfg configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java index 1834d7987da..a8a6e6bbac9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.webapp; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -49,10 +50,13 @@ protected void render(Block html) { Configuration conf = this.router.getConfig(); String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); + Client client = RouterWebServiceUtil.createJerseyClient(conf); - ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward( - webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, conf); + ClusterMetricsInfo metrics = RouterWebServiceUtil + .genericForward(webAppAddress, null, ClusterMetricsInfo.class, + HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + conf, client); boolean isEnabled = conf.getBoolean( YarnConfiguration.FEDERATION_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java index 921b859f2e3..87f20c81bb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -53,10 +54,12 @@ protected void render(Block html) { // Get the applications from the Resource Managers Configuration conf = this.router.getConfig(); + Client client = RouterWebServiceUtil.createJerseyClient(conf); String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); - AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null, - AppsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf); + AppsInfo apps = RouterWebServiceUtil + .genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf, + client); setTitle("Applications"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 00a8beb6684..f26cf190225 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Response; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -78,6 +79,10 @@ private String webAppAddress; private SubClusterId subClusterId = null; + // It is very expensive to create the client + // Jersey will spawn a thread for every client request + private Client client = null; + public void setWebAppAddress(String webAppAddress) { this.webAppAddress = webAppAddress; } @@ -109,7 +114,7 @@ public ClusterInfo getClusterInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, ClusterInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, null, null, - getConf()); + getConf(), client); } @Override @@ -117,7 +122,7 @@ public ClusterUserInfo getClusterUserInfo(HttpServletRequest hsr) { return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ClusterUserInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.CLUSTER_USER_INFO, null, - null, getConf()); + null, getConf(), client); } @Override @@ -125,7 +130,7 @@ public ClusterMetricsInfo getClusterMetricsInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, - getConf()); + getConf(), client); } @Override @@ -133,7 +138,7 @@ public SchedulerTypeInfo getSchedulerInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, SchedulerTypeInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER, null, null, - getConf()); + getConf(), client); } @Override @@ -143,7 +148,7 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, null, String.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_LOGS, null, null, - getConf()); + getConf(), client); } @Override @@ -156,7 +161,7 @@ public NodesInfo getNodes(String states) { return RouterWebServiceUtil.genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, - additionalParam, getConf()); + additionalParam, getConf(), client); } @Override @@ -164,7 +169,7 @@ public NodeInfo getNode(String nodeId) { return RouterWebServiceUtil.genericForward(webAppAddress, null, NodeInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId, null, - null, getConf()); + null, getConf(), client); } @Override @@ -172,9 +177,10 @@ public ResourceInfo updateNodeResource(HttpServletRequest hsr, String nodeId, ResourceOptionInfo resourceOption) { final String nodePath = RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId; - return RouterWebServiceUtil.genericForward(webAppAddress, hsr, - ResourceInfo.class, HTTPMethods.POST, - nodePath + "/resource", resourceOption, null, getConf()); + return RouterWebServiceUtil + .genericForward(webAppAddress, hsr, ResourceInfo.class, + HTTPMethods.POST, nodePath + "/resource", resourceOption, null, + getConf(), client); } @Override @@ -187,7 +193,7 @@ public AppsInfo getApps(HttpServletRequest hsr, String stateQuery, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, - getConf()); + getConf(), client); } @Override @@ -197,7 +203,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ActivitiesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_ACTIVITIES, null, - null, getConf()); + null, getConf(), client); } @Override @@ -206,7 +212,7 @@ public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, BulkActivitiesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_BULK_ACTIVITIES, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -218,7 +224,7 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppActivitiesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_APP_ACTIVITIES, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -228,7 +234,7 @@ public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ApplicationStatisticsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APP_STATISTICS, null, null, - getConf()); + getConf(), client); } @Override @@ -238,7 +244,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId, null, - null, getConf()); + null, getConf(), client); } @Override @@ -247,7 +253,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppState.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -257,7 +263,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, - targetState, null, getConf()); + targetState, null, getConf(), client); } @Override @@ -266,7 +272,7 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeToLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_TO_LABELS, null, - null, getConf()); + null, getConf(), client); } @Override @@ -281,7 +287,7 @@ public LabelsToNodesInfo getLabelsToNodes(Set labels) return RouterWebServiceUtil.genericForward(webAppAddress, null, LabelsToNodesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.LABEL_MAPPINGS, null, - additionalParam, getConf()); + additionalParam, getConf(), client); } @Override @@ -290,7 +296,7 @@ public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REPLACE_NODE_TO_LABELS, - newNodeToLabels, null, getConf()); + newNodeToLabels, null, getConf(), client); } @Override @@ -301,7 +307,7 @@ public Response replaceLabelsOnNode(Set newNodeLabelsName, .genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId + "/replace-labels", - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -310,7 +316,7 @@ public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_LABELS, null, - null, getConf()); + null, getConf(), client); } @Override @@ -319,7 +325,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.ADD_NODE_LABELS, - newNodeLabels, null, getConf()); + newNodeLabels, null, getConf(), client); } @Override @@ -329,7 +335,7 @@ public Response removeFromCluserNodeLabels(Set oldNodeLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REMOVE_NODE_LABELS, null, - null, getConf()); + null, getConf(), client); } @Override @@ -338,7 +344,7 @@ public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId + "/get-labels", - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -347,7 +353,7 @@ public AppPriority getAppPriority(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppPriority.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -357,7 +363,7 @@ public Response updateApplicationPriority(AppPriority targetPriority, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, - targetPriority, null, getConf()); + targetPriority, null, getConf(), client); } @Override @@ -366,7 +372,7 @@ public AppQueue getAppQueue(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppQueue.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -376,7 +382,7 @@ public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, - targetQueue, null, getConf()); + targetQueue, null, getConf(), client); } @Override @@ -385,7 +391,7 @@ public Response createNewApplication(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS_NEW_APPLICATION, null, - null, getConf()); + null, getConf(), client); } @Override @@ -395,7 +401,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, newApp, null, - getConf()); + getConf(), client); } @Override @@ -405,7 +411,7 @@ public Response postDelegationToken(DelegationToken tokenData, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, tokenData, - null, getConf()); + null, getConf(), client); } @Override @@ -415,7 +421,7 @@ public Response postDelegationTokenExpiration(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN_EXPIRATION, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -425,7 +431,7 @@ public Response cancelDelegationToken(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.DELETE, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, null, - null, getConf()); + null, getConf(), client); } @Override @@ -434,7 +440,7 @@ public Response createNewReservation(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_NEW, null, - null, getConf()); + null, getConf(), client); } @Override @@ -444,7 +450,7 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_SUBMIT, - resContext, null, getConf()); + resContext, null, getConf(), client); } @Override @@ -454,7 +460,7 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_UPDATE, - resContext, null, getConf()); + resContext, null, getConf(), client); } @Override @@ -464,7 +470,7 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_DELETE, - resContext, null, getConf()); + resContext, null, getConf(), client); } @Override @@ -476,7 +482,7 @@ public Response listReservation(String queue, String reservationId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_LIST, null, - null, getConf()); + null, getConf(), client); } @Override @@ -486,7 +492,7 @@ public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, .genericForward(webAppAddress, hsr, AppTimeoutInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS + "/" + type, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -495,7 +501,7 @@ public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppTimeoutsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -505,7 +511,7 @@ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUT, - appTimeout, null, getConf()); + appTimeout, null, getConf(), client); } @Override @@ -513,7 +519,7 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppAttemptsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -522,7 +528,7 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, RMQueueAclInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.QUEUES + "/" + queue - + "/access", null, null, getConf()); + + "/access", null, null, getConf(), client); } @Override @@ -532,7 +538,7 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req, AppAttemptInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -543,7 +549,7 @@ public ContainersInfo getContainers(HttpServletRequest req, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + RMWSConsts.CONTAINERS, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -555,7 +561,7 @@ public ContainerInfo getContainer(HttpServletRequest req, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + RMWSConsts.CONTAINERS + "/" + containerId, - null, null, getConf()); + null, null, getConf(), client); } @Override @@ -573,6 +579,6 @@ public Response signalToContainer(String containerId, String command, .genericForward(webAppAddress, req, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + "/" + RMWSConsts.CONTAINERS + "/" + containerId + "/" + RMWSConsts.SIGNAL + "/" + command, null, - null, getConf()); + null, getConf(), client); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index c3ab511d6fb..4734cf6bbf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.webapp; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; @@ -53,10 +54,12 @@ protected void render(Block html) { // Get the node info from the federation Configuration conf = this.router.getConfig(); + Client client = RouterWebServiceUtil.createJerseyClient(conf); String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); - NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null, - NodesInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf); + NodesInfo nodes = RouterWebServiceUtil + .genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf, + client); setTitle("Nodes"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index b4c09497bde..003aa627186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.HttpHeaders; @@ -92,13 +93,14 @@ private RouterWebServiceUtil() { * @param formParam the form parameters as input for a specific REST call * @param additionalParam the query parameters as input for a specific REST * call in case the call has no servlet request + * @param client same client used to reduce number of clients created * @return the retrieved entity from the REST call */ - protected static T genericForward( - final String webApp, final HttpServletRequest hsr, - final Class returnType, final HTTPMethods method, - final String targetPath, final Object formParam, - final Map additionalParam, Configuration conf) { + protected static T genericForward(final String webApp, + final HttpServletRequest hsr, final Class returnType, + final HTTPMethods method, final String targetPath, final Object formParam, + final Map additionalParam, Configuration conf, + Client client) { UserGroupInformation callerUGI = null; @@ -130,26 +132,34 @@ public T run() { paramMap = additionalParam; } - ClientResponse response = RouterWebServiceUtil.invokeRMWebService( - webApp, targetPath, method, - (hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam, - getMediaTypeFromHttpServletRequest(hsr, returnType), conf); + ClientResponse response = RouterWebServiceUtil + .invokeRMWebService(webApp, targetPath, method, + (hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam, + getMediaTypeFromHttpServletRequest(hsr, returnType), conf, + client); if (Response.class.equals(returnType)) { return (T) RouterWebServiceUtil.clientResponseToResponse(response); } - // YARN RM can answer with Status.OK or it throws an exception - if (response.getStatus() == SC_OK) { - return response.getEntity(returnType); - } - if (response.getStatus() == SC_NO_CONTENT) { - try { - return returnType.getConstructor().newInstance(); - } catch (RuntimeException | ReflectiveOperationException e) { - LOG.error("Cannot create empty entity for {}", returnType, e); + + try { + // YARN RM can answer with Status.OK or it throws an exception + if (response.getStatus() == SC_OK) { + return response.getEntity(returnType); + } + if (response.getStatus() == SC_NO_CONTENT) { + try { + return returnType.getConstructor().newInstance(); + } catch (RuntimeException | ReflectiveOperationException e) { + LOG.error("Cannot create empty entity for {}", returnType, e); + } + } + RouterWebServiceUtil.retrieveException(response); + return null; + } finally { + if (response != null) { + response.close(); } } - RouterWebServiceUtil.retrieveException(response); - return null; } }); } catch (InterruptedException e) { @@ -161,14 +171,21 @@ public T run() { /** * Performs an invocation of a REST call on a remote RMWebService. - * - * @param additionalParam + * @param webApp the address of the remote webap + * @param path to add to the webapp address + * @param method the HTTP method of the REST call + * @param additionalPath the servlet request path + * @param queryParams hsr of additional Param + * @param formParam the form parameters as input for a specific REST call + * @param mediaType Media type for Servlet request call + * @param conf to support http and https + * @param client same client used to reduce number of clients created + * @return Client response to REST call */ private static ClientResponse invokeRMWebService(String webApp, String path, HTTPMethods method, String additionalPath, Map queryParams, Object formParam, String mediaType, - Configuration conf) { - Client client = WebServiceClient.getWebServiceClient().createClient(); + Configuration conf, Client client) { InetSocketAddress socketAddress = NetUtils .getConnectAddress(NetUtils.createSocketAddr(webApp)); String scheme = YarnConfiguration.useHttps(conf) ? "https://" : "http://"; @@ -202,21 +219,25 @@ private static ClientResponse invokeRMWebService(String webApp, String path, ClientResponse response = null; - switch (method) { - case DELETE: - response = builder.delete(ClientResponse.class); - break; - case GET: - response = builder.get(ClientResponse.class); - break; - case POST: - response = builder.post(ClientResponse.class); - break; - case PUT: - response = builder.put(ClientResponse.class); - break; - default: - break; + try { + switch (method) { + case DELETE: + response = builder.delete(ClientResponse.class); + break; + case GET: + response = builder.get(ClientResponse.class); + break; + case POST: + response = builder.post(ClientResponse.class); + break; + case PUT: + response = builder.put(ClientResponse.class); + break; + default: + break; + } + } finally { + client.destroy(); } return response; @@ -316,6 +337,22 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, return allApps; } + /** + * Create a Jersey client instance. + */ + protected static Client createJerseyClient(Configuration conf) { + Client client = Client.create(); + client.setConnectTimeout((int) conf + .getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, + TimeUnit.MILLISECONDS)); + client.setReadTimeout((int) conf + .getTimeDuration(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT, + TimeUnit.MILLISECONDS)); + return client; + } + private static AppInfo mergeUAMWithUAM(AppInfo uam1, AppInfo uam2) { AppInfo partialReport = new AppInfo(); partialReport.setAppId(uam1.getAppId());