diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index be7cc89f5da..bdc6c7b0d10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3849,6 +3849,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + /** + * Connection and Read timeout from the Router to RM. + */ + public static final String ROUTER_WEBAPP_CONNECT_TIMEOUT = + ROUTER_WEBAPP_PREFIX + "connect-timeout"; + public static final long DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT = + TimeUnit.SECONDS.toMillis(30); + public static final String ROUTER_WEBAPP_READ_TIMEOUT = + ROUTER_WEBAPP_PREFIX + "read-timeout"; + public static final long DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT = + TimeUnit.SECONDS.toMillis(30); + //////////////////////////////// // CSI Volume configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 6f781fa7c0d..9fda8094c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -183,6 +183,10 @@ public void initializeMemberVariables() { .add(YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY); configurationPrefixToSkipCompare .add(YarnConfiguration.ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED); + configurationPrefixToSkipCompare + .add(YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT); // Set by container-executor.cfg configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5277be40b09..3be57fe3712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3962,6 +3962,24 @@ 0.0.0.0:8091 + + + It is client connection timeout interval for Router + web services. Time unit used is milliseconds. + + yarn.router.webapp.connect-timeout + 30s + + + + + It is client readout timeout interval for Router + web services. Time unit used is milliseconds. + + yarn.router.webapp.read-timeout + 30s + + It is TimelineClient 1.5 configuration whether to store active diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java index cd588fc4420..ef8d9c14b26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.webapp; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -48,11 +49,15 @@ @Override protected void render(Block html) { Configuration conf = this.router.getConfig(); + + Client client = RouterWebServiceUtil.createJerseyClient(conf); + String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward( webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + client); boolean isEnabled = conf.getBoolean( YarnConfiguration.FEDERATION_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java index 028bacd07c4..1278edea831 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -53,10 +54,13 @@ protected void render(Block html) { // Get the applications from the Resource Managers Configuration conf = this.router.getConfig(); + + Client client = RouterWebServiceUtil.createJerseyClient(conf); + String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, client); setTitle("Applications"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 3dc4fddb922..2a85b0e1b88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Response; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -77,6 +78,10 @@ private String webAppAddress; private SubClusterId subClusterId = null; + // It is very expensive to create the client + // Jersey will spawn a thread for every client request + private Client client = null; + public void setWebAppAddress(String webAppAddress) { this.webAppAddress = webAppAddress; } @@ -92,6 +97,7 @@ protected SubClusterId getSubClusterId() { @Override public void init(String user) { webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(getConf()); + client = RouterWebServiceUtil.createJerseyClient(getConf()); } @Override @@ -103,28 +109,32 @@ public ClusterInfo get() { public ClusterInfo getClusterInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, ClusterInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, null, null, + client); } @Override public ClusterUserInfo getClusterUserInfo(HttpServletRequest hsr) { return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ClusterUserInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.CLUSTER_USER_INFO, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.CLUSTER_USER_INFO, null, + null, client); } @Override public ClusterMetricsInfo getClusterMetricsInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, + null, client); } @Override public SchedulerTypeInfo getSchedulerInfo() { return RouterWebServiceUtil.genericForward(webAppAddress, null, SchedulerTypeInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER, null, + null, client); } @Override @@ -133,7 +143,8 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr) // time is specified inside hsr return RouterWebServiceUtil.genericForward(webAppAddress, null, String.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_LOGS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_LOGS, null, + null, client); } @Override @@ -146,7 +157,7 @@ public NodesInfo getNodes(String states) { return RouterWebServiceUtil.genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, - additionalParam); + additionalParam, client); } @Override @@ -154,7 +165,7 @@ public NodeInfo getNode(String nodeId) { return RouterWebServiceUtil.genericForward(webAppAddress, null, NodeInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId, null, - null); + null, client); } @Override @@ -164,7 +175,7 @@ public ResourceInfo updateNodeResource(HttpServletRequest hsr, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId; return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ResourceInfo.class, HTTPMethods.POST, - nodePath + "/resource", resourceOption, null); + nodePath + "/resource", resourceOption, null, client); } @Override @@ -176,7 +187,7 @@ public AppsInfo getApps(HttpServletRequest hsr, String stateQuery, // all the params are specified inside hsr return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, client); } @Override @@ -186,7 +197,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ActivitiesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_ACTIVITIES, null, - null); + null, client); } @Override @@ -198,7 +209,7 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppActivitiesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_APP_ACTIVITIES, - null, null); + null, null, client); } @Override @@ -207,7 +218,8 @@ public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, // stateQueries and typeQueries are specified inside hsr return RouterWebServiceUtil.genericForward(webAppAddress, hsr, ApplicationStatisticsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APP_STATISTICS, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APP_STATISTICS, null, + null, client); } @Override @@ -217,7 +229,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId, null, - null); + null, client); } @Override @@ -226,7 +238,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppState.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, - null, null); + null, null, client); } @Override @@ -236,7 +248,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, - targetState, null); + targetState, null, client); } @Override @@ -245,7 +257,7 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeToLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_TO_LABELS, null, - null); + null, client); } @Override @@ -260,7 +272,7 @@ public LabelsToNodesInfo getLabelsToNodes(Set labels) return RouterWebServiceUtil.genericForward(webAppAddress, null, LabelsToNodesInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.LABEL_MAPPINGS, null, - additionalParam); + additionalParam, client); } @Override @@ -269,7 +281,7 @@ public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REPLACE_NODE_TO_LABELS, - newNodeToLabels, null); + newNodeToLabels, null, client); } @Override @@ -280,7 +292,7 @@ public Response replaceLabelsOnNode(Set newNodeLabelsName, .genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId + "/replace-labels", - null, null); + null, null, client); } @Override @@ -289,7 +301,7 @@ public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_LABELS, null, - null); + null, client); } @Override @@ -298,7 +310,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.ADD_NODE_LABELS, - newNodeLabels, null); + newNodeLabels, null, client); } @Override @@ -308,7 +320,7 @@ public Response removeFromCluserNodeLabels(Set oldNodeLabels, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REMOVE_NODE_LABELS, null, - null); + null, client); } @Override @@ -317,7 +329,7 @@ public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId + "/get-labels", - null, null); + null, null, client); } @Override @@ -326,7 +338,7 @@ public AppPriority getAppPriority(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppPriority.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, - null, null); + null, null, client); } @Override @@ -336,7 +348,7 @@ public Response updateApplicationPriority(AppPriority targetPriority, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, - targetPriority, null); + targetPriority, null, client); } @Override @@ -345,7 +357,7 @@ public AppQueue getAppQueue(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppQueue.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, - null, null); + null, null, client); } @Override @@ -355,7 +367,7 @@ public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, - targetQueue, null); + targetQueue, null, client); } @Override @@ -364,7 +376,7 @@ public Response createNewApplication(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS_NEW_APPLICATION, null, - null); + null, client); } @Override @@ -373,7 +385,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, throws AuthorizationException, IOException, InterruptedException { return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, newApp, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, newApp, null, client); } @Override @@ -383,7 +395,7 @@ public Response postDelegationToken(DelegationToken tokenData, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, tokenData, - null); + null, client); } @Override @@ -393,7 +405,7 @@ public Response postDelegationTokenExpiration(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN_EXPIRATION, - null, null); + null, null, client); } @Override @@ -403,7 +415,7 @@ public Response cancelDelegationToken(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.DELETE, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, null, - null); + null, client); } @Override @@ -412,7 +424,7 @@ public Response createNewReservation(HttpServletRequest hsr) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_NEW, null, - null); + null, client); } @Override @@ -422,7 +434,7 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_SUBMIT, - resContext, null); + resContext, null, client); } @Override @@ -432,7 +444,7 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_UPDATE, - resContext, null); + resContext, null, client); } @Override @@ -442,7 +454,7 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_DELETE, - resContext, null); + resContext, null, client); } @Override @@ -454,7 +466,7 @@ public Response listReservation(String queue, String reservationId, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_LIST, null, - null); + null, client); } @Override @@ -464,7 +476,7 @@ public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, .genericForward(webAppAddress, hsr, AppTimeoutInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS + "/" + type, - null, null); + null, null, client); } @Override @@ -473,7 +485,7 @@ public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppTimeoutsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS, - null, null); + null, null, client); } @Override @@ -483,7 +495,7 @@ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUT, - appTimeout, null); + appTimeout, null, client); } @Override @@ -491,7 +503,7 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppAttemptsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS, - null, null); + null, null, client); } @Override @@ -500,7 +512,7 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username, return RouterWebServiceUtil.genericForward(webAppAddress, hsr, RMQueueAclInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.QUEUES + "/" + queue - + "/access", null, null); + + "/access", null, null, client); } @Override @@ -510,7 +522,7 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req, AppAttemptInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId, - null, null); + null, null, client); } @Override @@ -521,7 +533,7 @@ public ContainersInfo getContainers(HttpServletRequest req, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + RMWSConsts.CONTAINERS, - null, null); + null, null, client); } @Override @@ -533,7 +545,7 @@ public ContainerInfo getContainer(HttpServletRequest req, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + RMWSConsts.CONTAINERS + "/" + containerId, - null, null); + null, null, client); } @Override @@ -551,6 +563,6 @@ public Response signalToContainer(String containerId, String command, .genericForward(webAppAddress, req, Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + "/" + RMWSConsts.CONTAINERS + "/" + containerId + "/" + RMWSConsts.SIGNAL + "/" + command, null, - null); + null, client); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index 720302e5a0d..36d5209aa61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router.webapp; +import com.sun.jersey.api.client.Client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; @@ -53,10 +54,13 @@ protected void render(Block html) { // Get the node info from the federation Configuration conf = this.router.getConfig(); + + Client client = RouterWebServiceUtil.createJerseyClient(conf); + String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null); + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, client); setTitle("Nodes"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 40bdbd83c69..3bab903027d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.HttpHeaders; @@ -37,8 +38,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; @@ -87,13 +90,14 @@ private RouterWebServiceUtil() { * @param formParam the form parameters as input for a specific REST call * @param additionalParam the query parameters as input for a specific REST * call in case the call has no servlet request + * @param client same client used to reduce number of clients created * @return the retrieved entity from the REST call */ protected static T genericForward( final String webApp, final HttpServletRequest hsr, final Class returnType, final HTTPMethods method, final String targetPath, final Object formParam, - final Map additionalParam) { + final Map additionalParam, Client client) { UserGroupInformation callerUGI = null; @@ -128,23 +132,29 @@ public T run() { ClientResponse response = RouterWebServiceUtil.invokeRMWebService( webApp, targetPath, method, (hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam, - getMediaTypeFromHttpServletRequest(hsr, returnType)); + getMediaTypeFromHttpServletRequest(hsr, returnType), client); if (Response.class.equals(returnType)) { return (T) RouterWebServiceUtil.clientResponseToResponse(response); } - // YARN RM can answer with Status.OK or it throws an exception - if (response.getStatus() == SC_OK) { - return response.getEntity(returnType); - } - if (response.getStatus() == SC_NO_CONTENT) { - try { - return returnType.getConstructor().newInstance(); - } catch (RuntimeException | ReflectiveOperationException e) { - LOG.error("Cannot create empty entity for {}", returnType, e); + try { + // YARN RM can answer with Status.OK or it throws an exception + if (response.getStatus() == SC_OK) { + return response.getEntity(returnType); + } + if (response.getStatus() == SC_NO_CONTENT) { + try { + return returnType.getConstructor().newInstance(); + } catch (RuntimeException | ReflectiveOperationException e) { + LOG.error("Cannot create empty entity for {}", returnType, e); + } + } + RouterWebServiceUtil.retrieveException(response); + return null; + } finally { + if (response != null) { + response.close(); } } - RouterWebServiceUtil.retrieveException(response); - return null; } }); } catch (InterruptedException e) { @@ -156,14 +166,21 @@ public T run() { /** * Performs an invocation of a REST call on a remote RMWebService. - * - * @param additionalParam + * @param webApp the address of the remote webap + * @param path additional path to add to the webapp address + * @param method the HTTP method of the REST call + * @param additionalPath the servlet request path + * @param queryParams hsr of additional Param + * @param formParam the form parameters as input for a specific REST call + * @param mediaType Media type for Servlet request call + * @param client same client used to reduce number of clients created + * * @return the retrieved entity from the REST call + * @return Client response to REST call */ private static ClientResponse invokeRMWebService(String webApp, String path, HTTPMethods method, String additionalPath, - Map queryParams, Object formParam, String mediaType) { - Client client = Client.create(); - + Map queryParams, Object formParam, String mediaType, + Client client) { WebResource webResource = client.resource(webApp).path(path); if (additionalPath != null && !additionalPath.isEmpty()) { @@ -192,21 +209,25 @@ private static ClientResponse invokeRMWebService(String webApp, String path, ClientResponse response = null; - switch (method) { - case DELETE: - response = builder.delete(ClientResponse.class); - break; - case GET: - response = builder.get(ClientResponse.class); - break; - case POST: - response = builder.post(ClientResponse.class); - break; - case PUT: - response = builder.put(ClientResponse.class); - break; - default: - break; + try { + switch (method) { + case DELETE: + response = builder.delete(ClientResponse.class); + break; + case GET: + response = builder.get(ClientResponse.class); + break; + case POST: + response = builder.post(ClientResponse.class); + break; + case PUT: + response = builder.put(ClientResponse.class); + break; + default: + break; + } + } finally { + client.destroy(); } return response; @@ -461,4 +482,19 @@ public static void mergeMetrics(ClusterMetricsInfo metrics, return header; } + /** + * Create a Jersey client instance. + */ + protected static Client createJerseyClient(Configuration conf) { + Client client = Client.create(); + client.setConnectTimeout((int) conf.getTimeDuration( + YarnConfiguration.ROUTER_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_CONNECT_TIMEOUT, + TimeUnit.MILLISECONDS)); + client.setReadTimeout((int) conf.getTimeDuration( + YarnConfiguration.ROUTER_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT, + TimeUnit.MILLISECONDS)); + return client; + } }