From fddbd4a6f2f8a93bc3941cc182fafd3baf729c94 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Wed, 14 Apr 2021 15:30:35 +0200 Subject: [PATCH] YARN-10746 RmWebApp add default-node-label-expression to the queue info --- .../webapp/dao/CapacitySchedulerInfo.java | 3 ++ .../dao/CapacitySchedulerLeafQueueInfo.java | 12 ++--- .../dao/CapacitySchedulerQueueInfo.java | 8 ++- .../TestRMWebServicesCapacitySched.java | 52 +++++++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index fc18d4d6e44..78e2133a90b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,6 +61,7 @@ protected String queueType; protected String creationMethod; protected String autoCreationEligibility; + protected String defaultNodeLabelExpression; @XmlTransient static final float EPSILON = 1e-8f; @@ -114,6 +115,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent); autoCreationEligibility = CapacitySchedulerInfoHelper .getAutoCreationEligibility(parent); + + defaultNodeLabelExpression = parent.getDefaultNodeLabelExpression(); } public float getCapacity() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 6aef01bb3dd..16abb991df9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -52,7 +52,6 @@ protected ResourceInfo userAMResourceLimit; protected boolean preemptionDisabled; protected boolean intraQueuePreemptionDisabled; - protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; protected long maxApplicationLifetime; @@ -81,7 +80,6 @@ intraQueuePreemptionDisabled = q.getIntraQueuePreemptionDisabled(); orderingPolicyDisplayName = q.getOrderingPolicy().getInfo(); orderingPolicyInfo = q.getOrderingPolicy().getConfigName(); - defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); defaultPriority = q.getDefaultApplicationPriority().getPriority(); ArrayList usersList = users.getUsersList(); if (usersList.isEmpty()) { @@ -148,17 +146,17 @@ public float getUserLimitFactor() { public float getConfiguredMaxAMResourceLimit() { return configuredMaxAMResourceLimit; } - + public ResourceInfo getAMResourceLimit() { return AMResourceLimit; } - + public ResourceInfo getUsedAMResource() { return usedAMResource; } public ResourceInfo getUserAMResourceLimit() { - return userAMResourceLimit; + return userAMResourceLimit; } public boolean getPreemptionDisabled() { @@ -172,10 +170,6 @@ public boolean getIntraQueuePreemptionDisabled() { public String getOrderingPolicyDisplayName() { return orderingPolicyDisplayName; } - - public String getDefaultNodeLabelExpression() { - return defaultNodeLabelExpression; - } public int getDefaultApplicationPriority() { return defaultPriority; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 3865cce909f..31857bb94b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -91,6 +91,7 @@ protected String queueType; protected String creationMethod; protected String autoCreationEligibility; + protected String defaultNodeLabelExpression; CapacitySchedulerQueueInfo() { }; @@ -120,6 +121,7 @@ reservedContainers = q.getMetrics().getReservedContainers(); queueName = q.getQueueName(); state = q.getState(); + defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); resourcesUsed = new ResourceInfo(q.getUsedResources()); if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { hideReservationQueues = true; @@ -271,7 +273,7 @@ public ResourceInfo getResourcesUsed() { static float cap(float val, float low, float hi) { return Math.min(Math.max(val, low), hi); } - + public ArrayList getNodeLabels() { return this.nodeLabels; } @@ -335,4 +337,8 @@ public float getWeight() { public float getNormalizedWeight() { return normalizedWeight; } + + public String getDefaultNodeLabelExpression() { + return defaultNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index a0a088497b7..ab3294191bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -661,6 +661,58 @@ public void testPerUserResourcesXML() throws Exception { } } + @Test + public void testNodeLabelDefaultAPI() throws Exception { + CapacitySchedulerConfiguration config = + ((CapacityScheduler)rm.getResourceScheduler()).getConfiguration(); + + config.setDefaultNodeLabelExpression("root", "ROOT-INHERITED"); + config.setDefaultNodeLabelExpression("root.a", "root-a-default-label"); + rm.getResourceScheduler().reinitialize(config, rm.getRMContext()); + + //Start RM so that it accepts app submissions + rm.start(); + try { + //Get the XML from ws/v1/cluster/scheduler + WebResource r = resource(); + ClientResponse response = r.path("ws/v1/cluster/scheduler") + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + String xml = response.getEntity(String.class); + DocumentBuilder db = DocumentBuilderFactory.newInstance() + .newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + //Parse the XML we got + Document dom = db.parse(is); + + NodeList allQueues = dom.getElementsByTagName("queue"); + for (int i = 0; i < allQueues.getLength(); ++i) { + Node queueNode = allQueues.item(i); + Node queuePathNode = getChildNodeByName(queueNode, "queuePath"); + if (queuePathNode == null) { + continue; + } + + String queuePath = queuePathNode.getTextContent(); + if (queuePath != null) { + if (queuePath.startsWith("root.a")) { + assertEquals("root-a-default-label", + getChildNodeByName(queueNode, "defaultNodeLabelExpression") + .getTextContent()); + } else { + assertEquals("ROOT-INHERITED", + getChildNodeByName(queueNode, "defaultNodeLabelExpression") + .getTextContent()); + } + } + } + } finally { + rm.stop(); + } + } + private void checkResourcesUsed(JSONObject queue) throws JSONException { queue.getJSONObject("resourcesUsed").getInt("memory"); queue.getJSONObject("resourcesUsed").getInt("vCores"); -- 2.26.2