diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 52fff14..276fbfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -338,6 +338,11 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_RM_SCHEDULER = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; + /** resourcemanager scheduler description url parsing regex*/ + public static final String YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX = + "yarn.resourcemanager.scheduler.url.regex"; + public static final String DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX = ""; + /** RM set next Heartbeat interval for NM */ public static final String RM_NM_HEARTBEAT_INTERVAL_MS = RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 0ea7314..742ce2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -44,7 +44,8 @@ final Map maxQueueResources; // Sharing weights for each queue private final Map queueWeights; - + private final Map queueDescriptions; + // Max concurrent running applications for each queue and for each user; in addition, // for users that have no max specified, we use the userMaxJobsDefault. @VisibleForTesting @@ -98,6 +99,7 @@ public AllocationConfiguration(Map minQueueResources, Map maxQueueResources, Map queueMaxApps, Map userMaxApps, Map queueWeights, + Map queueDescriptions, Map queueMaxAMShares, int userMaxAppsDefault, int queueMaxAppsDefault, float queueMaxAMShareDefault, Map schedulingPolicies, @@ -116,6 +118,7 @@ public AllocationConfiguration(Map minQueueResources, this.userMaxApps = userMaxApps; this.queueMaxAMShares = queueMaxAMShares; this.queueWeights = queueWeights; + this.queueDescriptions = queueDescriptions; this.userMaxAppsDefault = userMaxAppsDefault; this.queueMaxAppsDefault = queueMaxAppsDefault; this.queueMaxAMShareDefault = queueMaxAMShareDefault; @@ -135,6 +138,7 @@ public AllocationConfiguration(Configuration conf) { minQueueResources = new HashMap(); maxQueueResources = new HashMap(); queueWeights = new HashMap(); + queueDescriptions = new HashMap(); queueMaxApps = new HashMap(); userMaxApps = new HashMap(); queueMaxAMShares = new HashMap(); @@ -212,6 +216,11 @@ public void setQueueWeight(String queue, ResourceWeights weight) { queueWeights.put(queue, weight); } + public String getQueueDescription(String name) { + String description = queueDescriptions.get(name); + return (description == null) ? "" : description; + } + public int getUserMaxApps(String user) { Integer maxApps = userMaxApps.get(user); return (maxApps == null) ? userMaxAppsDefault : maxApps; @@ -337,4 +346,4 @@ public void setReservationWindow(long window) { public void setAverageCapacity(int avgCapacity) { globalReservationQueueConfig.setAverageCapacity(avgCapacity); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index dab6d9f..1084711 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -215,6 +215,7 @@ public synchronized void reloadAllocations() throws IOException, Map userMaxApps = new HashMap(); Map queueMaxAMShares = new HashMap(); Map queueWeights = new HashMap(); + Map queueDescriptions = new HashMap(); Map queuePolicies = new HashMap(); Map minSharePreemptionTimeouts = new HashMap(); Map fairSharePreemptionTimeouts = new HashMap(); @@ -351,9 +352,9 @@ public synchronized void reloadAllocations() throws IOException, } loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, - queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, - fairSharePreemptionThresholds, queueAcls, configuredQueues, - reservableQueues); + queueDescriptions, queuePolicies, minSharePreemptionTimeouts, + fairSharePreemptionTimeouts, fairSharePreemptionThresholds, + queueAcls, configuredQueues, reservableQueues); } // Load placement policy and pass it configured queues @@ -397,10 +398,10 @@ public synchronized void reloadAllocations() throws IOException, AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueWeights, - queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, - queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, - minSharePreemptionTimeouts, fairSharePreemptionTimeouts, - fairSharePreemptionThresholds, queueAcls, + queueDescriptions, queueMaxAMShares, userMaxAppsDefault, + queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, + defaultSchedPolicy, minSharePreemptionTimeouts, + fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues); @@ -418,6 +419,7 @@ private void loadQueue(String parentName, Element element, Map maxQueueResources, Map queueMaxApps, Map userMaxApps, Map queueMaxAMShares, Map queueWeights, + Map queueDescriptions, Map queuePolicies, Map minSharePreemptionTimeouts, Map fairSharePreemptionTimeouts, @@ -473,6 +475,9 @@ private void loadQueue(String parentName, Element element, String text = ((Text)field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); queueWeights.put(queueName, new ResourceWeights((float)val)); + } else if("description".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + queueDescriptions.put(queueName, text); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; @@ -505,7 +510,7 @@ private void loadQueue(String parentName, Element element, "pool".equals(field.getTagName())) { loadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, - queuePolicies, minSharePreemptionTimeouts, + queueDescriptions, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues, reservableQueues); isLeaf = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index ade2880..fd46d90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -81,6 +81,10 @@ public String getQueueName() { return name; } + public String getDescription() { + return scheduler.getAllocationConfiguration().getQueueDescription(getName()); + } + public SchedulingPolicy getPolicy() { return policy; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java index d87fb5c..9c9950e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java @@ -79,6 +79,7 @@ protected void render(Block html) { } ri._(STEADY_FAIR_SHARE + ":", qinfo.getSteadyFairShare().toString()); ri._(INSTANTANEOUS_FAIR_SHARE + ":", qinfo.getFairShare().toString()); + ri._("Description:", WebUrlUtils.pullLinks(qinfo.getDescription())); html._(InfoBlock.class); // clear the info contents so this queue's info doesn't accumulate into another queue's info diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java new file mode 100644 index 0000000..129ca37 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + + +public class WebUrlUtils { + + private static Pattern validUrlRegex = null; + + static { + final Configuration conf = new YarnConfiguration(); + final String URL_REGEX = conf.get( + YarnConfiguration.YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX, + YarnConfiguration.DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX); + if(!URL_REGEX.equals( + YarnConfiguration.DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX)) { + try { + validUrlRegex = Pattern.compile((URL_REGEX)); + } catch (PatternSyntaxException e) { + validUrlRegex = null; + } + } + } + + public static String pullLinks(String text) { + if(validUrlRegex == null || text == null) { + return text; + } + return validUrlRegex.matcher(text).replaceAll( + "$1"); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 5fbfe51..a6a36a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -59,6 +59,7 @@ private ResourceInfo clusterResources; private String queueName; + private String description; private String schedulingPolicy; private Collection childQueues; @@ -70,6 +71,7 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueName = queue.getName(); + description = queue.getDescription(); schedulingPolicy = queue.getPolicy().getName(); clusterResources = new ResourceInfo(scheduler.getClusterResource()); @@ -139,6 +141,10 @@ public ResourceInfo getFairShare() { return fairResources; } + public String getDescription() { + return description; + } + public ResourceInfo getMinResources() { return minResources; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java new file mode 100644 index 0000000..481731f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestWebUrlUtils { + + @BeforeClass + public static void setup() throws IOException { + createConfig(); + } + + @AfterClass + public static void clean() { + restoreConfig(); + } + + @Test + public void testPullLinks(){ + assertEquals("pull links doesn't work correctly", + "http://go/jira/HADOOP-321" + + "http://go/jira/YARN-1321" + , WebUrlUtils.pullLinks("http://go/jira/HADOOP-321 http://go/jira/YARN-1321").trim()); + } + + private static File getYarnSiteXmlPath() { + File homeDir = new File("target/test-classes/").getAbsoluteFile(); + return new File(homeDir, "yarn-site.xml"); + } + + private static File getYarnSiteXmlBackUpPath() { + File homeDir = new File("target/test-classes/").getAbsoluteFile(); + return new File(homeDir, "yarn-site.xml.bak"); + } + + private static void restoreConfig() { + File coreSite = getYarnSiteXmlPath(); + File coreSiteBackUp = getYarnSiteXmlBackUpPath(); + if (coreSiteBackUp.exists()) { + coreSiteBackUp.renameTo(coreSite); + } else { + coreSite.delete(); + } + } + + private static void createConfig() throws IOException { + Configuration conf = new Configuration(false); + conf.set(YarnConfiguration.YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX, + "(((http://|https://|www[.])(\\S+))|(\\S+[.]com(/\\S+)?))(\\s|\\z)"); + + // replace the existing core-site.xml file (if any) + //for the duration of the test + File coreSite = getYarnSiteXmlPath(); + if (coreSite.exists()) { + coreSite.renameTo(getYarnSiteXmlBackUpPath()); + } + OutputStream os = new FileOutputStream(coreSite); + conf.writeXml(os); + os.close(); + } +}