Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java (revision 0) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebUrlUtils.java (revision 0) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestWebUrlUtils { + + @BeforeClass + public static void setup() throws IOException { + createConfig(); + } + + @AfterClass + public static void clean() { + restoreConfig(); + } + + @Test + public void testPullLinks(){ + Assert.assertEquals("pull links doesn't work correctly", + "http://go/jira/HADOOP-321" + + "http://go/jira/YARN-1321" + , WebUrlUtils.pullLinks("http://go/jira/HADOOP-321 http://go/jira/YARN-1321").trim()); + } + + private static File getYarnSiteXmlPath() { + File homeDir = new File("target/test-classes/").getAbsoluteFile(); + return new File(homeDir, "yarn-site.xml"); + } + + private static File getYarnSiteXmlBackUpPath() { + File homeDir = new File("target/test-classes/").getAbsoluteFile(); + return new File(homeDir, "yarn-site.xml.bak"); + } + + private static void restoreConfig() { + File coreSite = getYarnSiteXmlPath(); + File coreSiteBackUp = getYarnSiteXmlBackUpPath(); + if (coreSiteBackUp.exists()) { + coreSiteBackUp.renameTo(coreSite); + } else { + coreSite.delete(); + } + } + + private static void createConfig() throws IOException { + Configuration conf = new Configuration(false); + conf.set(YarnConfiguration.YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX, + "(((http://|https://|www[.])(\\S+))|(\\S+[.]com(/\\S+)?))(\\s|\\z)"); + + // replace the existing core-site.xml file (if any) + //for the duration of the test + File coreSite = getYarnSiteXmlPath(); + if (coreSite.exists()) { + coreSite.renameTo(getYarnSiteXmlBackUpPath()); + } + OutputStream os = new FileOutputStream(coreSite); + conf.writeXml(os); + os.close(); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java (working copy) @@ -74,7 +74,8 @@ ri._("Max Running Applications:", qinfo.getMaxApplications()); } ri._("Fair Share:", qinfo.getFairShare().toString()); - + ri._("Description:", WebUrlUtils.pullLinks(qinfo.getDescription())); + html._(InfoBlock.class); // clear the info contents so this queue's info doesn't accumulate into another queue's info Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java (revision 0) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/WebUrlUtils.java (revision 0) @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + + +public class WebUrlUtils { + + private static Pattern validUrlRegex = null; + + static { + final Configuration conf = new YarnConfiguration(); + final String URL_REGEX = conf.get( + YarnConfiguration.YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX, + YarnConfiguration.DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX); + if(!URL_REGEX.equals( + YarnConfiguration.DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX)) { + try { + validUrlRegex = Pattern.compile((URL_REGEX)); + } catch (PatternSyntaxException e) { + validUrlRegex = null; + } + } + } + + public static String pullLinks(String text) { + + if(validUrlRegex == null || text == null) { + return text; + } + return validUrlRegex.matcher(text).replaceAll( + "$1"); + } +} Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (working copy) @@ -57,6 +57,7 @@ private ResourceInfo clusterResources; private String queueName; + private String description; private String schedulingPolicy; private Collection childQueues; @@ -68,6 +69,7 @@ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueName = queue.getName(); + description = queue.getDescription(); schedulingPolicy = queue.getPolicy().getName(); clusterResources = new ResourceInfo(scheduler.getClusterCapacity()); @@ -113,6 +115,10 @@ public ResourceInfo getFairShare() { return fairResources; } + + public String getDescription() { + return description; + } public ResourceInfo getMinResources() { return minResources; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (working copy) @@ -68,6 +68,10 @@ return name; } + public String getDescription() { + return scheduler.getAllocationConfiguration().getQueueDescription(getName()); + } + public SchedulingPolicy getPolicy() { return policy; } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java (working copy) @@ -43,6 +43,7 @@ final Map maxQueueResources; // Sharing weights for each queue private final Map queueWeights; + private final Map queueDescriptions; // Max concurrent running applications for each queue and for each user; in addition, // for users that have no max specified, we use the userMaxJobsDefault. @@ -83,7 +84,8 @@ public AllocationConfiguration(Map minQueueResources, Map maxQueueResources, Map queueMaxApps, Map userMaxApps, - Map queueWeights, int userMaxAppsDefault, + Map queueWeights, + Map queueDescriptions, int userMaxAppsDefault, int queueMaxAppsDefault, Map schedulingPolicies, SchedulingPolicy defaultSchedulingPolicy, Map minSharePreemptionTimeouts, @@ -95,6 +97,7 @@ this.queueMaxApps = queueMaxApps; this.userMaxApps = userMaxApps; this.queueWeights = queueWeights; + this.queueDescriptions = queueDescriptions; this.userMaxAppsDefault = userMaxAppsDefault; this.queueMaxAppsDefault = queueMaxAppsDefault; this.defaultSchedulingPolicy = defaultSchedulingPolicy; @@ -111,6 +114,7 @@ minQueueResources = new HashMap(); maxQueueResources = new HashMap(); queueWeights = new HashMap(); + queueDescriptions = new HashMap(); queueMaxApps = new HashMap(); userMaxApps = new HashMap(); userMaxAppsDefault = Integer.MAX_VALUE; @@ -168,6 +172,11 @@ return (weight == null) ? ResourceWeights.NEUTRAL : weight; } + public String getQueueDescription(String name) { + String description = queueDescriptions.get(name); + return (description == null) ? "" : description; + } + public int getUserMaxApps(String user) { Integer maxApps = userMaxApps.get(user); return (maxApps == null) ? userMaxAppsDefault : maxApps; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java (working copy) @@ -201,6 +201,7 @@ Map queueMaxApps = new HashMap(); Map userMaxApps = new HashMap(); Map queueWeights = new HashMap(); + Map queueDescriptions = new HashMap(); Map queuePolicies = new HashMap(); Map minSharePreemptionTimeouts = new HashMap(); Map> queueAcls = @@ -290,8 +291,8 @@ parent = null; } loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, - userMaxApps, queueWeights, queuePolicies, minSharePreemptionTimeouts, - queueAcls, queueNamesInAllocFile); + userMaxApps, queueWeights, queueDescriptions, queuePolicies, + minSharePreemptionTimeouts, queueAcls, queueNamesInAllocFile); } // Load placement policy and pass it configured queues @@ -305,7 +306,7 @@ } AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, - queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault, + queueMaxApps, userMaxApps, queueWeights, queueDescriptions, userMaxAppsDefault, queueMaxAppsDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout, newPlacementPolicy, queueNamesInAllocFile); @@ -322,6 +323,7 @@ private void loadQueue(String parentName, Element element, Map minQueueResources, Map maxQueueResources, Map queueMaxApps, Map userMaxApps, Map queueWeights, + Map queueDescriptions, Map queuePolicies, Map minSharePreemptionTimeouts, Map> queueAcls, Set queueNamesInAllocFile) @@ -356,6 +358,9 @@ String text = ((Text)field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); queueWeights.put(queueName, new ResourceWeights((float)val)); + } else if("description".equals(field.getTagName())) { + String text = ((Text)field.getFirstChild()).getData().trim(); + queueDescriptions.put(queueName, text); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; @@ -374,7 +379,7 @@ } else if ("queue".endsWith(field.getTagName()) || "pool".equals(field.getTagName())) { loadQueue(queueName, field, minQueueResources, maxQueueResources, - queueMaxApps, userMaxApps, queueWeights, queuePolicies, + queueMaxApps, userMaxApps, queueWeights, queueDescriptions, queuePolicies, minSharePreemptionTimeouts, queueAcls, queueNamesInAllocFile); isLeaf = false; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (revision 1591726) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (working copy) @@ -289,6 +289,11 @@ public static final String DEFAULT_RM_SCHEDULER = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; + + /** resourcemanager scheduler description url parsing regex*/ + public static final String YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX = + "yarn.resourcemanager.scheduler.url.regex"; + public static final String DEFAULT_YARN_RM_SCHEDULER_DESCRIPTION_URL_REGEX = ""; /** RM set next Heartbeat interval for NM */ public static final String RM_NM_HEARTBEAT_INTERVAL_MS =