2347f6dbfae0a1e6ca0e10d85c18e0d806405b2c .../apache/hadoop/yarn/conf/YarnConfiguration.java | 20 +++++++++ .../hadoop/yarn/webapp/util/WebAppUtils.java | 10 +++++ .../GlobalPolicyGenerator.java | 42 +++++++++++++++++++ .../globalpolicygenerator/webapp/AboutBlock.java | 45 ++++++++++++++++++++ .../globalpolicygenerator/webapp/AboutPage.java | 33 +++++++++++++++ .../webapp/GPGController.java | 40 ++++++++++++++++++ .../globalpolicygenerator/webapp/GPGView.java | 49 ++++++++++++++++++++++ .../globalpolicygenerator/webapp/GPGWebApp.java | 43 +++++++++++++++++++ .../globalpolicygenerator/webapp/NavBlock.java | 39 +++++++++++++++++ 9 files changed, 321 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ed70e5e250e..c98aca77cc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3374,6 +3374,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FEDERATION_GPG_PREFIX = FEDERATION_PREFIX + "gpg."; + public static final String GPG_BIND_HOST = FEDERATION_GPG_PREFIX + "bind-host"; + + public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX + "webapp."; + // The number of threads to use for the GPG scheduled executor service public static final String GPG_SCHEDULED_EXECUTOR_THREADS = FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; @@ -3401,6 +3405,22 @@ public static boolean isAclEnabled(Configuration conf) { FEDERATION_GPG_PREFIX + "application.cleaner.interval-ms"; public static final long DEFAULT_GPG_APPCLEANER_INTERVAL_MS = -1; + /** The address of the GPG web application. */ + public static final String GPG_WEBAPP_ADDRESS = + GPG_WEBAPP_PREFIX + "address"; + + public static final int DEFAULT_GPG_WEBAPP_PORT = 8069; + public static final String DEFAULT_GPG_WEBAPP_ADDRESS = + "0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT; + + /** The https address of the GPG web application. */ + public static final String GPG_WEBAPP_HTTPS_ADDRESS = + GPG_WEBAPP_PREFIX + "https.address"; + + public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070; + public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT; + /** * Specifications on how (many times) to contact Router for apps. We need to * do this because Router might return partial application list because some diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index e62bf104ae4..37b25ed8caf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -143,6 +143,16 @@ public static String getRouterWebAppURLWithoutScheme(Configuration conf) { } } + public static String getGPGWebAppURLWithoutScheme(Configuration conf) { + if (YarnConfiguration.useHttps(conf)) { + return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS); + } else { + return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS); + } + } + public static List getProxyHostsAndPortsForAmFilter( Configuration conf) { List addrs = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index dc6a62e1027..dc85c195429 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -25,7 +25,9 @@ import org.apache.commons.lang.time.DurationFormatUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; @@ -34,9 +36,17 @@ import org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.ApplicationCleaner; import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator; import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner; +import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.hadoop.yarn.webapp.WebApps.Builder; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Global Policy Generator (GPG) is a Yarn Federation component. By tuning the * Federation policies in Federation State Store, GPG overlooks the entire @@ -57,6 +67,7 @@ public static final int SHUTDOWN_HOOK_PRIORITY = 30; private AtomicBoolean isStopping = new AtomicBoolean(false); private static final String METRICS_NAME = "Global Policy Generator"; + private static long gpgStartupTime = System.currentTimeMillis(); // Federation Variables private GPGContext gpgContext; @@ -66,6 +77,10 @@ private SubClusterCleaner subClusterCleaner; private ApplicationCleaner applicationCleaner; private PolicyGenerator policyGenerator; + protected String webAppAddress; + private JvmPauseMonitor pauseMonitor; + private Configuration conf; + private WebApp webApp; public GlobalPolicyGenerator() { super(GlobalPolicyGenerator.class.getName()); @@ -74,6 +89,7 @@ public GlobalPolicyGenerator() { @Override protected void serviceInit(Configuration conf) throws Exception { + this.conf = conf; // Set up the context this.gpgContext .setStateStoreFacade(FederationStateStoreFacade.getInstance()); @@ -95,7 +111,15 @@ protected void serviceInit(Configuration conf) throws Exception { this.policyGenerator = new PolicyGenerator(conf, this.gpgContext); + webAppAddress = WebAppUtils.getWebAppBindURL(conf, + YarnConfiguration.GPG_BIND_HOST, + WebAppUtils.getGPGWebAppURLWithoutScheme(conf)); + DefaultMetricsSystem.initialize(METRICS_NAME); + JvmMetrics jm = JvmMetrics.initSingleton("GPG", null); + pauseMonitor = new JvmPauseMonitor(); + addService(pauseMonitor); + jm.setPauseMonitor(pauseMonitor); // super.serviceInit after all services are added super.serviceInit(conf); @@ -137,6 +161,7 @@ protected void serviceStart() throws Exception { LOG.info("Scheduled policygenerator with interval: {}", DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis)); } + startWepApp(); } @Override @@ -155,6 +180,9 @@ protected void serviceStop() throws Exception { if (this.isStopping.getAndSet(true)) { return; } + if (webApp != null) { + webApp.stop(); + } DefaultMetricsSystem.shutdown(); super.serviceStop(); } @@ -181,6 +209,16 @@ private void initAndStart(Configuration conf, boolean hasToReboot) { this.start(); } + @VisibleForTesting + public void startWepApp() { + + RMWebAppUtil.setupSecurityAndFilters(conf, null); + + Builder builder = WebApps.$for("cluster", null, null, null).with(conf) + .at(webAppAddress); + webApp = builder.start(new GPGWebApp(this)); + } + @SuppressWarnings("resource") public static void startGPG(String[] argv, Configuration conf) { boolean federationEnabled = @@ -204,6 +242,10 @@ public static void startGPG(String[] argv, Configuration conf) { } } + public static long getGPGStartupTime() { + return gpgStartupTime; + } + public static void main(String[] argv) { startGPG(argv, new YarnConfiguration()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutBlock.java new file mode 100644 index 00000000000..ddd6a3020fa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutBlock.java @@ -0,0 +1,45 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import java.util.Date; + +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import com.google.inject.Inject; + +public class AboutBlock extends HtmlBlock{ + + @Inject + AboutBlock(GlobalPolicyGenerator gpg, ViewContext ctx) { + super(ctx); + } + + @Override + protected void render(Block html) { + info("GPG Details"). + __("GPG started on", new Date(GlobalPolicyGenerator.getGPGStartupTime())). + __("GPG Version", YarnVersionInfo.getVersion()). + __("Hadoop Version", VersionInfo.getVersion()); + +html.__(InfoBlock.class);} +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutPage.java new file mode 100644 index 00000000000..12336b5ec24 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/AboutPage.java @@ -0,0 +1,33 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.SubView; + +public class AboutPage extends GPGView{ + + @Override + protected void preHead(Page.HTML<__> html) { + commonPreHead(html); + } + + @Override + protected Class content() { + return AboutBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java new file mode 100644 index 00000000000..c1011d551ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java @@ -0,0 +1,40 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.Controller; +import com.google.inject.Inject; + +public class GPGController extends Controller { + + @Inject + GPGController(RequestContext ctx) { + super(ctx); + } + + @Override + public void index() { + setTitle("GPG"); + render(AboutPage.class); + } + + public void about() { + setTitle("About the Cluster"); + render(AboutPage.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGView.java new file mode 100644 index 00000000000..0f4c7e4646e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGView.java @@ -0,0 +1,49 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +public class GPGView extends TwoColumnLayout { + + @Override + protected void preHead(Page.HTML<__> html) { + commonPreHead(html); + setTitle("GPG"); + } + + protected void commonPreHead(Page.HTML<__> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override + protected Class nav() { + return NavBlock.class; + } + + @Override + protected Class content() { + return AboutBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java new file mode 100644 index 00000000000..a1e8e1b8f21 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java @@ -0,0 +1,43 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; + +public class GPGWebApp extends WebApp{ + private GlobalPolicyGenerator gpg; + + public GPGWebApp(GlobalPolicyGenerator gpg) { + this.gpg = gpg; + } + + @Override + public void setup() { + bind(JAXBContextResolver.class); + bind(GPGWebApp.class).toInstance(this); + bind(GenericExceptionHandler.class); + if (gpg != null) { + bind(GlobalPolicyGenerator.class).toInstance(gpg); + } + route("/", GPGController.class); + route("/cluster", GPGController.class, "about"); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java new file mode 100644 index 00000000000..a0762c1bb79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java @@ -0,0 +1,39 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +public class NavBlock extends HtmlBlock { + + @Override + public void render(Block html) { + html. + div("#nav"). + h3("Cluster"). + ul(). + li().a(url(""), "About").__(). + __(). + h3("Tools"). + ul(). + li().a("/conf", "Configuration").__(). + li().a("/logs", "Local logs").__(). + li().a("/stacks", "Server stacks").__(). + li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__(); + } +}