diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 911faf9..dced082 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1297,6 +1297,13 @@ + "admin.client.thread-count"; public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1; + /** The address of the SCM web application. */ + public static final String SCM_WEBAPP_ADDRESS = SCM_PREFIX + "webapp.address"; + + public static final int DEFAULT_SCM_WEBAPP_PORT = 8788; + public static final String DEFAULT_SCM_WEBAPP_ADDRESS = "0.0.0.0:" + + DEFAULT_SCM_WEBAPP_PORT; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 95245f9..fc8ca5d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1351,6 +1351,12 @@ yarn.sharedcache.manager.admin.thread-count 1 + + + The address of the web application in the SCM (shared cache manager) + yarn.sharedcache.manager.webapp.address + 0.0.0.0:8788 + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java index 426e625..e99f38f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.sharedcache.CacheStructureUtil; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; +import org.apache.hadoop.yarn.server.sharedcachemanager.webapp.SCMWebServer; import com.google.common.annotations.VisibleForTesting; @@ -60,9 +61,11 @@ private Configuration conf; private SCMStore store; + final SCMWebServer webServer; public SharedCacheManager() { super("SharedCacheManager"); + this.webServer = new SCMWebServer(); } @Override @@ -222,12 +225,19 @@ protected synchronized void serviceStart() throws Exception { DefaultMetricsSystem.initialize("SharedCacheManager"); JvmMetrics.initSingleton("SharedCacheManager", null); + // Start the web server + try { + webServer.start(this.conf, this); + } catch (IOException e) { + LOG.error("Cannot start the web server.", e); + } + super.serviceStart(); } @Override protected synchronized void serviceStop() throws Exception { - + webServer.stop(); DefaultMetricsSystem.shutdown(); super.serviceStop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java new file mode 100644 index 0000000..30047be --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/CleanerPage.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit; + +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedMap; +import java.util.TreeMap; + + +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetricsCollector; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.BODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.THEAD; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +import com.google.common.base.Joiner; + +public class CleanerPage extends TwoColumnLayout { + + private static final String TABLE_PREFIX = "table"; + + static class MetricTable extends HashMap { + private static final long serialVersionUID = -6664614528433518964L; + + public int rowCount() { + int rowCount = 0; + for (MetricColumn col : values()) + rowCount = Math.max(rowCount, col.size()); + return rowCount; + } + } + + static class MetricColumn extends LinkedList> { + private static final long serialVersionUID = 1163979877649143564L; + } + + /** + * Improvisation. TODO: send the CleanerList pointer as a constructor + * parameter to other classes + */ + static SortedMap cleanerTables = + new TreeMap(new Comparator() { + //reverse the order to have more recent tables on front (Long is Date in sec) + @Override + public int compare(Long arg0, Long arg1) { + return -1 * arg0.compareTo(arg1); + } + }); + + @Override + protected void preHead(Page.HTML<_> html) { + loadCleanerLists(); + commonPreHead(html); + setTitle("Cleaner Summary " + new Date()); + String tableNames = TABLE_PREFIX + Joiner.on(" " + TABLE_PREFIX).join(cleanerTables.keySet()); + set(DATATABLES_ID, tableNames); + set(initID(DATATABLES, tableNames), + tableInit().append(", aoColumns:[null, {bSearchable:false}]} ") + .toString()); + for (Long tableName : cleanerTables.keySet()) + setTableStyles(html, TABLE_PREFIX + Long.toString(tableName)); + } + + private void loadCleanerLists() { + CleanerMetricsCollector collector = new CleanerMetricsCollector(); + CleanerMetrics.getInstance().getMetricSource() + .getMetrics(collector, true); + String sessionId = collector.getSessionId(); + Long sessionIdLong = sessionId == null? 0: Long.parseLong(sessionId); + MetricTable cleanerTable = parseCleanerMetrics(collector.getMetrics(), sessionIdLong); + cleanerTables.put(sessionIdLong, cleanerTable); + } + + MetricTable parseCleanerMetrics(Map metricValueMap, Long sessionId) { + MetricTable cleanerTable = new MetricTable(); + MetricColumn metricCol = new MetricColumn(); + cleanerTable.put("Cleaner Started at: " + new Date(sessionId) + " " + "metric/value", metricCol); + for (Entry metricEntry : metricValueMap.entrySet()) { + metricCol.add(metricEntry); + } + return cleanerTable; + } + + protected void commonPreHead(Page.HTML<_> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override + protected Class nav() { + return CleanerNavBlock.class; + } + + @Override + protected Class content() { + return CleanerContentBlock.class; + } + private static class CleanerNavBlock extends HtmlBlock { + @Override + protected void render(Block html) { + html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._() + .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._() + .li().a("/metrics", "Metrics")._()._()._(); + } + } + + private static class CleanerContentBlock extends HtmlBlock { + + @Override + protected void render(Block html) { + for (Entry entry: CleanerPage.cleanerTables.entrySet()) + renderATable(html, TABLE_PREFIX + entry.getKey(), entry.getValue()); + } + + /** + * Render a table of top list in an html table. Each table represents an + * operation. + * + * @param html + * @param tableName + * @param metricTable + */ + protected void renderATable(Block html, String tableName, + MetricTable metricTable) { + TR>>> hr = + html.body().table('#' + tableName).thead().tr(); + for (String colName : metricTable.keySet()) + hr.td()._(colName)._(); + TBODY>> tableBody = hr._()._().tbody(); + + int rowCount = metricTable.rowCount(); + for (int rowId = 0; rowId < rowCount; rowId++) { + TR>>> tr = tableBody.tr(); + for (Entry colEntry : metricTable.entrySet()) { + MetricColumn col = colEntry.getValue(); + Entry cellEntry = col.get(rowId); + String cellContent = + cellEntry.getKey() + " = " + cellEntry.getValue(); + tr.td(cellContent); + } + tr._(); + } + tableBody._()._()._(); + } + + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java new file mode 100644 index 0000000..f32bee7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import org.apache.hadoop.yarn.webapp.Controller; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +public class SCMController extends Controller implements + YarnWebParams { + @Override + public void index() { + setTitle("Shared Cache Manager"); + } + + /** + * It is referenced in {@link SCMWebServer#SCMWebApp#setup()} + */ + @SuppressWarnings("unused") + public void cleaner() { + render(CleanerPage.class); + } + + /** + * It is referenced in {@link SCMWebServer#SCMWebApp#setup()} + */ + @SuppressWarnings("unused") + public void overview() { + render(SCMOverviewPage.class); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java new file mode 100644 index 0000000..0611d47 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java @@ -0,0 +1,45 @@ +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; + +// This class is used to summarize useful SCM metrics for webUI display +@XmlRootElement(name = "SCMMetrics") +@XmlAccessorType(XmlAccessType.FIELD) +public class SCMMetricsInfo { + protected long totalDeletedFiles; + protected long totalProcessedFiles; + protected long cacheHits; + protected long cacheMisses; + protected long cacheReleases; + protected long acceptedUploads; + protected long rejectedUploads; + + public SCMMetricsInfo() { + } + + public SCMMetricsInfo(CleanerMetrics cleanerMetrics, + ClientSCMMetrics clientSCMMetrics, + NMCacheUploaderSCMProtocolMetrics nmCacheUploaderSCMProtocolMetrics) { + totalDeletedFiles = cleanerMetrics.getTotalDeletedFiles(); + totalProcessedFiles = cleanerMetrics.getTotalProcessedFiles(); + cacheHits = clientSCMMetrics.getCacheHits(); + cacheMisses = clientSCMMetrics.getCacheMisses(); + cacheReleases = clientSCMMetrics.getCacheReleases(); + acceptedUploads = nmCacheUploaderSCMProtocolMetrics.getAcceptedUploads(); + rejectedUploads = nmCacheUploaderSCMProtocolMetrics.getRejectUploads(); + } + + public long getTotalDeletedFiles() { return totalDeletedFiles; } + public long getTotalProcessedFiles() { return totalProcessedFiles; } + public long getCacheHits() { return cacheHits; } + public long getCacheMisses() { return cacheMisses; } + public long getCacheReleases() { return cacheReleases; } + public long getAcceptedUploads() { return acceptedUploads; } + public long getRejectUploads() { return rejectedUploads; } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java new file mode 100644 index 0000000..fa8f575 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java @@ -0,0 +1,87 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import com.google.inject.Inject; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics; +import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.NMCacheUploaderSCMProtocolMetrics; +import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + +public class SCMOverviewPage extends TwoColumnLayout { + + @Override protected void preHead(Page.HTML<_> html) { + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + } + + @Override protected Class content() { + return SCMOverviewBlock.class; + } + + @Override + protected Class nav() { + return SCMOverviewNavBlock.class; + } + + static private class SCMOverviewNavBlock extends HtmlBlock { + @Override + protected void render(Block html) { + html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._() + .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._() + .li().a("/metrics", "Metrics")._()._()._(); + } + } + + static private class SCMOverviewBlock extends HtmlBlock { + final SharedCacheManager scm; + + @Inject + SCMOverviewBlock(SharedCacheManager scm, ViewContext ctx) { + super(ctx); + this.scm = scm; + } + + @Override + protected void render(Block html) { + SCMMetricsInfo metricsInfo = new SCMMetricsInfo( + CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(), + NMCacheUploaderSCMProtocolMetrics.getInstance()); + info("Shared Cache Manager overview"). + _("Started on:", Times.format(scm.getStartTime())). + _("Cache hits: ", metricsInfo.getCacheHits()). + _("Cache misses: ", metricsInfo.getCacheMisses()). + _("Cache releases: ", metricsInfo.getCacheReleases()). + _("Accepted uploads: ", metricsInfo.getAcceptedUploads()). + _("Rejected uploads: ", metricsInfo.getRejectUploads()). + _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()). + _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles()); + html._(InfoBlock.class); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java new file mode 100644 index 0000000..033ffbd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.sharedcachemanager.webapp; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; + +/** + * A very simple web interface for the metric reported by + * {@link org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager} + */ +public class SCMWebServer { + private static final Log LOG = LogFactory.getLog(SCMWebServer.class); + + private WebApp webApp; + + public void start(Configuration conf, SharedCacheManager scm) throws IOException { + String bindAddress = conf.get( + YarnConfiguration.SCM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_SCM_WEBAPP_ADDRESS); + try { + SCMWebApp scmWebApp = new SCMWebApp(scm); + this.webApp = WebApps.$for("cluster").at(bindAddress).start(scmWebApp); + LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress); + } catch (Exception e) { + String msg = SCMWebApp.class.getName() + " failed to start."; + LOG.error(msg, e); + throw new IOException(msg); + } + } + + public void stop() { + if (this.webApp != null) { + this.webApp.stop(); + } + } + + private static class SCMWebApp extends WebApp { + + private final SharedCacheManager scm; + + public SCMWebApp(SharedCacheManager scm) { + this.scm = scm; + } + + @Override + public void setup() { + if (scm != null) { + bind(SharedCacheManager.class).toInstance(scm); + } + route("/", SCMController.class, "overview"); + route("/cleaner", SCMController.class, "cleaner"); + } + } + +}