commit 68d302e4c2d8a4486627a674c2893256a4afd21a Author: Todd Lipcon Date: Fri Apr 29 15:20:29 2011 -0700 Move master.jsp page into Jamon diff --git pom.xml pom.xml index 6d62088..4c93515 100644 --- pom.xml +++ pom.xml @@ -512,6 +512,47 @@ + + org.jamon + jamon-maven-plugin + 2.3.4 + + + generate-sources + + translate + + + src/main/jamon + target/generated-jamon + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + + + org.jamon.project.jamonnature + + + org.jamon.project.templateBuilder + org.eclipse.jdt.core.javabuilder + org.jamon.project.markerUpdater + + + + .settings/org.jamon.prefs + # now + eclipse.preferences.version=1 + templateSourceDir=src/main/jamon + templateOutputDir=target/generated-jamon + + + + + @@ -772,6 +813,12 @@ + + org.jamon + jamon-runtime + 2.3.1 + + com.google.protobuf diff --git src/main/jamon/org/apache/hbase/tmpl/master/StatusTmpl.jamon src/main/jamon/org/apache/hbase/tmpl/master/StatusTmpl.jamon new file mode 100644 index 0000000..abbeebe --- /dev/null +++ src/main/jamon/org/apache/hbase/tmpl/master/StatusTmpl.jamon @@ -0,0 +1,194 @@ +<%doc> +Copyright 2011 The Apache Software Foundation + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +<%args> +HMaster master; +HBaseAdmin admin; +Map frags = null; +ServerName rootLocation = null; +ServerName metaLocation = null; +List servers = null; +boolean showAppendWarning = false; + +<%import> +java.util.*; +org.apache.hadoop.util.StringUtils; +org.apache.hadoop.hbase.util.Bytes; +org.apache.hadoop.hbase.util.JvmVersion; +org.apache.hadoop.hbase.util.FSUtils; +org.apache.hadoop.hbase.master.HMaster; +org.apache.hadoop.hbase.HConstants; +org.apache.hadoop.hbase.ServerName; +org.apache.hadoop.hbase.client.HBaseAdmin; +org.apache.hadoop.hbase.client.HConnectionManager; +org.apache.hadoop.hbase.HTableDescriptor; + + + + + +HBase Master: <% master.getServerName().getHostAndPort() %> + + + + +

Master: <% master.getServerName().getHostname() %>:<% master.getServerName().getPort() %>

+ + + +<%if JvmVersion.isBadJvmVersion() %> +
+ Your current JVM version <% System.getProperty("java.version") %> is known to be + unstable with HBase. Please see the + HBase wiki + for details. +
+ +<%if showAppendWarning %> +
+ You are currently running the HMaster without HDFS append support enabled. + This may result in data loss. + Please see the HBase wiki + for details. +
+ + +
+ +

Master Attributes

+ + + + + + + + + +<%if frags != null %> + + + +
Attribute NameValueDescription
HBase Version<% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and svn revision
HBase Compiled<% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
Hadoop Version<% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and svn revision
Hadoop Compiled<% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
HBase Root Directory<% FSUtils.getRootDir(master.getConfiguration()).toString() %>Location of HBase home directory
HBase Cluster ID<% master.getClusterId() != null ? master.getClusterId() : "Not set" %>Unique identifier generated for each HBase cluster
Load average<% StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>Average number of regions per regionserver. Naive computation.
Fragmentation<% frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including .META. and -ROOT-.
Zookeeper Quorum<% master.getZooKeeperWatcher().getQuorum() %>Addresses of all registered ZK servers. For more, see zk dump.
+<%if (rootLocation != null) %> +<& catalogTables &> + +<%if (metaLocation != null) %> +<& userTables &> + +<%if (servers != null) %> +<& regionServers &> + + + + + +<%def catalogTables> +

Catalog Tables

+ + + + <%if (frags != null) %> + + + + + + + <%if (frags != null)%> + + + + + <%if (metaLocation != null) %> + + + <%if (frags != null)%> + + + + + + +
TableFrag.Description
<% Bytes.toString(HConstants.ROOT_TABLE_NAME) %><% frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %>The -ROOT- table holds references to all .META. regions.
<% Bytes.toString(HConstants.META_TABLE_NAME) %><% frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %>The .META. table holds references to all User Table regions
+ + +<%def userTables> +

User Tables

+<%java> + HTableDescriptor[] tables = admin.listTables(); + HConnectionManager.deleteConnection(admin.getConfiguration(), false); + +<%if (tables != null && tables.length > 0)%> + + + +<%if (frags != null) %> + + + + +<%for HTableDescriptor htDesc : tables%> + + + <%if (frags != null) %> + + + + + + +

<% tables.length %> table(s) in set.

+
TableFrag.Description
><% htDesc.getNameAsString() %> <% frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %><% htDesc.toString() %>
+ + + +<%def regionServers> +

Region Servers

+<%if (servers != null && servers.size() > 0)%> +<%java> + int totalRegions = 0; + int totalRequests = 0; + + + + +<%java> + ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]); + Arrays.sort(serverNames); + for (ServerName serverName: serverNames) { + // HARDCODED FOR NOW; FIX -- READ FROM ZK + String hostname = serverName.getHostname() + ":60020"; + String url = "http://" + hostname + "/"; + // TODO: FIX + totalRegions += 0; + totalRequests += 0; + long startCode = serverName.getStartcode(); + + +<%java> + } + + +
AddressStart CodeLoad
<% hostname %><% startCode %><% 0 %>
Total: servers: <% servers.size() %> requests=<% totalRequests %>, regions=<% totalRegions %>
+ +

Load is requests per second and count of regions loaded

+ + diff --git src/main/java/org/apache/hadoop/hbase/master/HMaster.java src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f526411..f71c78b 100644 --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler; import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler; import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler; import org.apache.hadoop.hbase.master.metrics.MasterMetrics; +import org.apache.hadoop.hbase.master.web.MasterStatusServlet; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.replication.regionserver.Replication; import org.apache.hadoop.hbase.security.User; @@ -579,6 +580,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { if (port >= 0) { String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); this.infoServer = new InfoServer(MASTER, a, port, false); + this.infoServer.addServlet("index", "/status", MasterStatusServlet.class); this.infoServer.setAttribute(MASTER, this); this.infoServer.start(); } diff --git src/main/java/org/apache/hadoop/hbase/master/web/MasterStatusServlet.java src/main/java/org/apache/hadoop/hbase/master/web/MasterStatusServlet.java new file mode 100644 index 0000000..546daca --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/master/web/MasterStatusServlet.java @@ -0,0 +1,101 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.web; + +import java.io.IOException; + +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hbase.tmpl.master.StatusTmpl; + +/** + * The servlet responsible for rendering the index page of the + * master. + */ +public class MasterStatusServlet extends HttpServlet { + private static final Log LOG = LogFactory.getLog(MasterStatusServlet.class); + private static final long serialVersionUID = 1L; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException { + response.setContentType("text/html"); + + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + Configuration conf = master.getConfiguration(); + HBaseAdmin admin = new HBaseAdmin(conf); + + Map frags = getFragmentationInfo(master, conf); + + ServerName rootLocation = getRootLocationOrNull(master); + ServerName metaLocation = master.getCatalogTracker().getMetaLocation(); + List servers = master.getServerManager().getOnlineServersList(); + + new StatusTmpl() + .setFrags(frags) + .setShowAppendWarning(shouldShowAppendWarning(conf)) + .setRootLocation(rootLocation) + .setMetaLocation(metaLocation) + .setServers(servers) + .render(response.getWriter(), + master, admin); + } + + private ServerName getRootLocationOrNull(HMaster master) { + try { + return master.getCatalogTracker().getRootLocation(); + } catch (InterruptedException e) { + LOG.warn("Unable to get root location", e); + return null; + } + } + + private Map getFragmentationInfo( + HMaster master, Configuration conf) throws IOException { + boolean showFragmentation = conf.getBoolean( + "hbase.master.ui.fragmentation.enabled", false); + if (showFragmentation) { + return FSUtils.getTableFragmentation(master); + } else { + return null; + } + } + + static boolean shouldShowAppendWarning(Configuration conf) { + try { + return !FSUtils.isAppendSupported(conf) && FSUtils.isHDFS(conf); + } catch (IOException e) { + LOG.warn("Unable to determine if append is supported", e); + return false; + } + } +} diff --git src/main/resources/hbase-webapps/master/index.html src/main/resources/hbase-webapps/master/index.html index 6d301ab..0891ba1 100644 --- src/main/resources/hbase-webapps/master/index.html +++ src/main/resources/hbase-webapps/master/index.html @@ -1 +1 @@ - + diff --git src/main/resources/hbase-webapps/master/master.jsp src/main/resources/hbase-webapps/master/master.jsp index 2ed58d0..0891ba1 100644 --- src/main/resources/hbase-webapps/master/master.jsp +++ src/main/resources/hbase-webapps/master/master.jsp @@ -1,164 +1 @@ -<%@ page contentType="text/html;charset=UTF-8" - import="java.util.*" - import="org.apache.hadoop.conf.Configuration" - import="org.apache.hadoop.util.StringUtils" - import="org.apache.hadoop.hbase.util.Bytes" - import="org.apache.hadoop.hbase.util.JvmVersion" - import="org.apache.hadoop.hbase.util.FSUtils" - import="org.apache.hadoop.hbase.master.HMaster" - import="org.apache.hadoop.hbase.HConstants" - import="org.apache.hadoop.hbase.ServerName" - import="org.apache.hadoop.hbase.client.HBaseAdmin" - import="org.apache.hadoop.hbase.client.HConnectionManager" - import="org.apache.hadoop.hbase.HTableDescriptor" %><% - HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); - Configuration conf = master.getConfiguration(); - ServerName rootLocation = master.getCatalogTracker().getRootLocation(); - boolean metaOnline = master.getCatalogTracker().getMetaLocation() != null; - List servers = master.getServerManager().getOnlineServersList(); - int interval = conf.getInt("hbase.regionserver.msginterval", 1000)/1000; - if (interval == 0) { - interval = 1; - } - boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); - Map frags = null; - if (showFragmentation) { - frags = FSUtils.getTableFragmentation(master); - } -%> - - - -HBase Master: <%= master.getServerName().getHostAndPort() %> - - - - -

Master: <%=master.getServerName().getHostname()%>:<%=master.getServerName().getPort()%>

- - - -<% if (JvmVersion.isBadJvmVersion()) { %> -
- Your current JVM version <%= System.getProperty("java.version") %> is known to be - unstable with HBase. Please see the - HBase wiki - for details. -
-<% } %> -<% if (!FSUtils.isAppendSupported(conf) && FSUtils.isHDFS(conf)) { %> -
- You are currently running the HMaster without HDFS append support enabled. - This may result in data loss. - Please see the HBase wiki - for details. -
-<% } %> - -
- -

Master Attributes

- - - - - - - - - -<% if (showFragmentation) { %> - -<% } %> - -
Attribute NameValueDescription
HBase Version<%= org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<%= org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and svn revision
HBase Compiled<%= org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <%= org.apache.hadoop.hbase.util.VersionInfo.getUser() %>When HBase version was compiled and by whom
Hadoop Version<%= org.apache.hadoop.util.VersionInfo.getVersion() %>, r<%= org.apache.hadoop.util.VersionInfo.getRevision() %>Hadoop version and svn revision
Hadoop Compiled<%= org.apache.hadoop.util.VersionInfo.getDate() %>, <%= org.apache.hadoop.util.VersionInfo.getUser() %>When Hadoop version was compiled and by whom
HBase Root Directory<%= FSUtils.getRootDir(master.getConfiguration()).toString() %>Location of HBase home directory
HBase Cluster ID<%= master.getClusterId() != null ? master.getClusterId() : "Not set" %>Unique identifier generated for each HBase cluster
Load average<%= StringUtils.limitDecimalTo2(master.getServerManager().getAverageLoad()) %>Average number of regions per regionserver. Naive computation.
Fragmentation<%= frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including .META. and -ROOT-.
Zookeeper Quorum<%= master.getZooKeeperWatcher().getQuorum() %>Addresses of all registered ZK servers. For more, see zk dump.
- -

Catalog Tables

-<% - if (rootLocation != null) { %> - - - -<% if (showFragmentation) { %> - -<% } %> - - - - -<% if (showFragmentation) { %> - -<% } %> - - -<% - if (metaOnline) { %> - - -<% if (showFragmentation) { %> - -<% } %> - - - -<% } %> -
TableFrag.Description
<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %><%= frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %>The -ROOT- table holds references to all .META. regions.
<%= Bytes.toString(HConstants.META_TABLE_NAME) %><%= frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %>The .META. table holds references to all User Table regions
-<%} %> - -

User Tables

-<% - HBaseAdmin hba = new HBaseAdmin(conf); - HTableDescriptor[] tables = hba.listTables(); - HConnectionManager.deleteConnection(hba.getConfiguration(), false); - if(tables != null && tables.length > 0) { %> - - - -<% if (showFragmentation) { %> - -<% } %> - - -<% for(HTableDescriptor htDesc : tables ) { %> - - -<% if (showFragmentation) { %> - -<% } %> - - -<% } %> - -

<%= tables.length %> table(s) in set.

-
TableFrag.Description
><%= htDesc.getNameAsString() %> <%= frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %><%= htDesc.toString() %>
-<% } %> - -

Region Servers

-<% if (servers != null && servers.size() > 0) { %> -<% int totalRegions = 0; - int totalRequests = 0; -%> - - - -<% ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]); - Arrays.sort(serverNames); - for (ServerName serverName: serverNames) { - // HARDCODED FOR NOW; FIX -- READ FROM ZK - String hostname = serverName.getHostname() + ":60020"; - String url = "http://" + hostname + "/"; - // TODO: FIX - totalRegions += 0; - totalRequests += 0; - long startCode = serverName.getStartcode(); -%> - -<% } %> - -
AddressStart CodeLoad
<%= hostname %><%= startCode %><%= 0 %>
Total: servers: <%= servers.size() %> requests=<%= totalRequests %>, regions=<%= totalRegions %>
- -

Load is requests per second and count of regions loaded

-<% } %> - - + diff --git src/test/java/org/apache/hadoop/hbase/master/web/TestMasterStatusServlet.java src/test/java/org/apache/hadoop/hbase/master/web/TestMasterStatusServlet.java new file mode 100644 index 0000000..5fae744 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/master/web/TestMasterStatusServlet.java @@ -0,0 +1,123 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.web; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hbase.tmpl.master.StatusTmpl; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + +/** + * Tests for the master status page and its template. + */ +public class TestMasterStatusServlet { + + private HMaster master; + private Configuration conf; + private HBaseAdmin admin; + + @Before + public void setupBasicMocks() { + conf = HBaseConfiguration.create(); + + master = Mockito.mock(HMaster.class); + Mockito.doReturn(new ServerName("fakehost", 12345, 1234567890)) + .when(master).getServerName(); + Mockito.doReturn(conf).when(master).getConfiguration(); + + // Fake serverManager + ServerManager serverManager = Mockito.mock(ServerManager.class); + Mockito.doReturn(1.0).when(serverManager).getAverageLoad(); + Mockito.doReturn(serverManager).when(master).getServerManager(); + + // Fake ZKW + ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class); + Mockito.doReturn("fakequorum").when(zkw).getQuorum(); + Mockito.doReturn(zkw).when(master).getZooKeeperWatcher(); + + // Mock admin + admin = Mockito.mock(HBaseAdmin.class); + } + + + private void setupMockTables() throws IOException { + HTableDescriptor tables[] = new HTableDescriptor[] { + new HTableDescriptor("foo"), + new HTableDescriptor("bar") + }; + Mockito.doReturn(tables).when(admin).listTables(); + } + + @Test + public void testStatusTemplateNoTables() throws IOException { + new StatusTmpl().render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateRootAvailable() throws IOException { + new StatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateRootAndMetaAvailable() throws IOException { + setupMockTables(); + + new StatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .setMetaLocation(new ServerName("metaserver:123,12345")) + .render(new StringWriter(), + master, admin); + } + + @Test + public void testStatusTemplateWithServers() throws IOException { + setupMockTables(); + + List servers = Lists.newArrayList( + new ServerName("rootserver:123,12345"), + new ServerName("metaserver:123,12345")); + + new StatusTmpl() + .setRootLocation(new ServerName("rootserver:123,12345")) + .setMetaLocation(new ServerName("metaserver:123,12345")) + .setServers(servers) + .render(new StringWriter(), + master, admin); + } + +}