commit 988605b9b40318db3fa3cc619adcfbe1d8f90b8e Author: Todd Lipcon Date: Mon Aug 29 19:39:25 2011 -0700 HBASE-4292. MasterDumpServlet (v3) diff --git a/src/main/jamon/org/apache/hbase/tmpl/master/MasterStatusTmpl.jamon b/src/main/jamon/org/apache/hbase/tmpl/master/MasterStatusTmpl.jamon index 7e3d30c..e3e13e7 100644 --- a/src/main/jamon/org/apache/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/src/main/jamon/org/apache/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -51,7 +51,12 @@ org.apache.hadoop.hbase.HTableDescriptor;

Master: <% master.getServerName().getHostname() %>:<% master.getServerName().getPort() %>

- + <%if JvmVersion.isBadJvmVersion() %> diff --git a/src/main/jamon/org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/src/main/jamon/org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 04c33a0..864e62c 100644 --- a/src/main/jamon/org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/src/main/jamon/org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -56,7 +56,12 @@ org.apache.hadoop.hbase.HRegionInfo;

Region Server: <% serverInfo.getServerAddress().getHostname() %>:<% serverInfo.getServerAddress().getPort() %>

- +

Region Server Attributes

@@ -103,4 +108,4 @@ If region has both an empty start and an empty end key, its the only region in t

Not serving regions

- \ No newline at end of file + diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 0265444..6529a6f 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -675,6 +675,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { String a = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0"); this.infoServer = new InfoServer(MASTER, a, port, false, this.conf); this.infoServer.addServlet("status", "/master-status", MasterStatusServlet.class); + this.infoServer.addServlet("dump", "/dump", MasterDumpServlet.class); this.infoServer.setAttribute(MASTER, this); this.infoServer.start(); } diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java new file mode 100644 index 0000000..876eda4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java @@ -0,0 +1,121 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.Date; +import java.util.Map; +import java.util.NavigableMap; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HServerInfo; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.monitoring.LogMonitoring; +import org.apache.hadoop.hbase.monitoring.StateDumpServlet; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.util.ReflectionUtils; + +public class MasterDumpServlet extends StateDumpServlet { + private static final long serialVersionUID = 1L; + private static final String LINE = + "==========================================================="; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException { + HMaster master = (HMaster) getServletContext().getAttribute(HMaster.MASTER); + assert master != null : "No Master in context!"; + + response.setContentType("text/plain"); + OutputStream os = response.getOutputStream(); + PrintWriter out = new PrintWriter(os); + + out.println("Master status for " + master.getServerName() + + " as of " + new Date()); + + out.println("\n\nVersion Info:"); + out.println(LINE); + dumpVersionInfo(out); + + out.println("\n\nTasks:"); + out.println(LINE); + TaskMonitor.get().dumpAsText(out); + + out.println("\n\nServers:"); + out.println(LINE); + dumpServers(master, out); + + out.println("\n\nRegions-in-transition:"); + out.println(LINE); + dumpRIT(master, out); + + out.println("\n\nExecutors:"); + out.println(LINE); + dumpExecutors(master.getExecutorService(), out); + + out.println("\n\nStacks:"); + out.println(LINE); + ReflectionUtils.printThreadInfo(out, ""); + + out.println("\n\nMaster configuration:"); + out.println(LINE); + Configuration conf = master.getConfiguration(); + out.flush(); + conf.writeXml(os); + os.flush(); + + out.println("\n\nRecent regionserver aborts:"); + out.println(LINE); + master.getRegionServerFatalLogBuffer().dumpTo(out); + + out.println("\n\nLogs"); + out.println(LINE); + long tailKb = getTailKbParam(request); + LogMonitoring.dumpTailOfLogs(out, tailKb); + + out.flush(); + } + + + private void dumpRIT(HMaster master, PrintWriter out) { + NavigableMap regionsInTransition = + master.getAssignmentManager().getRegionsInTransition(); + for (Map.Entry e : regionsInTransition.entrySet()) { + String rid = e.getKey(); + RegionState rs = e.getValue(); + out.println("Region " + rid + ": " + rs.toDescriptiveString()); + } + } + + private void dumpServers(HMaster master, PrintWriter out) { + Map servers = + master.getServerManager().getOnlineServers(); + for (Map.Entry e : servers.entrySet()) { + out.println(e.getKey() + ": " + e.getValue()); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java new file mode 100644 index 0000000..d121ee1 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java @@ -0,0 +1,95 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.nio.channels.FileChannel; +import java.util.Enumeration; +import java.util.Set; + +import org.apache.hadoop.io.IOUtils; +import org.apache.log4j.Appender; +import org.apache.log4j.FileAppender; +import org.apache.log4j.Logger; + +import com.google.common.collect.Sets; + +/** + * Utility functions for reading the log4j logs that are + * being written by HBase. + */ +public abstract class LogMonitoring { + public static Set getActiveLogFiles() throws IOException { + Set ret = Sets.newHashSet(); + Appender a; + @SuppressWarnings("unchecked") + Enumeration e = Logger.getRootLogger().getAllAppenders(); + while (e.hasMoreElements()) { + a = e.nextElement(); + if (a instanceof FileAppender) { + FileAppender fa = (FileAppender) a; + String filename = fa.getFile(); + ret.add(new File(filename)); + } + } + return ret; + } + + + public static void dumpTailOfLogs( + PrintWriter out, long tailKb) throws IOException { + Set logs = LogMonitoring.getActiveLogFiles(); + for (File f : logs) { + out.println("+++++++++++++++++++++++++++++++"); + out.println(f.getAbsolutePath()); + out.println("+++++++++++++++++++++++++++++++"); + try { + dumpTailOfLog(f, out, tailKb); + } catch (IOException ioe) { + out.println("Unable to dump log at " + f); + ioe.printStackTrace(out); + } + out.println("\n\n"); + } + } + + private static void dumpTailOfLog(File f, PrintWriter out, long tailKb) + throws IOException { + FileInputStream fis = new FileInputStream(f); + try { + FileChannel channel = fis.getChannel(); + channel.position(Math.max(0, channel.size() - tailKb*1024)); + BufferedReader r = new BufferedReader( + new InputStreamReader(fis)); + r.readLine(); // skip the first partial line + String line; + while ((line = r.readLine()) != null) { + out.println(line); + } + } finally { + IOUtils.closeStream(fis); + } + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java new file mode 100644 index 0000000..604f10d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java @@ -0,0 +1,62 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Map; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.hbase.executor.ExecutorService; +import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorStatus; +import org.apache.hadoop.hbase.util.VersionInfo; + +public abstract class StateDumpServlet extends HttpServlet { + static final long DEFAULT_TAIL_KB = 100; + private static final long serialVersionUID = 1L; + + protected void dumpVersionInfo(PrintWriter out) { + VersionInfo.writeTo(out); + + out.println("Hadoop " + org.apache.hadoop.util.VersionInfo.getVersion()); + out.println("Subversion " + org.apache.hadoop.util.VersionInfo.getUrl() + " -r " + + org.apache.hadoop.util.VersionInfo.getRevision()); + out.println("Compiled by " + org.apache.hadoop.util.VersionInfo.getUser() + + " on " + org.apache.hadoop.util.VersionInfo.getDate()); + } + + protected long getTailKbParam(HttpServletRequest request) { + String param = request.getParameter("tailkb"); + if (param == null) { + return DEFAULT_TAIL_KB; + } + return Long.parseLong(param); + } + + protected void dumpExecutors(ExecutorService service, PrintWriter out) + throws IOException { + Map statuses = service.getAllExecutorStatuses(); + for (ExecutorStatus status : statuses.values()) { + status.dumpTo(out, " "); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java index 7eab85f..d994e54 100644 --- a/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java +++ b/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hbase.monitoring; +import java.io.PrintWriter; import java.lang.ref.WeakReference; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; @@ -120,6 +121,28 @@ public class TaskMonitor { return (cts > 0 && System.currentTimeMillis() - cts > EXPIRATION_TIME); } + + public void dumpAsText(PrintWriter out) { + long now = System.currentTimeMillis(); + + List tasks = TaskMonitor.get().getTasks(); + for (MonitoredTask task : tasks) { + out.println("Task: " + task.getDescription()); + out.println("Status: " + task.getState() + ":" + task.getStatus()); + long running = (now - task.getStartTime())/1000; + if (task.getCompletionTimestamp() != -1) { + long completed = (now - task.getCompletionTimestamp()) / 1000; + out.println("Completed " + completed + "s ago"); + out.println("Ran for " + + (task.getCompletionTimestamp() - task.getStartTime())/1000 + + "s"); + } else { + out.println("Running for " + running + "s"); + } + out.println(); + } + } + /** * This class encapsulates an object as well as a weak reference to a proxy * that passes through calls to that object. In art form: diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 1dc8a0e..8ff6e62 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1360,6 +1360,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, try { this.infoServer = new InfoServer("regionserver", addr, port, false, this.conf); this.infoServer.addServlet("status", "/rs-status", RSStatusServlet.class); + this.infoServer.addServlet("dump", "/dump", RSDumpServlet.class); this.infoServer.setAttribute(REGIONSERVER, this); this.infoServer.start(); break; @@ -3003,6 +3004,10 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, public Set getRegionsInTransitionInRS() { return this.regionsInTransitionInRS; } + + public ExecutorService getExecutorService() { + return service; + } // // Main program and support routines diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java new file mode 100644 index 0000000..df33d82 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java @@ -0,0 +1,85 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.Date; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.monitoring.LogMonitoring; +import org.apache.hadoop.hbase.monitoring.StateDumpServlet; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.util.ReflectionUtils; + +public class RSDumpServlet extends StateDumpServlet { + private static final long serialVersionUID = 1L; + private static final String LINE = + "==========================================================="; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException { + HRegionServer hrs = (HRegionServer)getServletContext().getAttribute( + HRegionServer.REGIONSERVER); + assert hrs != null : "No RS in context!"; + + response.setContentType("text/plain"); + OutputStream os = response.getOutputStream(); + PrintWriter out = new PrintWriter(os); + + out.println("Master status for " + hrs.getServerName() + + " as of " + new Date()); + + out.println("\n\nVersion Info:"); + out.println(LINE); + dumpVersionInfo(out); + + out.println("\n\nTasks:"); + out.println(LINE); + TaskMonitor.get().dumpAsText(out); + + out.println("\n\nExecutors:"); + out.println(LINE); + dumpExecutors(hrs.getExecutorService(), out); + + out.println("\n\nStacks:"); + out.println(LINE); + ReflectionUtils.printThreadInfo(out, ""); + + out.println("\n\nRS Configuration:"); + out.println(LINE); + Configuration conf = hrs.getConfiguration(); + out.flush(); + conf.writeXml(os); + os.flush(); + + out.println("\n\nLogs"); + out.println(LINE); + long tailKb = getTailKbParam(request); + LogMonitoring.dumpTailOfLogs(out, tailKb); + + out.flush(); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java index e13f197..b516e16 100644 --- a/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java +++ b/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.util; import org.apache.commons.logging.LogFactory; +import java.io.PrintWriter; + import org.apache.hadoop.hbase.VersionAnnotation; import org.apache.hadoop.hbase.master.HMaster; import org.apache.commons.logging.Log; @@ -87,15 +89,27 @@ public class VersionInfo { return version != null ? version.url() : "Unknown"; } - public static void logVersion(){ - LOG.info("HBase " + getVersion()); - LOG.info("Subversion " + getUrl() + " -r " + getRevision()); - LOG.info("Compiled by " + getUser() + " on " + getDate()); + static String[] versionReport() { + return new String[] { + "HBase " + getVersion(), + "Subversion " + getUrl() + " -r " + getRevision(), + "Compiled by " + getUser() + " on " + getDate() + }; + } + + public static void writeTo(PrintWriter out) { + for (String line : versionReport()) { + out.println(line); + } + } + + public static void logVersion() { + for (String line : versionReport()) { + LOG.info(line); + } } public static void main(String[] args) { - System.out.println("HBase " + getVersion()); - System.out.println("Subversion " + getUrl() + " -r " + getRevision()); - System.out.println("Compiled by " + getUser() + " on " + getDate()); + logVersion(); } }