### Eclipse Workspace Patch 1.0 #P hbase-0.20-ro Index: src/webapps/master/table.jsp =================================================================== --- src/webapps/master/table.jsp (revision 819224) +++ src/webapps/master/table.jsp (working copy) @@ -27,6 +27,7 @@ master.getServerAddressToServerInfo(); String tableHeader = "

Table Regions

"; HServerAddress rootLocation = master.getRootRegionLocation(); + Map frags = master.getTableFragmentation(); %> @@ -120,6 +121,7 @@
NameRegion ServerEncoded NameStart KeyEnd Key
+
Attribute NameValueDescription
Enabled<%= hbadmin.isTableEnabled(table.getTableName()) %>Is the table enabled
Fragmentation<%= frags.get(tableName) != null ? frags.get(tableName).intValue() + "%" : "n/a" %>How fragmented is the table. After a major compaction it is 0%.
<% Map regions = table.getRegionsInfo(); Index: src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (revision 894780) +++ src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (working copy) @@ -108,6 +108,12 @@ new MetricsIntValue("memstoreSizeMB", registry); /** + * Size of the compaction queue. + */ + public final MetricsIntValue compactionQueueSize = + new MetricsIntValue("compactionQueueSize", registry); + + /** * filesystem read latency */ public final MetricsTimeVaryingRate fsReadLatency = @@ -158,7 +164,7 @@ this.memstoreSizeMB.pushMetric(this.metricsRecord); this.regions.pushMetric(this.metricsRecord); this.requests.pushMetric(this.metricsRecord); - + this.compactionQueueSize.pushMetric(this.metricsRecord); this.blockCacheSize.pushMetric(this.metricsRecord); this.blockCacheFree.pushMetric(this.metricsRecord); this.blockCacheCount.pushMetric(this.metricsRecord); @@ -218,6 +224,8 @@ Integer.valueOf(this.storefileIndexSizeMB.get())); sb = Strings.appendKeyValue(sb, "memstoreSize", Integer.valueOf(this.memstoreSizeMB.get())); + sb = Strings.appendKeyValue(sb, "compactionQueueSize", + Integer.valueOf(this.compactionQueueSize.get())); // Duplicate from jvmmetrics because metrics are private there so // inaccessible. MemoryUsage memory = Index: src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (revision 892652) +++ src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (working copy) @@ -227,4 +227,14 @@ this.interrupt(); } } + + /** + * Returns the current size of the queue containing regions that are + * processed. + * + * @return The current size of the regions queue. + */ + public int getCompactionQueueSize() { + return compactionQueue.size(); + } } \ No newline at end of file Index: src/java/org/apache/hadoop/hbase/util/FSUtils.java =================================================================== --- src/java/org/apache/hadoop/hbase/util/FSUtils.java (revision 894524) +++ src/java/org/apache/hadoop/hbase/util/FSUtils.java (working copy) @@ -20,10 +20,11 @@ package org.apache.hadoop.hbase.util; import java.io.DataInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RemoteExceptionHandler; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.FSConstants; @@ -368,6 +370,98 @@ } /** + * Returns the total overall fragmentation percentage. Includes .META. and + * -ROOT- as well. + * + * @param master The master defining the HBase root and file system. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static int getTotalTableFragmentation(final HMaster master) + throws IOException { + Map map = getTableFragmentation(master); + return map != null && map.size() > 0 ? map.get("-TOTAL-").intValue() : -1; + } + + /** + * Runs through the HBase rootdir and checks how many stores for each table + * have more than one file in them. Checks -ROOT- and .META. too. The total + * percentage across all tables is stored under the special key "-TOTAL-". + * + * @param master The master defining the HBase root and file system. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableFragmentation( + final HMaster master) + throws IOException { + Path path = master.getRootDir(); + // since HMaster.getFileSystem() is package private + FileSystem fs = path.getFileSystem(master.getConfiguration()); + return getTableFragmentation(fs, path); + } + + /** + * Runs through the HBase rootdir and checks how many stores for each table + * have more than one file in them. Checks -ROOT- and .META. too. The total + * percentage across all tables is stored under the special key "-TOTAL-". + * + * @param fs The file system to use. + * @param hbaseRootDir The root directory to scan. + * @return A map for each table and its percentage. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableFragmentation( + final FileSystem fs, final Path hbaseRootDir) + throws IOException { + Map frags = new HashMap(); + int cfCountTotal = 0; + int cfFragTotal = 0; + DirFilter df = new DirFilter(fs); + // presumes any directory under hbase.rootdir is a table + FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df); + for (int i = 0; i < tableDirs.length; i++) { + // Skip the .log directory. All others should be tables. Inside a table, + // there are compaction.dir directories to skip. Otherwise, all else + // should be regions. Then in each region, should only be family + // directories. Under each of these, should be one file only. + Path d = tableDirs[i].getPath(); + if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { + continue; + } + int cfCount = 0; + int cfFrag = 0; + FileStatus [] regionDirs = fs.listStatus(d, df); + for (int j = 0; j < regionDirs.length; j++) { + Path dd = regionDirs[j].getPath(); + if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) { + continue; + } + // else its a region name, now look in region for families + FileStatus [] familyDirs = fs.listStatus(dd, df); + for (int k = 0; k < familyDirs.length; k++) { + cfCount++; + cfCountTotal++; + Path family = familyDirs[k].getPath(); + // now in family make sure only one file + FileStatus [] familyStatus = fs.listStatus(family); + if (familyStatus.length > 1) { + cfFrag++; + cfFragTotal++; + } + } + } + // compute percentage per table and store in result list + frags.put(d.getName(), new Integer( + Math.round((float) cfFrag / cfCount * 100))); + } + // set overall percentage for all tables + frags.put("-TOTAL-", new Integer( + Math.round((float) cfFragTotal / cfCountTotal * 100))); + return frags; + } + + /** * Expects to find -ROOT- directory. * @param fs * @param hbaseRootDir Index: src/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- src/java/org/apache/hadoop/hbase/master/HMaster.java (revision 894780) +++ src/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -159,6 +159,9 @@ private MasterMetrics metrics; + private long lastFragmentationQuery = -1L; + private Map fragmentation = null; + /** * Build the HMaster out of a raw configuration item. * @param conf configuration @@ -1178,6 +1181,17 @@ } } + public Map getTableFragmentation() throws IOException { + long now = System.currentTimeMillis(); + // only check every two minutes, make this a constant or config? lg + int check = 120 * 1000; + if (lastFragmentationQuery == -1 || now - lastFragmentationQuery > check) { + fragmentation = FSUtils.getTableFragmentation(this); + lastFragmentationQuery = now; + } + return fragmentation; + } + /* * Main program */ Index: src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 892652) +++ src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -1085,6 +1085,8 @@ this.metrics.storefiles.set(storefiles); this.metrics.memstoreSizeMB.set((int)(memstoreSize/(1024*1024))); this.metrics.storefileIndexSizeMB.set((int)(storefileIndexSize/(1024*1024))); + this.metrics.compactionQueueSize.set(compactSplitThread. + getCompactionQueueSize()); LruBlockCache lruBlockCache = (LruBlockCache)StoreFile.getBlockCache(conf); if (lruBlockCache != null) { Index: src/webapps/master/master.jsp =================================================================== --- src/webapps/master/master.jsp (revision 892652) +++ src/webapps/master/master.jsp (working copy) @@ -3,6 +3,7 @@ import="java.net.URLEncoder" import="org.apache.hadoop.io.Text" import="org.apache.hadoop.hbase.util.Bytes" + import="org.apache.hadoop.hbase.util.FSUtils" import="org.apache.hadoop.hbase.master.HMaster" import="org.apache.hadoop.hbase.HConstants" import="org.apache.hadoop.hbase.master.MetaRegion" @@ -23,6 +24,7 @@ if (interval == 0) { interval = 1; } + Map frags = master.getTableFragmentation(); %> @@ -48,6 +50,7 @@ HBase Root Directory<%= master.getRootDir().toString() %>Location of HBase home directory Load average<%= master.getAverageLoad() %>Average number of regions per regionserver. Naive computation. Regions On FS<%= master.countRegionsOnFS() %>Number of regions on FileSystem. Rough count. +Fragmentation<%= frags.get("-TOTAL-") != null ? frags.get("-TOTAL-").intValue() + "%" : "n/a" %>Overall fragmentation of all tables, including .META. and -ROOT-. Zookeeper Quorum<%= master.getZooKeeperWrapper().getQuorumServers() %>Addresses of all registered ZK servers. For more, see zk dump. @@ -55,11 +58,17 @@ <% if (rootLocation != null) { %> - - + + + + + <% if (onlineRegions != null && onlineRegions.size() > 0) { %> - + + + + <% } %>
TableDescription
<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %>The -ROOT- table holds references to all .META. regions.
TableFrag.Description
<%= Bytes.toString(HConstants.ROOT_TABLE_NAME) %><%= frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %>The -ROOT- table holds references to all .META. regions.
<%= Bytes.toString(HConstants.META_TABLE_NAME) %>The .META. table holds references to all User Table regions
<%= Bytes.toString(HConstants.META_TABLE_NAME) %><%= frags.get(".META.") != null ? frags.get(".META.").intValue() + "%" : "n/a" %>The .META. table holds references to all User Table regions
@@ -69,9 +78,11 @@ <% HTableDescriptor[] tables = new HBaseAdmin(conf).listTables(); if(tables != null && tables.length > 0) { %> - + <% for(HTableDescriptor htDesc : tables ) { %> - + + + <% } %>

<%= tables.length %> table(s) in set.

TableDescription
TableFrag.Description
><%= htDesc.getNameAsString() %> <%= htDesc.toString() %>
><%= htDesc.getNameAsString() %><%= frags.get(htDesc.getNameAsString()) != null ? frags.get(htDesc.getNameAsString()).intValue() + "%" : "n/a" %><%= htDesc.toString() %>