diff --git a/lib/hadoop-0.19.1-core.jar b/lib/hadoop-0.19.1-core.jar deleted file mode 100644 index 440f9d8..0000000 Binary files a/lib/hadoop-0.19.1-core.jar and /dev/null differ diff --git a/lib/hadoop-0.19.1-test.jar b/lib/hadoop-0.19.1-test.jar deleted file mode 100644 index d817731..0000000 Binary files a/lib/hadoop-0.19.1-test.jar and /dev/null differ diff --git a/lib/hadoop-0.20.0-core.jar b/lib/hadoop-0.20.0-core.jar new file mode 100644 index 0000000..c99ce6d Binary files /dev/null and b/lib/hadoop-0.20.0-core.jar differ diff --git a/lib/hadoop-0.20.0-test.jar b/lib/hadoop-0.20.0-test.jar new file mode 100644 index 0000000..02b17d4 Binary files /dev/null and b/lib/hadoop-0.20.0-test.jar differ diff --git a/lib/jetty-5.1.4.jar b/lib/jetty-5.1.4.jar deleted file mode 100644 index dcbd99e..0000000 Binary files a/lib/jetty-5.1.4.jar and /dev/null differ diff --git a/lib/jetty-6.1.14.jar b/lib/jetty-6.1.14.jar new file mode 100644 index 0000000..8c503be Binary files /dev/null and b/lib/jetty-6.1.14.jar differ diff --git a/lib/jetty-util-6.1.14.jar b/lib/jetty-util-6.1.14.jar new file mode 100644 index 0000000..8f924bb Binary files /dev/null and b/lib/jetty-util-6.1.14.jar differ diff --git a/src/java/org/apache/hadoop/hbase/io/hfile/Compression.java b/src/java/org/apache/hadoop/hbase/io/hfile/Compression.java index 6994844..29da283 100644 --- a/src/java/org/apache/hadoop/hbase/io/hfile/Compression.java +++ b/src/java/org/apache/hadoop/hbase/io/hfile/Compression.java @@ -33,7 +33,6 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; import org.apache.hadoop.io.compress.GzipCodec; -import org.apache.hadoop.io.compress.LzoCodec; /** * Compression related stuff. @@ -73,61 +72,20 @@ public final class Compression { */ public static enum Algorithm { LZO("lzo") { - private LzoCodec codec; @Override CompressionCodec getCodec() { - if (codec == null) { - Configuration conf = new Configuration(); - conf.setBoolean("hadoop.native.lib", true); - codec = new LzoCodec(); - codec.setConf(conf); - } - - return codec; + throw new UnsupportedOperationException("LZO compression is disabled for now"); } - @Override - public synchronized InputStream createDecompressionStream( - InputStream downStream, Decompressor decompressor, - int downStreamBufferSize) throws IOException { - InputStream bis1 = null; - if (downStreamBufferSize > 0) { - bis1 = new BufferedInputStream(downStream, downStreamBufferSize); - } - else { - bis1 = downStream; - } - codec.getConf() - .setInt("io.compression.codec.lzo.buffersize", 64 * 1024); - CompressionInputStream cis = - codec.createInputStream(bis1, decompressor); - BufferedInputStream bis2 = new BufferedInputStream(cis, DATA_IBUF_SIZE); - return bis2; + public InputStream createDecompressionStream(InputStream downStream, Decompressor decompressor, int downStreamBufferSize) throws IOException { + throw new UnsupportedOperationException("LZO compression is disabled for now"); } - @Override - public synchronized OutputStream createCompressionStream( - OutputStream downStream, Compressor compressor, - int downStreamBufferSize) throws IOException { - OutputStream bos1 = null; - if (downStreamBufferSize > 0) { - bos1 = new BufferedOutputStream(downStream, downStreamBufferSize); - } - else { - bos1 = downStream; - } - codec.getConf() - .setInt("io.compression.codec.lzo.buffersize", 64 * 1024); - CompressionOutputStream cos = - codec.createOutputStream(bos1, compressor); - BufferedOutputStream bos2 = - new BufferedOutputStream(new FinishOnFlushCompressionStream(cos), - DATA_OBUF_SIZE); - return bos2; + public OutputStream createCompressionStream(OutputStream downStream, Compressor compressor, int downStreamBufferSize) throws IOException { + throw new UnsupportedOperationException("LZO compression is disabled for now"); } }, - GZ("gz") { private GzipCodec codec; diff --git a/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 9281f68..2a42364 100644 --- a/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -1341,10 +1341,10 @@ public class HFile { private BlockIndex() { this(null); } - + + /** - * Constructor - * @param trailer File tail structure with index stats. + * @param c comparator used to compare keys. */ BlockIndex(final RawComparatorc) { this.comparator = c; diff --git a/src/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java b/src/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java index 77009ce..3ddfda8 100644 --- a/src/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java +++ b/src/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java @@ -638,18 +638,9 @@ public class HBaseRPC { rpcMetrics.rpcQueueTime.inc(qTime); rpcMetrics.rpcProcessingTime.inc(processingTime); - MetricsTimeVaryingRate m = rpcMetrics.metricsList.get(call.getMethodName()); - - if (m != null) { - m.inc(processingTime); - } - else { - rpcMetrics.metricsList.put(call.getMethodName(), new MetricsTimeVaryingRate(call.getMethodName())); - m = rpcMetrics.metricsList.get(call.getMethodName()); - m.inc(processingTime); - } - - if (verbose) log("Return: "+value); + rpcMetrics.inc(call.getMethodName(), processingTime); + + if (verbose) log("Return: " + value); return new HbaseObjectWritable(method.getReturnType(), value); diff --git a/src/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java b/src/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java index 57ceff4..fcfd139 100644 --- a/src/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java +++ b/src/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java @@ -30,6 +30,7 @@ import org.apache.hadoop.metrics.MetricsRecord; import org.apache.hadoop.metrics.MetricsUtil; import org.apache.hadoop.metrics.Updater; import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; +import org.apache.hadoop.metrics.util.MetricsRegistry; /** * @@ -65,14 +66,29 @@ public class HBaseRpcMetrics implements Updater { * - they can be set directly by calling their set/inc methods * -they can also be read directly - e.g. JMX does this. */ - - public MetricsTimeVaryingRate rpcQueueTime = new MetricsTimeVaryingRate("RpcQueueTime"); - public MetricsTimeVaryingRate rpcProcessingTime = new MetricsTimeVaryingRate("RpcProcessingTime"); + public MetricsRegistry registry = new MetricsRegistry(); - public Map metricsList = Collections.synchronizedMap(new HashMap()); + public MetricsTimeVaryingRate rpcQueueTime = new MetricsTimeVaryingRate("RpcQueueTime", registry); + public MetricsTimeVaryingRate rpcProcessingTime = new MetricsTimeVaryingRate("RpcProcessingTime", registry); + + //public Map metricsList = Collections.synchronizedMap(new HashMap()); + + private MetricsTimeVaryingRate get(String key) { + return (MetricsTimeVaryingRate) registry.get(key); + } + private MetricsTimeVaryingRate create(String key) { + MetricsTimeVaryingRate newMetric = new MetricsTimeVaryingRate(key, this.registry); + return newMetric; + } + + public synchronized void inc(String name, int amt) { + MetricsTimeVaryingRate m = get(name); + if (m == null) { + m = create(name); + } + m.inc(amt); + } - - /** * Push the metrics to the monitoring subsystem on doUpdate() call. * @param context @@ -81,18 +97,14 @@ public class HBaseRpcMetrics implements Updater { rpcQueueTime.pushMetric(metricsRecord); rpcProcessingTime.pushMetric(metricsRecord); - synchronized (metricsList) { - // Iterate through the rpcMetrics hashmap to propogate the different rpc metrics. - Set keys = metricsList.keySet(); - - Iterator keyIter = keys.iterator(); + synchronized (registry) { + // Iterate through the registry to propogate the different rpc metrics. - while (keyIter.hasNext()) { - Object key = keyIter.next(); - MetricsTimeVaryingRate value = metricsList.get(key); + for (String metricName : registry.getKeyList() ) { + MetricsTimeVaryingRate value = (MetricsTimeVaryingRate) registry.get(metricName); - value.pushMetric(metricsRecord); - } + value.pushMetric(metricsRecord); + } } metricsRecord.update(); } diff --git a/src/java/org/apache/hadoop/hbase/master/HMaster.java b/src/java/org/apache/hadoop/hbase/master/HMaster.java index 7c0e827..806b9b7 100644 --- a/src/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/java/org/apache/hadoop/hbase/master/HMaster.java @@ -410,7 +410,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface, LOG.info("Stopping infoServer"); try { this.infoServer.stop(); - } catch (InterruptedException ex) { + } catch (Exception ex) { ex.printStackTrace(); } } diff --git a/src/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/java/org/apache/hadoop/hbase/master/ServerManager.java index 890d09d..eadd102 100644 --- a/src/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/src/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -125,6 +125,9 @@ class ServerManager implements HConstants { String serverName = HServerInfo.getServerName(info); if (serversToServerInfo.containsKey(serverName) || deadServers.contains(serverName)) { + LOG.debug("Server start was rejected: " + serverInfo); + LOG.debug("serversToServerInfo.containsKey: " + serversToServerInfo.containsKey(serverName)); + LOG.debug("deadServers.contains: " + deadServers.contains(serverName)); throw new Leases.LeaseStillHeldException(serverName); } Watcher watcher = new ServerExpirer(serverName, info.getServerAddress()); diff --git a/src/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java b/src/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java index 9f635c0..4d527b0 100644 --- a/src/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java +++ b/src/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java @@ -25,6 +25,7 @@ import org.apache.hadoop.metrics.MetricsUtil; import org.apache.hadoop.metrics.Updater; import org.apache.hadoop.metrics.jvm.JvmMetrics; import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; /** @@ -37,12 +38,12 @@ import org.apache.hadoop.metrics.util.MetricsIntValue; public class MasterMetrics implements Updater { private final Log LOG = LogFactory.getLog(this.getClass()); private final MetricsRecord metricsRecord; - + private final MetricsRegistry registry = new MetricsRegistry(); /* * Count of requests to the cluster since last call to metrics update */ private final MetricsIntValue cluster_requests = - new MetricsIntValue("cluster_requests"); + new MetricsIntValue("cluster_requests", registry); public MasterMetrics() { MetricsContext context = MetricsUtil.getContext("hbase"); @@ -90,7 +91,7 @@ public class MasterMetrics implements Updater { */ public void incrementRequests(final int inc) { synchronized(this.cluster_requests) { - this.cluster_requests.inc(inc); + this.cluster_requests.set(this.cluster_requests.get() + inc); } } } \ No newline at end of file diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 8f8d6d2..45c5ded 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -524,8 +524,8 @@ public class HRegionServer implements HConstants, HRegionInterface, LOG.info("Stopping infoServer"); try { this.infoServer.stop(); - } catch (InterruptedException ex) { - ex.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); } } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java index 94216fa..8838e89 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java @@ -32,6 +32,7 @@ import org.apache.hadoop.metrics.jvm.JvmMetrics; import org.apache.hadoop.metrics.util.MetricsIntValue; import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; /** * This class is for maintaining the various regionserver statistics @@ -45,33 +46,36 @@ public class RegionServerMetrics implements Updater { private final MetricsRecord metricsRecord; private long lastUpdate = System.currentTimeMillis(); private static final int MB = 1024*1024; + + private MetricsRegistry registry = new MetricsRegistry(); - public final MetricsTimeVaryingRate atomicIncrementTime = new MetricsTimeVaryingRate("hbase_atomicIncrementTime"); + public final MetricsTimeVaryingRate atomicIncrementTime = + new MetricsTimeVaryingRate("hbase_atomicIncrementTime", registry); /** * Count of regions carried by this regionserver */ - public final MetricsIntValue regions = new MetricsIntValue("hbase_regions"); + public final MetricsIntValue regions = new MetricsIntValue("hbase_regions", registry); /** * Block cache size. */ - public final MetricsLongValue blockCacheSize = new MetricsLongValue("hbase_blockCacheSize"); + public final MetricsLongValue blockCacheSize = new MetricsLongValue("hbase_blockCacheSize", registry); /** * Block cache free size. */ - public final MetricsLongValue blockCacheFree = new MetricsLongValue("hbase_blockCacheFree"); + public final MetricsLongValue blockCacheFree = new MetricsLongValue("hbase_blockCacheFree", registry); /** * Block cache item count. */ - public final MetricsLongValue blockCacheCount = new MetricsLongValue("hbase_blockCacheCount"); + public final MetricsLongValue blockCacheCount = new MetricsLongValue("hbase_blockCacheCount", registry); /** * Block hit ratio. */ - public final MetricsIntValue blockCacheHitRatio = new MetricsIntValue("hbase_blockCacheHitRatio"); + public final MetricsIntValue blockCacheHitRatio = new MetricsIntValue("hbase_blockCacheHitRatio", registry); /* * Count of requests to the regionservers since last call to metrics update @@ -81,24 +85,24 @@ public class RegionServerMetrics implements Updater { /** * Count of stores open on the regionserver. */ - public final MetricsIntValue stores = new MetricsIntValue("hbase_stores"); + public final MetricsIntValue stores = new MetricsIntValue("hbase_stores", registry); /** * Count of storefiles open on the regionserver. */ - public final MetricsIntValue storefiles = new MetricsIntValue("hbase_storefiles"); + public final MetricsIntValue storefiles = new MetricsIntValue("hbase_storefiles", registry); /** * Sum of all the storefile index sizes in this regionserver in MB */ public final MetricsIntValue storefileIndexSizeMB = - new MetricsIntValue("hbase_storefileIndexSizeMB"); + new MetricsIntValue("hbase_storefileIndexSizeMB", registry); /** * Sum of all the memcache sizes in this regionserver in MB */ public final MetricsIntValue memcacheSizeMB = - new MetricsIntValue("hbase_memcacheSizeMB"); + new MetricsIntValue("hbase_memcacheSizeMB", registry); public RegionServerMetrics() { MetricsContext context = MetricsUtil.getContext("hbase"); diff --git a/src/java/org/apache/hadoop/hbase/rest/Dispatcher.java b/src/java/org/apache/hadoop/hbase/rest/Dispatcher.java index 7369268..e71a4fe 100644 --- a/src/java/org/apache/hadoop/hbase/rest/Dispatcher.java +++ b/src/java/org/apache/hadoop/hbase/rest/Dispatcher.java @@ -19,15 +19,6 @@ */ package org.apache.hadoop.hbase.rest; -import java.io.BufferedReader; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -38,8 +29,20 @@ import org.apache.hadoop.hbase.rest.parser.IHBaseRestParser; import org.apache.hadoop.hbase.rest.serializer.RestSerializerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.InfoServer; -import org.mortbay.http.NCSARequestLog; -import org.mortbay.http.SocketListener; +import org.mortbay.jetty.Connector; +import org.mortbay.jetty.NCSARequestLog; +import org.mortbay.jetty.bio.SocketConnector; +import org.mortbay.jetty.handler.RequestLogHandler; +import org.mortbay.jetty.webapp.WebAppContext; +import org.mortbay.thread.QueuedThreadPool; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; /** * Servlet implementation class for hbase REST interface. Presumes container @@ -466,15 +469,30 @@ public class Dispatcher extends javax.servlet.http.HttpServlet { printUsageAndExit(); } org.mortbay.jetty.Server webServer = new org.mortbay.jetty.Server(); - SocketListener listener = new SocketListener(); - listener.setPort(port); - listener.setHost(bindAddress); - listener.setMaxThreads(numThreads); - webServer.addListener(listener); + + Connector connector = new SocketConnector(); + connector.setPort(port); + connector.setHost(bindAddress); + + QueuedThreadPool pool = new QueuedThreadPool(); + pool.setMaxThreads(numThreads); + + webServer.addConnector(connector); + webServer.setThreadPool(pool); + + WebAppContext wac = new WebAppContext(); + wac.setContextPath("/"); + wac.setWar(InfoServer.getWebAppDir("rest")); + NCSARequestLog ncsa = new NCSARequestLog(); ncsa.setLogLatency(true); - webServer.setRequestLog(ncsa); - webServer.addWebApplication("/", InfoServer.getWebAppDir("rest")); + + RequestLogHandler rlh = new RequestLogHandler(); + rlh.setRequestLog(ncsa); + rlh.setHandler(wac); + + webServer.addHandler(rlh); + webServer.start(); } diff --git a/src/java/org/apache/hadoop/hbase/util/InfoServer.java b/src/java/org/apache/hadoop/hbase/util/InfoServer.java index eef5083..7d567cc 100644 --- a/src/java/org/apache/hadoop/hbase/util/InfoServer.java +++ b/src/java/org/apache/hadoop/hbase/util/InfoServer.java @@ -21,8 +21,11 @@ import java.io.IOException; import java.net.URL; import org.apache.hadoop.http.HttpServer; -import org.mortbay.http.HttpContext; -import org.mortbay.http.handler.ResourceHandler; +import org.mortbay.jetty.handler.ResourceHandler; +import org.mortbay.jetty.handler.ContextHandler; +import org.mortbay.jetty.handler.ContextHandlerCollection; +import org.mortbay.jetty.handler.HandlerCollection; +import org.mortbay.jetty.webapp.WebAppContext; /** * Create a Jetty embedded server to answer http requests. The primary goal @@ -46,21 +49,32 @@ public class InfoServer extends HttpServer { public InfoServer(String name, String bindAddress, int port, boolean findPort) throws IOException { super(name, bindAddress, port, findPort); - + + HandlerCollection handlers = + new ContextHandlerCollection(); + // Set up the context for "/logs/" if "hbase.log.dir" property is defined. String logDir = System.getProperty("hbase.log.dir"); if (logDir != null) { - HttpContext logContext = new HttpContext(); + ContextHandler logContext = new ContextHandler(); logContext.setContextPath("/logs/*"); logContext.setResourceBase(logDir); logContext.addHandler(new ResourceHandler()); - webServer.addContext(logContext); + + handlers.addHandler(logContext); } if (name.equals("master")) { // Put up the rest webapp. - webServer.addWebApplication("/api", getWebAppDir("rest")); + WebAppContext wac = new WebAppContext(); + wac.setContextPath("/api"); + wac.setWar(getWebAppDir("rest")); + + handlers.addHandler(wac); } + + webServer.addHandler(handlers); + } /** diff --git a/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java index e06007e..54f64c0 100644 --- a/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -233,7 +233,7 @@ public class TestHFile extends TestCase { * Make sure the orginals for our compression libs doesn't change on us. */ public void testCompressionOrdinance() { - assertTrue(Compression.Algorithm.LZO.ordinal() == 0); + //assertTrue(Compression.Algorithm.LZO.ordinal() == 0); assertTrue(Compression.Algorithm.GZ.ordinal() == 1); assertTrue(Compression.Algorithm.NONE.ordinal() == 2); }