Index: src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java
===================================================================
--- src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java (revision 1175430)
+++ src/test/java/org/apache/hadoop/hbase/client/TestHTableUtil.java (working copy)
@@ -27,6 +27,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HBaseTestingUtilityFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -38,7 +39,7 @@
*/
public class TestHTableUtil {
final Log LOG = LogFactory.getLog(getClass());
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static byte [] ROW = Bytes.toBytes("testRow");
private static byte [] FAMILY = Bytes.toBytes("testFamily");
private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
@@ -49,7 +50,7 @@
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL = HBaseTestingUtilityFactory.getMiniCluster(1);
}
/**
@@ -57,7 +58,7 @@
*/
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
+ HBaseTestingUtilityFactory.returnMiniCluster(TEST_UTIL);;
}
/**
Index: src/test/java/org/apache/hadoop/hbase/HBaseTestingUtilityFactory.java
===================================================================
--- src/test/java/org/apache/hadoop/hbase/HBaseTestingUtilityFactory.java (revision 0)
+++ src/test/java/org/apache/hadoop/hbase/HBaseTestingUtilityFactory.java (revision 0)
@@ -0,0 +1,304 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+/**
+ * HBaseTestingUtilityFactory
+ *
+ * Initializing and shutting down HBaseTestingUtility instances are expensive. This factory
+ * is intended to create and cache HBaseTestingUtility instances, and also tear them down
+ * after a configured period of inactivity.
+ *
+ */
+public class HBaseTestingUtilityFactory {
+
+ private static final Integer ONE = new Integer(1);
+ private static final Integer ZERO = new Integer(0);
+
+ /**
+ * The "linger" time in seconds. This factory will wait this many seconds
+ * when it has detected that none of the cached MiniClusters are being used
+ * before tearing them down.
+ * This property can be set as a Java System Property
+ *
+ */
+ public static final String LINGER_PROPERTY = "hbase.testing.utility.factory.linger";
+
+ /**
+ * "linger" default in seconds.
+ */
+ public final static int LINGER_DEFAULT = 10;
+
+ private long lingerSeconds = LINGER_DEFAULT;
+
+ private final static Log LOG = LogFactory.getLog(HBaseTestingUtilityFactory.class);
+
+ /**
+ * # of RS to Testing Util (miniclusters)
+ * The reason this structure exists is we might want to keep a number of MiniClusters active, with
+ * varying numbers of RS.
+ */
+ private Map typeToMiniCluster = new HashMap();
+
+ /**
+ * Testing Util instance to minicluster usage count. This is the reference counter.
+ *
+ */
+ private Map mcUsageCount = new HashMap();
+
+ /**
+ * Cleaner thread of HBaseTestingUtility instance
+ */
+ private UtilityCleaner cleanerThread = null;
+
+ private HBaseTestingUtilityFactory() {
+
+ // check for linger time in system property
+ String val = System.getProperty(LINGER_PROPERTY);
+ if (val != null) {
+ lingerSeconds = Long.parseLong(val);
+ LOG.info("lingerSeconds set to "+lingerSeconds);
+ }
+ }
+
+ private static final HBaseTestingUtilityFactory INSTANCE = new HBaseTestingUtilityFactory();
+
+ protected synchronized Map getMiniClusterUsageCountMap() {
+ return mcUsageCount;
+ }
+
+ //
+ // these static methods are the public interface for the factory..
+ //
+
+ /**
+ * Obtains a MiniCluster initialized with variable number of slaves..
+ * This instance MUST be returned via the 'returnMiniCluster' method in this factory to accurately
+ * keep track of usage.
+ *
+ * @param slaves the number of MiniCluster slaves
+ * @return HBaseTestingUtility
+ * @throws Exception exception
+ */
+ public static HBaseTestingUtility getMiniCluster(int slaves) throws Exception {
+ return INSTANCE.getMiniClusterImpl(slaves);
+ }
+
+ /**
+ * This method must be invoked when the MiniCluster is no longer needed.
+ * Returning the MiniCluster will also cause all the tables to be removed so it can be reused.
+ *
+ * @param utilInstance HBaseTestingUtility instance no longer in use
+ * @throws IOException exception
+ */
+ public static void returnMiniCluster(HBaseTestingUtility utilInstance) throws IOException {
+ INSTANCE.returnMiniClusterImpl(utilInstance);
+ }
+
+ //
+ // these instance methods are the implementation of the factory..
+ //
+
+ /**
+ * Obtains a MiniCluster initialized with variable number of slaves..
+ * This instance MUST be returned via the 'returnMiniCluster' method in this factory to accurately
+ * keep track of usage.
+ *
+ * @return HBaseTestingUtility
+ * @throws Exception exception
+ */
+ protected synchronized HBaseTestingUtility getMiniClusterImpl(int slaves) throws Exception {
+ HBaseTestingUtility tu = typeToMiniCluster.get(slaves); // 1 RS
+ if (tu == null) {
+ tu = new HBaseTestingUtility();
+ typeToMiniCluster.put(slaves,tu);
+ LOG.info("starting MiniCluster("+slaves+")");
+ tu.startMiniCluster(slaves);
+ } else {
+ LOG.info("returning pre-created MiniCluster("+slaves+")");
+ }
+
+ incrementUsage(tu);
+
+ // start cleaner thread if it's not started yet
+ if (cleanerThread == null) {
+ cleanerThread = new UtilityCleaner();
+ cleanerThread.setLingerMs(lingerSeconds * 1000);
+ cleanerThread.setFactory(INSTANCE);
+ Thread t = new Thread(cleanerThread);
+ t.start();
+ }
+ return tu;
+ }
+
+ private synchronized void incrementUsage(HBaseTestingUtility tu) {
+ Integer i = mcUsageCount.get(tu);
+ if (i == null) {
+ i = new Integer(ONE);
+ } else {
+ int j = i.intValue() + 1;
+ i = new Integer(j);
+ }
+ mcUsageCount.put(tu, i);
+ }
+
+ private synchronized void decrementUsage(HBaseTestingUtility tu) {
+ Integer i = mcUsageCount.get(tu);
+ if (i != null) {
+ int j = i.intValue() - 1;
+ i = new Integer(j);
+ mcUsageCount.put(tu, i);
+ } else {
+ // the count is null
+ throw new RuntimeException("Decrement for unknown HBaseTestingUtility instance");
+ }
+ }
+
+ protected synchronized void shutdownAllMiniClusters() throws IOException {
+ LOG.info("Shutting down all MiniClusters");
+ for (HBaseTestingUtility tu : mcUsageCount.keySet()) {
+ tu.shutdownMiniCluster();
+ }
+ mcUsageCount.clear();
+ typeToMiniCluster.clear();
+
+ // reset the utility cleaner thread.
+ // the assumption is that the only thing that should be calling
+ // this method is the cleaner thread right before it exits.
+ if (cleanerThread != null) {
+ cleanerThread = null;
+ }
+ }
+
+ /**
+ * This method must be invoked when the MiniCluster is no longer needed.
+ * Returning the MiniCluster will also cause all the tables to be removed so it can be reused.
+ *
+ * @param utilInstance HBaseTestingUtility instance no longer in use
+ *
+ */
+ protected synchronized void returnMiniClusterImpl(HBaseTestingUtility utilInstance) throws IOException {
+ decrementUsage(utilInstance);
+ clearTables(utilInstance);
+ }
+
+ /**
+ * Drop all HBase tables in the instance
+ *
+ * @param utilInstance
+ * @throws IOException
+ */
+ private void clearTables(HBaseTestingUtility utilInstance) throws IOException {
+
+ HBaseAdmin admin = utilInstance.getHBaseAdmin();
+ HTableDescriptor[] tables = admin.listTables();
+ for (HTableDescriptor table: tables) {
+ LOG.debug("Deleting table " + table.getNameAsString());
+ long start = System.currentTimeMillis();
+ admin.disableTable(table.getName());
+ admin.deleteTable(table.getName());
+ start = System.currentTimeMillis() - start;
+ LOG.info("Deleted table " + table.getNameAsString() + " in " + start + " (ms)");
+ }
+ }
+
+
+ /**
+ * UtilityCleaner will check the reference counts of allocated HBaseTestingUtility instances.
+ *
+ * When no instances are in use it will wait for a configurable period of time, then call the
+ * appropriate shutdown methods.
+ *
+ */
+ private static class UtilityCleaner implements Runnable {
+
+ private long lingerMs = 0; // the 'wait time' in MS.
+ private HBaseTestingUtilityFactory factory;
+
+ public UtilityCleaner() {
+
+ }
+
+ public void setLingerMs(long lingerMs) {
+ this.lingerMs = lingerMs;
+ }
+
+ public void setFactory(HBaseTestingUtilityFactory factory) {
+ this.factory = factory;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("UtilityCleaner starting...");
+
+ long lingerStart = 0;
+
+ try {
+ int inUseCount = 0;
+ Thread.sleep(2000); // check every so often activity
+ Map utilUsageCount = factory.getMiniClusterUsageCountMap();
+ for (HBaseTestingUtility tu: factory.getMiniClusterUsageCountMap().keySet()) {
+ Integer i = utilUsageCount.get(tu);
+ if (! i.equals(ZERO)) {
+ inUseCount++;
+ }
+ }
+ if (inUseCount == 0) {
+ // nothing is being used...
+
+ if (lingerStart == 0) {
+ // this is the first time we have detected that nothing is being used.
+ lingerStart = System.currentTimeMillis();
+ } else {
+ // we've noticed before.
+ if ((System.currentTimeMillis() - lingerStart) > lingerMs) {
+ // we've waited long enough.
+ // shut down the instances and stop.
+ LOG.info("UtilityCleaner shutting down all shared MiniClusters");
+ factory.shutdownAllMiniClusters();
+
+ LOG.info("UtilityCleaner");
+ return;
+ }
+ }
+
+ } else {
+ // there is at least one instance in use...
+ lingerStart = 0;
+ }
+
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ }
+
+}