From f8c49f30446c894ad0c633eed7a0d4f9920e4491 Mon Sep 17 00:00:00 2001 From: Mikhail Bautin Date: Tue, 28 Aug 2012 21:01:16 +0000 Subject: [PATCH] [HBASE-6640] [0.89-fb] Allow multiple regions to be opened simultaneously, from a thread pool Author: aaiyer Summary: Although master can assign multiple regions to the RS simultaneously, regionservers currently only process this one by one. This causes the time taken for region opening to be ~5-10 mins during a rack failure. A simple change to parallize region opens. This should help us cut down this time. Test Plan: test on mr do a rack failure test and check the time taken to assign+open regions Reviewers: kannan, kranganathan, avf Reviewed By: avf CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D545919 git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/0.89-fb@1378342 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/hbase/HConstants.java | 13 ++++++ .../hadoop/hbase/regionserver/HRegionServer.java | 42 +++++++++++++++----- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java index 639fdaa..bac1c1a 100644 --- a/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -236,6 +236,19 @@ public final class HConstants { */ public static final int DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX = 8; + /** + * The max number of threads used for opening and closing regions + * in parallel + */ + public static final String HREGION_OPEN_AND_CLOSE_THREADS_MAX = + "hbase.region.open.and.close.threads.max"; + + /** + * The default number for the max number of threads used for opening and + * closing regions in parallel + */ + public static final int DEFAULT_HREGION_OPEN_AND_CLOSE_THREADS_MAX = 8; + /** Default maximum file size */ public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024; diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 8d44279..842a682 100755 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -54,6 +54,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -287,6 +288,7 @@ public class HRegionServer implements HRegionInterface, private ZooKeeperWrapper zooKeeperWrapper; private final ExecutorService logCloseThreadPool; + private final ExecutorService regionOpenCloseThreadPool; // Log Splitting Worker private SplitLogWorker splitLogWorker; @@ -413,6 +415,20 @@ public class HRegionServer implements HRegionInterface, logCloseThreadPool = Executors.newFixedThreadPool(logCloseThreads, new DaemonThreadFactory("hregionserver-split-logClose-thread-")); + + int maxRegionOpenCloseThreads = Math.max(1, + conf.getInt(HConstants.HREGION_OPEN_AND_CLOSE_THREADS_MAX, + HConstants.DEFAULT_HREGION_OPEN_AND_CLOSE_THREADS_MAX)); + regionOpenCloseThreadPool = Threads + .getBoundedCachedThreadPool(maxRegionOpenCloseThreads, 30L, TimeUnit.SECONDS, + new ThreadFactory() { + private int count = 1; + + public Thread newThread(Runnable r) { + Thread t = new Thread(r, "regionOpenCloseThread-" + count++); + return t; + } + }); } /** @@ -1797,11 +1813,12 @@ public class HRegionServer implements HRegionInterface, } } if (!requeued) { + String favouredNodes = null; if (e.msg.getMessage() != null && e.msg.getMessage().length > 0) { - openRegion(info, new String(e.msg.getMessage())); - } else { - openRegion(info, null); + favouredNodes = new String(e.msg.getMessage()); } + regionOpenCloseThreadPool.submit( + createRegionOpenCallable(info, favouredNodes)); } break; @@ -2072,17 +2089,12 @@ public class HRegionServer implements HRegionInterface, } // Then, we close the regions - ExecutorService closingPoolExecutor = - new ThreadPoolExecutor(1, Integer.MAX_VALUE, - 60, TimeUnit.SECONDS, - new SynchronousQueue(), - new DaemonThreadFactory("regionserver-closing-")); - List> futures = new ArrayList>(regionsToClose.size()); for (int i = 0; i < regionsToClose.size(); i++ ) { - futures.add(closingPoolExecutor.submit(createRegionCloseCallable(regionsToClose.get(i)))); + futures.add(regionOpenCloseThreadPool.submit( + createRegionCloseCallable(regionsToClose.get(i)))); } ArrayList regionsClosed = new ArrayList(); @@ -2101,6 +2113,16 @@ public class HRegionServer implements HRegionInterface, return regionsClosed; } + private Callable createRegionOpenCallable(final HRegionInfo rinfo, + final String favouredNodes) { + return new Callable() { + public Object call() throws IOException { + openRegion(rinfo, favouredNodes); + return null; + } + }; + } + private Callable createRegionCloseCallable(final HRegion region) { return new Callable() { public Object call() throws IOException { -- 1.7.0.4