Index: hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java =================================================================== --- hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java (revision 1442047) +++ hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestBulkDeleteProtocol.java (working copy) @@ -95,6 +95,7 @@ rows++; } assertEquals(0, rows); + ht.close(); } @Test @@ -118,6 +119,7 @@ rows++; } assertEquals(0, rows); + ht.close(); } private long invokeBulkDeleteProtocol(byte[] tableName, final Scan scan, final int rowBatchSize, @@ -147,6 +149,7 @@ for (BulkDeleteResponse response : result.values()) { noOfDeletedRows += response.getRowsDeleted(); } + ht.close(); return noOfDeletedRows; } @@ -177,6 +180,7 @@ rows++; } assertEquals(90, rows); + ht.close(); } @Test @@ -205,6 +209,7 @@ rows++; } assertEquals(100, rows); + ht.close(); } @Test @@ -235,6 +240,7 @@ rows++; } assertEquals(100, rows); + ht.close(); } @Test @@ -282,6 +288,7 @@ rows++; } assertEquals(100, rows); + ht.close(); } @Test @@ -328,6 +335,7 @@ rows++; } assertEquals(100, rows); + ht.close(); } @Test @@ -412,6 +420,7 @@ rows++; } assertEquals(100, rows); + ht.close(); } private HTable createTable(byte[] tableName) throws IOException { Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (revision 1442047) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (working copy) @@ -547,14 +547,14 @@ List scanners = scanFixture(kvs); Scan scan = new Scan(); scan.setMaxVersions(2); - HStore.ScanInfo scanInfo = new HStore.ScanInfo(Bytes.toBytes("cf"), + HStore.ScanInfo newScanInfo = new HStore.ScanInfo(Bytes.toBytes("cf"), 0 /* minVersions */, 2 /* maxVersions */, 500 /* ttl */, false /* keepDeletedCells */, 200, /* timeToPurgeDeletes */ KeyValue.COMPARATOR); StoreScanner scanner = - new StoreScanner(scan, scanInfo, + new StoreScanner(scan, newScanInfo, ScanType.MAJOR_COMPACT, null, scanners, HConstants.OLDEST_TIMESTAMP); List results = new ArrayList(); @@ -568,6 +568,7 @@ assertEquals(kvs[14], results.get(5)); assertEquals(kvs[15], results.get(6)); assertEquals(7, results.size()); + scanner.close(); }finally{ EnvironmentEdgeManagerTestHelper.reset(); } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java (revision 1442047) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java (working copy) @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; @@ -499,4 +500,9 @@ // TODO Auto-generated method stub return null; } + + @Override + public ExecutorService getExecutorService() { + return null; + } } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java (revision 1442047) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java (working copy) @@ -19,7 +19,10 @@ // this is deliberately not in the o.a.h.h.regionserver package // in order to make sure all required classes/method are available +import static org.junit.Assert.assertEquals; + import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -44,21 +47,22 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.ScanType; -import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; -import static org.junit.Assert.*; - @Category(MediumTests.class) +@RunWith(Parameterized.class) public class TestCoprocessorScanPolicy { final Log LOG = LogFactory.getLog(getClass()); protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -66,7 +70,6 @@ private static final byte[] Q = Bytes.toBytes("qual"); private static final byte[] R = Bytes.toBytes("row"); - @BeforeClass public static void setUpBeforeClass() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); @@ -80,9 +83,22 @@ TEST_UTIL.shutdownMiniCluster(); } + @Parameters + public static Collection parameters() { + return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; + } + + public TestCoprocessorScanPolicy(boolean parallelSeekEnable) { + TEST_UTIL.getMiniHBaseCluster().getConf() + .setBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, parallelSeekEnable); + } + @Test public void testBaseCases() throws Exception { byte[] tableName = Bytes.toBytes("baseCases"); + if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) { + TEST_UTIL.deleteTable(tableName); + } HTable t = TEST_UTIL.createTable(tableName, F, 1); // set the version override to 2 Put p = new Put(R); @@ -129,6 +145,9 @@ @Test public void testTTL() throws Exception { byte[] tableName = Bytes.toBytes("testTTL"); + if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) { + TEST_UTIL.deleteTable(tableName); + } HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(F) .setMaxVersions(10) Index: hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java (revision 1442047) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java (working copy) @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.CompactionRequestor; @@ -172,4 +173,9 @@ // TODO Auto-generated method stub return null; } + + @Override + public ExecutorService getExecutorService() { + return null; + } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (revision 1442047) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (working copy) @@ -76,7 +76,7 @@ private EventHandlerListener listener; // Time to wait for events to happen, should be kept short - protected final int waitingTimeForEvents; + protected int waitingTimeForEvents; private final Span parent; @@ -143,8 +143,11 @@ // Master controlled events to be executed on the master M_SERVER_SHUTDOWN (70, ExecutorType.MASTER_SERVER_OPERATIONS), // Master is processing shutdown of a RS - M_META_SERVER_SHUTDOWN (72, ExecutorType.MASTER_META_SERVER_OPERATIONS); // Master is processing shutdown of RS hosting a meta region (-ROOT- or .META.). + M_META_SERVER_SHUTDOWN (72, ExecutorType.MASTER_META_SERVER_OPERATIONS), // Master is processing shutdown of RS hosting a meta region (-ROOT- or .META.). + // RS controlled events to be executed on the RS + RS_PARALLEL_SEEK (80, ExecutorType.RS_PARALLEL_SEEK); + private final int code; private final ExecutorService.ExecutorType executor; @@ -190,8 +193,10 @@ this.server = server; this.eventType = eventType; seqid = seqids.incrementAndGet(); - this.waitingTimeForEvents = server.getConfiguration(). - getInt("hbase.master.event.waiting.time", 1000); + if (server != null) { + this.waitingTimeForEvents = server.getConfiguration(). + getInt("hbase.master.event.waiting.time", 1000); + } } public void run() { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java (revision 1442047) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java (working copy) @@ -92,7 +92,8 @@ RS_OPEN_META (22), RS_CLOSE_REGION (23), RS_CLOSE_ROOT (24), - RS_CLOSE_META (25); + RS_CLOSE_META (25), + RS_PARALLEL_SEEK (26); ExecutorType(int value) {} Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java (revision 0) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java (revision 0) @@ -0,0 +1,73 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.handler; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl; + +/** + * Handler to seek storefiles in parallel. + */ +@InterfaceAudience.Private +public class ParallelSeekHandler extends EventHandler { + private static final Log LOG = LogFactory.getLog(ParallelSeekHandler.class); + private KeyValueScanner scanner; + private KeyValue keyValue; + private long readPoint; + private CountDownLatch latch; + private Throwable err = null; + + public ParallelSeekHandler(KeyValueScanner scanner,KeyValue keyValue, + long readPoint, CountDownLatch latch) { + super(null, EventType.RS_PARALLEL_SEEK); + this.scanner = scanner; + this.keyValue = keyValue; + this.readPoint = readPoint; + this.latch = latch; + } + + @Override + public void process() { + try { + MultiVersionConsistencyControl.setThreadReadPoint(readPoint); + scanner.seek(keyValue); + } catch (IOException e) { + LOG.error("", e); + setErr(e); + } finally { + latch.countDown(); + } + } + + public Throwable getErr() { + return err; + } + + public void setErr(Throwable err) { + this.err = err; + } +} Property changes on: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/ParallelSeekHandler.java ___________________________________________________________________ Added: svn:mime-type + text/plain Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (revision 1442047) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (working copy) @@ -20,9 +20,11 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; +import java.util.concurrent.CountDownLatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,8 +33,10 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.regionserver.HStore.ScanInfo; +import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -59,6 +63,11 @@ protected final boolean isGet; protected final boolean explicitColumnQuery; protected final boolean useRowColBloom; + /** + * A flag that enables StoreFileScanner parallel-seeking + */ + protected boolean isParallelSeekEnabled = false; + protected ExecutorService executor; protected final Scan scan; protected final NavigableSet columns; protected final long oldestUnexpiredTS; @@ -66,6 +75,8 @@ /** We don't ever expect to change this, the constant is just for clarity. */ static final boolean LAZY_SEEK_ENABLED_BY_DEFAULT = true; + public static final String STORESCANNER_PARALLEL_SEEK_ENABLE = + "hbase.storescanner.parallel.seek.enable"; /** Used during unit testing to ensure that lazy seek does save seek ops */ protected static boolean lazySeekEnabledGlobally = @@ -92,6 +103,17 @@ // for multi-row (non-"get") scans because this is not done in // StoreFile.passesBloomFilter(Scan, SortedSet). useRowColBloom = numCol > 1 || (!isGet && numCol == 1); + // The parallel-seeking is on : + // 1) the config value is *true* + // 2) store has more than one store file + if (store != null && store.getHRegion() != null + && store.getStorefilesCount() > 1) { + RegionServerServices rsService = store.getHRegion().getRegionServerServices(); + if (rsService == null || !rsService.getConfiguration().getBoolean( + STORESCANNER_PARALLEL_SEEK_ENABLE, false)) return; + isParallelSeekEnabled = true; + executor = rsService.getExecutorService(); + } } /** @@ -127,8 +149,12 @@ scanner.requestSeek(matcher.getStartKey(), false, true); } } else { - for (KeyValueScanner scanner : scanners) { - scanner.seek(matcher.getStartKey()); + if (!isParallelSeekEnabled) { + for (KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + } else { + parallelSeek(scanners, matcher.getStartKey()); } } @@ -166,8 +192,12 @@ scanners = selectScannersFrom(scanners); // Seek all scanners to the initial key - for(KeyValueScanner scanner : scanners) { - scanner.seek(matcher.getStartKey()); + if (!isParallelSeekEnabled) { + for (KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + } else { + parallelSeek(scanners, matcher.getStartKey()); } // Combine all seeked scanners with a heap @@ -193,8 +223,12 @@ Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS); // Seek all scanners to the initial key - for (KeyValueScanner scanner : scanners) { - scanner.seek(matcher.getStartKey()); + if (!isParallelSeekEnabled) { + for (KeyValueScanner scanner : scanners) { + scanner.seek(matcher.getStartKey()); + } + } else { + parallelSeek(scanners, matcher.getStartKey()); } heap = new KeyValueHeap(scanners, scanInfo.getComparator()); } @@ -512,8 +546,12 @@ * could have done it now by storing the scan object from the constructor */ List scanners = getScannersNoCompaction(); - for(KeyValueScanner scanner : scanners) { - scanner.seek(lastTopKey); + if (!isParallelSeekEnabled) { + for (KeyValueScanner scanner : scanners) { + scanner.seek(lastTopKey); + } + } else { + parallelSeek(scanners, lastTopKey); } // Combine all seeked scanners with a heap @@ -544,9 +582,8 @@ checkReseek(); if (explicitColumnQuery && lazySeekEnabledGlobally) { return heap.requestSeek(kv, true, useRowColBloom); - } else { - return heap.reseek(kv); } + return heap.reseek(kv); } @Override @@ -555,6 +592,44 @@ } /** + * Seek storefiles in parallel to optimize IO latency as much as possible + * @param scanners the list {@link KeyValueScanner}s to be read from + * @param kv the KeyValue on which the operation is being requested + * @throws IOException + */ + private void parallelSeek(final List + scanners, final KeyValue kv) throws IOException { + if (scanners.isEmpty()) return; + int storeFileScannerCount = scanners.size(); + CountDownLatch latch = new CountDownLatch(storeFileScannerCount); + List handlers = + new ArrayList(storeFileScannerCount); + for (KeyValueScanner scanner : scanners) { + if (scanner instanceof StoreFileScanner) { + ParallelSeekHandler seekHandler = new ParallelSeekHandler(scanner, kv, + MultiVersionConsistencyControl.getThreadReadPoint(), latch); + executor.submit(seekHandler); + handlers.add(seekHandler); + } else { + scanner.seek(kv); + latch.countDown(); + } + } + + try { + latch.await(); + } catch (InterruptedException ie) { + throw new InterruptedIOException(ie.getMessage()); + } + + for (ParallelSeekHandler handler : handlers) { + if (handler.getErr() != null) { + throw new IOException(handler.getErr()); + } + } + } + + /** * Used in testing. * @return all scanners in no particular order */ Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java (revision 1442047) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java (working copy) @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.zookeeper.KeeperException; @@ -92,4 +93,9 @@ * @return The RegionServer's "Leases" service */ public Leases getLeases(); + + /** + * @return hbase executor service + */ + public ExecutorService getExecutorService(); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1442047) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -29,14 +29,12 @@ import java.net.BindException; import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -53,7 +51,6 @@ import javax.management.ObjectName; -import com.google.protobuf.Message; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -66,9 +63,9 @@ import org.apache.hadoop.hbase.FailedSanityCheckException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HealthCheckChore; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.HealthCheckChore; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.OutOfOrderScannerNextException; @@ -214,6 +211,7 @@ import com.google.common.base.Function; import com.google.protobuf.ByteString; +import com.google.protobuf.Message; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -1510,6 +1508,10 @@ conf.getInt("hbase.regionserver.executor.closeroot.threads", 1)); this.service.startExecutorService(ExecutorType.RS_CLOSE_META, conf.getInt("hbase.regionserver.executor.closemeta.threads", 1)); + if (conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false)) { + this.service.startExecutorService(ExecutorType.RS_PARALLEL_SEEK, + conf.getInt("hbase.storescanner.parallel.seek.threads", 10)); + } Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller", uncaughtExceptionHandler); Index: hbase-server/src/main/resources/hbase-default.xml =================================================================== --- hbase-server/src/main/resources/hbase-default.xml (revision 1442047) +++ hbase-server/src/main/resources/hbase-default.xml (working copy) @@ -442,6 +442,21 @@ + hbase.storescanner.parallel.seek.enable + false + + Enables StoreFileScanner parallel-seeking in StoreScanner, + a feature which can reduce response latency under special conditions. + + + + hbase.storescanner.parallel.seek.threads + 10 + + The default thread pool size if parallel-seeking feature enabled. + + + hbase.mapreduce.hfileoutputformat.blocksize 65536 The mapreduce HFileOutputFormat writes storefiles/hfiles.