From dc6612f3b28bd77b48655882f2a882289921d382 Mon Sep 17 00:00:00 2001 From: huaxiangsun Date: Thu, 16 Nov 2017 16:50:18 -0800 Subject: [PATCH] HBASE-19163 Maximum lock count exceeded from region server's batch processing --- .../apache/hadoop/hbase/regionserver/HRegion.java | 23 ++++++++++++++++++ .../hadoop/hbase/client/TestFromClientSide3.java | 27 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 3a3cb03..f517813 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -226,6 +226,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; public static final int DEFAULT_MAX_CELL_SIZE = 10485760; + public static final String HBASE_REGIONSERVER_MINIBATCH_SIZE = + "hbase.regionserver.keyvalue.maxsize"; + public static final int DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE = 10000; + /** * This is the global default value for durability. All tables/mutations not * defining a durability or using USE_DEFAULT will default to this value. @@ -338,6 +342,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // in bytes final long maxCellSize; + // Number of mutations for minibatch processing. + private final int miniBatchSize; + // negative number indicates infinite timeout static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L; final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool(); @@ -809,6 +816,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE); this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE); + this.miniBatchSize = conf.getInt(HBASE_REGIONSERVER_MINIBATCH_SIZE, + DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE); } void setHTableSpecificConf() { @@ -3150,6 +3159,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi int readyToWriteCount = 0; int lastIndexExclusive = 0; for (; lastIndexExclusive < size(); lastIndexExclusive++) { + // It reaches the miniBatchSize, stop here and process the miniBatch + if (readyToWriteCount == region.miniBatchSize) { + break; + } + if (!isOperationPending(lastIndexExclusive)) { continue; } @@ -3177,6 +3191,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } else { acquiredRowLocks.add(rowLock); } + readyToWriteCount++; } return createMiniBatch(lastIndexExclusive, readyToWriteCount); @@ -5619,6 +5634,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi TraceUtil.addTimelineAnnotation("Interrupted exception getting row lock"); Thread.currentThread().interrupt(); throw iie; + } catch (Error error) { + // The maximum lock count for read lock is 64K (hardcoded), when this maximum count + // is reached, it will throw out an Error. This Error needs to be caught so it can + // go ahead to process the minibatch with lock acquired. + IOException ioe = new IOException(); + ioe.initCause(error); + TraceUtil.addTimelineAnnotation("Error getting row lock"); + throw ioe; } finally { // Clean up the counts just in case this was the thing keeping the context alive. if (!success && rowLockContext != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index 5f7622a..e5d5324 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -411,6 +411,33 @@ public class TestFromClientSide3 { } } + // Test Table.batch with large amount of mutations against the same key. + // It used to trigger read lock's "Maximum lock count exceeded" Error. + @Test + public void testHTableWithLargeBatch() throws Exception { + Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), + new byte[][] { FAMILY }); + int sixtyFourK = 64 * 1024; + try { + List actions = new ArrayList(); + Object[] results = new Object[(sixtyFourK + 1) * 2]; + + for (int i = 0; i < sixtyFourK + 1; i ++) { + Put put1 = new Put(ROW); + put1.addColumn(FAMILY, QUALIFIER, VALUE); + actions.add(put1); + + Put put2 = new Put(ANOTHERROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE); + actions.add(put2); + } + + table.batch(actions, results); + } finally { + table.close(); + } + } + @Test public void testBatchWithRowMutation() throws Exception { LOG.info("Starting testBatchWithRowMutation"); -- 2.5.2