From ccdc275eb931f35ac356ef77c033858573f54d21 Mon Sep 17 00:00:00 2001 From: Jonathan Lawlor Date: Tue, 21 Apr 2015 17:59:09 -0700 Subject: [PATCH] HBASE-13527 The default value for hbase.client.scanner.max.result.size is never actually set on Scans --- .../apache/hadoop/hbase/client/ClientScanner.java | 5 +++ .../org/apache/hadoop/hbase/client/HTable.java | 6 ++- .../hadoop/hbase/client/TableConfiguration.java | 10 +++++ .../hbase/client/TestScannersFromClientSide.java | 50 ++++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 6a129c71865e94f5a248b7294cdb1874cd959efb..ecbadb7ee5053d1b98cdb81e4b69a8d70bb12483 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -208,6 +208,11 @@ public class ClientScanner extends AbstractClientScanner { return lastNext; } + @VisibleForTesting + protected long getMaxResultSize() { + return maxScannerResultSize; + } + // returns true if the passed region endKey protected boolean checkScanStopRow(final byte [] endKey) { if (this.scan.getStopRow().length > 0) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 434e32fd258328029a4d4ad1cc8f0d3aed615de7..d3bad280fa9504c4f0e175e77b4a3cec00ae4fc7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -116,6 +116,7 @@ public class HTable implements HTableInterface { private boolean autoFlush = true; private boolean closed = false; protected int scannerCaching; + protected long scannerMaxResultSize; private ExecutorService pool; // For Multi & Scan private int operationTimeout; private final boolean cleanupPoolOnClose; // shutdown the pool in close() @@ -217,7 +218,7 @@ public class HTable implements HTableInterface { this.operationTimeout = tableName.isSystemTable() ? tableConfiguration.getMetaOperationTimeout() : tableConfiguration.getOperationTimeout(); this.scannerCaching = tableConfiguration.getScannerCaching(); - + this.scannerMaxResultSize = tableConfiguration.getScannerMaxResultSize(); if (this.rpcCallerFactory == null) { this.rpcCallerFactory = connection.getNewRpcRetryingCallerFactory(configuration); } @@ -615,6 +616,9 @@ public class HTable implements HTableInterface { if (scan.getCaching() <= 0) { scan.setCaching(scannerCaching); } + if (scan.getMaxResultSize() <= 0) { + scan.setMaxResultSize(scannerMaxResultSize); + } if (scan.isReversed()) { if (scan.isSmall()) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java index 70ad179bfaf890f043419d7a9de5c696e1e0514a..901e86d3043507f7d6f6a9893792ae48ffcaed30 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java @@ -37,6 +37,7 @@ public class TableConfiguration { private final int metaOperationTimeout; private final int operationTimeout; private final int scannerCaching; + private final long scannerMaxResultSize; private final int primaryCallTimeoutMicroSecond; private final int replicaCallTimeoutMicroSecondScan; private final int retries; @@ -59,6 +60,10 @@ public class TableConfiguration { this.scannerCaching = conf.getInt( HConstants.HBASE_CLIENT_SCANNER_CACHING, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING); + this.scannerMaxResultSize = + conf.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, + HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); + this.primaryCallTimeoutMicroSecond = conf.getInt("hbase.client.primaryCallTimeout.get", 10000); // 10ms @@ -82,6 +87,7 @@ public class TableConfiguration { this.metaOperationTimeout = HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT; this.operationTimeout = HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT; this.scannerCaching = HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING; + this.scannerMaxResultSize = HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE; this.primaryCallTimeoutMicroSecond = 10000; this.replicaCallTimeoutMicroSecondScan = 1000000; this.retries = HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER; @@ -119,4 +125,8 @@ public class TableConfiguration { public int getMaxKeyValueSize() { return maxKeyValueSize; } + + public long getScannerMaxResultSize() { + return scannerMaxResultSize; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java index 1e939335a048784a2b6eb7b3d4aaac1fb91b4896..fc04bf0674ba6ef5a4c2bd16de2cd812c0ecc8b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java @@ -175,6 +175,56 @@ public class TestScannersFromClientSide { } @Test + public void testMaxResultSizeIsSetToDefault() throws Exception { + TableName TABLE = TableName.valueOf("testMaxResultSizeIsSetToDefault"); + Table ht = TEST_UTIL.createTable(TABLE, FAMILY); + + // The max result size we expect the scan to use by default. + long expectedMaxResultSize = + TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, + HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); + + int numRows = 5; + byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); + + int numQualifiers = 10; + byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); + + // Specify the cell size such that a single row will be larger than the default + // value of maxResultSize. This means that Scan RPCs should return at most a single + // result back to the client. + int cellSize = (int) (expectedMaxResultSize / (numQualifiers - 1)); + byte[] cellValue = Bytes.createMaxByteArray(cellSize); + + Put put; + List puts = new ArrayList(); + for (int row = 0; row < ROWS.length; row++) { + put = new Put(ROWS[row]); + for (int qual = 0; qual < QUALIFIERS.length; qual++) { + KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue); + put.add(kv); + } + puts.add(put); + } + ht.put(puts); + + // Create a scan with the default configuration. + Scan scan = new Scan(); + + ResultScanner scanner = ht.getScanner(scan); + assertTrue(scanner instanceof ClientScanner); + ClientScanner clientScanner = (ClientScanner) scanner; + + // Call next to issue a single RPC to the server + scanner.next(); + + // The scanner should have, at most, a single result in its cache. If there more results exists + // in the cache it means that more than the expected max result size was fetched. + assertTrue("The cache contains: " + clientScanner.getCacheSize() + " results", + clientScanner.getCacheSize() <= 1); + } + + @Test public void testSmallScan() throws Exception { TableName TABLE = TableName.valueOf("testSmallScan"); -- 1.9.3 (Apple Git-50)