diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index 09de2ac..b698836 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -520,8 +520,8 @@ public class HConnectionManager implements HConstants { HRegionInfo currentRegion; Scan scan = new Scan(startKey); scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER); - int rows = this.conf.getInt("hbase.meta.scanner.caching", 100); - scan.setCaching(rows); + long bufferSize = this.conf.getLong("hbase.meta.scan.buffer", 100000); + scan.setBufferSize(bufferSize); ScannerCallable s = new ScannerCallable(this, (Bytes.equals(tableName, HConstants.META_TABLE_NAME) ? HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME), scan); diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 7ec95cb..bb8ebc3 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -55,26 +55,23 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Used to communicate with a single HBase table. - *

- * This class is not thread safe for writes. - * Gets, puts, and deletes take out a row lock for the duration - * of their operation. Scans (currently) do not respect - * row locking. + *

+ * This class is not thread safe for writes. Gets, puts, and deletes take out a + * row lock for the duration of their operation. Scans (currently) do not + * respect row locking. */ public class HTable implements HTableInterface { private final HConnection connection; - private final byte [] tableName; + private final byte[] tableName; protected final int scannerTimeout; private volatile Configuration configuration; private final ArrayList writeBuffer = new ArrayList(); private long writeBufferSize; private boolean autoFlush; private long currentWriteBufferSize; - protected int scannerCaching; + protected long scanBufferSize; private int maxKeyValueSize; - private long maxScannerResultSize; - /** * Creates an object to access a HBase table. * @@ -82,7 +79,7 @@ public class HTable implements HTableInterface { * @throws IOException if a remote or network exception occurs */ public HTable(final String tableName) - throws IOException { + throws IOException { this(HBaseConfiguration.create(), Bytes.toBytes(tableName)); } @@ -92,20 +89,20 @@ public class HTable implements HTableInterface { * @param tableName Name of the table. * @throws IOException if a remote or network exception occurs */ - public HTable(final byte [] tableName) - throws IOException { + public HTable(final byte[] tableName) + throws IOException { this(HBaseConfiguration.create(), tableName); } /** * Creates an object to access a HBase table. * - * @param conf Configuration object to use. + * @param conf Configuration object to use. * @param tableName Name of the table. * @throws IOException if a remote or network exception occurs */ public HTable(Configuration conf, final String tableName) - throws IOException { + throws IOException { this(conf, Bytes.toBytes(tableName)); } @@ -113,12 +110,12 @@ public class HTable implements HTableInterface { /** * Creates an object to access a HBase table. * - * @param conf Configuration object to use. + * @param conf Configuration object to use. * @param tableName Name of the table. * @throws IOException if a remote or network exception occurs */ - public HTable(Configuration conf, final byte [] tableName) - throws IOException { + public HTable(Configuration conf, final byte[] tableName) + throws IOException { this.tableName = tableName; if (conf == null) { this.scannerTimeout = 0; @@ -127,17 +124,15 @@ public class HTable implements HTableInterface { } this.connection = HConnectionManager.getConnection(conf); this.scannerTimeout = - (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD); + (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, + HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD); this.configuration = conf; this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW); this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152); this.autoFlush = true; this.currentWriteBufferSize = 0; - this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1); + this.scanBufferSize = conf.getLong("hbase.client.scan.buffer", 1048576); - this.maxScannerResultSize = conf.getLong( - HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, - HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1); int nrHRS = getCurrentNrHRS(); @@ -150,9 +145,9 @@ public class HTable implements HTableInterface { // Unfortunately Executors.newCachedThreadPool does not allow us to // set the maximum size of the pool, so we have to do it ourselves. this.pool = new ThreadPoolExecutor(0, nrThreads, - 60, TimeUnit.SECONDS, - new LinkedBlockingQueue(), - new DaemonThreadFactory()); + 60, TimeUnit.SECONDS, + new LinkedBlockingQueue(), + new DaemonThreadFactory()); } public Configuration getConfiguration() { @@ -160,8 +155,9 @@ public class HTable implements HTableInterface { } /** - * TODO Might want to change this to public, would be nice if the number - * of threads would automatically change when servers were added and removed + * TODO Might want to change this to public, would be nice if the number of + * threads would automatically change when servers were added and removed + * * @return the number of region servers that are currently running * @throws IOException if a remote or network exception occurs */ @@ -175,6 +171,7 @@ public class HTable implements HTableInterface { /** * Tells whether or not a table is enabled or not. + * * @param tableName Name of table to check. * @return {@code true} if table is online. * @throws IOException if a remote or network exception occurs @@ -185,6 +182,7 @@ public class HTable implements HTableInterface { /** * Tells whether or not a table is enabled or not. + * * @param tableName Name of table to check. * @return {@code true} if table is online. * @throws IOException if a remote or network exception occurs @@ -195,57 +193,62 @@ public class HTable implements HTableInterface { /** * Tells whether or not a table is enabled or not. - * @param conf The Configuration object to use. + * + * @param conf The Configuration object to use. * @param tableName Name of table to check. * @return {@code true} if table is online. * @throws IOException if a remote or network exception occurs */ public static boolean isTableEnabled(Configuration conf, String tableName) - throws IOException { + throws IOException { return isTableEnabled(conf, Bytes.toBytes(tableName)); } /** * Tells whether or not a table is enabled or not. - * @param conf The Configuration object to use. + * + * @param conf The Configuration object to use. * @param tableName Name of table to check. * @return {@code true} if table is online. * @throws IOException if a remote or network exception occurs */ public static boolean isTableEnabled(Configuration conf, byte[] tableName) - throws IOException { + throws IOException { return HConnectionManager.getConnection(conf).isTableEnabled(tableName); } /** * Find region location hosting passed row using cached info + * * @param row Row to find. * @return The location of the given row. * @throws IOException if a remote or network exception occurs */ public HRegionLocation getRegionLocation(final String row) - throws IOException { + throws IOException { return connection.getRegionLocation(tableName, Bytes.toBytes(row), false); } /** * Finds the region on which the given row is being served. + * * @param row Row to find. * @return Location of the row. * @throws IOException if a remote or network exception occurs */ - public HRegionLocation getRegionLocation(final byte [] row) - throws IOException { + public HRegionLocation getRegionLocation(final byte[] row) + throws IOException { return connection.getRegionLocation(tableName, row, false); } - public byte [] getTableName() { + public byte[] getTableName() { return this.tableName; } /** * INTERNAL Used by unit tests and tools to do low-level * manipulations. + * * @return An HConnection instance. */ // TODO(tsuna): Remove this. Unit tests shouldn't require public helpers. @@ -254,26 +257,21 @@ public class HTable implements HTableInterface { } /** - * Gets the number of rows that a scanner will fetch at once. - *

- * The default value comes from {@code hbase.client.scanner.caching}. + * Get the number of bytes to buffer that will be passed to scanners. + * + * @return the number of bytes to buffer */ - public int getScannerCaching() { - return scannerCaching; + public long getScanBufferSize() { + return scanBufferSize; } /** - * Sets the number of rows that a scanner will fetch at once. - *

- * This will override the value specified by - * {@code hbase.client.scanner.caching}. - * Increasing this value will reduce the amount of work needed each time - * {@code next()} is called on a scanner, at the expense of memory use - * (since more rows will need to be maintained in memory by the scanners). - * @param scannerCaching the number of rows a scanner will fetch at once. + * Set the number of bytes to buffer that will be passed to scanners. + * + * @param scanBufferSize the number of bytes to buffer */ - public void setScannerCaching(int scannerCaching) { - this.scannerCaching = scannerCaching; + public void setScanBufferSize(long scanBufferSize) { + this.scanBufferSize = scanBufferSize; } public HTableDescriptor getTableDescriptor() throws IOException { @@ -283,19 +281,21 @@ public class HTable implements HTableInterface { /** * Gets the starting row key for every region in the currently open table. - *

+ *

* This is mainly useful for the MapReduce integration. + * * @return Array of region starting row keys * @throws IOException if a remote or network exception occurs */ - public byte [][] getStartKeys() throws IOException { + public byte[][] getStartKeys() throws IOException { return getStartEndKeys().getFirst(); } /** * Gets the ending row key for every region in the currently open table. - *

+ *

* This is mainly useful for the MapReduce integration. + * * @return Array of region ending row keys * @throws IOException if a remote or network exception occurs */ @@ -306,20 +306,21 @@ public class HTable implements HTableInterface { /** * Gets the starting and ending row keys for every region in the currently * open table. - *

+ *

* This is mainly useful for the MapReduce integration. + * * @return Pair of arrays of region starting and ending row keys * @throws IOException if a remote or network exception occurs */ @SuppressWarnings("unchecked") - public Pair getStartEndKeys() throws IOException { + public Pair getStartEndKeys() throws IOException { final List startKeyList = new ArrayList(); final List endKeyList = new ArrayList(); MetaScannerVisitor visitor = new MetaScannerVisitor() { public boolean processRow(Result rowResult) throws IOException { HRegionInfo info = Writables.getHRegionInfo( - rowResult.getValue(HConstants.CATALOG_FAMILY, - HConstants.REGIONINFO_QUALIFIER)); + rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); if (Bytes.equals(info.getTableDesc().getName(), getTableName())) { if (!(info.isOffline() || info.isSplit())) { startKeyList.add(info.getStartKey()); @@ -331,13 +332,14 @@ public class HTable implements HTableInterface { }; MetaScanner.metaScan(configuration, visitor, this.tableName); return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]), - endKeyList.toArray(new byte[endKeyList.size()][])); + endKeyList.toArray(new byte[endKeyList.size()][])); } /** * Gets all the regions and their address for this table. - *

+ *

* This is mainly useful for the MapReduce integration. + * * @return A map of HRegionInfo with it's server address * @throws IOException if a remote or network exception occurs */ @@ -348,16 +350,16 @@ public class HTable implements HTableInterface { MetaScannerVisitor visitor = new MetaScannerVisitor() { public boolean processRow(Result rowResult) throws IOException { HRegionInfo info = Writables.getHRegionInfo( - rowResult.getValue(HConstants.CATALOG_FAMILY, - HConstants.REGIONINFO_QUALIFIER)); + rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER)); if (!(Bytes.equals(info.getTableDesc().getName(), getTableName()))) { return false; } HServerAddress server = new HServerAddress(); - byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY, - HConstants.SERVER_QUALIFIER); + byte[] value = rowResult.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); if (value != null && value.length > 0) { String address = Bytes.toString(value); server = new HServerAddress(address); @@ -374,16 +376,16 @@ public class HTable implements HTableInterface { return regionMap; } - public Result getRowOrBefore(final byte[] row, final byte[] family) - throws IOException { - return connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, row) { - public Result call() throws IOException { - return server.getClosestRowBefore(location.getRegionInfo().getRegionName(), - row, family); - } - }); - } + public Result getRowOrBefore(final byte[] row, final byte[] family) + throws IOException { + return connection.getRegionServerWithRetries( + new ServerCallable(connection, tableName, row) { + public Result call() throws IOException { + return server.getClosestRowBefore(location.getRegionInfo().getRegionName(), + row, family); + } + }); + } public ResultScanner getScanner(final Scan scan) throws IOException { ClientScanner s = new ClientScanner(scan); @@ -391,14 +393,14 @@ public class HTable implements HTableInterface { return s; } - public ResultScanner getScanner(byte [] family) throws IOException { + public ResultScanner getScanner(byte[] family) throws IOException { Scan scan = new Scan(); scan.addFamily(family); return getScanner(scan); } - public ResultScanner getScanner(byte [] family, byte [] qualifier) - throws IOException { + public ResultScanner getScanner(byte[] family, byte[] qualifier) + throws IOException { Scan scan = new Scan(); scan.addColumn(family, qualifier); return getScanner(scan); @@ -406,28 +408,28 @@ public class HTable implements HTableInterface { public Result get(final Get get) throws IOException { return connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, get.getRow()) { - public Result call() throws IOException { - return server.get(location.getRegionInfo().getRegionName(), get); - } + new ServerCallable(connection, tableName, get.getRow()) { + public Result call() throws IOException { + return server.get(location.getRegionInfo().getRegionName(), get); } + } ); } public void delete(final Delete delete) - throws IOException { + throws IOException { connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, delete.getRow()) { - public Boolean call() throws IOException { - server.delete(location.getRegionInfo().getRegionName(), delete); - return null; // FindBugs NP_BOOLEAN_RETURN_NULL - } + new ServerCallable(connection, tableName, delete.getRow()) { + public Boolean call() throws IOException { + server.delete(location.getRegionInfo().getRegionName(), delete); + return null; // FindBugs NP_BOOLEAN_RETURN_NULL } + } ); } public void delete(final List deletes) - throws IOException { + throws IOException { int last = 0; try { last = connection.processBatchOfDeletes(deletes, this.tableName); @@ -455,16 +457,17 @@ public class HTable implements HTableInterface { } } - public long incrementColumnValue(final byte [] row, final byte [] family, - final byte [] qualifier, final long amount) - throws IOException { + public long incrementColumnValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long amount) + throws IOException { return incrementColumnValue(row, family, qualifier, amount, true); } @SuppressWarnings({"ThrowableInstanceNeverThrown"}) - public long incrementColumnValue(final byte [] row, final byte [] family, - final byte [] qualifier, final long amount, final boolean writeToWAL) - throws IOException { + public long incrementColumnValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long amount, + final boolean writeToWAL) + throws IOException { NullPointerException npe = null; if (row == null) { npe = new NullPointerException("row is null"); @@ -473,72 +476,75 @@ public class HTable implements HTableInterface { } if (npe != null) { throw new IOException( - "Invalid arguments to incrementColumnValue", npe); + "Invalid arguments to incrementColumnValue", npe); } return connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, row) { - public Long call() throws IOException { - return server.incrementColumnValue( - location.getRegionInfo().getRegionName(), row, family, - qualifier, amount, writeToWAL); - } + new ServerCallable(connection, tableName, row) { + public Long call() throws IOException { + return server.incrementColumnValue( + location.getRegionInfo().getRegionName(), row, family, + qualifier, amount, writeToWAL); } + } ); } /** * Atomically checks if a row/family/qualifier value match the expectedValue. - * If it does, it adds the put. If value == null, checks for non-existence - * of the value. + * If it does, it adds the put. If value == null, checks for non-existence of + * the value. * - * @param row to check - * @param family column family + * @param row to check + * @param family column family * @param qualifier column qualifier - * @param value the expected value - * @param put put to execute if value matches. - * @throws IOException + * @param value the expected value + * @param put put to execute if value matches. * @return true if the new put was execute, false otherwise + * @throws IOException */ - public boolean checkAndPut(final byte [] row, - final byte [] family, final byte [] qualifier, final byte [] value, - final Put put) - throws IOException { + public boolean checkAndPut(final byte[] row, + final byte[] family, final byte[] qualifier, + final byte[] value, final Put put) + throws IOException { return connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, row) { - public Boolean call() throws IOException { - return server.checkAndPut(location.getRegionInfo().getRegionName(), - row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE; - } + new ServerCallable(connection, tableName, row) { + public Boolean call() throws IOException { + return server.checkAndPut(location.getRegionInfo().getRegionName(), + row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE; } + } ); } /** - * Test for the existence of columns in the table, as specified in the Get.

- * - * This will return true if the Get matches one or more keys, false if not.

+ * Test for the existence of columns in the table, as specified in the + * Get.

+ *

+ * This will return true if the Get matches one or more keys, false if + * not.

+ *

+ * This is a server-side call so it prevents any data from being transfered to + * the client. * - * This is a server-side call so it prevents any data from being transfered - * to the client. * @param get param to check for * @return true if the specified Get matches one or more keys, false if not * @throws IOException */ public boolean exists(final Get get) throws IOException { return connection.getRegionServerWithRetries( - new ServerCallable(connection, tableName, get.getRow()) { - public Boolean call() throws IOException { - return server. - exists(location.getRegionInfo().getRegionName(), get); - } + new ServerCallable(connection, tableName, get.getRow()) { + public Boolean call() throws IOException { + return server. + exists(location.getRegionInfo().getRegionName(), get); } + } ); } public void flushCommits() throws IOException { try { connection.processBatchOfPuts(writeBuffer, - tableName, pool); + tableName, pool); } finally { // the write buffer was adjusted by processBatchOfPuts currentWriteBufferSize = 0; @@ -553,7 +559,7 @@ public class HTable implements HTableInterface { } // validate for well-formedness - private void validatePut(final Put put) throws IllegalArgumentException{ + private void validatePut(final Put put) throws IllegalArgumentException { if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert"); } @@ -568,26 +574,26 @@ public class HTable implements HTableInterface { } } - public RowLock lockRow(final byte [] row) - throws IOException { + public RowLock lockRow(final byte[] row) + throws IOException { return connection.getRegionServerWithRetries( new ServerCallable(connection, tableName, row) { public RowLock call() throws IOException { long lockId = - server.lockRow(location.getRegionInfo().getRegionName(), row); - return new RowLock(row,lockId); + server.lockRow(location.getRegionInfo().getRegionName(), row); + return new RowLock(row, lockId); } } ); } public void unlockRow(final RowLock rl) - throws IOException { + throws IOException { connection.getRegionServerWithRetries( new ServerCallable(connection, tableName, rl.getRow()) { public Boolean call() throws IOException { server.unlockRow(location.getRegionInfo().getRegionName(), - rl.getLockId()); + rl.getLockId()); return null; // FindBugs NP_BOOLEAN_RETURN_NULL } } @@ -600,18 +606,18 @@ public class HTable implements HTableInterface { /** * Turns 'auto-flush' on or off. - *

+ *

* When enabled (default), {@link Put} operations don't get buffered/delayed * and are immediately executed. This is slower but safer. - *

+ *

* Turning this off means that multiple {@link Put}s will be accepted before * any RPC is actually sent to do the write operations. If the application - * dies before pending writes get flushed to HBase, data will be lost. - * Other side effects may include the fact that the application thinks a - * {@link Put} was executed successfully whereas it was in fact only - * buffered and the operation may fail when attempting to flush all pending - * writes. In that case though, the code will retry the failed {@link Put} - * upon its next attempt to flush the buffer. + * dies before pending writes get flushed to HBase, data will be lost. Other + * side effects may include the fact that the application thinks a {@link Put} + * was executed successfully whereas it was in fact only buffered and the + * operation may fail when attempting to flush all pending writes. In that + * case though, the code will retry the failed {@link Put} upon its next + * attempt to flush the buffer. * * @param autoFlush Whether or not to enable 'auto-flush'. * @see #flushCommits @@ -622,9 +628,10 @@ public class HTable implements HTableInterface { /** * Returns the maximum size in bytes of the write buffer for this HTable. - *

- * The default value comes from the configuration parameter - * {@code hbase.client.write.buffer}. + *

+ * The default value comes from the configuration parameter {@code + * hbase.client.write.buffer}. + * * @return The size of the write buffer in bytes. */ public long getWriteBufferSize() { @@ -633,21 +640,23 @@ public class HTable implements HTableInterface { /** * Sets the size of the buffer in bytes. - *

- * If the new size is less than the current amount of data in the - * write buffer, the buffer gets flushed. + *

+ * If the new size is less than the current amount of data in the write + * buffer, the buffer gets flushed. + * * @param writeBufferSize The new write buffer size, in bytes. * @throws IOException if a remote or network exception occurs. */ public void setWriteBufferSize(long writeBufferSize) throws IOException { this.writeBufferSize = writeBufferSize; - if(currentWriteBufferSize > writeBufferSize) { + if (currentWriteBufferSize > writeBufferSize) { flushCommits(); } } /** * Returns the write buffer. + * * @return The current write buffer. */ public ArrayList getWriteBuffer() { @@ -655,9 +664,8 @@ public class HTable implements HTableInterface { } /** - * Implements the scanner interface for the HBase client. - * If there are multiple regions in a table, this scanner will iterate - * through them all. + * Implements the scanner interface for the HBase client. If there are + * multiple regions in a table, this scanner will iterate through them all. */ protected class ClientScanner implements ResultScanner { private final Log CLIENT_LOG = LogFactory.getLog(this.getClass()); @@ -669,7 +677,7 @@ public class HTable implements HTableInterface { private HRegionInfo currentRegion = null; private ScannerCallable callable = null; private final LinkedList cache = new LinkedList(); - private final int caching; + private final long bufferSize; private long lastNext; // Keep lastResult returned successfully in case we have to reset scanner. private Result lastResult = null; @@ -677,17 +685,17 @@ public class HTable implements HTableInterface { protected ClientScanner(final Scan scan) { if (CLIENT_LOG.isDebugEnabled()) { CLIENT_LOG.debug("Creating scanner over " - + Bytes.toString(getTableName()) - + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'"); + + Bytes.toString(getTableName()) + + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'"); } this.scan = scan; this.lastNext = System.currentTimeMillis(); - // Use the caching from the Scan. If not set, use the default cache setting for this table. - if (this.scan.getCaching() > 0) { - this.caching = this.scan.getCaching(); + // Use the buffer size from the Scan. If not set, use the default scan buffer size setting for this table. + if (this.scan.getBufferSize() > 0) { + this.bufferSize = this.scan.getBufferSize(); } else { - this.caching = HTable.this.scannerCaching; + this.bufferSize = HTable.this.scanBufferSize; } // Removed filter validation. We have a new format now, only one of all @@ -699,7 +707,7 @@ public class HTable implements HTableInterface { } public void initialize() throws IOException { - nextScanner(this.caching, false); + nextScanner(this.bufferSize, false); } protected Scan getScan() { @@ -711,10 +719,11 @@ public class HTable implements HTableInterface { } // returns true if the passed region endKey - private boolean checkScanStopRow(final byte [] endKey) { + + private boolean checkScanStopRow(final byte[] endKey) { if (this.scan.getStopRow().length > 0) { // there is a stop row, check to see if we are past it. - byte [] stopRow = scan.getStopRow(); + byte[] stopRow = scan.getStopRow(); int cmp = Bytes.compareTo(stopRow, 0, stopRow.length, endKey, 0, endKey.length); if (cmp <= 0) { @@ -735,8 +744,9 @@ public class HTable implements HTableInterface { * @param nbRows * @param done Server-side says we're done scanning. */ - private boolean nextScanner(int nbRows, final boolean done) - throws IOException { + + private boolean nextScanner(long bufferSize, final boolean done) + throws IOException { // Close the previous scanner if it's open if (this.callable != null) { this.callable.setClose(); @@ -745,15 +755,15 @@ public class HTable implements HTableInterface { } // Where to start the next scanner - byte [] localStartKey; + byte[] localStartKey; // if we're at end of table, close and return false to stop iterating if (this.currentRegion != null) { - byte [] endKey = this.currentRegion.getEndKey(); + byte[] endKey = this.currentRegion.getEndKey(); if (endKey == null || - Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) || - checkScanStopRow(endKey) || - done) { + Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) || + checkScanStopRow(endKey) || + done) { close(); if (CLIENT_LOG.isDebugEnabled()) { CLIENT_LOG.debug("Finished with scanning at " + this.currentRegion); @@ -773,7 +783,7 @@ public class HTable implements HTableInterface { Bytes.toStringBinary(localStartKey) + "'"); } try { - callable = getScannerCallable(localStartKey, nbRows); + callable = getScannerCallable(localStartKey, bufferSize); // Open a scanner on the region server starting at the // beginning of the region getConnection().getRegionServerWithRetries(callable); @@ -785,12 +795,12 @@ public class HTable implements HTableInterface { return true; } - protected ScannerCallable getScannerCallable(byte [] localStartKey, - int nbRows) { + protected ScannerCallable getScannerCallable(byte[] localStartKey, + long bufferSize) { scan.setStartRow(localStartKey); ScannerCallable s = new ScannerCallable(getConnection(), getTableName(), scan); - s.setCaching(nbRows); + s.setBufferSize(bufferSize); return s; } @@ -801,12 +811,11 @@ public class HTable implements HTableInterface { return null; } if (cache.size() == 0) { - Result [] values = null; - long remainingResultSize = maxScannerResultSize; - int countdown = this.caching; + Result[] values = null; + long remainingBufferSize = this.bufferSize; // We need to reset it if it's a new callable that was created // with a countdown in nextScanner - callable.setCaching(this.caching); + callable.setBufferSize(this.bufferSize); // This flag is set when we want to skip the result returned. We do // this when we reset scanner because it split under us. boolean skipFirst = false; @@ -824,11 +833,11 @@ public class HTable implements HTableInterface { } catch (DoNotRetryIOException e) { long timeout = lastNext + scannerTimeout; if (e instanceof UnknownScannerException && - timeout < System.currentTimeMillis()) { + timeout < System.currentTimeMillis()) { long elapsed = System.currentTimeMillis() - lastNext; ScannerTimeoutException ex = new ScannerTimeoutException( - elapsed + "ms passed since the last invocation, " + - "timeout is currently set to " + scannerTimeout); + elapsed + "ms passed since the last invocation, " + + "timeout is currently set to " + scannerTimeout); ex.initCause(e); throw ex; } @@ -853,14 +862,13 @@ public class HTable implements HTableInterface { for (Result rs : values) { cache.add(rs); for (KeyValue kv : rs.raw()) { - remainingResultSize -= kv.heapSize(); + remainingBufferSize -= kv.heapSize(); } - countdown--; this.lastResult = rs; } } // Values == null means server-side filter has determined we must STOP - } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null)); + } while (remainingBufferSize > 0 && nextScanner(remainingBufferSize, values == null)); } if (cache.size() > 0) { @@ -870,18 +878,19 @@ public class HTable implements HTableInterface { } /** - * Get nbRows rows. - * How many RPCs are made is determined by the {@link Scan#setCaching(int)} - * setting (or hbase.client.scanner.caching in hbase-site.xml). + * Get nbRows rows. How many RPCs are made is determined by + * the {@link Scan#setBufferSize(long)} setting (or hbase.client.scanner.caching + * in hbase-site.xml). + * * @param nbRows number of rows to return * @return Between zero and nbRows RowResults. Scan is done - * if returned array is of zero-length (We never return null). + * if returned array is of zero-length (We never return null). * @throws IOException */ - public Result [] next(int nbRows) throws IOException { + public Result[] next(int nbRows) throws IOException { // Collect values to be returned here ArrayList resultSets = new ArrayList(nbRows); - for(int i = 0; i < nbRows; i++) { + for (int i = 0; i < nbRows; i++) { Result next = next(); if (next != null) { resultSets.add(next); @@ -955,28 +964,28 @@ public class HTable implements HTableInterface { static class DaemonThreadFactory implements ThreadFactory { static final AtomicInteger poolNumber = new AtomicInteger(1); - final ThreadGroup group; - final AtomicInteger threadNumber = new AtomicInteger(1); - final String namePrefix; - - DaemonThreadFactory() { - SecurityManager s = System.getSecurityManager(); - group = (s != null)? s.getThreadGroup() : - Thread.currentThread().getThreadGroup(); - namePrefix = "pool-" + - poolNumber.getAndIncrement() + - "-thread-"; - } + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + + DaemonThreadFactory() { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : + Thread.currentThread().getThreadGroup(); + namePrefix = "pool-" + + poolNumber.getAndIncrement() + + "-thread-"; + } - public Thread newThread(Runnable r) { - Thread t = new Thread(group, r, - namePrefix + threadNumber.getAndIncrement(), - 0); - if (!t.isDaemon()) - t.setDaemon(true); - if (t.getPriority() != Thread.NORM_PRIORITY) - t.setPriority(Thread.NORM_PRIORITY); - return t; - } + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, + namePrefix + threadNumber.getAndIncrement(), + 0); + if (!t.isDaemon()) + t.setDaemon(true); + if (t.getPriority() != Thread.NORM_PRIORITY) + t.setPriority(Thread.NORM_PRIORITY); + return t; + } } } diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java index 3de661e..d5c6041 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java +++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -67,14 +67,14 @@ class MetaScanner implements HConstants { // Scan over each meta region ScannerCallable callable; - int rows = configuration.getInt("hbase.meta.scanner.caching", 100); + long bufferSize = configuration.getLong("hbase.meta.scan.buffer", 100000); do { Scan scan = new Scan(startRow).addFamily(CATALOG_FAMILY); callable = new ScannerCallable(connection, META_TABLE_NAME, scan); // Open scanner connection.getRegionServerWithRetries(callable); try { - callable.setCaching(rows); + callable.setBufferSize(bufferSize); done: do { //we have all the rows here Result [] rrs = connection.getRegionServerWithRetries(callable); diff --git a/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/src/main/java/org/apache/hadoop/hbase/client/Scan.java index 29b3cb0..1ad186d 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/Scan.java +++ b/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -48,7 +48,7 @@ import java.util.TreeSet; *

* To scan everything for each row, instantiate a Scan object. *

- * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}. + * To modify scanner buffer for just this scan, use {@link #setBufferSize(long)}}. *

* To further define the scope of what to get when scanning, perform additional * methods as outlined below. @@ -82,7 +82,7 @@ public class Scan implements Writable { private byte [] stopRow = HConstants.EMPTY_END_ROW; private int maxVersions = 1; private int batch = -1; - private int caching = -1; + private long bufferSize = -1; private boolean cacheBlocks = true; private Filter filter = null; private TimeRange tr = new TimeRange(); @@ -131,7 +131,7 @@ public class Scan implements Writable { stopRow = scan.getStopRow(); maxVersions = scan.getMaxVersions(); batch = scan.getBatch(); - caching = scan.getCaching(); + bufferSize = scan.getBufferSize(); cacheBlocks = scan.getCacheBlocks(); filter = scan.getFilter(); // clone? TimeRange ctr = scan.getTimeRange(); @@ -290,13 +290,13 @@ public class Scan implements Writable { } /** - * Set the number of rows for caching that will be passed to scanners. - * If not set, the default setting from {@link HTable#getScannerCaching()} will apply. - * Higher caching values will enable faster scanners but will use more memory. - * @param caching the number of rows for caching + * Set the number of bytes of data to buffer for each call to the region server. + * If not set, the default setting from {@link HTable#getScanBufferSize()} will apply. + * Higher values will enable faster scanners but will use more memory. + * @param bufferSize the number of bytes for the scan buffer */ - public void setCaching(int caching) { - this.caching = caching; + public void setBufferSize(long bufferSize) { + this.bufferSize = bufferSize; } /** @@ -383,10 +383,10 @@ public class Scan implements Writable { } /** - * @return caching the number of rows fetched when calling next on a scanner + * @return the number of bytes of data to buffer for each call to the region server. */ - public int getCaching() { - return this.caching; + public long getBufferSize() { + return this.bufferSize; } /** @@ -447,8 +447,8 @@ public class Scan implements Writable { sb.append(this.maxVersions); sb.append(", batch="); sb.append(this.batch); - sb.append(", caching="); - sb.append(this.caching); + sb.append(", bufferSize="); + sb.append(this.bufferSize); sb.append(", cacheBlocks="); sb.append(this.cacheBlocks); sb.append(", timeRange="); @@ -512,7 +512,7 @@ public class Scan implements Writable { this.stopRow = Bytes.readByteArray(in); this.maxVersions = in.readInt(); this.batch = in.readInt(); - this.caching = in.readInt(); + this.bufferSize = in.readLong(); this.cacheBlocks = in.readBoolean(); if(in.readBoolean()) { this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in))); @@ -542,7 +542,7 @@ public class Scan implements Writable { Bytes.writeByteArray(out, this.stopRow); out.writeInt(this.maxVersions); out.writeInt(this.batch); - out.writeInt(this.caching); + out.writeLong(this.bufferSize); out.writeBoolean(this.cacheBlocks); if(this.filter == null) { out.writeBoolean(false); diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 2fff71e..34715e4 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -39,7 +39,7 @@ public class ScannerCallable extends ServerCallable { private boolean instantiated = false; private boolean closed = false; private Scan scan; - private int caching = 1; + private long bufferSize = 0; /** * @param connection which connection @@ -74,7 +74,7 @@ public class ScannerCallable extends ServerCallable { } else { Result [] rrs = null; try { - rrs = server.next(scannerId, caching); + rrs = server.next(scannerId, bufferSize); } catch (IOException e) { IOException ioe = null; if (e instanceof RemoteException) { @@ -135,18 +135,18 @@ public class ScannerCallable extends ServerCallable { } /** - * Get the number of rows that will be fetched on next - * @return the number of rows for caching + * Get the number of bytes that will be fetched on next + * @return the number of bytes for caching */ - public int getCaching() { - return caching; + public long getBufferSize() { + return bufferSize; } /** - * Set the number of rows that will be fetched on next - * @param caching the number of rows for caching + * Set the number of bytes that will be fetched on next + * @param the number of bytes for caching */ - public void setCaching(int caching) { - this.caching = caching; + public void setBufferSize(long bufferSize) { + this.bufferSize = bufferSize; } } diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java index 4cbe52a..769f99c 100644 --- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java +++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java @@ -197,13 +197,13 @@ public interface HRegionInterface extends HBaseRPCProtocolVersion { /** * Get the next set of values * @param scannerId clientId passed to openScanner - * @param numberOfRows the number of rows to fetch + * @param bufferSize the number of bytes to fetch * @return Array of Results (map of values); array is empty if done with this * region and null if we are NOT to go to the next region (happens when a * filter rules that the scan is done). * @throws IOException e */ - public Result [] next(long scannerId, int numberOfRows) throws IOException; + public Result [] next(long scannerId, long bufferSize) throws IOException; /** * Close a scanner diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java index 89674a6..9b2cd12 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java @@ -19,8 +19,6 @@ */ package org.apache.hadoop.hbase.mapreduce; -import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; @@ -31,6 +29,8 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.StringUtils; +import java.io.IOException; + /** * Convert HBase tabular data into a format that is consumable by Map/Reduce. */ @@ -60,7 +60,7 @@ implements Configurable { /** Set to false to disable server-side caching of blocks for this scan. */ public static final String SCAN_CACHEBLOCKS = "hbase.mapreduce.scan.cacheblocks"; /** The number of rows for caching that will be passed to scanners. */ - public static final String SCAN_CACHEDROWS = "hbase.mapreduce.scan.cachedrows"; + public static final String SCAN_BUFFERSIZE = "hbase.mapreduce.scan.buffersize"; /** The configuration. */ private Configuration conf = null; @@ -128,8 +128,8 @@ implements Configurable { scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS))); } - if (conf.get(SCAN_CACHEDROWS) != null) { - scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS))); + if (conf.get(SCAN_BUFFERSIZE) != null) { + scan.setBufferSize(Long.parseLong(conf.get(SCAN_BUFFERSIZE))); } // false by default, full table scans generate too much BC churn diff --git a/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java b/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java index ef349af..31ef4b5 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java +++ b/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java @@ -175,7 +175,7 @@ abstract class BaseScanner extends Chore implements HConstants { this.master.getServerConnection().getHRegionConnection(region.getServer()); Scan s = new Scan().addFamily(HConstants.CATALOG_FAMILY); // Make this scan do a row at a time otherwise, data can be stale. - s.setCaching(1); + s.setBufferSize(0); scannerId = regionServer.openScanner(region.getRegionName(), s); while (true) { Result values = regionServer.next(scannerId); diff --git a/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java b/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java index 60c1fd7..448d3d0 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java +++ b/src/main/java/org/apache/hadoop/hbase/master/TableOperation.java @@ -83,9 +83,9 @@ abstract class TableOperation implements HConstants { Bytes.toBytes(Bytes.toString(tableName) + ",,"); Scan scan = new Scan(tableNameMetaStart).addFamily(CATALOG_FAMILY); long scannerId = this.server.openScanner(m.getRegionName(), scan); - int rows = this.master.getConfiguration(). - getInt("hbase.meta.scanner.caching", 100); - scan.setCaching(rows); + long bufferSize = this.master.getConfiguration(). + getLong("hbase.meta.scan.buffer", 100000); + scan.setBufferSize(bufferSize); List emptyRows = new ArrayList(); try { while (true) { diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 78f3223..1fd96c1 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -163,8 +163,6 @@ public class HRegionServer implements HConstants, HRegionInterface, protected final int numRegionsToReport; - private final long maxScannerResultSize; - // Remote HMaster private HMasterRegionInterface hbaseMaster; @@ -265,10 +263,6 @@ public class HRegionServer implements HConstants, HRegionInterface, sleeper = new Sleeper(this.msgInterval, this.stopRequested); - this.maxScannerResultSize = conf.getLong( - HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, - HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); - // Task thread to process requests from Master this.worker = new Worker(); @@ -1745,14 +1739,14 @@ public class HRegionServer implements HConstants, HRegionInterface, } public Result next(final long scannerId) throws IOException { - Result [] res = next(scannerId, 1); + Result [] res = next(scannerId, 0); if(res == null || res.length == 0) { return null; } return res[0]; } - public Result [] next(final long scannerId, int nbRows) throws IOException { + public Result [] next(final long scannerId, long bufferSize) throws IOException { try { String scannerName = String.valueOf(scannerId); InternalScanner s = this.scanners.get(scannerName); @@ -1768,16 +1762,15 @@ public class HRegionServer implements HConstants, HRegionInterface, throw e; } this.leases.renewLease(scannerName); - List results = new ArrayList(nbRows); - long currentScanResultSize = 0; + List results = new ArrayList(); List values = new ArrayList(); - for (int i = 0; i < nbRows && currentScanResultSize < maxScannerResultSize; i++) { + for (long currentBufferSize = 0; currentBufferSize <= bufferSize; ) { requestCount.incrementAndGet(); // Collect values to be returned here boolean moreRows = s.next(values); if (!values.isEmpty()) { for (KeyValue kv : values) { - currentScanResultSize += kv.heapSize(); + currentBufferSize += kv.heapSize(); } results.add(new Result(values)); } diff --git a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java index f97781e..fe4b7aa 100644 --- a/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java +++ b/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java @@ -20,23 +20,18 @@ package org.apache.hadoop.hbase.rest.model; -import java.io.IOException; -import java.io.Serializable; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; - -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - +import com.google.protobuf.ByteString; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.api.json.JSONUnmarshaller; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.BinaryPrefixComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; @@ -52,18 +47,20 @@ import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.filter.ValueFilter; import org.apache.hadoop.hbase.filter.WhileMatchFilter; import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.rest.ProtobufMessageHandler; import org.apache.hadoop.hbase.rest.protobuf.generated.ScannerMessage.Scanner; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.ByteString; - -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.api.json.JSONJAXBContext; -import com.sun.jersey.api.json.JSONMarshaller; -import com.sun.jersey.api.json.JSONUnmarshaller; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.IOException; +import java.io.Serializable; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; /** * A representation of Scanner parameters. @@ -357,10 +354,6 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable { } model.setStartTime(scan.getTimeRange().getMin()); model.setEndTime(scan.getTimeRange().getMax()); - int caching = scan.getCaching(); - if (caching > 0) { - model.setBatch(caching); - } int maxVersions = scan.getMaxVersions(); if (maxVersions > 0) { model.setMaxVersions(maxVersions); diff --git a/src/main/resources/hbase-default.xml b/src/main/resources/hbase-default.xml index e3a9669..c76dadc 100644 --- a/src/main/resources/hbase-default.xml +++ b/src/main/resources/hbase-default.xml @@ -128,12 +128,12 @@ - hbase.client.scanner.caching - 1 - Number of rows that will be fetched when calling next - on a scanner if it is not served from memory. Higher caching values + hbase.client.scan.buffer + 1048576 + Number of bytes that will be fetched when calling next + on a scanner if it is not served from memory. Higher values will enable faster scanners but will eat up more memory and some - calls of next may take longer and longer times when the cache is empty. + calls of next may take longer and longer times when the buffer is empty. diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index d76c75e..431ded7 100644 --- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -19,23 +19,6 @@ */ package org.apache.hadoop.hbase; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.PrintStream; -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.TreeMap; -import java.util.Arrays; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.lang.reflect.Constructor; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -50,12 +33,12 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.PageFilter; -import org.apache.hadoop.hbase.filter.WhileMatchFilter; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.WhileMatchFilter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Hash; @@ -77,6 +60,23 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; import org.apache.hadoop.util.LineReader; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.Constructor; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * Script used evaluating HBase performance and scalability. Runs a HBase * client that steps through one of a set of hardcoded tests or 'experiments' @@ -744,7 +744,7 @@ public class PerformanceEvaluation implements HConstants { this.table = new HTable(conf, tableName); this.table.setAutoFlush(false); this.table.setWriteBufferSize(1024*1024*12); - this.table.setScannerCaching(30); + this.table.setScanBufferSize(100000); } void testTakedown() throws IOException { diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 95e494a..3eda8dc 100644 --- a/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -19,23 +19,9 @@ */ package org.apache.hadoop.hbase.client; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.UUID; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -45,6 +31,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.PrefixFilter; @@ -53,7 +40,6 @@ import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.WhileMatchFilter; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.AfterClass; @@ -62,6 +48,19 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}. * Sets up the HBase mini cluster once at start and runs through all client tests. @@ -166,9 +165,9 @@ public class TestFromClientSide { 0, getNumberOfRows(keyPrefix2, value2, table)); assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, getNumberOfRows(keyPrefix3, value2, table)); - ht.setScannerCaching(0); + ht.setScanBufferSize(0); assertEquals("Got back incorrect number of rows from scan", 0, - getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100); + getNumberOfRows(keyPrefix1, value2, table)); ht.setScanBufferSize(100000); assertEquals("Got back incorrect number of rows from scan", 0, getNumberOfRows(keyPrefix2, value2, table)); }