Index: src/java/org/apache/hadoop/hbase/client/HTable.java =================================================================== --- src/java/org/apache/hadoop/hbase/client/HTable.java (revision 658259) +++ src/java/org/apache/hadoop/hbase/client/HTable.java (working copy) @@ -35,15 +35,14 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.filter.RowFilterInterface; import org.apache.hadoop.hbase.filter.StopRowFilter; import org.apache.hadoop.hbase.filter.WhileMatchRowFilter; import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.io.Cell; import org.apache.hadoop.hbase.io.RowResult; -import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Text; /** @@ -57,6 +56,7 @@ protected final long pause; protected final int numRetries; protected Random rand; + protected HBaseConfiguration configuration; protected volatile boolean tableDoesNotExist; @@ -96,6 +96,7 @@ public HTable(HBaseConfiguration conf, final byte [] tableName) throws IOException { this.connection = HConnectionManager.getConnection(conf); + this.configuration = conf; this.tableName = tableName; this.pause = conf.getLong("hbase.client.pause", 10 * 1000); this.numRetries = conf.getInt("hbase.client.retries.number", 5); @@ -142,145 +143,70 @@ /** * Gets the starting row key for every region in the currently open table + * * @return Array of region starting row keys * @throws IOException */ @SuppressWarnings("null") - public byte [][] getStartKeys() throws IOException { - List keyList = new ArrayList(); + public byte[][] getStartKeys() throws IOException { + final List keyList = new ArrayList(); - long scannerId = -1L; - byte [] startRow = - HRegionInfo.createRegionName(this.tableName, null, NINES); - HRegionLocation metaLocation = null; - HRegionInterface server; - - // scan over the each meta region - do { - try{ - // turn the start row into a location - metaLocation = - connection.locateRegion(META_TABLE_NAME, startRow); + MetaScannerVisitor visitor = new MetaScannerVisitor() { - // connect to the server hosting the .META. region - server = - connection.getHRegionConnection(metaLocation.getServerAddress()); + public boolean processRow(RowResult rowResult, + HRegionLocation metaLocation, HRegionInfo info) throws IOException { - // open a scanner over the meta region - scannerId = server.openScanner( - metaLocation.getRegionInfo().getRegionName(), - new byte[][]{COL_REGIONINFO}, tableName, LATEST_TIMESTAMP, - null); - - // iterate through the scanner, accumulating unique region names - while (true) { - RowResult values = server.next(scannerId); - if (values == null || values.size() == 0) { - break; - } - - HRegionInfo info = new HRegionInfo(); - info = (HRegionInfo) Writables.getWritable( - values.get(COL_REGIONINFO).getValue(), info); - - if (!Bytes.equals(info.getTableDesc().getName(), this.tableName)) { - break; - } - - if (info.isOffline() || info.isSplit()) { - continue; - } + if (!(Bytes.toString(info.getTableDesc().getName()).equals(Bytes + .toString(tableName)))) { + return false; + } + if (!(info.isOffline() || info.isSplit())) { keyList.add(info.getStartKey()); } - - // close that remote scanner - server.close(scannerId); - - // advance the startRow to the end key of the current region - startRow = metaLocation.getRegionInfo().getEndKey(); - } catch (IOException e) { - // need retry logic? - throw e; + return true; } - } while (Bytes.compareTo(startRow, EMPTY_START_ROW) != 0); - - return keyList.toArray(new byte [keyList.size()][]); + + }; + MetaScanner.metaScan(configuration, visitor, this.tableName); + + return keyList.toArray(new byte[keyList.size()][]); } - + /** * Get all the regions and their address for this table + * * @return A map of HRegionInfo with it's server address * @throws IOException */ @SuppressWarnings("null") public Map getRegionsInfo() throws IOException { - // TODO This code is a near exact copy of getStartKeys. To be refactored HBASE-626 - HashMap regionMap = new HashMap(); - - long scannerId = -1L; - byte [] startRow = - HRegionInfo.createRegionName(this.tableName, null, NINES); - HRegionLocation metaLocation = null; - HRegionInterface server; - - // scan over the each meta region - do { - try{ - // turn the start row into a location - metaLocation = - connection.locateRegion(META_TABLE_NAME, startRow); - // connect to the server hosting the .META. region - server = - connection.getHRegionConnection(metaLocation.getServerAddress()); + final HashMap regionMap = new HashMap(); - // open a scanner over the meta region - scannerId = server.openScanner( - metaLocation.getRegionInfo().getRegionName(), - new byte [][]{COL_REGIONINFO}, tableName, LATEST_TIMESTAMP, - null); - - // iterate through the scanner, accumulating regions and their regionserver - while (true) { - RowResult values = server.next(scannerId); - if (values == null || values.size() == 0) { - break; - } - - HRegionInfo info = new HRegionInfo(); - info = (HRegionInfo) Writables.getWritable( - values.get(COL_REGIONINFO).getValue(), info); + MetaScannerVisitor visitor = new MetaScannerVisitor() { - if (!Bytes.equals(info.getTableDesc().getName(), this.tableName)) { - break; - } - - if (info.isOffline() || info.isSplit()) { - continue; - } + public boolean processRow(RowResult rowResult, + HRegionLocation metaLocation, HRegionInfo info) throws IOException { + + if (!(Bytes.toString(info.getTableDesc().getName()).equals(Bytes + .toString(tableName)))) { + return false; + } + if (!(info.isOffline() || info.isSplit())) { regionMap.put(info, metaLocation.getServerAddress()); } - - // close that remote scanner - server.close(scannerId); - - // advance the startRow to the end key of the current region - startRow = metaLocation.getRegionInfo().getEndKey(); - - // turn the start row into a location - metaLocation = - connection.locateRegion(META_TABLE_NAME, startRow); - } catch (IOException e) { - // need retry logic? - throw e; + return true; } - } while (Bytes.compareTo(startRow, EMPTY_START_ROW) != 0); - + + }; + MetaScanner.metaScan(configuration, visitor, tableName); + return regionMap; } - /** + + /** * Get a single value for the specified row and column - * + * * @param row row key * @param column column name * @return value for specified row/column Index: src/java/org/apache/hadoop/hbase/client/MetaScanner.java =================================================================== --- src/java/org/apache/hadoop/hbase/client/MetaScanner.java (revision 0) +++ src/java/org/apache/hadoop/hbase/client/MetaScanner.java (revision 0) @@ -0,0 +1,101 @@ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.io.RowResult; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; + +/** + * Scanner class that contains the .META. table scanning logic + * and uses a Retryable scanner. Provided visitors will be called + * for each rows. + */ + +class MetaScanner implements HConstants { + + /** + * Scans the meta table and calls a visitor on each RowResult and uses a empty + * start row value as table name. + * + * @param configuration + * @param a custom visitor + * @throws IOException + */ + + public static void metaScan(HBaseConfiguration configuration, + MetaScannerVisitor visitor) throws IOException { + metaScan(configuration, visitor, EMPTY_START_ROW); + } + + /** + * Scans the meta table and calls a visitor on each RowResult. Uses a table + * name to locate meta regions. + * + * @param configuration + * @param visitor + * @param tableName + * @throws IOException + */ + public static void metaScan(HBaseConfiguration configuration, + MetaScannerVisitor visitor, byte[] tableName) throws IOException { + HConnection connection = HConnectionManager.getConnection(configuration); + HRegionLocation metaLocation = null; + boolean toContinue = true; + byte[] startRow = Bytes.toString(tableName).equals( + Bytes.toString(EMPTY_START_ROW)) ? tableName : HRegionInfo + .createRegionName(tableName, null, NINES); + + // scan over the each meta region + do { + ScannerCallable callable = new ScannerCallable(connection, + META_TABLE_NAME, COL_REGIONINFO_ARRAY, tableName, LATEST_TIMESTAMP, + null); + try { + // open scanner + connection.getRegionServerWithRetries(callable); + metaLocation = connection.locateRegion(META_TABLE_NAME, startRow); + while (toContinue) { + RowResult rowResult = connection.getRegionServerWithRetries(callable); + if (rowResult == null || rowResult.size() == 0) { + break; + } + HRegionInfo info = Writables.getHRegionInfo(rowResult + .get(COL_REGIONINFO)); + toContinue = visitor.processRow(rowResult, metaLocation, info); + } + // advance the startRow to the end key of the current region + startRow = callable.getHRegionInfo().getEndKey(); + } finally { + // close scanner + callable.setClose(); + connection.getRegionServerWithRetries(callable); + } + } while (Bytes.compareTo(startRow, LAST_ROW) != 0); + } + + /** + * Visitor class called to process each row of the .META. table + * + */ + protected interface MetaScannerVisitor { + + /** + * Visitor method that accepts a RowResult and the meta region location. + * Implementations can return false to stop the region's loop if it becomes + * unnecessary for some reason. + * + * @param rowResult + * @param metaLocation + * @return A boolean to know if it should continue to loop in the region + * @throws IOException + */ + public boolean processRow(RowResult rowResult, + HRegionLocation metaLocation, HRegionInfo info) throws IOException; + } + +} Index: src/java/org/apache/hadoop/hbase/client/HConnectionManager.java =================================================================== --- src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (revision 658259) +++ src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (working copy) @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.NoServerForRegionException; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.io.Cell; import org.apache.hadoop.hbase.io.RowResult; import org.apache.hadoop.hbase.ipc.HMasterInterface; @@ -157,16 +158,19 @@ this.masterChecked = false; this.servers = new ConcurrentHashMap(); } - + /** {@inheritDoc} */ public HMasterInterface getMaster() throws MasterNotRunningException { - HServerAddress masterLocation = null; synchronized (this.masterLock) { - for (int tries = 0; !this.closed && - !this.masterChecked && this.master == null && tries < numRetries; - tries++) { - String m = this.conf.get(MASTER_ADDRESS, DEFAULT_MASTER_ADDRESS); - masterLocation = new HServerAddress(m); + for (int tries = 0; + !this.closed && + !this.masterChecked && this.master == null && + tries < numRetries; + tries++) { + + HServerAddress masterLocation = new HServerAddress(this.conf.get( + MASTER_ADDRESS, DEFAULT_MASTER_ADDRESS)); + try { HMasterInterface tryMaster = (HMasterInterface)HbaseRPC.getProxy( HMasterInterface.class, HMasterInterface.versionID, @@ -178,12 +182,12 @@ } } catch (IOException e) { - if (tries == numRetries - 1) { + if(tries == numRetries - 1) { // This was our last chance - don't bother sleeping break; } LOG.info("Attempt " + tries + " of " + this.numRetries + - " failed with <" + e + ">. Retrying after sleep of " + this.pause); + " failed with <" + e + ">. Retrying after sleep of " + this.pause); } // We either cannot connect to master or it is not running. Sleep & retry @@ -197,8 +201,7 @@ this.masterChecked = true; } if (this.master == null) { - throw new MasterNotRunningException(masterLocation == null? "": - masterLocation.toString()); + throw new MasterNotRunningException(); } return this.master; } @@ -257,40 +260,23 @@ /** {@inheritDoc} */ public HTableDescriptor[] listTables() throws IOException { - HashSet uniqueTables = new HashSet(); - byte [] startRow = EMPTY_START_ROW; + final HashSet uniqueTables = new HashSet(); - // scan over the each meta region - do { - ScannerCallable callable = new ScannerCallable(this, META_TABLE_NAME, - COL_REGIONINFO_ARRAY, startRow, LATEST_TIMESTAMP, null); - try { - // open scanner - getRegionServerWithRetries(callable); - // iterate through the scanner, accumulating unique table names - while (true) { - RowResult values = getRegionServerWithRetries(callable); - if (values == null || values.size() == 0) { - break; - } + MetaScannerVisitor visitor = new MetaScannerVisitor() { - HRegionInfo info = - Writables.getHRegionInfo(values.get(COL_REGIONINFO)); + public boolean processRow(RowResult rowResult, + HRegionLocation metaLocation, HRegionInfo info) throws IOException { - // Only examine the rows where the startKey is zero length - if (info.getStartKey().length == 0) { - uniqueTables.add(info.getTableDesc()); - } + // Only examine the rows where the startKey is zero length + if (info.getStartKey().length == 0) { + uniqueTables.add(info.getTableDesc()); } - // advance the startRow to the end key of the current region - startRow = callable.getHRegionInfo().getEndKey(); - } finally { - // close scanner - callable.setClose(); - getRegionServerWithRetries(callable); + return true; } - } while (Bytes.compareTo(startRow, LAST_ROW) != 0); - + + }; + MetaScanner.metaScan(conf, visitor); + return uniqueTables.toArray(new HTableDescriptor[uniqueTables.size()]); } @@ -321,11 +307,12 @@ // This block guards against two threads trying to find the root // region at the same time. One will go do the find while the // second waits. The second thread will not do find. + if (!useCache || rootRegionLocation == null) { return locateRootRegion(); } return rootRegionLocation; - } + } } else if (Bytes.equals(tableName, META_TABLE_NAME)) { synchronized (metaRegionLock) { // This block guards against two threads trying to load the meta @@ -652,29 +639,30 @@ */ private HRegionLocation locateRootRegion() throws IOException { + getMaster(); + HServerAddress rootRegionAddress = null; + for (int tries = 0; tries < numRetries; tries++) { int localTimeouts = 0; - // Ask the master which server has the root region + + // ask the master which server has the root region while (rootRegionAddress == null && localTimeouts < numRetries) { rootRegionAddress = master.findRootRegion(); if (rootRegionAddress == null) { - // Increment and then only sleep if retries left. - if (++localTimeouts < numRetries) { - try { - if (LOG.isDebugEnabled()) { - LOG.debug("Sleeping " + pause + "ms. Waiting for root " - + "region. Attempt " + tries + " of " + numRetries); - } - Thread.sleep(pause); - if (LOG.isDebugEnabled()) { - LOG.debug("Wake. Retry finding root region."); - } - } catch (InterruptedException iex) { - // continue + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Sleeping. Waiting for root region."); } + Thread.sleep(pause); + if (LOG.isDebugEnabled()) { + LOG.debug("Wake. Retry finding root region."); + } + } catch (InterruptedException iex) { + // continue } + localTimeouts++; } }