diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index 80e22b9..2fd45e8 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -1027,7 +1027,7 @@ class ConnectionManager { public List locateRegions(final TableName tableName, final boolean useCache, final boolean offlined) throws IOException { NavigableMap regions = MetaScanner.allTableRegions(conf, this, - tableName, offlined); + tableName); final List locations = new ArrayList(); for (HRegionInfo regionInfo : regions.keySet()) { RegionLocations list = locateRegion(tableName, regionInfo.getStartKey(), useCache, true); diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index e19272b..fc44cc8 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.Callable; @@ -605,7 +606,24 @@ public class HTable implements HTableInterface, RegionLocator { @Deprecated public NavigableMap getRegionLocations() throws IOException { // TODO: Odd that this returns a Map of HRI to SN whereas getRegionLocator, singular, returns an HRegionLocation. - return MetaScanner.allTableRegions(getConfiguration(), this.connection, getName(), false); + return MetaScanner.allTableRegions(getConfiguration(), this.connection, getName()); + } + + /** + * Gets all the regions and their address for this table. + *

+ * This is mainly useful for the MapReduce integration. + * @return A map of HRegionInfo with it's server address + * @throws IOException if a remote or network exception occurs + */ + @Override + public List getAllRegionLocations() throws IOException { + NavigableMap locations = getRegionLocations(); + ArrayList regions = new ArrayList<>(locations.size()); + for (Entry entry : locations.entrySet()) { + regions.add(new HRegionLocation(entry.getKey(), entry.getValue())); + } + return regions; } /** diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java index cd16d17..966663b 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -81,7 +81,7 @@ public class MetaScanner { * null if not interested in a particular table. * @throws IOException e */ - public static void metaScan(Configuration configuration, ClusterConnection connection, + public static void metaScan(Configuration configuration, Connection connection, MetaScannerVisitor visitor, TableName userTableName) throws IOException { metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE, TableName.META_TABLE_NAME); @@ -127,27 +127,37 @@ public class MetaScanner { * @param metaTableName Meta table to scan, root or meta. * @throws IOException e */ - static void metaScan(Configuration configuration, ClusterConnection connection, + static void metaScan(Configuration configuration, Connection connection, final MetaScannerVisitor visitor, final TableName tableName, final byte[] row, final int rowLimit, final TableName metaTableName) throws IOException { + boolean closeConnection = false; if (connection == null){ - connection = ConnectionManager.getConnectionInternal(configuration); + connection = ConnectionFactory.createConnection(configuration); + closeConnection = true; } int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE; // Calculate startrow for scan. byte[] startRow; ResultScanner scanner = null; - HTable metaTable = null; + Table metaTable = null; try { - metaTable = new HTable(TableName.META_TABLE_NAME, connection, null); + // TODO: This fails lots of tests because the connection needs to be managed. A lot of tests + // have unmanaged tests. Those tests should be fixed to have a managed connection. + // metaTable = connection.getTable(TableName.META_TABLE_NAME); + metaTable = new HTable(TableName.META_TABLE_NAME, connection); if (row != null) { // Scan starting at a particular row in a particular table byte[] searchRow = HRegionInfo.createRegionName(tableName, row, HConstants.NINES, false); - Result startRowResult = metaTable.getRowOrBefore(searchRow, HConstants.CATALOG_FAMILY); + Scan scan = new Scan(searchRow); + scan.setReversed(true); + scan.addFamily(HConstants.CATALOG_FAMILY); + + ResultScanner metaScanner = metaTable.getScanner(scan); + Result startRowResult = metaScanner.next(); if (startRowResult == null) { throw new TableNotFoundException("Cannot find row in "+ TableName @@ -217,7 +227,9 @@ public class MetaScanner { LOG.debug("Got exception in closing meta table", t); } } - + if (closeConnection) { + connection.close(); + } } } @@ -272,16 +284,12 @@ public class MetaScanner { /** * Lists all of the table regions currently in META. * @param conf - * @param offlined True if we are to include offlined regions, false and we'll - * leave out offlined regions from returned list. * @return Map of all user-space regions to servers * @throws IOException */ public static NavigableMap allTableRegions(Configuration conf, - ClusterConnection connection, final TableName tableName, - final boolean offlined) throws IOException { - final NavigableMap regions = - new TreeMap(); + Connection connection, final TableName tableName) throws IOException { + final NavigableMap regions = new TreeMap<>(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override public boolean processRowInternal(Result result) throws IOException { @@ -304,7 +312,7 @@ public class MetaScanner { * Lists table regions and locations grouped by region range from META. */ public static List listTableRegionLocations(Configuration conf, - ClusterConnection connection, final TableName tableName) throws IOException { + Connection connection, final TableName tableName) throws IOException { final List regions = new ArrayList(); MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { @Override diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java index 505e841..7a39537 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java @@ -18,14 +18,17 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Pair; import java.io.Closeable; import java.io.IOException; +import java.util.List; /** * Used to view region location information for a single HBase table. @@ -55,6 +58,14 @@ public interface RegionLocator extends Closeable { throws IOException; /** + * Retrieves all of the regions associated with this table. + * @return a {@link List} of all regions associated with this table. + * @throws IOException if a remote or network exception occurs + */ + public List getAllRegionLocations() + throws IOException; + + /** * Gets the starting row key for every region in the currently open table. *

* This is mainly useful for the MapReduce integration. @@ -86,4 +97,18 @@ public interface RegionLocator extends Closeable { * Gets the fully qualified table name instance of this table. */ TableName getName(); + + /** + * Returns the {@link org.apache.hadoop.conf.Configuration} object used by this instance. + *

+ * The reference returned is not a copy, so any change made to it will + * affect this instance. + */ + Configuration getConfiguration(); + + /** + * Gets the {@link org.apache.hadoop.hbase.HTableDescriptor table descriptor} for this table. + * @throws java.io.IOException if a remote or network exception occurs. + */ + HTableDescriptor getTableDescriptor() throws IOException; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java index ddc2f56..1ecb7c6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RegionsResource.java @@ -77,7 +77,7 @@ public class RegionsResource extends ResourceBase { TableName tableName = TableName.valueOf(tableResource.getName()); TableInfoModel model = new TableInfoModel(tableName.getNameAsString()); Map regions = MetaScanner.allTableRegions( - servlet.getConfiguration(), null, tableName, false); + servlet.getConfiguration(), null, tableName); for (Map.Entry e: regions.entrySet()) { HRegionInfo hri = e.getKey(); ServerName addr = e.getValue(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSizeCalculator.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSizeCalculator.java index f9c4c92..a1cc86e 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSizeCalculator.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSizeCalculator.java @@ -23,18 +23,19 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.RegionLocator; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -60,26 +61,26 @@ public class RegionSizeCalculator { /** * Computes size of each region for table and given column families. * */ - public RegionSizeCalculator(HTable table) throws IOException { + public RegionSizeCalculator(RegionLocator table) throws IOException { this(table, new HBaseAdmin(table.getConfiguration())); } /** ctor for unit testing */ - RegionSizeCalculator (HTable table, Admin admin) throws IOException { + RegionSizeCalculator (RegionLocator regionLocator, Admin admin) throws IOException { try { - if (!enabled(table.getConfiguration())) { + if (!enabled(regionLocator.getConfiguration())) { LOG.info("Region size calculation disabled."); return; } - LOG.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\"."); + LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\"."); //get regions for table - Set tableRegionInfos = table.getRegionLocations().keySet(); + List tableRegionInfos = regionLocator.getAllRegionLocations(); Set tableRegions = new TreeSet(Bytes.BYTES_COMPARATOR); - for (HRegionInfo regionInfo : tableRegionInfos) { - tableRegions.add(regionInfo.getRegionName()); + for (HRegionLocation regionInfo : tableRegionInfos) { + tableRegions.add(regionInfo.getRegionInfo().getRegionName()); } ClusterStatus clusterStatus = admin.getClusterStatus(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java index 8333954..3f5a1e1 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java @@ -212,8 +212,7 @@ public class TestMultiVersions { NavigableMap locations = table.getRegionLocations(); assertEquals(2, locations.size()); int index = 0; - for (Map.Entry e: locations.entrySet()) { - HRegionInfo hri = e.getKey(); + for (HRegionInfo hri: locations.keySet()) { if (index == 0) { assertTrue(Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())); assertTrue(Bytes.equals(hri.getEndKey(), splitRows[0])); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java index f0bc1ec..91028a9 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java @@ -190,7 +190,7 @@ public class TestMetaScanner { while(!isStopped()) { try { NavigableMap regions = - MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), null, TABLENAME, false); + MetaScanner.allTableRegions(TEST_UTIL.getConfiguration(), null, TABLENAME); LOG.info("-------"); byte[] lastEndKey = HConstants.EMPTY_START_ROW; diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 67a8f19..df56e5a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -237,7 +237,7 @@ public class TestEndToEndSplitTransaction { Random random = new Random(); for (int i=0; i< 5; i++) { NavigableMap regions = MetaScanner.allTableRegions(conf, null, - tableName, false); + tableName); if (regions.size() == 0) { continue; } @@ -310,7 +310,7 @@ public class TestEndToEndSplitTransaction { //MetaScanner.allTableRegions() NavigableMap regions = MetaScanner.allTableRegions(conf, null, - tableName, false); + tableName); verifyTableRegions(regions.keySet()); //MetaScanner.listAllRegions() diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index e2fa54d..448cccd 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -1262,7 +1262,7 @@ public class TestHBaseFsck { setupTableWithRegionReplica(table, 2); assertEquals(ROWKEYS.length, countRows()); NavigableMap map = MetaScanner.allTableRegions(conf, null, - tbl.getName(), false); + tbl.getName()); int i = 0; // store the HRIs of the regions we will mess up for (Map.Entry m : map.entrySet()) { @@ -1294,7 +1294,7 @@ public class TestHBaseFsck { i = 0; HRegionInfo[] newHris = new HRegionInfo[2]; // get all table's regions from meta - map = MetaScanner.allTableRegions(conf, null, tbl.getName(), false); + map = MetaScanner.allTableRegions(conf, null, tbl.getName()); // get the HRIs of the new regions (hbck created new regions for fixing the hdfs mess-up) for (Map.Entry m : map.entrySet()) { if (m.getKey().getStartKey().length > 0 && diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java index a8e0a55..4886edc 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java @@ -20,14 +20,17 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.RegionLocator; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -36,7 +39,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.TreeMap; import static org.junit.Assert.assertEquals; @@ -52,7 +54,7 @@ public class TestRegionSizeCalculator { @Test public void testSimpleTestCase() throws Exception { - HTable table = mockTable("region1", "region2", "region3"); + RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3"); Admin admin = mockAdmin( mockServer( @@ -65,7 +67,7 @@ public class TestRegionSizeCalculator { ) ); - RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin); + RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes())); assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes())); @@ -84,7 +86,7 @@ public class TestRegionSizeCalculator { @Test public void testLargeRegion() throws Exception { - HTable table = mockTable("largeRegion"); + RegionLocator regionLocator = mockRegionLocator("largeRegion"); Admin admin = mockAdmin( mockServer( @@ -92,7 +94,7 @@ public class TestRegionSizeCalculator { ) ); - RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin); + RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); assertEquals(((long) Integer.MAX_VALUE) * megabyte, calculator.getRegionSize("largeRegion".getBytes())); } @@ -101,7 +103,7 @@ public class TestRegionSizeCalculator { @Test public void testDisabled() throws Exception { String regionName = "cz.goout:/index.html"; - HTable table = mockTable(regionName); + RegionLocator table = mockRegionLocator(regionName); Admin admin = mockAdmin( mockServer( @@ -124,17 +126,17 @@ public class TestRegionSizeCalculator { /** * Makes some table with given region names. * */ - private HTable mockTable(String... regionNames) throws IOException { - HTable mockedTable = Mockito.mock(HTable.class); + private RegionLocator mockRegionLocator(String... regionNames) throws IOException { + RegionLocator mockedTable = Mockito.mock(HTable.class); when(mockedTable.getConfiguration()).thenReturn(configuration); - when(mockedTable.getTableName()).thenReturn("sizeTestTable".getBytes()); - NavigableMap regionLocations = new TreeMap(); - when(mockedTable.getRegionLocations()).thenReturn(regionLocations); + when(mockedTable.getName()).thenReturn(TableName.valueOf("sizeTestTable")); + List regionLocations = new ArrayList<>(); + when(mockedTable.getAllRegionLocations()).thenReturn(regionLocations); for (String regionName : regionNames) { HRegionInfo info = Mockito.mock(HRegionInfo.class); when(info.getRegionName()).thenReturn(regionName.getBytes()); - regionLocations.put(info, null);//we are not interested in values + regionLocations.add(new HRegionLocation(info, null));//we are not interested in values } return mockedTable;