diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 6859cb3..0e86274 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -139,7 +139,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable { private final boolean hostnamesCanChange; private final long pause; private final long pauseForCQTBE;// pause for CallQueueTooBigException, if specified - private final boolean useMetaReplicas; + private boolean useMetaReplicas; private final int numTries; final int rpcTimeout; @@ -306,6 +306,14 @@ class ConnectionImplementation implements ClusterConnection, Closeable { } /** + * @param useMetaReplicas + */ + @VisibleForTesting + void setUseMetaReplicas(final boolean useMetaReplicas) { + this.useMetaReplicas = useMetaReplicas; + } + + /** * @param conn The connection for which to replace the generator. * @param cnm Replaces the nonce generator used, for testing. * @return old nonce generator. diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java index 2c77541..df4d446 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java @@ -39,6 +39,9 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; @@ -157,6 +160,34 @@ public class TestReplicaWithCluster { } } + /** + * This copro is used to simulate region server hosting meta down exception for Get and Scan + */ + public static class RegionServerHostingPrimayMetaRegionSlowCopro implements RegionObserver { + static boolean slowDownPrimaryMetaScan = false; + + @Override + public RegionScanner preScannerOpen(final ObserverContext e, + final Scan scan, final RegionScanner s) throws IOException { + + int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId(); + + // Slow down with the primary meta region scan + if (slowDownPrimaryMetaScan && (e.getEnvironment().getRegion().getRegionInfo().isMetaRegion() + && (replicaId == 0))) { + LOG.info("Scan with primary meta region, slow down a bit"); + try { + Thread.sleep(50); + } catch (InterruptedException ie) { + // Ingore + } + + } + return null; + } + } + + @BeforeClass public static void beforeClass() throws Exception { // enable store file refreshing @@ -178,6 +209,16 @@ public class TestReplicaWithCluster { // Retry less so it can fail faster HTU.getConfiguration().setInt("hbase.client.retries.number", 1); + // Enable meta replica at server side + HTU.getConfiguration().setInt("hbase.meta.replica.count", 2); + + // Make sure master does not host system tables. + HTU.getConfiguration().set("hbase.balancer.tablesOnMaster", "none"); + + // Set system coprocessor so it can be applied to meta regions + HTU.getConfiguration().set("hbase.coprocessor.region.classes", + RegionServerHostingPrimayMetaRegionSlowCopro.class.getName()); + HTU.startMiniCluster(NB_SERVERS); HTU.getHBaseCluster().startMaster(); } @@ -570,4 +611,71 @@ public class TestReplicaWithCluster { HTU.deleteTable(hdt.getTableName()); } } + + @Test + public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException { + HTU.getAdmin().setBalancerRunning(false, true); + + ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(true); + + // Create table then get the single region for our new table. + HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion"); + hdt.setRegionReplication(2); + try { + + Table table = HTU.createTable(hdt, new byte[][] { f }, null); + + // Get Meta location + RegionLocations mrl = ((ClusterConnection) HTU.getConnection()) + .locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); + + // Get user table location + RegionLocations url = ((ClusterConnection) HTU.getConnection()) + .locateRegion(hdt.getTableName(), row, false, false); + + // Make sure that user primary region is co-hosted with the meta region + if (!url.getDefaultRegionLocation().getServerName() + .equals(mrl.getDefaultRegionLocation().getServerName())) { + HTU.moveRegionAndWait(url.getDefaultRegionLocation().getRegionInfo(), + mrl.getDefaultRegionLocation().getServerName()); + } + + // Make sure that the user replica region is not hosted by the same region server with + // primary + if (url.getRegionLocation(1).getServerName() + .equals(mrl.getDefaultRegionLocation().getServerName())) { + HTU.moveRegionAndWait(url.getRegionLocation(1).getRegionInfo(), + url.getDefaultRegionLocation().getServerName()); + } + + RegionServerHostingPrimayMetaRegionSlowCopro.slowDownPrimaryMetaScan = true; + + // Wait until the meta table is updated with new location info + while (true) { + mrl = ((ClusterConnection) HTU.getConnection()) + .locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); + + // Get user table location + url = ((ClusterConnection) HTU.getConnection()) + .locateRegion(hdt.getTableName(), row, false, true); + + LOG.info("meta locations " + mrl); + LOG.info("table locations " + url); + ServerName a = url.getDefaultRegionLocation().getServerName(); + ServerName b = mrl.getDefaultRegionLocation().getServerName(); + if (a.equals(b)) { + break; + } else { + LOG.info("Waiting for new region info to be updated in meta table"); + Thread.sleep(100); + } + } + } finally { + RegionServerHostingPrimayMetaRegionSlowCopro.slowDownPrimaryMetaScan = false; + ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(false); + HTU.getAdmin().setBalancerRunning(true, true); + HTU.getAdmin().disableTable(hdt.getTableName()); + HTU.deleteTable(hdt.getTableName()); + } + } }