From 1bf6fdbd19da5bdca9cb594051e450d10c65e620 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Tue, 21 Aug 2018 21:47:37 +0800 Subject: [PATCH] HBASE-21086 NPE in TestTableResource --- .../hadoop/hbase/rest/TestTableResource.java | 17 ++-- .../TestEndToEndSplitTransaction.java | 85 +++++++++---------- 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java index 5fa3072281..7a86d2fde2 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableResource.java @@ -31,18 +31,19 @@ import javax.xml.bind.JAXBException; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction; import org.apache.hadoop.hbase.rest.client.Client; import org.apache.hadoop.hbase.rest.client.Cluster; @@ -98,8 +99,8 @@ public class TestTableResource { if (admin.tableExists(TABLE)) { return; } - HTableDescriptor htd = new HTableDescriptor(TABLE); - htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + TableDescriptor htd = TableDescriptorBuilder.newBuilder(TABLE) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(COLUMN_FAMILY)).build(); admin.createTable(htd); byte[] k = new byte[3]; byte [][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(COLUMN)); @@ -131,8 +132,8 @@ public class TestTableResource { admin.split(TABLE); // give some time for the split to happen - TestEndToEndSplitTransaction.blockUntilRegionSplit(TEST_UTIL.getConfiguration(), 60000, - m.get(0).getRegionInfo().getRegionName(), true); + TestEndToEndSplitTransaction.blockUntilRegionSplit(TEST_UTIL, 60000, + m.get(0).getRegion().getRegionName(), true); long timeout = System.currentTimeMillis() + (15 * 1000); while (System.currentTimeMillis() < timeout && m.size()!=2){ try { @@ -179,7 +180,7 @@ public class TestTableResource { TableRegionModel region = regions.next(); boolean found = false; for (HRegionLocation e: regionMap) { - HRegionInfo hri = e.getRegionInfo(); + RegionInfo hri = e.getRegion(); String hriRegionName = hri.getRegionNameAsString(); String regionName = region.getName(); if (hriRegionName.equals(regionName)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 85e9d3098d..6887064c17 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -30,7 +30,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ChoreService; @@ -63,8 +62,6 @@ import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.hadoop.hbase.util.StoppableImplementation; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hbase.thirdparty.com.google.common.collect.Iterators; -import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -76,6 +73,9 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.Iterators; +import org.apache.hbase.thirdparty.com.google.common.collect.Maps; + @Category(LargeTests.class) public class TestEndToEndSplitTransaction { @@ -217,14 +217,12 @@ public class TestEndToEndSplitTransaction { TableName tableName; byte[] family; Admin admin; - HRegionServer rs; RegionSplitter(Table table) throws IOException { this.table = table; this.tableName = table.getName(); - this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next(); + this.family = table.getDescriptor().getColumnFamilies()[0].getName(); admin = TEST_UTIL.getAdmin(); - rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); connection = TEST_UTIL.getConnection(); } @@ -257,15 +255,14 @@ public class TestEndToEndSplitTransaction { addData(start); addData(mid); - flushAndBlockUntilDone(admin, rs, region.getRegionName()); - compactAndBlockUntilDone(admin, rs, region.getRegionName()); + flushAndBlockUntilDone(TEST_UTIL, region.getRegionName()); + compactAndBlockUntilDone(TEST_UTIL, region.getRegionName()); log("Initiating region split for:" + region.getRegionNameAsString()); try { - admin.splitRegion(region.getRegionName(), splitPoint); + admin.splitRegionAsync(region.getRegionName(), splitPoint); // wait until the split is complete - blockUntilRegionSplit(CONF, 50000, region.getRegionName(), true); - + blockUntilRegionSplit(TEST_UTIL, 50000, region.getRegionName(), true); } catch (NotServingRegionException ex) { // ignore } @@ -326,7 +323,7 @@ public class TestEndToEndSplitTransaction { Set regions = new TreeSet<>(RegionInfo.COMPARATOR); for (HRegionLocation loc : rl.getAllRegionLocations()) { - regions.add(loc.getRegionInfo()); + regions.add(loc.getRegion()); } verifyTableRegions(regions); } @@ -396,46 +393,44 @@ public class TestEndToEndSplitTransaction { } /* some utility methods for split tests */ + private static HRegion getRegion(HBaseTestingUtility util, byte[] regionName) { + return util.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(t -> t.getRegionServer().getOnlineRegion(regionName)).filter(r -> r != null).findFirst() + .orElse(null); + } - public static void flushAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName) + private static void flushAndBlockUntilDone(HBaseTestingUtility util, byte[] regionName) throws IOException, InterruptedException { log("flushing region: " + Bytes.toStringBinary(regionName)); - admin.flushRegion(regionName); + util.getAdmin().flushRegion(regionName); log("blocking until flush is complete: " + Bytes.toStringBinary(regionName)); Threads.sleepWithoutInterrupt(500); - while (rs.getOnlineRegion(regionName).getMemStoreDataSize() > 0) { - Threads.sleep(50); - } + + HRegion region = getRegion(util, regionName); + util.waitFor(60000, () -> region.getMemStoreDataSize() <= 0); } - public static void compactAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName) + private static void compactAndBlockUntilDone(HBaseTestingUtility util, byte[] regionName) throws IOException, InterruptedException { log("Compacting region: " + Bytes.toStringBinary(regionName)); - admin.majorCompactRegion(regionName); + util.getAdmin().majorCompactRegion(regionName); log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName)); Threads.sleepWithoutInterrupt(500); - outer: for (;;) { - for (Store store : rs.getOnlineRegion(regionName).getStores()) { - if (store.getStorefilesCount() > 1) { - Threads.sleep(50); - continue outer; - } - } - break; - } + HRegion region = getRegion(util, regionName); + util.waitFor(60000, + () -> region.getStores().stream().allMatch(s -> s.getStorefilesCount() <= 1)); } /** * Blocks until the region split is complete in hbase:meta and region server opens the daughters */ - public static void blockUntilRegionSplit(Configuration conf, long timeout, + public static void blockUntilRegionSplit(HBaseTestingUtility util, long timeout, final byte[] regionName, boolean waitForDaughters) throws IOException, InterruptedException { long start = System.currentTimeMillis(); log("blocking until region is split:" + Bytes.toStringBinary(regionName)); RegionInfo daughterA = null, daughterB = null; - try (Connection conn = ConnectionFactory.createConnection(conf); - Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) { + try (Table metaTable = util.getConnection().getTable(TableName.META_TABLE_NAME)) { Result result = null; RegionInfo region = null; while ((System.currentTimeMillis() - start) < timeout) { @@ -463,33 +458,31 @@ public class TestEndToEndSplitTransaction { //if we are here, this means the region split is complete or timed out if (waitForDaughters) { long rem = timeout - (System.currentTimeMillis() - start); - blockUntilRegionIsInMeta(conn, rem, daughterA); + blockUntilRegionIsInMeta(util.getConnection(), rem, daughterA); rem = timeout - (System.currentTimeMillis() - start); - blockUntilRegionIsInMeta(conn, rem, daughterB); + blockUntilRegionIsInMeta(util.getConnection(), rem, daughterB); rem = timeout - (System.currentTimeMillis() - start); - blockUntilRegionIsOpened(conf, rem, daughterA); + blockUntilRegionIsOpened(util.getConfiguration(), rem, daughterA); rem = timeout - (System.currentTimeMillis() - start); - blockUntilRegionIsOpened(conf, rem, daughterB); + blockUntilRegionIsOpened(util.getConfiguration(), rem, daughterB); // Compacting the new region to make sure references can be cleaned up - compactAndBlockUntilDone(TEST_UTIL.getAdmin(), - TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterA.getRegionName()); - compactAndBlockUntilDone(TEST_UTIL.getAdmin(), - TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterB.getRegionName()); + compactAndBlockUntilDone(util, daughterA.getRegionName()); + compactAndBlockUntilDone(util, daughterB.getRegionName()); - removeCompactedFiles(conn, timeout, daughterA); - removeCompactedFiles(conn, timeout, daughterB); + removeCompactedFiles(util, timeout, daughterA); + removeCompactedFiles(util, timeout, daughterB); } } } - public static void removeCompactedFiles(Connection conn, long timeout, RegionInfo hri) + private static void removeCompactedFiles(HBaseTestingUtility util, long timeout, RegionInfo hri) throws IOException, InterruptedException { log("remove compacted files for : " + hri.getRegionNameAsString()); - List regions = TEST_UTIL.getHBaseCluster().getRegions(hri.getTable()); + List regions = util.getHBaseCluster().getRegions(hri.getTable()); regions.stream().forEach(r -> { try { r.getStores().get(0).closeAndArchiveCompactedFiles(); @@ -499,13 +492,13 @@ public class TestEndToEndSplitTransaction { }); } - public static void blockUntilRegionIsInMeta(Connection conn, long timeout, RegionInfo hri) + private static void blockUntilRegionIsInMeta(Connection conn, long timeout, RegionInfo hri) throws IOException, InterruptedException { log("blocking until region is in META: " + hri.getRegionNameAsString()); long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < timeout) { HRegionLocation loc = MetaTableAccessor.getRegionLocation(conn, hri); - if (loc != null && !loc.getRegionInfo().isOffline()) { + if (loc != null && !loc.getRegion().isOffline()) { log("found region in META: " + hri.getRegionNameAsString()); break; } @@ -513,7 +506,7 @@ public class TestEndToEndSplitTransaction { } } - public static void blockUntilRegionIsOpened(Configuration conf, long timeout, RegionInfo hri) + private static void blockUntilRegionIsOpened(Configuration conf, long timeout, RegionInfo hri) throws IOException, InterruptedException { log("blocking until region is opened for reading:" + hri.getRegionNameAsString()); long start = System.currentTimeMillis(); -- 2.17.1