From 18db0b65fb788b6d8f6fa50f2cdb8acc9a021d6b Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Wed, 30 May 2018 16:30:12 +0900 Subject: [PATCH] HBASE-20648 HBASE-19364 "Truncate_preserve fails with table when replica region > 1" for master branch --- .../master/procedure/TruncateTableProcedure.java | 5 +- .../procedure/TestTruncateTableProcedure.java | 67 ++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java index 4b2c21f..808d90b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; @@ -89,6 +90,7 @@ public class TruncateTableProcedure // TODO: Move out... in the acquireLock() LOG.debug("waiting for '" + getTableName() + "' regions in transition"); regions = env.getAssignmentManager().getRegionStates().getRegionsOfTable(getTableName()); + RegionReplicaUtil.removeNonDefaultRegions(regions); assert regions != null && !regions.isEmpty() : "unexpected 0 regions"; ProcedureSyncWait.waitRegionInTransition(env, regions); @@ -148,7 +150,8 @@ public class TruncateTableProcedure if (isRollbackSupported(state)) { setFailure("master-truncate-table", e); } else { - LOG.warn("Retriable error trying to truncate table=" + getTableName() + " state=" + state, e); + LOG.warn("Retriable error trying to truncate table=" + getTableName() + + " state=" + state, e); } } return Flow.HAS_MORE_STATE; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java index acd883d..35e8788 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java @@ -20,17 +20,24 @@ package org.apache.hadoop.hbase.master.procedure; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ModifyRegionUtils; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -39,6 +46,7 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + @Category({MasterTests.class, MediumTests.class}) public class TestTruncateTableProcedure extends TestTableDDLProcedureBase { @@ -217,4 +225,63 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase { UTIL.getConnection(), tableName, 50, splitKeys, families); assertEquals(50, UTIL.countRows(tableName)); } + + @Test(timeout = 60000) + public void testTruncateWithPreserveAfterSplit() throws Exception { + String[] families = new String[] { "f1", "f2" }; + byte[][] splitKeys = + new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; + TableName tableName = TableName.valueOf(name.getMethodName()); + RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), + tableName, splitKeys, families); + splitAndTruncate(tableName, regions); + } + + @Test(timeout = 60000) + public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception { + String[] families = new String[] { "f1", "f2" }; + byte[][] splitKeys = + new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; + TableName tableName = TableName.valueOf(name.getMethodName()); + + // create a table with region replications + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) + .setRegionReplication(3) + .setColumnFamilies( + Arrays.stream(families) + .map(fam -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fam)).build()) + .collect(Collectors.toList())) + .build(); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); + ProcedureExecutor procExec = getMasterProcedureExecutor(); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); + + splitAndTruncate(tableName, regions); + } + + private void splitAndTruncate(TableName tableName, RegionInfo[] regions) throws IOException, + InterruptedException { + + // split a region + UTIL.getAdmin().split(tableName, new byte[]{'0'}); + UTIL.waitUntilAllRegionsAssigned(tableName); + + // wait until split really happens + while (UTIL.getAdmin().getRegions(tableName).size() <= regions.length) { + Thread.sleep(50); + } + + // disable the table + UTIL.getAdmin().disableTable(tableName); + + // truncate the table + ProcedureExecutor procExec = getMasterProcedureExecutor(); + long procId = ProcedureTestingUtility.submitAndWait(procExec, + new TruncateTableProcedure(procExec.getEnvironment(), tableName, true)); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + UTIL.waitUntilAllRegionsAssigned(tableName); + } } -- 2.10.1 (Apple Git-78)