From 3332e9f4488dce8d65e2bf434a98ca322a834d36 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 10 Apr 2019 19:56:39 +0100 Subject: [PATCH] HBASE-22200 - WALSplitter.hasRecoveredEdits should use same FS instance from WAL region dir Change-Id: I34429b04bc28e8ae7ca48238aba49ac6ff2712da --- .../assignment/AssignmentManagerUtil.java | 3 +- .../apache/hadoop/hbase/wal/WALSplitter.java | 7 ++- .../apache/hadoop/hbase/wal/TestWALSplit.java | 44 ++++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java index 97ae7eadd6..4f9343cf22 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManagerUtil.java @@ -229,8 +229,7 @@ final class AssignmentManagerUtil { } static void checkClosedRegion(MasterProcedureEnv env, RegionInfo regionInfo) throws IOException { - if (WALSplitter.hasRecoveredEdits(env.getMasterServices().getFileSystem(), - env.getMasterConfiguration(), regionInfo)) { + if (WALSplitter.hasRecoveredEdits(env.getMasterConfiguration(), regionInfo)) { throw new IOException("Recovered.edits are found in Region: " + regionInfo + ", abort split/merge to prevent data loss"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 66795978f2..3959ec7b74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -539,14 +539,13 @@ public class WALSplitter { /** * Check whether there is recovered.edits in the region dir - * @param walFS FileSystem * @param conf conf * @param regionInfo the region to check * @throws IOException IOException * @return true if recovered.edits exist in the region dir */ - public static boolean hasRecoveredEdits(final FileSystem walFS, - final Configuration conf, final RegionInfo regionInfo) throws IOException { + public static boolean hasRecoveredEdits(final Configuration conf, + final RegionInfo regionInfo) throws IOException { // No recovered.edits for non default replica regions if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { return false; @@ -555,7 +554,7 @@ public class WALSplitter { //directly without converting it to default replica's regioninfo. Path regionDir = FSUtils.getWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName()); - NavigableSet files = getSplitEditFilesSorted(walFS, regionDir); + NavigableSet files = getSplitEditFilesSorted(FSUtils.getWALFileSystem(conf), regionDir); return files != null && !files.isEmpty(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index e6644f07dd..b4e25336a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -383,18 +383,7 @@ public class TestWALSplit { */ @Test public void testRecoveredEditsPathForMeta() throws IOException { - byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); - Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); - Path regiondir = new Path(tdir, - RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); - fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + Path p = createRecoveredEditsPathForRegion(); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); } @@ -405,27 +394,40 @@ public class TestWALSplit { */ @Test public void testOldRecoveredEditsFileSidelined() throws IOException { - byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + Path p = createRecoveredEditsPathForRegion(); Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); Path regiondir = new Path(tdir, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName()); fs.createNewFile(parent); // create a recovered.edits file - - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close(); } + private Path createRecoveredEditsPathForRegion() throws IOException{ + byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + long now = System.currentTimeMillis(); + Entry entry = + new Entry(new WALKeyImpl(encoded, + TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), + new WALEdit()); + Path p = WALSplitter.getRegionSplitEditsPath(entry, + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + return p; + } + + @Test + public void testHasRecoveredEdits() throws IOException { + Path p = createRecoveredEditsPathForRegion(); + assertFalse(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + String renamedEdit = p.getName().split("-")[0]; + fs.createNewFile(new Path(p.getParent(), renamedEdit)); + assertTrue(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + } + private void useDifferentDFSClient() throws IOException { // make fs act as a different client now // initialize will create a new DFSClient with a new client ID -- 2.17.2 (Apple Git-113)