From 4132ce2a9877c5056e57866247cd3cc3c2e01ccb Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 10 Apr 2019 13:41:36 +0100 Subject: [PATCH] HBASE-22200 - WALSplitter.hasRecoveredEdits should use same FS instance from WAL region dir Change-Id: Ib4f909d69ded18522543dbfc8b0c50bc7b6d2dff --- .../assignment/SplitTableRegionProcedure.java | 3 +- .../apache/hadoop/hbase/wal/WALSplitter.java | 5 ++- .../apache/hadoop/hbase/wal/TestWALSplit.java | 44 ++++++++++--------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 938dffa42b..0b72ebb846 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -149,8 +149,7 @@ public class SplitTableRegionProcedure * @throws IOException IOException */ static boolean hasRecoveredEdits(MasterProcedureEnv env, RegionInfo ri) throws IOException { - return WALSplitter.hasRecoveredEdits(env.getMasterServices().getFileSystem(), - env.getMasterConfiguration(), ri); + return WALSplitter.hasRecoveredEdits(env.getMasterConfiguration(), ri); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 638c5740d6..0a5520d14a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -544,8 +544,8 @@ public class WALSplitter { * @throws IOException IOException * @return true if recovered.edits exist in the region dir */ - public static boolean hasRecoveredEdits(final FileSystem walFS, - final Configuration conf, final RegionInfo regionInfo) throws IOException { + public static boolean hasRecoveredEdits(final Configuration conf, + final RegionInfo regionInfo) throws IOException { // No recovered.edits for non default replica regions if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { return false; @@ -554,6 +554,7 @@ public class WALSplitter { //directly without converting it to default replica's regioninfo. Path regionDir = FSUtils.getWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName()); + FileSystem walFS = regionDir.getFileSystem(conf); NavigableSet files = getSplitEditFilesSorted(walFS, regionDir); return files != null && !files.isEmpty(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index e6644f07dd..b4e25336a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -383,18 +383,7 @@ public class TestWALSplit { */ @Test public void testRecoveredEditsPathForMeta() throws IOException { - byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); - Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); - Path regiondir = new Path(tdir, - RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); - fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + Path p = createRecoveredEditsPathForRegion(); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); } @@ -405,27 +394,40 @@ public class TestWALSplit { */ @Test public void testOldRecoveredEditsFileSidelined() throws IOException { - byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + Path p = createRecoveredEditsPathForRegion(); Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); Path regiondir = new Path(tdir, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName()); fs.createNewFile(parent); // create a recovered.edits file - - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close(); } + private Path createRecoveredEditsPathForRegion() throws IOException{ + byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + long now = System.currentTimeMillis(); + Entry entry = + new Entry(new WALKeyImpl(encoded, + TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), + new WALEdit()); + Path p = WALSplitter.getRegionSplitEditsPath(entry, + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + return p; + } + + @Test + public void testHasRecoveredEdits() throws IOException { + Path p = createRecoveredEditsPathForRegion(); + assertFalse(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + String renamedEdit = p.getName().split("-")[0]; + fs.createNewFile(new Path(p.getParent(), renamedEdit)); + assertTrue(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + } + private void useDifferentDFSClient() throws IOException { // make fs act as a different client now // initialize will create a new DFSClient with a new client ID -- 2.17.2 (Apple Git-113)