From a67b2307a6dedd4701c3efb38879caad6e0ef676 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 10 Apr 2019 19:25:33 +0100 Subject: [PATCH] HBASE-22200 - WALSplitter.hasRecoveredEdits should use same FS instance from WAL region dir Change-Id: I26924b45bc1dd49d33dcea706d489a8e55cbe2e7 --- .../assignment/SplitTableRegionProcedure.java | 3 +- .../apache/hadoop/hbase/wal/WALSplitter.java | 7 ++- .../apache/hadoop/hbase/wal/TestWALSplit.java | 48 +++++++++++-------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 938dffa42b..0b72ebb846 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -149,8 +149,7 @@ public class SplitTableRegionProcedure * @throws IOException IOException */ static boolean hasRecoveredEdits(MasterProcedureEnv env, RegionInfo ri) throws IOException { - return WALSplitter.hasRecoveredEdits(env.getMasterServices().getFileSystem(), - env.getMasterConfiguration(), ri); + return WALSplitter.hasRecoveredEdits(env.getMasterConfiguration(), ri); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 638c5740d6..c436db2b7a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -538,14 +538,13 @@ public class WALSplitter { /** * Check whether there is recovered.edits in the region dir - * @param walFS FileSystem * @param conf conf * @param regionInfo the region to check * @throws IOException IOException * @return true if recovered.edits exist in the region dir */ - public static boolean hasRecoveredEdits(final FileSystem walFS, - final Configuration conf, final RegionInfo regionInfo) throws IOException { + public static boolean hasRecoveredEdits(final Configuration conf, + final RegionInfo regionInfo) throws IOException { // No recovered.edits for non default replica regions if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { return false; @@ -554,7 +553,7 @@ public class WALSplitter { //directly without converting it to default replica's regioninfo. Path regionDir = FSUtils.getWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName()); - NavigableSet files = getSplitEditFilesSorted(walFS, regionDir); + NavigableSet files = getSplitEditFilesSorted(FSUtils.getWALFileSystem(conf), regionDir); return files != null && !files.isEmpty(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index e6644f07dd..938fb31e39 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -383,18 +383,7 @@ public class TestWALSplit { */ @Test public void testRecoveredEditsPathForMeta() throws IOException { - byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); - Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); - Path regiondir = new Path(tdir, - RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); - fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + Path p = createRecoveredEditsPathForRegion(); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); } @@ -405,27 +394,44 @@ public class TestWALSplit { */ @Test public void testOldRecoveredEditsFileSidelined() throws IOException { - byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + Path p = createRecoveredEditsPathForRegion(); Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); Path regiondir = new Path(tdir, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); fs.mkdirs(regiondir); - long now = System.currentTimeMillis(); - Entry entry = - new Entry(new WALKeyImpl(encoded, - TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), - new WALEdit()); Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName()); fs.createNewFile(parent); // create a recovered.edits file - - Path p = WALSplitter.getRegionSplitEditsPath(entry, - FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close(); } + private Path createRecoveredEditsPathForRegion() throws IOException{ + byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); + long now = System.currentTimeMillis(); + Entry entry = + new Entry(new WALKeyImpl(encoded, + TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), + new WALEdit()); + Path p = WALSplitter.getRegionSplitEditsPath(entry, + FILENAME_BEING_SPLIT, TMPDIRNAME, conf); + return p; + } + + /** + * Test hasRecoveredEdits correctly identifies proper recovered edits file on related dir. + * @throws IOException + */ + @Test + public void testHasRecoveredEdits() throws IOException { + Path p = createRecoveredEditsPathForRegion(); + assertFalse(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + String renamedEdit = p.getName().split("-")[0]; + fs.createNewFile(new Path(p.getParent(), renamedEdit)); + assertTrue(WALSplitter.hasRecoveredEdits(conf, RegionInfoBuilder.FIRST_META_REGIONINFO)); + } + private void useDifferentDFSClient() throws IOException { // make fs act as a different client now // initialize will create a new DFSClient with a new client ID -- 2.17.2 (Apple Git-113)