From 840f2f673662e5d08fd408ab53cfaafdb3e44079 Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Sun, 4 Mar 2018 14:30:07 +0900 Subject: [PATCH] HBASE-20006 TestRestoreSnapshotFromClientWithRegionReplicas is flakey --- .../hadoop/hbase/regionserver/StoreFileInfo.java | 37 +++++++++++++++------- .../hadoop/hbase/util/ServerRegionReplicaUtil.java | 27 ++++++++++++---- .../regionserver/TestHRegionReplayEvents.java | 10 +++--- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index e7419d5..c9f5f8d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -157,13 +157,12 @@ public class StoreFileInfo { /** * Create a Store File Info from an HFileLink - * @param conf the {@link Configuration} to use - * @param fs The current file system to use. + * @param conf The {@link Configuration} to use + * @param fs The current file system to use * @param fileStatus The {@link FileStatus} of the file */ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, - final HFileLink link) - throws IOException { + final HFileLink link) { this.fs = fs; this.conf = conf; // initialPath can be null only if we get a link. @@ -175,15 +174,13 @@ public class StoreFileInfo { /** * Create a Store File Info from an HFileLink - * @param conf - * @param fs - * @param fileStatus - * @param reference - * @throws IOException + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + * @param reference The reference instance */ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, - final Reference reference) - throws IOException { + final Reference reference) { this.fs = fs; this.conf = conf; this.initialPath = fileStatus.getPath(); @@ -193,6 +190,24 @@ public class StoreFileInfo { } /** + * Create a Store File Info from an HFileLink and a Reference + * @param conf The {@link Configuration} to use + * @param fs The current file system to use + * @param fileStatus The {@link FileStatus} of the file + * @param reference The reference instance + * @param link The link instance + */ + public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, + final Reference reference, final HFileLink link) { + this.fs = fs; + this.conf = conf; + this.initialPath = fileStatus.getPath(); + this.createdTimestamp = fileStatus.getModificationTime(); + this.reference = reference; + this.link = link; + } + + /** * Sets the region coprocessor env. * @param coprocessorHost */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java index 8a43d17..8580794 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java @@ -127,15 +127,28 @@ public class ServerRegionReplicaUtil extends RegionReplicaUtil { } // else create a store file link. The link file does not exists on filesystem though. - HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(), - regionInfoForFs.getEncodedName(), familyName, path.getName()); - - if (StoreFileInfo.isReference(path)) { + if (HFileLink.isHFileLink(path) || StoreFileInfo.isHFile(path)) { + HFileLink link = HFileLink + .build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, + path.getName()); + return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); + } else if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); - return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); + Path referencePath = StoreFileInfo.getReferredToFile(path); + if (HFileLink.isHFileLink(referencePath)) { + // HFileLink Reference + HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, referencePath); + return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference, link); + } else { + // Reference + HFileLink link = HFileLink + .build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, + path.getName()); + return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); + } + } else { + throw new IOException("path=" + path + " doesn't look like a valid StoreFile"); } - - return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index 26b57b0..81b77890 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -1578,7 +1578,7 @@ public class TestHRegionReplayEvents { .addStoreFlushes(StoreFlushDescriptor.newBuilder() .setFamilyName(UnsafeByteOperations.unsafeWrap(families[0])) .setStoreHomeDir("/store_home_dir") - .addFlushOutput("/foo/baz/bar") + .addFlushOutput("/foo/baz/123") .build()) .build()); } @@ -1593,8 +1593,8 @@ public class TestHRegionReplayEvents { .setEncodedRegionName( UnsafeByteOperations.unsafeWrap(primaryRegion.getRegionInfo().getEncodedNameAsBytes())) .setFamilyName(UnsafeByteOperations.unsafeWrap(families[0])) - .addCompactionInput("/foo") - .addCompactionOutput("/bar") + .addCompactionInput("/123") + .addCompactionOutput("/456") .setStoreHomeDir("/store_home_dir") .setRegionName(UnsafeByteOperations.unsafeWrap(primaryRegion.getRegionInfo().getRegionName())) .build() @@ -1617,7 +1617,7 @@ public class TestHRegionReplayEvents { .addStores(StoreDescriptor.newBuilder() .setFamilyName(UnsafeByteOperations.unsafeWrap(families[0])) .setStoreHomeDir("/store_home_dir") - .addStoreFile("/foo") + .addStoreFile("/123") .build()) .build()); } @@ -1634,7 +1634,7 @@ public class TestHRegionReplayEvents { .addStores(StoreDescriptor.newBuilder() .setFamilyName(UnsafeByteOperations.unsafeWrap(families[0])) .setStoreHomeDir("/store_home_dir") - .addStoreFile("/foo") + .addStoreFile("/123") .build()) .build()); } -- 2.10.1 (Apple Git-78)