From a55637098fbc563572b45fe54013fef5d49a081f Mon Sep 17 00:00:00 2001 From: zhangduo Date: Sat, 2 Apr 2016 20:53:35 +0800 Subject: [PATCH] HBASE-15537 Add multi WAL support for AsyncFSWAL --- .../hadoop/hbase/wal/AbstractFSWALProvider.java | 2 +- .../hadoop/hbase/wal/RegionGroupingProvider.java | 138 +++++++++------------ .../org/apache/hadoop/hbase/wal/WALFactory.java | 30 +++-- ...estReplicationEndpointWithMultipleAsyncWAL.java | 36 ++++++ .../TestReplicationEndpointWithMultipleWAL.java | 2 + ...KillMasterRSCompressedWithMultipleAsyncWAL.java | 37 ++++++ ...ationKillMasterRSCompressedWithMultipleWAL.java | 2 + ...tReplicationSyncUpToolWithMultipleAsyncWAL.java | 37 ++++++ .../TestReplicationSyncUpToolWithMultipleWAL.java | 2 + .../wal/TestBoundedRegionGroupingStrategy.java | 131 ++++++++++--------- 10 files changed, 268 insertions(+), 149 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationEndpointWithMultipleAsyncWAL.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationKillMasterRSCompressedWithMultipleAsyncWAL.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java index 2f5c299..15b2950 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java @@ -82,7 +82,7 @@ public abstract class AbstractFSWALProvider> implemen * @param factory factory that made us, identity used for FS layout. may not be null * @param conf may not be null * @param listeners may be null - * @param providerId differentiate between providers from one facotry, used for FS layout. may be + * @param providerId differentiate between providers from one factory, used for FS layout. may be * null */ @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java index 0aeaccf..e5b0acd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java @@ -18,34 +18,31 @@ */ package org.apache.hadoop.hbase.wal; -import static org.apache.hadoop.hbase.wal.DefaultWALProvider.META_WAL_PROVIDER_ID; -import static org.apache.hadoop.hbase.wal.DefaultWALProvider.WAL_FILE_NAME_DELIMITER; +import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.META_WAL_PROVIDER_ID; +import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.WAL_FILE_NAME_DELIMITER; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.regionserver.wal.FSHLog; -import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; - +import org.apache.hadoop.hbase.classification.InterfaceAudience; // imports for classes still in regionserver.wal import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.IdReadWriteLock; /** * A WAL Provider that returns a WAL per group of regions. * + * This provider follows the decorator pattern and mainly holds the logic for WAL grouping. + * WAL creation/roll/close is delegated to {@link #DELEGATE_PROVIDER} + * * Region grouping is handled via {@link RegionGroupingStrategy} and can be configured via the * property "hbase.wal.regiongrouping.strategy". Current strategy choices are *