From add1a5b38e6177d68c68687d9f6336b68c86bae0 Mon Sep 17 00:00:00 2001 From: Guanghao Zhang Date: Mon, 8 Jan 2018 14:56:13 +0800 Subject: [PATCH] HBASE-19139 Create Async Admin methods for Clear Block Cache --- .../apache/hadoop/hbase/CacheEvictionStats.java | 4 +- .../hadoop/hbase/CacheEvictionStatsBuilder.java | 21 +++---- .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 13 +++++ .../hadoop/hbase/client/AsyncHBaseAdmin.java | 7 +++ .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 51 +++++++++++++++++ .../hadoop/hbase/client/TestInterfaceAlign.java | 2 - .../regionserver/TestClearRegionBlockCache.java | 65 ++++++++++++++++++++-- 7 files changed, 144 insertions(+), 19 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java index 91cedd6..4d1214e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStats.java @@ -33,8 +33,8 @@ public final class CacheEvictionStats { private final Map exceptions; CacheEvictionStats(CacheEvictionStatsBuilder builder) { - this.evictedBlocks = builder.evictedBlocks; - this.maxCacheSize = builder.maxCacheSize; + this.evictedBlocks = builder.evictedBlocks.get(); + this.maxCacheSize = builder.maxCacheSize.get(); this.exceptions = builder.exceptions; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java index d9e1400..3bf5005 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CacheEvictionStatsBuilder.java @@ -18,27 +18,28 @@ */ package org.apache.hadoop.hbase; -import java.util.HashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public final class CacheEvictionStatsBuilder { - long evictedBlocks = 0; - long maxCacheSize = 0; - Map exceptions = new HashMap<>(); + AtomicLong evictedBlocks = new AtomicLong(0); + AtomicLong maxCacheSize = new AtomicLong(0); + ConcurrentMap exceptions = new ConcurrentHashMap<>(); CacheEvictionStatsBuilder() { } public CacheEvictionStatsBuilder withEvictedBlocks(long evictedBlocks) { - this.evictedBlocks = evictedBlocks; + this.evictedBlocks.set(evictedBlocks); return this; } public CacheEvictionStatsBuilder withMaxCacheSize(long maxCacheSize) { - this.maxCacheSize = maxCacheSize; + this.maxCacheSize.set(maxCacheSize); return this; } @@ -46,9 +47,9 @@ public final class CacheEvictionStatsBuilder { exceptions.put(regionName, ie); } - public CacheEvictionStatsBuilder append(CacheEvictionStats stats) { - this.evictedBlocks += stats.getEvictedBlocks(); - this.maxCacheSize += stats.getMaxCacheSize(); + public synchronized CacheEvictionStatsBuilder append(CacheEvictionStats stats) { + this.evictedBlocks.getAndAdd(stats.getEvictedBlocks()); + this.maxCacheSize.getAndAdd(stats.getMaxCacheSize()); this.exceptions.putAll(stats.getExceptions()); return this; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index af39f17..a375265 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.client; import com.google.protobuf.RpcChannel; + import java.util.Collection; import java.util.EnumSet; import java.util.List; @@ -27,6 +28,8 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.NamespaceDescriptor; @@ -1211,4 +1214,14 @@ public interface AsyncAdmin { * @return - returns a list of servers that not cleared wrapped by a {@link CompletableFuture}. */ CompletableFuture> clearDeadServers(final List servers); + + /** + * Clear all the blocks corresponding to this table from BlockCache. For expert-admins. Calling + * this API will drop all the cached blocks specific to a table from BlockCache. This can + * significantly impact the query performance as the subsequent queries will have to retrieve the + * blocks from underlying filesystem. + * @param tableName table to clear block cache + * @return CacheEvictionStats related to the eviction wrapped by a {@link CompletableFuture}. + */ + CompletableFuture clearBlockCache(final TableName tableName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 3ca5d69..d0d19c1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -27,6 +27,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.regex.Pattern; + +import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.NamespaceDescriptor; @@ -734,4 +736,9 @@ class AsyncHBaseAdmin implements AsyncAdmin { public CompletableFuture> clearDeadServers(List servers) { return wrap(rawAdmin.clearDeadServers(servers)); } + + @Override + public CompletableFuture clearBlockCache(TableName tableName) { + return wrap(rawAdmin.clearBlockCache(tableName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index ac00234..5285caa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -42,6 +42,8 @@ import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.CacheEvictionStats; +import org.apache.hadoop.hbase.CacheEvictionStatsBuilder; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.ClusterMetricsBuilder; @@ -97,6 +99,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; @@ -3387,4 +3391,51 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { }); return future; } + + @Override + public CompletableFuture clearBlockCache(TableName tableName) { + CompletableFuture future = new CompletableFuture<>(); + getTableHRegionLocations(tableName).whenComplete((locations, err) -> { + if (err != null) { + future.completeExceptionally(err); + return; + } + Map> regionInfoByServerName = + locations.stream().filter(l -> l.getRegion() != null) + .filter(l -> !l.getRegion().isOffline()).filter(l -> l.getServerName() != null) + .collect(Collectors.groupingBy(l -> l.getServerName(), + Collectors.mapping(l -> l.getRegion(), Collectors.toList()))); + List> futures = new ArrayList<>(); + CacheEvictionStatsBuilder builder = CacheEvictionStats.builder(); + for (Map.Entry> entry : regionInfoByServerName.entrySet()) { + futures + .add(clearBlockCache(entry.getKey(), entry.getValue()).whenComplete((stats, err2) -> { + if (err2 != null) { + future.completeExceptionally(err2); + } else { + builder.append(stats); + } + })); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])) + .whenComplete((ret, err3) -> { + if (err3 != null) { + future.completeExceptionally(err3); + } else { + future.complete(builder.build()); + } + }); + }); + return future; + } + + private CompletableFuture clearBlockCache(ServerName serverName, + List hris) { + return this. newAdminCaller().action((controller, stub) -> this + . adminCall( + controller, stub, RequestConverter.buildClearRegionBlockCacheRequest(hris), + (s, c, req, done) -> s.clearRegionBlockCache(controller, req, done), + resp -> ProtobufUtil.toCacheEvictionStats(resp.getStats()))) + .serverName(serverName).call(); + } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java index 2266d06..8ddb392 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestInterfaceAlign.java @@ -55,8 +55,6 @@ public class TestInterfaceAlign { adminMethodNames.remove("getConfiguration"); adminMethodNames.removeAll(getMethodNames(Abortable.class)); adminMethodNames.removeAll(getMethodNames(Closeable.class)); - // TODO: Remove this after HBASE-19139 - adminMethodNames.remove("clearBlockCache"); adminMethodNames.forEach(method -> { boolean contains = asyncAdminMethodNames.contains(method); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java index b9d38f1..98eff20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java @@ -19,13 +19,18 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.AsyncAdmin; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TestAsyncAdminBase; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -35,6 +40,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -44,6 +51,7 @@ import static org.junit.Assert.assertEquals; @Category(MediumTests.class) @RunWith(Parameterized.class) public class TestClearRegionBlockCache { + private static final Logger LOG = LoggerFactory.getLogger(TestClearRegionBlockCache.class); private static final TableName TABLE_NAME = TableName.valueOf("testClearRegionBlockCache"); private static final byte[] FAMILY = Bytes.toBytes("family"); private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("5") }; @@ -77,6 +85,9 @@ public class TestClearRegionBlockCache { // Create table table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY); + + HTU.loadNumericRows(table, FAMILY, 1, 10); + HTU.flush(TABLE_NAME); } @After @@ -86,9 +97,6 @@ public class TestClearRegionBlockCache { @Test public void testClearBlockCache() throws Exception { - HTU.loadNumericRows(table, FAMILY, 1, 10); - HTU.flush(TABLE_NAME); - BlockCache blockCache1 = rs1.getCacheConfig().getBlockCache(); BlockCache blockCache2 = rs2.getCacheConfig().getBlockCache(); @@ -98,18 +106,65 @@ public class TestClearRegionBlockCache { // scan will cause blocks to be added in BlockCache scanAllRegionsForRS(rs1); assertEquals(blockCache1.getBlockCount() - initialBlockCount1, - HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); + HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); clearRegionBlockCache(rs1); scanAllRegionsForRS(rs2); assertEquals(blockCache2.getBlockCount() - initialBlockCount2, - HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); clearRegionBlockCache(rs2); assertEquals(initialBlockCount1, blockCache1.getBlockCount()); assertEquals(initialBlockCount2, blockCache2.getBlockCount()); } + @Test + public void testClearBlockCacheFromAdmin() throws Exception { + Admin admin = HTU.getAdmin(); + + // All RS run in a same process, so the block cache is same for rs1 and rs2 + BlockCache blockCache = rs1.getCacheConfig().getBlockCache(); + long initialBlockCount = blockCache.getBlockCount(); + + // scan will cause blocks to be added in BlockCache + scanAllRegionsForRS(rs1); + assertEquals(blockCache.getBlockCount() - initialBlockCount, + HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); + scanAllRegionsForRS(rs2); + assertEquals(blockCache.getBlockCount() - initialBlockCount, + HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); + + CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME); + assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); + assertEquals(initialBlockCount, blockCache.getBlockCount()); + } + + @Test + public void testClearBlockCacheFromAsyncAdmin() throws Exception { + AsyncAdmin admin = + ConnectionFactory.createAsyncConnection(HTU.getConfiguration()).get().getAdmin(); + + // All RS run in a same process, so the block cache is same for rs1 and rs2 + BlockCache blockCache = rs1.getCacheConfig().getBlockCache(); + long initialBlockCount = blockCache.getBlockCount(); + + // scan will cause blocks to be added in BlockCache + scanAllRegionsForRS(rs1); + assertEquals(blockCache.getBlockCount() - initialBlockCount, + HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); + scanAllRegionsForRS(rs2); + assertEquals(blockCache.getBlockCount() - initialBlockCount, + HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); + + CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME).get(); + assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); + assertEquals(initialBlockCount, blockCache.getBlockCount()); + } + private void scanAllRegionsForRS(HRegionServer rs) throws IOException { for (Region region : rs.getRegions(TABLE_NAME)) { RegionScanner scanner = region.getScanner(new Scan()); -- 1.9.1