From d3caaa0abd8a95f430f105672f8165602bea9ebe Mon Sep 17 00:00:00 2001 From: Vincent Date: Mon, 2 May 2016 17:30:16 -0700 Subject: [PATCH] HBASE-15617 Canary in regionserver mode might not enumerate all regionservers --- .../java/org/apache/hadoop/hbase/tool/Canary.java | 42 ++++- .../apache/hadoop/hbase/tool/TestCanaryTool.java | 172 +++++++++++++++++++++ 2 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 9ad7242..f2e5b0f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -23,6 +23,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -77,9 +78,12 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.RegionSplitter; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.KeeperException; /** * HBase Canary Tool, that that can be used to do @@ -93,6 +97,8 @@ import org.apache.hadoop.util.ToolRunner; * selected randomly and outputs some information about failure or latency. */ public final class Canary implements Tool { + public static final String CANARY_TOOL = "CANARY_TOOL"; + // Sink interface used by the canary to outputs information public interface Sink { public long getReadFailureCount(); @@ -590,7 +596,7 @@ public final class Canary implements Tool { // for more details. final ScheduledChore authChore = AuthUtil.getAuthChore(conf); if (authChore != null) { - choreService = new ChoreService("CANARY_TOOL"); + choreService = new ChoreService(CANARY_TOOL); choreService.scheduleChore(authChore); } @@ -1106,7 +1112,9 @@ public final class Canary implements Tool { String serverName = entry.getKey(); AtomicLong successes = new AtomicLong(0); successMap.put(serverName, successes); - if (this.allRegions) { + if (entry.getValue().isEmpty()) { + LOG.error(String.format("Regionserver not serving any regions - %s", serverName)); + } else if (this.allRegions) { for (HRegionInfo region : entry.getValue()) { tasks.add(new RegionServerTask(this.connection, serverName, @@ -1182,6 +1190,14 @@ public final class Canary implements Tool { table.close(); } + //get any live regionservers not serving any regions, other than draining servers + Set drainingServers = getDrainingServers(); + for (ServerName rs : this.admin.getClusterStatus().getServers()) { + String rsName = rs.getHostname(); + if (!rsAndRMap.containsKey(rsName) && !drainingServers.contains(rsName)) { + rsAndRMap.put(rsName, Collections.emptyList()); + } + } } catch (IOException e) { String msg = "Get HTables info failed"; LOG.error(msg, e); @@ -1199,6 +1215,28 @@ public final class Canary implements Tool { return rsAndRMap; } + private Set getDrainingServers() { + Configuration conf = this.connection.getConfiguration(); + String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + String drainingZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.draining.rs", "draining")); + Set drainingServers = new HashSet<>(); + try { + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, CANARY_TOOL + connection.toString(), + null); + List servers = ZKUtil.listChildrenNoWatch(zkw, drainingZNode); + for (String n : servers) { + ServerName sn = ServerName.valueOf(ZKUtil.getNodeName(n)); + drainingServers.add(sn.getHostname()); + } + } catch (IOException | KeeperException e) { + LOG.error("failed to get draining regionservers", e); + this.errorCode = INIT_ERROR_EXIT_CODE; + } + return drainingServers; + } + private Map> doFilterRegionServerByName( Map> fullRsAndRMap) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java new file mode 100644 index 0000000..a701b92 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java @@ -0,0 +1,172 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.tool; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Appender; +import org.apache.log4j.LogManager; +import org.apache.log4j.spi.LoggingEvent; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.never; + +@RunWith(MockitoJUnitRunner.class) +@Category({MediumTests.class}) +public class TestCanaryTool { + + private HBaseTestingUtility testingUtility; + private static final byte[] FAMILY = Bytes.toBytes("f"); + private static final byte[] COLUMN = Bytes.toBytes("col"); + + @Before + public void setUp() throws Exception { + testingUtility = new HBaseTestingUtility(); + testingUtility.startMiniCluster(); + LogManager.getRootLogger().addAppender(mockAppender); + } + + @After + public void tearDown() throws Exception { + testingUtility.shutdownMiniCluster(); + LogManager.getRootLogger().removeAppender(mockAppender); + } + + @Mock + Appender mockAppender; + + @Test + public void testBasicCanaryWorks() throws Exception { + TableName tableName = TableName.valueOf("testTable"); + HTable table = testingUtility.createTable(tableName, new byte[][] { FAMILY }); + // insert some test rows + for (int i=0; i<1000; i++) { + byte[] iBytes = Bytes.toBytes(i); + Put p = new Put(iBytes); + p.addColumn(FAMILY, COLUMN, iBytes); + table.put(p); + } + ExecutorService executor = new ScheduledThreadPoolExecutor(1); + Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink()); + Canary canary = new Canary(executor, sink); + String[] args = { "-t", "10000", "testTable" }; + ToolRunner.run(testingUtility.getConfiguration(), canary, args); + verify(sink, atLeastOnce()) + .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + } + + //no table created, so there should be no regions + @Test + public void testRegionserverNoRegions() throws Exception { + runRegionserverCanary(); + verify(mockAppender).doAppend(argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return ((LoggingEvent) argument).getRenderedMessage().contains("Regionserver not serving any regions"); + } + })); + } + + //by creating a table, there shouldn't be any region servers not serving any regions + @Test + public void testRegionserverWithRegions() throws Exception { + TableName tableName = TableName.valueOf("testTable"); + testingUtility.createTable(tableName, new byte[][] { FAMILY }); + runRegionserverCanary(); + verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return ((LoggingEvent) argument).getRenderedMessage().contains("Regionserver not serving any regions"); + } + })); + } + + //no table created, so no regions, but regionserver is marked as draining + @Test + public void testDrainingRegionServer() throws Exception { + //mark the regionserver as draining in ZK + ServerName rs = testingUtility.getHBaseCluster().getRegionServer(0).getServerName(); + Configuration conf = testingUtility.getConfiguration(); + String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + String drainingZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.draining.rs", "draining")); + String rsZNode = ZKUtil.joinZNode(drainingZNode, rs.getServerName()); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "draining_servers", null); + ZKUtil.createAndFailSilent(zkw, rsZNode); + + //make sure master watcher was triggered and updated its list of draining servers + Thread.sleep(500); + HMaster master = testingUtility.getHBaseCluster().getMaster(); + List drainingServersList = master.getServerManager().getDrainingServersList(); + assertEquals(1, drainingServersList.size()); + assertEquals(rs, drainingServersList.get(0)); + + //canary shouldn't log anything since regionserver is draining + runRegionserverCanary(); + verify(mockAppender, never()).doAppend(argThat(new ArgumentMatcher() { + @Override + public boolean matches(Object argument) { + return ((LoggingEvent) argument).getRenderedMessage().contains("Regionserver not serving any regions"); + } + })); + } + + private void runRegionserverCanary() throws Exception { + ExecutorService executor = new ScheduledThreadPoolExecutor(1); + Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink()); + String[] args = { "-t", "10000", "-regionserver"}; + ToolRunner.run(testingUtility.getConfiguration(), canary, args); + } + +} + -- 2.7.4