From 9e5c43bea0ed17be66d7e299863869991636b502 Mon Sep 17 00:00:00 2001 From: Ali Yousafzai Date: Tue, 30 May 2017 11:24:38 -0700 Subject: [PATCH] HBASE-18054 log when we add/remove failed servers in client --- .../hadoop/hbase/ipc/BlockingRpcConnection.java | 2 +- .../org/apache/hadoop/hbase/ipc/FailedServers.java | 11 ++- .../hadoop/hbase/ipc/NettyRpcConnection.java | 2 +- .../hadoop/hbase/ipc/TestFailedServersLog.java | 82 ++++++++++++++++++++++ .../apache/hadoop/hbase/ipc/TestHBaseClient.java | 11 +-- 5 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java index 1012ad0350..df6a7b5c14 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java @@ -488,7 +488,7 @@ class BlockingRpcConnection extends RpcConnection implements Runnable { closeSocket(); IOException e = ExceptionUtil.asInterrupt(t); if (e == null) { - this.rpcClient.failedServers.addToFailedServers(remoteId.address); + this.rpcClient.failedServers.addToFailedServers(remoteId.address, t); if (t instanceof LinkageError) { // probably the hbase hadoop version does not match the running hadoop version e = new DoNotRetryIOException(t); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java index 868cdc681b..ede796395d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java @@ -21,6 +21,8 @@ import java.net.InetSocketAddress; import java.util.Iterator; import java.util.LinkedList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -33,6 +35,7 @@ import org.apache.hadoop.hbase.util.Pair; public class FailedServers { private final LinkedList> failedServers = new LinkedList<>(); private final int recheckServersTimeout; + private static Log LOG = LogFactory.getLog(FailedServers.class); public FailedServers(Configuration conf) { this.recheckServersTimeout = conf.getInt( @@ -42,11 +45,15 @@ public class FailedServers { /** * Add an address to the list of the failed servers list. */ - public synchronized void addToFailedServers(InetSocketAddress address) { + public synchronized void addToFailedServers(InetSocketAddress address, Throwable throwable) { final long expiry = EnvironmentEdgeManager.currentTime() + recheckServersTimeout; failedServers.addFirst(new Pair<>(expiry, address.toString())); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Added failed server with address " + address.toString() + " to list caused by " + + throwable.toString()); + } } - /** * Check if the server should be considered as bad. Clean the old entries of the list. * diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java index 204b812f17..c879990f94 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcConnection.java @@ -266,7 +266,7 @@ class NettyRpcConnection extends RpcConnection { Channel ch = future.channel(); if (!future.isSuccess()) { failInit(ch, toIOE(future.cause())); - rpcClient.failedServers.addToFailedServers(remoteId.address); + rpcClient.failedServers.addToFailedServers(remoteId.address, future.cause()); return; } ch.writeAndFlush(connectionHeaderPreamble.retainedDuplicate()); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java new file mode 100644 index 0000000000..5d4a63956d --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestFailedServersLog.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThat; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.log4j.Appender; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.spi.LoggingEvent; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@Category({ ClientTests.class, SmallTests.class }) +public class TestFailedServersLog { + static final int TEST_PORT = 9999; + private InetSocketAddress addr; + + @Mock + private Appender mockAppender; + + @Captor + private ArgumentCaptor captorLoggingEvent; + + @Before + public void setup() { + LogManager.getRootLogger().addAppender(mockAppender); + } + + @After + public void teardown() { + LogManager.getRootLogger().removeAppender(mockAppender); + } + + @Test + public void testAddToFailedServersLogging() { + Throwable nullException = new NullPointerException(); + Throwable iOException = new IOException(); + + FailedServers fs = new FailedServers(new Configuration()); + addr = new InetSocketAddress(TEST_PORT); + + fs.addToFailedServers(addr, nullException); + + Mockito.verify(mockAppender).doAppend((LoggingEvent) captorLoggingEvent.capture()); + LoggingEvent loggingEvent = (LoggingEvent) captorLoggingEvent.getValue(); + assertThat(loggingEvent.getLevel(), is(Level.DEBUG)); + assertEquals(nullException.toString(), "java.lang.NullPointerException"); + assertEquals(loggingEvent.getRenderedMessage(), "Added failed server with address " + + addr.toString() + " to list caused by " + nullException.toString()); + assertNotEquals(loggingEvent.getRenderedMessage(), "Added failed server with address " + + addr.toString() + " to list caused by " + iOException.toString()); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseClient.java index 26488cf947..b2bccd6e7d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestHBaseClient.java @@ -37,6 +37,7 @@ public class TestHBaseClient { ManualEnvironmentEdge ee = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge( ee ); FailedServers fs = new FailedServers(new Configuration()); + Throwable testThrowable = new Throwable();//throwable already tested in TestFailedServers.java InetSocketAddress ia = InetSocketAddress.createUnresolved("bad", 12); InetSocketAddress ia2 = InetSocketAddress.createUnresolved("bad", 12); // same server as ia @@ -46,7 +47,7 @@ public class TestHBaseClient { Assert.assertFalse( fs.isFailedServer(ia) ); - fs.addToFailedServers(ia); + fs.addToFailedServers(ia,testThrowable); Assert.assertTrue( fs.isFailedServer(ia) ); Assert.assertTrue( fs.isFailedServer(ia2) ); @@ -58,9 +59,9 @@ public class TestHBaseClient { Assert.assertFalse( fs.isFailedServer(ia) ); Assert.assertFalse( fs.isFailedServer(ia2) ); - fs.addToFailedServers(ia); - fs.addToFailedServers(ia3); - fs.addToFailedServers(ia4); + fs.addToFailedServers(ia,testThrowable); + fs.addToFailedServers(ia3,testThrowable); + fs.addToFailedServers(ia4,testThrowable); Assert.assertTrue( fs.isFailedServer(ia) ); Assert.assertTrue( fs.isFailedServer(ia2) ); @@ -74,7 +75,7 @@ public class TestHBaseClient { Assert.assertFalse( fs.isFailedServer(ia4) ); - fs.addToFailedServers(ia3); + fs.addToFailedServers(ia3,testThrowable); Assert.assertFalse( fs.isFailedServer(ia4) ); } } -- 2.11.0 (Apple Git-81)