From eff3e68f9df7f85251132d24859ee974f73b2a67 Mon Sep 17 00:00:00 2001 From: "krish.dey" Date: Fri, 24 Aug 2018 11:45:30 +0800 Subject: [PATCH] For the Patch for HBASE-20942 --- .../org/apache/hadoop/hbase/ipc/RpcServer.java | 26 +++- .../hbase/ipc/TestRpcServerTraceLogging.java | 169 +++++++++++++++++++++ 2 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcServerTraceLogging.java diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index fb2ee4040c..b1cc3071b5 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -179,6 +179,10 @@ public abstract class RpcServer implements RpcServerInterface, protected static final String WARN_RESPONSE_TIME = "hbase.ipc.warn.response.time"; protected static final String WARN_RESPONSE_SIZE = "hbase.ipc.warn.response.size"; + protected static final int DEFAULT_TRACE_LOG_MAX_LENGTH = 1000; + protected static final String TRACE_LOG_MAX_LENGTH = "hbase.ipc.trace.log.max.length"; + protected final int traceLogMaxLength; + protected static final String KEY_WORD_TRUNCATED = " "; /** * Minimum allowable timeout (in milliseconds) in rpc request's header. This * configuration exists to prevent the rpc service regarding this request as timeout immediately. @@ -301,6 +305,7 @@ public abstract class RpcServer implements RpcServerInterface, this.minClientRequestTimeout = conf.getInt(MIN_CLIENT_REQUEST_TIMEOUT, DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT); this.maxRequestSize = conf.getInt(MAX_REQUEST_SIZE, DEFAULT_MAX_REQUEST_SIZE); + this.traceLogMaxLength = conf.getInt(TRACE_LOG_MAX_LENGTH, DEFAULT_TRACE_LOG_MAX_LENGTH); this.metrics = new MetricsHBaseServer(name, new MetricsHBaseServerWrapperImpl(this)); this.tcpNoDelay = conf.getBoolean("hbase.ipc.server.tcpnodelay", true); @@ -499,9 +504,7 @@ public abstract class RpcServer implements RpcServerInterface, // The params could be really big, make sure they don't kill us at WARN String stringifiedParam = ProtobufUtil.getShortTextFormat(param); if (stringifiedParam.length() > 150) { - // Truncate to 1000 chars if TRACE is on, else to 150 chars - stringifiedParam = stringifiedParam.subSequence( - 0, LOG.isTraceEnabled() ? 1000 : 150) + " "; + stringifiedParam = truncateTraceLogLength(stringifiedParam); } responseInfo.put("param", stringifiedParam); if (param instanceof ClientProtos.ScanRequest && rsRpcServices != null) { @@ -517,6 +520,23 @@ public abstract class RpcServer implements RpcServerInterface, LOG.warn("(response" + tag + "): " + MAPPER.writeValueAsString(responseInfo)); } + /** + * Truncate to number of chars decided by conf hbase.ipc.trace.log.max.length + * if TRACE is on else to 150 chars Refer to Jira HBASE-20826 and HBASE-20942 + * @param strParam stringifiedParam to be truncated + * @return truncated trace log string + */ + @VisibleForTesting + String truncateTraceLogLength(String strParam) { + if (LOG.isTraceEnabled()) { + int truncatedLength = + strParam.length() < traceLogMaxLength ? strParam.length() : traceLogMaxLength; + String truncatedFlag = truncatedLength == strParam.length() ? "" : KEY_WORD_TRUNCATED; + return strParam.subSequence(0, truncatedLength) + truncatedFlag; + } + return strParam.subSequence(0, 150) + KEY_WORD_TRUNCATED; + } + /** * Set the handler for calling out of RPC for error conditions. * @param handler the handler implementation diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcServerTraceLogging.java hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcServerTraceLogging.java new file mode 100644 index 0000000000..d65dbc8bea --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcServerTraceLogging.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.ipc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.mockito.Mockito; +import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService; +import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; + +@Category(SmallTests.class) +public class TestRpcServerTraceLogging { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule + .forClass(TestRpcServerTraceLogging.class); + + @Rule + public TestName name = new TestName(); + + Logger rpcServerLog = Logger.getLogger("org.apache.hadoop.hbase.ipc.RpcServer"); + + private static HBaseTestingUtility hbaseTestingUtility = new HBaseTestingUtility(); + + static final String TRACE_LOG_MSG = + "This is dummy message for testing:: region { type: REGION_NAME value: \"hbase:meta,,1\" }" + + " scan { column { family: \"info\" } time_range { from: 0 to: 9223372036854775807 } " + + "max_versions: 1 cache_blocks: true max_result_size: 2097152 caching: 2147483647 } " + + "number_of_rows: 2147483647 close_scanner: false client_handles_partials: " + + "true client_handles_heartbeats: true track_scan_metrics: false"; + + static final int TRACE_LOG_LENGTH = TRACE_LOG_MSG.length(); + + Configuration conf = hbaseTestingUtility.getConfiguration(); + + private class RpcServerImplForTesting extends RpcServer { + + RpcServerImplForTesting(Server server, String name, + List services, InetSocketAddress bindAddress, + Configuration conf, RpcScheduler scheduler) throws IOException { + super(server, name, services, bindAddress, conf, scheduler, true); + } + + @Override + public void start() { + // Do nothing + } + + @Override + public void stop() { + // Do nothing + } + + @Override + public void join() throws InterruptedException { + // Do nothing + } + + @Override + public void setSocketSendBufSize(int size) { + // Do nothing + } + + @Override + public InetSocketAddress getListenerAddress() { + return null; + } + + @Override + public Pair call(BlockingService service, MethodDescriptor md, + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status) + throws IOException { + return null; + } + + @Override + public Pair call(BlockingService service, MethodDescriptor md, + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status, + long startTime, int timeout) throws IOException { + return null; + } + + @Override + public int getNumOpenConnections() { + return 0; + } + + } + + @Test + public void testLoggingWithTraceOff() throws Exception { + conf.setInt("hbase.ipc.trace.log.max.length", 250); + rpcServerLog.setLevel(Level.DEBUG); + RpcServerImplForTesting rpcServer = + new RpcServerImplForTesting(Mockito.mock(Server.class), name.getMethodName(), + new ArrayList(), + Mockito.mock(InetSocketAddress.class), conf, Mockito.mock(RpcScheduler.class)); + String truncatedString = rpcServer.truncateTraceLogLength(TRACE_LOG_MSG); + + assertEquals(150 + RpcServer.KEY_WORD_TRUNCATED.length(), truncatedString.length()); + assertTrue(truncatedString.contains("")); + } + + @Test + public void testLoggingWithTraceOn() throws Exception { + conf.setInt("hbase.ipc.trace.log.max.length", 250); + rpcServerLog.setLevel(Level.TRACE); + RpcServerImplForTesting rpcServer = + new RpcServerImplForTesting(Mockito.mock(Server.class), name.getMethodName(), + new ArrayList(), + Mockito.mock(InetSocketAddress.class), conf, Mockito.mock(RpcScheduler.class)); + String truncatedString = rpcServer.truncateTraceLogLength(TRACE_LOG_MSG); + + assertEquals(250 + RpcServer.KEY_WORD_TRUNCATED.length(), truncatedString.length()); + assertTrue(truncatedString.contains("")); + } + + @Test + public void testLoggingWithTraceOnLargeMax() throws Exception { + conf.setInt("hbase.ipc.trace.log.max.length", 2000); + rpcServerLog.setLevel(Level.TRACE); + RpcServerImplForTesting rpcServer = + new RpcServerImplForTesting(Mockito.mock(Server.class), name.getMethodName(), + new ArrayList(), + Mockito.mock(InetSocketAddress.class), conf, Mockito.mock(RpcScheduler.class)); + String truncatedString = rpcServer.truncateTraceLogLength(TRACE_LOG_MSG); + + assertEquals(TRACE_LOG_LENGTH, truncatedString.length()); + assertFalse(rpcServer.truncateTraceLogLength(TRACE_LOG_MSG).contains("")); + } +} -- 2.15.2 (Apple Git-101.1)