From d202db54b63f58296c362e1915e8ceefffe3655a Mon Sep 17 00:00:00 2001 From: "krish.dey" Date: Wed, 22 Aug 2018 21:23:37 -0400 Subject: [PATCH] Patch for https://issues.apache.org/jira/browse/HBASE-20942 Patch for https://issues.apache.org/jira/browse/HBASE-21108 --- .../org/apache/hadoop/hbase/ipc/RpcServer.java | 13 ++- .../apache/hadoop/hbase/ipc/TestTraceLogging.java | 129 +++++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestTraceLogging.java diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index fb2ee4040c..358c162ea7 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -191,6 +191,10 @@ public abstract class RpcServer implements RpcServerInterface, protected static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds protected static final int DEFAULT_WARN_RESPONSE_SIZE = 100 * 1024 * 1024; + protected static final int DEFAULT_TRACE_PARAM_SIZE = 1000; + protected static final String TRACE_PARAM_SIZE = "hbase.ipc.trace.param.size"; + protected final int traceParamSize; + protected static final ObjectMapper MAPPER = new ObjectMapper(); protected final int maxRequestSize; @@ -302,6 +306,8 @@ public abstract class RpcServer implements RpcServerInterface, DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT); this.maxRequestSize = conf.getInt(MAX_REQUEST_SIZE, DEFAULT_MAX_REQUEST_SIZE); + this.traceParamSize = conf.getInt(TRACE_PARAM_SIZE, DEFAULT_TRACE_PARAM_SIZE); + this.metrics = new MetricsHBaseServer(name, new MetricsHBaseServerWrapperImpl(this)); this.tcpNoDelay = conf.getBoolean("hbase.ipc.server.tcpnodelay", true); this.tcpKeepAlive = conf.getBoolean("hbase.ipc.server.tcpkeepalive", true); @@ -500,8 +506,13 @@ public abstract class RpcServer implements RpcServerInterface, String stringifiedParam = ProtobufUtil.getShortTextFormat(param); if (stringifiedParam.length() > 150) { // Truncate to 1000 chars if TRACE is on, else to 150 chars + // Refer to Jira HBASE-20826 + int truncatedLength = stringifiedParam.length() < traceParamSize ? + stringifiedParam.length(): traceParamSize; + String truncatedFlag = (truncatedLength == stringifiedParam.length()) + ? "" : " "; stringifiedParam = stringifiedParam.subSequence( - 0, LOG.isTraceEnabled() ? 1000 : 150) + " "; + 0, LOG.isTraceEnabled() ? truncatedLength : 150) + truncatedFlag; } responseInfo.put("param", stringifiedParam); if (param instanceof ClientProtos.ScanRequest && rsRpcServices != null) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestTraceLogging.java hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestTraceLogging.java new file mode 100644 index 0000000000..7f569dbfc9 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestTraceLogging.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.ipc; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +@Category(MediumTests.class) +public class TestTraceLogging { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule + .forClass(TestTraceLogging.class); + + private HBaseTestingUtility hbtu; + @Rule + public TestName testName = new TestName(); + private TableName tableName; + private final byte[] family = Bytes.toBytes("f"); + private final byte[] qualifier = Bytes.toBytes("q"); + TestAppender appender = new TestAppender(); + + @Before + public void setUp() { + hbtu = new HBaseTestingUtility(); + tableName = TableName.valueOf("Table-" + testName.getMethodName()); + Logger log = Logger.getLogger("org.apache.hadoop.hbase.ipc.RpcServer"); + log.setLevel(Level.TRACE); + log.addAppender(appender); + } + + private void writeSomeData(Table table) throws IOException { + int VALUE_SIZE = 2 * 1024; + Random rand = new Random(); + byte[] value = new byte[VALUE_SIZE]; + rand.nextBytes(value); + table.put(new Put(Bytes.toBytes(0)).addColumn(family, qualifier, value)); + } + + @Test + public void testTraceLoggingWithTruncate() throws Exception { + Configuration conf = hbtu.getConfiguration(); + conf.setInt("hbase.ipc.warn.response.size", 1024); + conf.setInt("hbase.ipc.trace.param.size", 250); + hbtu.startMiniCluster(1); + Table table = hbtu.createTable(tableName, family); + writeSomeData(table); + assertTrue(appender.events.get(0).getMessage().toString().contains("")); + hbtu.deleteTable(tableName); + hbtu.shutdownMiniCluster(); + } + + @Test + public void testTraceLoggingWithOutTruncate() throws Exception { + Configuration conf = hbtu.getConfiguration(); + conf.setInt("hbase.ipc.warn.response.size", 1024); + conf.setInt("hbase.ipc.trace.param.size", 2000); + hbtu.startMiniCluster(1); + Table table = hbtu.createTable(tableName, family); + writeSomeData(table); + assertTrue(!appender.events.get(0).getMessage().toString().contains("")); + hbtu.deleteTable(tableName); + hbtu.shutdownMiniCluster(); + } +} + +/** + * Inner class to capture Logging Event for the trace + */ +class TestAppender extends AppenderSkeleton { + protected List events = new ArrayList(); + + @Override + public void close() { + } + + @Override + public boolean requiresLayout() { + // TODO Auto-generated method stub + return false; + } + + @Override + protected void append(LoggingEvent event) { + // Since we are only interested in the logging of + // RpcServer#logResponse warn message + if (event.getMessage().toString().startsWith("(response")) { + events.add(event); + } + } +} -- 2.15.2 (Apple Git-101.1)