From 3214e909102b2f45331b3115c9572548a76225d3 Mon Sep 17 00:00:00 2001 From: dvdreddy Date: Sun, 3 Apr 2016 22:10:46 -0700 Subject: [PATCH] HBASE-15437 Response size calculated in RPCServer for warning tooLarge responses does NOT count CellScanner payload --- .../org/apache/hadoop/hbase/ipc/BufferChain.java | 4 ++ .../org/apache/hadoop/hbase/ipc/RpcServer.java | 51 ++++++++++++++++------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java index babd2f8..7899157 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java @@ -66,6 +66,10 @@ class BufferChain { return remaining > 0; } + int getRemaining() { + return remaining; + } + /** * Write out our chain of buffers in chunks * @param channel Where to write diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index f0aed2e..7d9047b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -321,6 +321,12 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver { private long responseBlockSize = 0; private boolean retryImmediatelySupported; + // These values are used in the warn logging whenever a call takes too long or the + // response is too big and are set later during the process of the call and + // are currently initialized to -1, to avoid logging when not set + private int processingTime = -1; + private int queueTime = -1; + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", justification="Can't figure why this complaint is happening... see below") Call(int id, final BlockingService service, final MethodDescriptor md, RequestHeader header, @@ -464,6 +470,30 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver { LOG.warn("Exception while running the Rpc Callback.", e); } } + // A successful call without any unknown errors + if (response != null) { + // This is the point where we exactly know the full response + // so updating the response metrics + int responseSize = response.getRemaining(); + metrics.sentResponse(responseSize); + // log any RPC responses that are slower than the configured warn + // response time or larger than configured warning size + boolean tooSlow = (processingTime > warnResponseTime && warnResponseTime > -1); + boolean tooLarge = (responseSize > warnResponseSize && warnResponseSize > -1); + if (tooSlow || tooLarge) { + // when tagging, we let TooLarge trump TooSmall to keep output simple + // note that large responses will often also be slow. + try { + logResponse(new Object[] {param}, + md.getName(), md.getName() + "(" + param.getClass().getName() + ")", + (tooLarge ? "TooLarge" : "TooSlow"), + connection.toString(), timestamp, processingTime, queueTime, + responseSize); + } catch (IOException e) { + LOG.warn("Exception while logging warn for tooLarge/tooSlow RPCs", e); + } + } + } } private byte[] createHeaderAndMessageBytes(Message result, Message header) @@ -2260,24 +2290,17 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver { " totalTime: " + totalTime); } long requestSize = param.getSerializedSize(); - long responseSize = result.getSerializedSize(); metrics.dequeuedCall(qTime); metrics.processedCall(processingTime); metrics.totalCall(totalTime); metrics.receivedRequest(requestSize); - metrics.sentResponse(responseSize); - // log any RPC responses that are slower than the configured warn - // response time or larger than configured warning size - boolean tooSlow = (processingTime > warnResponseTime && warnResponseTime > -1); - boolean tooLarge = (responseSize > warnResponseSize && warnResponseSize > -1); - if (tooSlow || tooLarge) { - // when tagging, we let TooLarge trump TooSmall to keep output simple - // note that large responses will often also be slow. - logResponse(new Object[]{param}, - md.getName(), md.getName() + "(" + param.getClass().getName() + ")", - (tooLarge ? "TooLarge" : "TooSlow"), - status.getClient(), startTime, processingTime, qTime, - responseSize); + // Storing the response times and queue times for warn logging + // when they are tooLarge / tooSlow in Call#setResponse method + RpcCallContext callContext = RpcServer.getCurrentCall(); + if (callContext instanceof Call) { + Call currentCall = (Call) callContext; + currentCall.processingTime = processingTime; + currentCall.queueTime = qTime; } return new Pair(result, controller.cellScanner()); } catch (Throwable e) { -- 2.2.1