From 1abffc56ab84c9a69a32e9255f892b1be6bc45c5 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 7 Nov 2014 15:40:52 -0800 Subject: [PATCH] HBASE-12448 Fix rate reporting in compaction progress DEBUG logging --- .../hbase/regionserver/compactions/Compactor.java | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index 6b07d73..b637270 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils; /** @@ -229,42 +230,38 @@ public abstract class Compactor { protected boolean performCompaction(InternalScanner scanner, CellSink writer, long smallestReadPoint, boolean cleanSeqId) throws IOException { int bytesWritten = 0; + int bytesWrittenProgress = 0; // Since scanner.next() can return 'false' but still be delivering data, // we have to use a do/while loop. List cells = new ArrayList(); int closeCheckInterval = HStore.getCloseCheckInterval(); - long lastMillis; + long lastMillis = 0; if (LOG.isDebugEnabled()) { - lastMillis = System.currentTimeMillis(); - } else { - lastMillis = 0; + lastMillis = EnvironmentEdgeManager.currentTime(); } + long now = 0; boolean hasMore; do { hasMore = scanner.next(cells, compactionKVMax); + if (LOG.isDebugEnabled()) { + now = EnvironmentEdgeManager.currentTime(); + } // output to writer: for (Cell c : cells) { if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) { CellUtil.setSequenceId(c, 0); } writer.append(c); + int len = KeyValueUtil.length(c); ++progress.currentCompactedKVs; - progress.totalCompactedSize += KeyValueUtil.length(c); - + progress.totalCompactedSize += len; + if (LOG.isDebugEnabled()) { + bytesWrittenProgress += len; + } // check periodically to see if a system stop is requested if (closeCheckInterval > 0) { - bytesWritten += KeyValueUtil.length(c); + bytesWritten += len; if (bytesWritten > closeCheckInterval) { - // Log the progress of long running compactions every minute if - // logging at DEBUG level - if (LOG.isDebugEnabled()) { - long now = System.currentTimeMillis(); - if ((now - lastMillis) >= 60 * 1000) { - LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", - (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0))); - lastMillis = now; - } - } bytesWritten = 0; if (!store.areWritesEnabled()) { progress.cancel(); @@ -273,6 +270,16 @@ public abstract class Compactor { } } } + // Log the progress of long running compactions every minute if + // logging at DEBUG level + if (LOG.isDebugEnabled()) { + if ((now - lastMillis) >= 60 * 1000) { + LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", + (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0))); + lastMillis = now; + bytesWrittenProgress = 0; + } + } cells.clear(); } while (hasMore); progress.complete(); -- 1.7.12.4 (Apple Git-37)