From 2064fe807ce8bee4d52afa4cd2b929da9fae8ead Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 7 Nov 2014 15:40:52 -0800 Subject: [PATCH] HBASE-12448 Fix rate reporting in compaction progress DEBUG logging --- .../hbase/regionserver/compactions/Compactor.java | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index df44899..47ee27d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils; /** @@ -207,19 +208,22 @@ public abstract class Compactor { protected boolean performCompaction(InternalScanner scanner, CellSink writer, long smallestReadPoint) throws IOException { int bytesWritten = 0; + int bytesWrittenProgress = 0; // Since scanner.next() can return 'false' but still be delivering data, // we have to use a do/while loop. List kvs = new ArrayList(); int closeCheckInterval = HStore.getCloseCheckInterval(); - long lastMillis; + long lastMillis = 0; if (LOG.isDebugEnabled()) { - lastMillis = System.currentTimeMillis(); - } else { - lastMillis = 0; + lastMillis = EnvironmentEdgeManager.currentTimeMillis(); } + long now = 0; boolean hasMore; do { hasMore = scanner.next(kvs, compactionKVMax); + if (LOG.isDebugEnabled()) { + now = EnvironmentEdgeManager.currentTimeMillis(); + } // output to writer: for (Cell c : kvs) { KeyValue kv = KeyValueUtil.ensureKeyValue(c); @@ -227,23 +231,17 @@ public abstract class Compactor { kv.setMvccVersion(0); } writer.append(kv); + int len = kv.getLength(); ++progress.currentCompactedKVs; - progress.totalCompactedSize += kv.getLength(); + progress.totalCompactedSize += len; + if (LOG.isDebugEnabled()) { + bytesWrittenProgress += len; + } // check periodically to see if a system stop is requested if (closeCheckInterval > 0) { - bytesWritten += kv.getLength(); + bytesWritten += len; if (bytesWritten > closeCheckInterval) { - // Log the progress of long running compactions every minute if - // logging at DEBUG level - if (LOG.isDebugEnabled()) { - long now = System.currentTimeMillis(); - if ((now - lastMillis) >= 60 * 1000) { - LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", - (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0))); - lastMillis = now; - } - } bytesWritten = 0; if (!store.areWritesEnabled()) { progress.cancel(); @@ -252,6 +250,16 @@ public abstract class Compactor { } } } + // Log the progress of long running compactions every minute if + // logging at DEBUG level + if (LOG.isDebugEnabled()) { + if ((now - lastMillis) >= 60 * 1000) { + LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", + (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0))); + lastMillis = now; + bytesWrittenProgress = 0; + } + } kvs.clear(); } while (hasMore); progress.complete(); -- 1.7.12.4 (Apple Git-37)