From 8696e71d10cfd7f43840176c8f27bcf1e3ecf9e1 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Fri, 7 Nov 2014 15:40:52 -0800 Subject: [PATCH] HBASE-12448 Fix rate reporting in compaction progress DEBUG logging --- .../hbase/regionserver/compactions/Compactor.java | 44 +++++++++++++--------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java index df44899..ff20f6f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.StringUtils; /** @@ -206,20 +207,23 @@ public abstract class Compactor { */ protected boolean performCompaction(InternalScanner scanner, CellSink writer, long smallestReadPoint) throws IOException { - int bytesWritten = 0; + long bytesWritten = 0; + long bytesWrittenProgress = 0; // Since scanner.next() can return 'false' but still be delivering data, // we have to use a do/while loop. List kvs = new ArrayList(); - int closeCheckInterval = HStore.getCloseCheckInterval(); - long lastMillis; + long closeCheckInterval = HStore.getCloseCheckInterval(); + long lastMillis = 0; if (LOG.isDebugEnabled()) { - lastMillis = System.currentTimeMillis(); - } else { - lastMillis = 0; + lastMillis = EnvironmentEdgeManager.currentTimeMillis(); } + long now = 0; boolean hasMore; do { hasMore = scanner.next(kvs, compactionKVMax); + if (LOG.isDebugEnabled()) { + now = EnvironmentEdgeManager.currentTimeMillis(); + } // output to writer: for (Cell c : kvs) { KeyValue kv = KeyValueUtil.ensureKeyValue(c); @@ -227,23 +231,17 @@ public abstract class Compactor { kv.setMvccVersion(0); } writer.append(kv); + int len = kv.getLength(); ++progress.currentCompactedKVs; - progress.totalCompactedSize += kv.getLength(); + progress.totalCompactedSize += len; + if (LOG.isDebugEnabled()) { + bytesWrittenProgress += len; + } // check periodically to see if a system stop is requested if (closeCheckInterval > 0) { - bytesWritten += kv.getLength(); + bytesWritten += len; if (bytesWritten > closeCheckInterval) { - // Log the progress of long running compactions every minute if - // logging at DEBUG level - if (LOG.isDebugEnabled()) { - long now = System.currentTimeMillis(); - if ((now - lastMillis) >= 60 * 1000) { - LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", - (bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0))); - lastMillis = now; - } - } bytesWritten = 0; if (!store.areWritesEnabled()) { progress.cancel(); @@ -252,6 +250,16 @@ public abstract class Compactor { } } } + // Log the progress of long running compactions every minute if + // logging at DEBUG level + if (LOG.isDebugEnabled()) { + if ((now - lastMillis) >= 60 * 1000) { + LOG.debug("Compaction progress: " + progress + String.format(", rate=%.2f kB/sec", + (bytesWrittenProgress / 1024.0) / ((now - lastMillis) / 1000.0))); + lastMillis = now; + bytesWrittenProgress = 0; + } + } kvs.clear(); } while (hasMore); progress.complete(); -- 1.7.12.4 (Apple Git-37)