From 54ff6a623e2df416ff7a7dd3a70f370f6f68f329 Mon Sep 17 00:00:00 2001 From: Umesh Agashe Date: Wed, 21 Feb 2018 13:59:59 -0800 Subject: [PATCH] HBASE-19767 Fix for Master web UI shows negative values for Remaining KVs Negative Remaining KVs and progress percent greater than 100 is because CompactionProgress#totalCompactingKVs is sometimes less than CompactionProgress#currentCompactedKVs. Changes add a getter to CompactionProgress#totalCompactingKVs and from inside getter warning is logged. currentCompactedKVs are return when totalCompactingKVs are less than current. --- .../apache/hadoop/hbase/regionserver/HRegionServer.java | 2 +- .../org/apache/hadoop/hbase/regionserver/HStore.java | 4 ++-- .../regionserver/compactions/CompactionProgress.java | 16 ++++++++++++---- .../hadoop/hbase/regionserver/TestMajorCompaction.java | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 49b7b806cd44daa602ae32b02b48917acf8351d1..a76dec20770298a2be761ac30dfda1d61d552a90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1623,7 +1623,7 @@ public class HRegionServer extends HasThread implements storefileIndexSizeKB += store.getStorefilesRootLevelIndexSize() / 1024; CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { - totalCompactingKVs += progress.totalCompactingKVs; + totalCompactingKVs += progress.getTotalCompactingKVs(); currentCompactedKVs += progress.currentCompactedKVs; } rootLevelIndexSizeKB += (int) (store.getStorefilesRootLevelIndexSize() / 1024); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index bef50b4878e03e23ab5450d1beeab259a4b3a27c..c0ef3ef2968d5b75b3492a4ffa2d6eb69708df7c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1373,10 +1373,10 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat writeCompactionWalRecord(filesToCompact, sfs); replaceStoreFiles(filesToCompact, sfs); if (cr.isMajor()) { - majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs; + majorCompactedCellsCount += getCompactionProgress().getTotalCompactingKVs(); majorCompactedCellsSize += getCompactionProgress().totalCompactedSize; } else { - compactedCellsCount += getCompactionProgress().totalCompactingKVs; + compactedCellsCount += getCompactionProgress().getTotalCompactingKVs(); compactedCellsSize += getCompactionProgress().totalCompactedSize; } long outputBytes = getTotalSize(sfs); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java index d40651aa9d7a9c9c18b7c93926438772a148f356..577276e85ed48c4e980cc12b5300bc0cc9e34fc6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionProgress.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.regionserver.compactions; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class holds information relevant for tracking the progress of a @@ -32,9 +34,10 @@ import org.apache.yetus.audience.InterfaceAudience; */ @InterfaceAudience.Private public class CompactionProgress { + private static final Logger LOG = LoggerFactory.getLogger(CompactionProgress.class); /** the total compacting key values in currently running compaction */ - public long totalCompactingKVs; + private long totalCompactingKVs; /** the completed count of key values in currently running compaction */ public long currentCompactedKVs = 0; /** the total size of data processed by the currently running compaction, in bytes */ @@ -51,7 +54,7 @@ public class CompactionProgress { * @return float */ public float getProgressPct() { - return (float)currentCompactedKVs / totalCompactingKVs; + return (float)currentCompactedKVs / getTotalCompactingKVs(); } /** @@ -72,7 +75,12 @@ public class CompactionProgress { /** * @return the total compacting key values in currently running compaction */ - public long getTotalCompactingKvs() { + public long getTotalCompactingKVs() { + if (totalCompactingKVs < currentCompactedKVs) { + LOG.warn("totalCompactingKVs={} less than currentCompactedKVs={}", + totalCompactingKVs, currentCompactedKVs); + return currentCompactedKVs; + } return totalCompactingKVs; } @@ -92,7 +100,7 @@ public class CompactionProgress { @Override public String toString() { - return String.format("%d/%d (%.2f%%)", currentCompactedKVs, totalCompactingKVs, + return String.format("%d/%d (%.2f%%)", currentCompactedKVs, getTotalCompactingKVs(), 100 * getProgressPct()); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java index ee717f911bffa3346200ed22c54805b7e714bb2e..4fdd6b337f67cadd6cfd5b7a74ded454380a5e65 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java @@ -221,7 +221,7 @@ public class TestMajorCompaction { if( progress != null ) { ++storeCount; assertTrue(progress.currentCompactedKVs > 0); - assertTrue(progress.totalCompactingKVs > 0); + assertTrue(progress.getTotalCompactingKVs() > 0); } assertTrue(storeCount > 0); } -- 2.14.3 (Apple Git-98)