From 9e1ec2048f52ba90a709cb821fa5793e1565a0ef Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 23 May 2019 11:18:20 -0700 Subject: [PATCH] HBASE-22459 Expose store reader reference count --- .../org/apache/hadoop/hbase/RegionLoad.java | 8 ++++++++ .../org/apache/hadoop/hbase/RegionMetrics.java | 4 ++++ .../hadoop/hbase/RegionMetricsBuilder.java | 18 ++++++++++++++++++ .../MetricsRegionServerSource.java | 2 ++ .../regionserver/MetricsRegionWrapper.java | 5 +++++ .../regionserver/MetricsRegionSourceImpl.java | 4 ++++ .../TestMetricsRegionSourceImpl.java | 5 +++++ .../src/main/protobuf/ClusterStatus.proto | 3 +++ .../src/main/protobuf/ClusterStatus.proto | 3 +++ .../hadoop/hbase/regionserver/HStore.java | 7 +++++++ .../regionserver/MetricsRegionWrapperImpl.java | 9 +++++++++ .../hadoop/hbase/regionserver/Store.java | 5 +++++ .../regionserver/MetricsRegionWrapperStub.java | 5 +++++ 13 files changed, 78 insertions(+) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java index ef92291aff..efcd20ba73 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java @@ -363,6 +363,13 @@ public class RegionLoad implements RegionMetrics { return metrics.getLastMajorCompactionTimestamp(); } + /** + * @return the reference count for the stores of this region + */ + public int getStoreRefCount() { + return metrics.getStoreRefCount(); + } + /** * @see java.lang.Object#toString() */ @@ -371,6 +378,7 @@ public class RegionLoad implements RegionMetrics { StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores", this.getStores()); Strings.appendKeyValue(sb, "numberOfStorefiles", this.getStorefiles()); + Strings.appendKeyValue(sb, "storeRefCount", this.getStoreRefCount()); Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", this.getStoreUncompressedSizeMB()); Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp", diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java index a75ddba7e5..3905f89166 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java @@ -150,4 +150,8 @@ public interface RegionMetrics { */ long getLastMajorCompactionTimestamp(); + /** + * @return the reference count for the stores of this region + */ + int getStoreRefCount(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java index 9011c2a1b6..6bd0f056aa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java @@ -65,6 +65,7 @@ public final class RegionMetricsBuilder { Size.Unit.KILOBYTE)) .setStoreCount(regionLoadPB.getStores()) .setStoreFileCount(regionLoadPB.getStorefiles()) + .setStoreRefCount(regionLoadPB.getStoreRefCount()) .setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE)) .setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream() .collect(Collectors.toMap( @@ -111,6 +112,7 @@ public final class RegionMetricsBuilder { .get(Size.Unit.KILOBYTE)) .setStores(regionMetrics.getStoreCount()) .setStorefiles(regionMetrics.getStoreCount()) + .setStoreRefCount(regionMetrics.getStoreRefCount()) .setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE)) .addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId())) .setStoreUncompressedSizeMB( @@ -125,6 +127,7 @@ public final class RegionMetricsBuilder { private final byte[] name; private int storeCount; private int storeFileCount; + private int storeRefCount; private long compactingCellCount; private long compactedCellCount; private Size storeFileSize = Size.ZERO; @@ -154,6 +157,10 @@ public final class RegionMetricsBuilder { this.storeFileCount = value; return this; } + public RegionMetricsBuilder setStoreRefCount(int value) { + this.storeRefCount = value; + return this; + } public RegionMetricsBuilder setCompactingCellCount(long value) { this.compactingCellCount = value; return this; @@ -227,6 +234,7 @@ public final class RegionMetricsBuilder { return new RegionMetricsImpl(name, storeCount, storeFileCount, + storeRefCount, compactingCellCount, compactedCellCount, storeFileSize, @@ -250,6 +258,7 @@ public final class RegionMetricsBuilder { private final byte[] name; private final int storeCount; private final int storeFileCount; + private final int storeRefCount; private final long compactingCellCount; private final long compactedCellCount; private final Size storeFileSize; @@ -270,6 +279,7 @@ public final class RegionMetricsBuilder { RegionMetricsImpl(byte[] name, int storeCount, int storeFileCount, + int storeRefCount, final long compactingCellCount, long compactedCellCount, Size storeFileSize, @@ -290,6 +300,7 @@ public final class RegionMetricsBuilder { this.name = Preconditions.checkNotNull(name); this.storeCount = storeCount; this.storeFileCount = storeFileCount; + this.storeRefCount = storeRefCount; this.compactingCellCount = compactingCellCount; this.compactedCellCount = compactedCellCount; this.storeFileSize = Preconditions.checkNotNull(storeFileSize); @@ -324,6 +335,11 @@ public final class RegionMetricsBuilder { return storeFileCount; } + @Override + public int getStoreRefCount() { + return storeRefCount; + } + @Override public Size getStoreFileSize() { return storeFileSize; @@ -415,6 +431,8 @@ public final class RegionMetricsBuilder { this.getStoreCount()); Strings.appendKeyValue(sb, "storeFileCount", this.getStoreFileCount()); + Strings.appendKeyValue(sb, "storeRefCount", + this.getStoreRefCount()); Strings.appendKeyValue(sb, "uncompressedStoreFileSize", this.getUncompressedStoreFileSize()); Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp", diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 6d9ce54486..8a7e647226 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -231,6 +231,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String WALFILE_SIZE_DESC = "Size of all WAL Files"; String STOREFILE_COUNT = "storeFileCount"; String STOREFILE_COUNT_DESC = "Number of Store Files"; + String STORE_REF_COUNT = "storeRefCount"; + String STORE_REF_COUNT_DESC = "Store reference count"; String MEMSTORE_SIZE = "memStoreSize"; String MEMSTORE_SIZE_DESC = "Size of the memstore"; String STOREFILE_SIZE = "storeFileSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index df776b1d6a..6f8f60ba4e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -159,4 +159,9 @@ public interface MetricsRegionWrapper { * Get the replica id of this region. */ int getReplicaId(); + + /** + * @return the number of references active on the store + */ + long getStoreRefCount(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 1831062c28..265df31449 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -213,6 +213,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { regionNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT, MetricsRegionServerSource.STOREFILE_COUNT_DESC), this.regionWrapper.getNumStoreFiles()); + mrb.addGauge(Interns.info( + regionNamePrefix + MetricsRegionServerSource.STORE_REF_COUNT, + MetricsRegionServerSource.STORE_REF_COUNT), + this.regionWrapper.getStoreRefCount()); mrb.addGauge(Interns.info( regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE, MetricsRegionServerSource.MEMSTORE_SIZE_DESC), diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 6ff83759ab..78fc397f0d 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -94,6 +94,11 @@ public class TestMetricsRegionSourceImpl { return 0; } + @Override + public long getStoreRefCount() { + return 0; + } + @Override public long getMemStoreSize() { return 0; diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto index efdfd0c8dc..0880549489 100644 --- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto @@ -146,6 +146,9 @@ message RegionLoad { /** the current total coprocessor requests made to region */ optional uint64 cp_requests_count = 20; + + /** the number of references active on the store */ + optional int32 store_ref_count = 21 [default = 0]; } /* Server-level protobufs */ diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto index 009d172c36..4e89fe6d27 100644 --- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto @@ -142,6 +142,9 @@ message RegionLoad { /** the current total coprocessor requests made to region */ optional uint64 cp_requests_count = 20; + + /** the number of references active on the store */ + optional int32 store_ref_count = 21 [default = 0]; } /* Server-level protobufs */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 2f40bcbc72..0d2771162d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -2139,6 +2139,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat .map(t -> EnvironmentEdgeManager.currentTime() - t); } + @Override + public int getStoreRefCount() { + return this.storeEngine.getStoreFileManager().getStorefiles().stream() + .filter(sf -> sf.getReader() != null).filter(HStoreFile::isHFile) + .mapToInt(HStoreFile::getRefCount).sum(); + } + @Override public OptionalLong getMaxStoreFileAge() { return getStoreFileAgeStream().max(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index 871dbd8ee0..08ff0d0476 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -48,6 +48,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable private ScheduledExecutorService executor; private Runnable runnable; private long numStoreFiles; + private long storeRefCount; private long memstoreSize; private long storeFileSize; private long maxStoreFileAge; @@ -119,6 +120,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable return storeFileSize; } + @Override + public long getStoreRefCount() { + return storeRefCount; + } + @Override public long getReadRequestCount() { return this.region.getReadRequestsCount(); @@ -226,6 +232,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable @Override public void run() { long tempNumStoreFiles = 0; + int tempStoreRefCount = 0; long tempMemstoreSize = 0; long tempStoreFileSize = 0; long tempMaxStoreFileAge = 0; @@ -239,6 +246,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable if (region.stores != null) { for (Store store : region.stores.values()) { tempNumStoreFiles += store.getStorefilesCount(); + tempStoreRefCount += store.getStoreRefCount(); tempMemstoreSize += store.getMemStoreSize().getDataSize(); tempStoreFileSize += store.getStorefilesSize(); OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); @@ -265,6 +273,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable } numStoreFiles = tempNumStoreFiles; + storeRefCount = tempStoreRefCount; memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; maxStoreFileAge = tempMaxStoreFileAge; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 6eb9f18b70..6b8daea83c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -126,6 +126,11 @@ public interface Store { */ int getStorefilesCount(); + /** + * @return Reference count over store + */ + int getStoreRefCount(); + /** * @return Count of compacted store files */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index e544206673..acdb7321ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -60,6 +60,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper { return 102; } + @Override + public long getStoreRefCount() { + return 0; + } + @Override public long getMemStoreSize() { return 103; -- 2.21.0