diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index b0d8c24..a6c46d6 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -340,6 +340,8 @@ String REPLAY_KEY = "replay"; String SCAN_SIZE_KEY = "scanSize"; String SCAN_TIME_KEY = "scanTime"; + String STRIPE_KEY = "stripeInfo"; + String STRIPE_DESC = "The stripe information of the region"; String SLOW_MUTATE_KEY = "slowPutCount"; String SLOW_GET_KEY = "slowGetCount"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index a7c7096..fa9618b 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -44,6 +44,7 @@ */ String getRegionName(); + String getStripeInfo(); /** * Get the number of stores hosted on this region server. */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 05f1126..8bdf39a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -287,6 +287,11 @@ mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.REPLICA_ID, MetricsRegionSource.REPLICA_ID_DESC), this.regionWrapper.getReplicaId()); + String stripeInfo = this.regionWrapper.getStripeInfo(); + if(stripeInfo != null && !"".equals(stripeInfo.trim())){ + mrb.tag(Interns.info(regionNamePrefix + MetricsRegionServerSource.STRIPE_KEY, + MetricsRegionServerSource.STRIPE_DESC), stripeInfo); + } } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 4f5a8bd..47a112d 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; - import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.MetricsTests; @@ -163,5 +162,10 @@ public int getReplicaId() { return 0; } + + @Override + public String getStripeInfo() { + return null; + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index f7ad4ae..897c927 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -125,6 +125,8 @@ protected final MemStore memstore; // This stores directory in the filesystem. protected final HRegion region; + private StripeState stripes; + private final HColumnDescriptor family; private final HRegionFileSystem fs; protected Configuration conf; @@ -2486,4 +2488,13 @@ lock.writeLock().unlock(); } } + + @Override + public StripeState getStripes() { + return this.stripes; + } + + public void setStripes(StripeState stripes) { + this.stripes = stripes; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index f9e01cd..1f7af23 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.metrics2.MetricsExecutor; +import com.google.common.collect.ImmutableList; + @InterfaceAudience.Private public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable { @@ -47,6 +49,7 @@ private long minStoreFileAge; private long avgStoreFileAge; private long numReferenceFiles; + private String stripeInfo; private ScheduledFuture regionMetricsUpdateTask; @@ -178,6 +181,7 @@ long avgAgeNumerator = 0; long numHFiles = 0; + StringBuilder stripeStr = new StringBuilder(); if (region.stores != null) { for (Store store : region.stores.values()) { tempNumStoreFiles += store.getStorefilesCount(); @@ -196,6 +200,18 @@ avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles; numHFiles += storeHFiles; tempNumReferenceFiles += store.getNumReferenceFiles(); + + StripeState state = store.getStripes(); + if(state != null){ + stripeStr.append(store.getColumnFamilyName()).append("_level_0,") + .append(state.level0Files.size()).append(";"); + int index = 0; + for(ImmutableList stripe : state.stripeFiles){ + stripeStr.append(store.getColumnFamilyName()).append("_stripe_") + .append(index).append(",").append(stripe.size()).append(";"); + index ++; + } + } } } @@ -203,6 +219,8 @@ memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; maxStoreFileAge = tempMaxStoreFileAge; + stripeInfo = stripeStr.toString(); + if (tempMinStoreFileAge != Long.MAX_VALUE) { minStoreFileAge = tempMinStoreFileAge; } @@ -228,4 +246,9 @@ return region.getRegionInfo().getReplicaId(); } + @Override + public String getStripeInfo() { + return stripeInfo; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 4cca4a1..f7986e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -63,6 +63,8 @@ Collection getStorefiles(); + StripeState getStripes(); + /** * Close all the readers We don't need to worry about subsequent requests because the Region * holds a write lock that will prevent any more reads or writes. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeState.java new file mode 100644 index 0000000..4f8075f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeState.java @@ -0,0 +1,31 @@ +package org.apache.hadoop.hbase.regionserver; + +import java.util.ArrayList; + +import com.google.common.collect.ImmutableList; + +/** + * The state class. Used solely to replace results atomically during + * compactions and avoid complicated error handling. + */ +public class StripeState { + /** + * The end rows of each stripe. The last stripe end is always open-ended, so it's not stored + * here. It is invariant that the start row of the stripe is the end row of the previous one + * (and is an open boundary for the first one). + */ + public byte[][] stripeEndRows = new byte[0][]; + + /** + * Files by stripe. Each element of the list corresponds to stripeEndRow element with the + * same index, except the last one. Inside each list, the files are in reverse order by + * seqNum. Note that the length of this is one higher than that of stripeEndKeys. + */ + public ArrayList> stripeFiles = new ArrayList>(); + /** Level 0. The files are in reverse order by seqNum. */ + public ImmutableList level0Files = ImmutableList.of(); + + /** Cached list of all files in the structure, to return from some calls */ + public ImmutableList allFilesCached = ImmutableList.of(); + public ImmutableList allCompactedFilesCached = ImmutableList.of(); +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java index 9255634..0ddbd93 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreEngine.java @@ -61,7 +61,7 @@ Configuration conf, Store store, CellComparator comparator) throws IOException { this.config = new StripeStoreConfig(conf, store); this.compactionPolicy = new StripeCompactionPolicy(conf, store, config); - this.storeFileManager = new StripeStoreFileManager(comparator, conf, this.config); + this.storeFileManager = new StripeStoreFileManager(store, comparator, conf, this.config); this.storeFlusher = new StripeStoreFlusher( conf, store, this.compactionPolicy, this.storeFileManager); this.compactor = new StripeCompactor(conf, store); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java index ef2c282..c80e791 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java @@ -82,33 +82,7 @@ public final static byte[] OPEN_KEY = HConstants.EMPTY_BYTE_ARRAY; final static byte[] INVALID_KEY = null; - /** - * The state class. Used solely to replace results atomically during - * compactions and avoid complicated error handling. - */ - private static class State { - /** - * The end rows of each stripe. The last stripe end is always open-ended, so it's not stored - * here. It is invariant that the start row of the stripe is the end row of the previous one - * (and is an open boundary for the first one). - */ - public byte[][] stripeEndRows = new byte[0][]; - - /** - * Files by stripe. Each element of the list corresponds to stripeEndRow element with the - * same index, except the last one. Inside each list, the files are in reverse order by - * seqNum. Note that the length of this is one higher than that of stripeEndKeys. - */ - public ArrayList> stripeFiles - = new ArrayList>(); - /** Level 0. The files are in reverse order by seqNum. */ - public ImmutableList level0Files = ImmutableList.of(); - - /** Cached list of all files in the structure, to return from some calls */ - public ImmutableList allFilesCached = ImmutableList.of(); - private ImmutableList allCompactedFilesCached = ImmutableList.of(); - } - private State state = null; + private StripeState state = null; /** Cached file metadata (or overrides as the case may be) */ private HashMap fileStarts = new HashMap(); @@ -119,13 +93,15 @@ private static final byte[] INVALID_KEY_IN_MAP = new byte[0]; private final CellComparator cellComparator; + private Store store; private StripeStoreConfig config; private final int blockingFileCount; public StripeStoreFileManager( - CellComparator kvComparator, Configuration conf, StripeStoreConfig config) { + Store store, CellComparator kvComparator, Configuration conf, StripeStoreConfig config) { this.cellComparator = kvComparator; + this.store = store; this.config = config; this.blockingFileCount = conf.getInt( HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT); @@ -157,7 +133,7 @@ @Override public ImmutableCollection clearFiles() { ImmutableCollection result = state.allFilesCached; - this.state = new State(); + this.state = new StripeState(); this.fileStarts.clear(); this.fileEnds.clear(); return result; @@ -166,7 +142,7 @@ @Override public ImmutableCollection clearCompactedFiles() { ImmutableCollection result = state.allCompactedFilesCached; - this.state = new State(); + this.state = new StripeState(); return result; } @@ -474,7 +450,7 @@ } // Copy the results into the fields. - State state = new State(); + StripeState state = new StripeState(); state.level0Files = ImmutableList.copyOf(level0Files); state.stripeFiles = new ArrayList>(candidateStripes.size()); state.stripeEndRows = new byte[Math.max(0, candidateStripes.size() - 1)][]; @@ -490,6 +466,7 @@ } state.allFilesCached = ImmutableList.copyOf(newAllFiles); this.state = state; + ((HStore)this.store).setStripes(state); debugDumpState("Files loaded"); } @@ -732,7 +709,7 @@ processNewCandidateStripes(newStripes); } // Create new state and update parent. - State state = createNewState(false); + StripeState state = createNewState(false); StripeStoreFileManager.this.state = state; updateMetadataMaps(); } @@ -740,16 +717,16 @@ private void deleteResults(Collection compactedFiles) throws IOException { this.compactedFiles = compactedFiles; // Create new state and update parent. - State state = createNewState(true); + StripeState state = createNewState(true); StripeStoreFileManager.this.state = state; updateMetadataMaps(); } - private State createNewState(boolean delCompactedFiles) { - State oldState = StripeStoreFileManager.this.state; + private StripeState createNewState(boolean delCompactedFiles) { + StripeState oldState = StripeStoreFileManager.this.state; // Stripe count should be the same unless the end rows changed. assert oldState.stripeFiles.size() == this.stripeFiles.size() || this.stripeEndRows != null; - State newState = new State(); + StripeState newState = new StripeState(); newState.level0Files = (this.level0Files == null) ? oldState.level0Files : ImmutableList.copyOf(this.level0Files); newState.stripeEndRows = (this.stripeEndRows == null) ? oldState.stripeEndRows @@ -776,6 +753,7 @@ } newState.allFilesCached = ImmutableList.copyOf(newAllFiles); newState.allCompactedFilesCached = ImmutableList.copyOf(newAllCompactedFiles); + ((HStore)StripeStoreFileManager.this.store).setStripes(newState); return newState; } @@ -1013,7 +991,7 @@ public Collection getUnneededFiles(long maxTs, List filesCompacting) { // 1) We can never get rid of the last file which has the maximum seqid in a stripe. // 2) Files that are not the latest can't become one due to (1), so the rest are fair game. - State state = this.state; + StripeState state = this.state; Collection expiredStoreFiles = null; for (ImmutableList stripe : state.stripeFiles) { expiredStoreFiles = findExpiredFiles(stripe, maxTs, filesCompacting, expiredStoreFiles); @@ -1043,7 +1021,7 @@ @Override public double getCompactionPressure() { - State stateLocal = this.state; + StripeState stateLocal = this.state; if (stateLocal.allFilesCached.size() > blockingFileCount) { // just a hit to tell others that we have reached the blocking file count. return 2.0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 1ab44ec..f3170e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -132,4 +132,9 @@ public int getReplicaId() { return replicaid; } + + @Override + public String getStripeInfo() { + return null; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java index e9d34ed..f2ba79a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java @@ -607,9 +607,10 @@ private static StripeStoreFileManager createManager( ArrayList sfs, Configuration conf) throws Exception { + Store store = Mockito.mock(HStore.class); StripeStoreConfig config = new StripeStoreConfig( - conf, Mockito.mock(StoreConfigInformation.class)); - StripeStoreFileManager result = new StripeStoreFileManager(CellComparator.COMPARATOR, conf, + conf, store); + StripeStoreFileManager result = new StripeStoreFileManager(store, CellComparator.COMPARATOR, conf, config); result.loadFiles(sfs); return result;