From 8514c04673a71b7dc71fd42222d022bfd56281a2 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Tue, 19 Apr 2016 11:51:18 +0800 Subject: [PATCH] HBASE-15665 Support using different StoreFileComparators for different CompactionPolicies --- .../hbase/regionserver/DateTieredStoreEngine.java | 5 +++-- .../hbase/regionserver/DefaultStoreEngine.java | 4 +++- .../regionserver/DefaultStoreFileManager.java | 22 +++++++++++++++------- .../apache/hadoop/hbase/regionserver/HStore.java | 5 +++-- .../hbase/regionserver/StoreFileManager.java | 14 ++++++++++---- .../hbase/regionserver/StripeStoreFileManager.java | 15 +++++++++------ .../compactions/DateTieredCompactionPolicy.java | 2 -- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 1bd0f1c..f067819 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -58,8 +58,9 @@ public class DateTieredStoreEngine extends protected void createComponents(Configuration conf, Store store, KVComparator kvComparator) throws IOException { this.compactionPolicy = new DateTieredCompactionPolicy(conf, store); - this.storeFileManager = new DefaultStoreFileManager(kvComparator, conf, - compactionPolicy.getConf()); + this.storeFileManager = + new DefaultStoreFileManager(kvComparator, StoreFile.Comparators.SEQ_ID_MAX_TIMESTAMP, conf, + compactionPolicy.getConf()); this.storeFlusher = new DefaultStoreFlusher(conf, store); this.compactor = new DateTieredCompactor(conf, store); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java index 727152c..d9b48d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java @@ -81,7 +81,9 @@ public class DefaultStoreEngine extends StoreEngine< } catch (Exception e) { throw new IOException("Unable to load configured compaction policy '" + className + "'", e); } - storeFileManager = new DefaultStoreFileManager(kvComparator, conf, compactionPolicy.getConf()); + storeFileManager = + new DefaultStoreFileManager(kvComparator, StoreFile.Comparators.SEQ_ID, conf, + compactionPolicy.getConf()); className = conf.get( DEFAULT_STORE_FLUSHER_CLASS_KEY, DEFAULT_STORE_FLUSHER_CLASS.getName()); try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFileManager.java index 6475a87..1547c13 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFileManager.java @@ -22,9 +22,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -33,10 +38,6 @@ import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - /** * Default implementation of StoreFileManager. Not thread-safe. */ @@ -47,16 +48,18 @@ class DefaultStoreFileManager implements StoreFileManager { private final KVComparator kvComparator; private final CompactionConfiguration comConf; private final int blockingFileCount; - + private final Comparator storeFileComparator; /** * List of store files inside this store. This is an immutable list that * is atomically replaced when its contents change. */ private volatile ImmutableList storefiles = null; - public DefaultStoreFileManager(KVComparator kvComparator, Configuration conf, + public DefaultStoreFileManager(KVComparator kvComparator, + Comparator storeFileComparator, Configuration conf, CompactionConfiguration comConf) { this.kvComparator = kvComparator; + this.storeFileComparator = storeFileComparator; this.comConf = comConf; this.blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT); @@ -160,7 +163,7 @@ class DefaultStoreFileManager implements StoreFileManager { } private void sortAndSetStoreFiles(List storeFiles) { - Collections.sort(storeFiles, StoreFile.Comparators.SEQ_ID); + Collections.sort(storeFiles, storeFileComparator); storefiles = ImmutableList.copyOf(storeFiles); } @@ -173,5 +176,10 @@ class DefaultStoreFileManager implements StoreFileManager { } return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact); } + + @Override + public Comparator getStoreFileComparator() { + return storeFileComparator; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 17682ff..14143ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1363,7 +1363,8 @@ public class HStore implements Store { filesToCompact = filesToCompact.subList(count - N, count); isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); - Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID); + Collections.sort(filesCompacting, storeEngine.getStoreFileManager() + .getStoreFileComparator()); } } finally { this.lock.readLock().unlock(); @@ -1554,7 +1555,7 @@ public class HStore implements Store { Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting); } filesCompacting.addAll(filesToAdd); - Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID); + Collections.sort(filesCompacting, storeEngine.getStoreFileManager().getStoreFileComparator()); } private void removeUnneededFiles() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java index 8196b1f..cd3a946 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java @@ -18,16 +18,16 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.common.collect.ImmutableCollection; + import java.io.IOException; import java.util.Collection; +import java.util.Comparator; import java.util.Iterator; import java.util.List; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; - -import com.google.common.collect.ImmutableCollection; +import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * Manages the store files and basic metadata about that that determines the logical structure @@ -140,4 +140,10 @@ public interface StoreFileManager { * @see Store#getCompactionPressure() */ double getCompactionPressure(); + + /** + * @return the comparator used to sort storefiles. Usually, the + * {@link StoreFile#getMaxSequenceId()} is the first priority. + */ + Comparator getStoreFileComparator(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java index b023b6a..87608f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java @@ -18,17 +18,19 @@ */ package org.apache.hadoop.hbase.regionserver; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.TreeMap; import org.apache.commons.logging.Log; @@ -43,10 +45,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ConcatenatedLists; import org.apache.hadoop.util.StringUtils; -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - /** * Stripe implementation of StoreFileManager. @@ -993,4 +991,9 @@ public class StripeStoreFileManager } return max; } + + @Override + public Comparator getStoreFileComparator() { + return StoreFile.Comparators.SEQ_ID; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java index aefcd8f..64b7779 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java @@ -194,10 +194,8 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy { long now = EnvironmentEdgeManager.currentTimeMillis(); long oldestToCompact = getOldestToCompact(comConf.getMaxStoreFileAgeMillis(), now); - // Make sure the store files is sorted by SeqId then maxTimestamp List storeFileList = Lists.newArrayList(filterOldStoreFiles(candidateSelection, oldestToCompact)); - Collections.sort(storeFileList, StoreFile.Comparators.SEQ_ID_MAX_TIMESTAMP); List> storefileMaxTimestampPairs = Lists.newArrayListWithCapacity(Iterables.size(storeFileList)); -- 1.9.1