Index: conf/hbase-default.xml
===================================================================
--- conf/hbase-default.xml (revision 686453)
+++ conf/hbase-default.xml (working copy)
@@ -245,6 +245,18 @@
+ hbase.hstore.compaction.max
+ 10
+ Max number of HStoreFiles to compact at one time while doing a minor compaction.
+
+
+
+ hbase.hregion.majorcompaction
+ 86400000
+ The time (in miliseconds) between compactions of all HStoreFiles in to one default 1 day.
+
+
+
hbase.regionserver.thread.splitcompactcheckfrequency
20000
How often a region server runs the split/compaction check.
Index: src/java/org/apache/hadoop/hbase/regionserver/HStore.java
===================================================================
--- src/java/org/apache/hadoop/hbase/regionserver/HStore.java (revision 686453)
+++ src/java/org/apache/hadoop/hbase/regionserver/HStore.java (working copy)
@@ -86,7 +86,8 @@
final FileSystem fs;
private final HBaseConfiguration conf;
protected long ttl;
-
+ private long majorCompactionTime;
+ private int maxFilesToCompact;
private final long desiredMaxFileSize;
private volatile long storeSize;
@@ -170,6 +171,7 @@
this.ttl *= 1000;
this.memcache = new Memcache(this.ttl);
this.compactionDir = HRegion.getCompactionDir(basedir);
+ //this.path = HRegion.getRegionDir(basedir, info);
this.storeName = Bytes.toBytes(this.info.getEncodedName() + "/" +
Bytes.toString(this.family.getName()));
this.storeNameStr = Bytes.toString(this.storeName);
@@ -187,6 +189,8 @@
}
this.desiredMaxFileSize = maxFileSize;
+ this.majorCompactionTime = conf.getLong("hbase.hregion.majorcompaction", 86400000);
+ this.maxFilesToCompact = conf.getInt("hbase.hstore.compaction.max", 10);
this.storeSize = 0L;
if (family.getCompression() == HColumnDescriptor.CompressionType.BLOCK) {
@@ -730,7 +734,7 @@
* @return mid key if a split is needed, null otherwise
* @throws IOException
*/
- StoreSize compact(final boolean force) throws IOException {
+ StoreSize compact(boolean force) throws IOException {
synchronized (compactLock) {
long maxId = -1;
int nrows = -1;
@@ -740,7 +744,20 @@
return null;
}
filesToCompact = new ArrayList(this.storefiles.values());
-
+
+ // check to see if we need to do a major compaction on this region
+ // if so then change force to true to skip the incremental compacting below
+ // only check if force is not true if true major compaction will happen anyways
+ if (!force){
+ Path mapdir = HStoreFile.getMapDir(basedir, info.getEncodedName(), family.getName());
+ long lowTimestamp = HRegion.getLowTimestamp(fs, mapdir);
+ if (lowTimestamp > System.currentTimeMillis() - majorCompactionTime){
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Major compaction triggered on store: " + this.storeNameStr);
+ }
+ force = true;
+ }
+ }
// The max-sequenceID in any of the to-be-compacted TreeMaps is the
// last key of storefiles.
maxId = this.storefiles.lastKey().longValue();
@@ -777,7 +794,7 @@
// size of the second, skip the largest, and continue to next...,
// until we meet the compactionThreshold limit.
for (point = 0; point < compactionThreshold - 1; point++) {
- if (fileSizes[point] < fileSizes[point + 1] * 2) {
+ if (fileSizes[point] < fileSizes[point + 1] * 2 && maxFilesToCompact < (countOfFiles - point)) {
break;
}
skipped += fileSizes[point];
Index: src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
===================================================================
--- src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 686453)
+++ src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy)
@@ -308,7 +308,29 @@
}
}
}
+ /*
+ * Gets lowest timestamp from files in a dir
+ *
+ * @param fs
+ * @param dir
+ * @throws IOException
+ */
+ static long getLowTimestamp(FileSystem fs, Path dir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ if (stats == null || stats.length == 0) {
+ return 0l;
+ }
+ long lowTimestamp = Long.MAX_VALUE;
+ for (int i = 0; i < stats.length; i++) {
+ long timestamp = stats[i].getModificationTime();
+ if (timestamp < lowTimestamp){
+ lowTimestamp = timestamp;
+ }
+ }
+ return lowTimestamp;
+ }
+
//////////////////////////////////////////////////////////////////////////////
// Members
//////////////////////////////////////////////////////////////////////////////
@@ -859,7 +881,7 @@
* @throws IOException
*/
public byte [] compactStores() throws IOException {
- return compactStores(false);
+ return compactStores(false);
}
/**