Index: src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (revision 1331204) +++ src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.CancelableProgressable; import org.apache.hadoop.hbase.zookeeper.ZKAssign; @@ -343,8 +344,17 @@ // We failed open. Our caller will see the 'null' return value // and transition the node back to FAILED_OPEN. If that fails, // we rely on the Timeout Monitor in the master to reassign. - LOG.error("Failed open of region=" + - this.regionInfo.getRegionNameAsString(), t); + LOG.error( + "Failed open of region=" + this.regionInfo.getRegionNameAsString() + + ", starting to roll back the global memstore size.", t); + // Decrease the global memstore size. + if (this.rsServices != null) { + RegionServerAccounting rsAccounting = + this.rsServices.getRegionServerAccounting(); + if (rsAccounting != null) { + rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName()); + } + } } return region; } Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1331204) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -299,6 +299,7 @@ final long timestampSlop; private volatile long lastFlushTime; final RegionServerServices rsServices; + private RegionServerAccounting rsAccounting; private List> recentFlushes = new ArrayList>(); private long blockingMemStoreSize; final long threadWakeFrequency; @@ -403,9 +404,10 @@ "hbase.hregion.keyvalue.timestamp.slop.millisecs", HConstants.LATEST_TIMESTAMP); - // don't initialize coprocessors if not running within a regionserver - // TODO: revisit if coprocessors should load in other cases if (rsServices != null) { + this.rsAccounting = this.rsServices.getRegionServerAccounting(); + // don't initialize coprocessors if not running within a regionserver + // TODO: revisit if coprocessors should load in other cases this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf); } if (LOG.isDebugEnabled()) { @@ -662,14 +664,9 @@ * @return the size of memstore in this region */ public long addAndGetGlobalMemstoreSize(long memStoreSize) { - if (this.rsServices != null) { - RegionServerAccounting rsAccounting = - this.rsServices.getRegionServerAccounting(); - - if (rsAccounting != null) { - rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize); - } - } + if (this.rsAccounting != null) { + rsAccounting.addAndGetGlobalMemstoreSize(memStoreSize); + } return this.memstoreSize.getAndAdd(memStoreSize); } @@ -2653,6 +2650,11 @@ throw e; } } + // The edits size added into rsAccounting during this replaying will not + // be required any more. So just clear it. + if (this.rsAccounting != null) { + this.rsAccounting.clearRegionReplayEditsSize(this.regionInfo.getRegionName()); + } } if (seqid > minSeqId) { // Then we added some edits to memory. Flush and cleanup split edit files. @@ -2834,7 +2836,11 @@ * @return True if we should flush. */ protected boolean restoreEdit(final Store s, final KeyValue kv) { - return isFlushSize(this.addAndGetGlobalMemstoreSize(s.add(kv))); + long kvSize = s.add(kv); + if (this.rsAccounting != null) { + rsAccounting.addAndGetRegionReplayEditsSize(this.regionInfo.getRegionName(), kvSize); + } + return isFlushSize(this.addAndGetGlobalMemstoreSize(kvSize)); } /* @@ -4611,7 +4617,7 @@ public static final long FIXED_OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.ARRAY + - 30 * ClassSize.REFERENCE + Bytes.SIZEOF_INT + + 31 * ClassSize.REFERENCE + Bytes.SIZEOF_INT + (6 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN); Index: src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java (revision 1331204) +++ src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java (working copy) @@ -19,8 +19,12 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.util.Bytes; + /** * RegionServerAccounting keeps record of some basic real time information about * the Region Server. Currently, it only keeps record the global memstore size. @@ -29,6 +33,11 @@ private final AtomicLong atomicGlobalMemstoreSize = new AtomicLong(0); + // Store the edits size during replaying HLog. Use this to roll back the + // global memstore size once a region opening failed. + private final ConcurrentMap replayEditsPerRegion = + new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + /** * @return the global Memstore size in the RegionServer */ @@ -44,5 +53,46 @@ public long addAndGetGlobalMemstoreSize(long memStoreSize) { return atomicGlobalMemstoreSize.addAndGet(memStoreSize); } - + + /*** + * Add memStoreSize to replayEditsPerRegion. + * + * @param regionName region name. + * @param memStoreSize the Memstore size will be added to replayEditsPerRegion. + * @return the replay edits size for region hri. + */ + public long addAndGetRegionReplayEditsSize(byte[] regionName, long memStoreSize) { + AtomicLong replayEdistsSize = replayEditsPerRegion.get(regionName); + if (replayEdistsSize == null) { + replayEdistsSize = new AtomicLong(0); + replayEditsPerRegion.put(regionName, replayEdistsSize); + } + return replayEdistsSize.addAndGet(memStoreSize); + } + + /** + * Roll back the global MemStore size for a specified region when this region + * can't be opened. + * + * @param regionName the region which could not open. + * @return the global Memstore size in the RegionServer + */ + public long rollbackRegionReplayEditsSize(byte[] regionName) { + AtomicLong replayEditsSize = replayEditsPerRegion.get(regionName); + long editsSizeLong = 0L; + if (replayEditsSize != null) { + editsSizeLong = -replayEditsSize.get(); + clearRegionReplayEditsSize(regionName); + } + return addAndGetGlobalMemstoreSize(editsSizeLong); + } + + /** + * Clear a region from replayEditsPerRegion. + * + * @param regionName region name. + */ + public void clearRegionReplayEditsSize(byte[] regionName) { + replayEditsPerRegion.remove(regionName); + } }