diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java index 20a5896..d57d243 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java @@ -564,20 +564,20 @@ public class HLog implements Syncable { */ private byte [][] cleanOldLogs() throws IOException { Long oldestOutstandingSeqNum = getOldestOutstandingSeqNum(); - // Get the set of all log files whose final ID is older than or - // equal to the oldest pending region operation + // Get the set of all log files whose last sequence number is smaller than + // the oldest edit's sequence number. TreeSet sequenceNumbers = new TreeSet(this.outputfiles.headMap( - (Long.valueOf(oldestOutstandingSeqNum.longValue() + 1L))).keySet()); + (Long.valueOf(oldestOutstandingSeqNum.longValue()))).keySet()); // Now remove old log files (if any) int logsToRemove = sequenceNumbers.size(); if (logsToRemove > 0) { if (LOG.isDebugEnabled()) { // Find associated region; helps debugging. byte [] oldestRegion = getOldestRegion(oldestOutstandingSeqNum); - LOG.debug("Found " + logsToRemove + " hlogs to remove " + - " out of total " + this.outputfiles.size() + "; " + - "oldest outstanding sequenceid is " + oldestOutstandingSeqNum + + LOG.debug("Found " + logsToRemove + " hlogs to remove" + + " out of total " + this.outputfiles.size() + ";" + + " oldest outstanding sequenceid is " + oldestOutstandingSeqNum + " from region " + Bytes.toString(oldestRegion)); } for (Long seq : sequenceNumbers) { @@ -668,7 +668,7 @@ public class HLog implements Syncable { } if (currentfilenum >= 0) { oldFile = computeFilename(currentfilenum); - this.outputfiles.put(Long.valueOf(this.logSeqNum.get() - 1), oldFile); + this.outputfiles.put(Long.valueOf(this.logSeqNum.get()), oldFile); } } return oldFile; diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java index 741db3d..39e4a90 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java @@ -587,6 +587,64 @@ public class TestHLog { assertEquals(COL_COUNT, visitor.increments); } + @Test + public void testLogCleaning() throws Exception { + LOG.info("testLogCleaning"); + final byte [] tableName = Bytes.toBytes("testLogCleaning"); + final byte [] tableName2 = Bytes.toBytes("testLogCleaning2"); + + HLog log = new HLog(fs, dir, oldLogDir, conf, null); + HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + HRegionInfo hri2 = new HRegionInfo(new HTableDescriptor(tableName2), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); + + // Add a single edit and make sure that rolling won't remove the file + // Before HBASE-3198 it used to delete it + addEdits(log, hri, tableName, 1); + log.rollWriter(); + assertEquals(1, log.getNumLogFiles()); + + // See if there's anything wrong with more than 1 edit + addEdits(log, hri, tableName, 2); + log.rollWriter(); + assertEquals(2, log.getNumLogFiles()); + + // Now mix edits from 2 regions, still no flushing + addEdits(log, hri, tableName, 1); + addEdits(log, hri2, tableName2, 1); + addEdits(log, hri, tableName, 1); + addEdits(log, hri2, tableName2, 1); + log.rollWriter(); + assertEquals(3, log.getNumLogFiles()); + + // Flush the first region, we expect to see the first two files getting + // archived + long seqId = log.startCacheFlush(); + log.completeCacheFlush(hri.getRegionName(), tableName, seqId, false); + log.rollWriter(); + assertEquals(2, log.getNumLogFiles()); + + // Flush the second region, which removes all the remaining output files + // since the oldest was completely flushed and the two others only contain + // flush information + seqId = log.startCacheFlush(); + log.completeCacheFlush(hri2.getRegionName(), tableName2, seqId, false); + log.rollWriter(); + assertEquals(0, log.getNumLogFiles()); + } + + private void addEdits(HLog log, HRegionInfo hri, byte [] tableName, + int times) throws IOException { + final byte [] row = Bytes.toBytes("row"); + for (int i = 0; i < times; i++) { + long timestamp = System.currentTimeMillis(); + WALEdit cols = new WALEdit(); + cols.add(new KeyValue(row, row, row, timestamp, row)); + log.append(hri, tableName, cols, timestamp); + } + } + static class DumbLogEntriesVisitor implements LogEntryVisitor { int increments = 0;