From 5c4917e6f32bb750b1ccf510be23c5fd6209e534 Mon Sep 17 00:00:00 2001 From: Guanghao Zhang Date: Thu, 10 Jan 2019 10:22:36 +0800 Subject: [PATCH] HBASE-21691 Fix flaky test TestRecoveredEdits --- .../hbase/regionserver/TestRecoveredEdits.java | 67 +++++++++++++--------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java index 17bc01d..11f8f59 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java @@ -21,13 +21,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparatorImpl; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -38,8 +39,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -168,46 +168,61 @@ public class TestRecoveredEdits { * @throws IOException */ private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf, - final Path edits, final HRegion region) - throws IOException { + final Path edits, final HRegion region) throws IOException { int count = 0; - // Based on HRegion#replayRecoveredEdits - WAL.Reader reader = null; - try { - reader = WALFactory.createReader(fs, edits, conf); + // Read all cells from recover edits + List walCells = new ArrayList<>(); + try (WAL.Reader reader = WALFactory.createReader(fs, edits, conf)) { WAL.Entry entry; while ((entry = reader.next()) != null) { WALKey key = entry.getKey(); WALEdit val = entry.getEdit(); count++; // Check this edit is for this region. - if (!Bytes.equals(key.getEncodedRegionName(), - region.getRegionInfo().getEncodedNameAsBytes())) { + if (!Bytes + .equals(key.getEncodedRegionName(), region.getRegionInfo().getEncodedNameAsBytes())) { continue; } Cell previous = null; - for (Cell cell: val.getCells()) { + for (Cell cell : val.getCells()) { if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue; if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) continue; previous = cell; - Get g = new Get(CellUtil.cloneRow(cell)); - Result r = region.get(g); - boolean found = false; - for (CellScanner scanner = r.cellScanner(); scanner.advance();) { - Cell current = scanner.current(); - if (PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, cell, - current) == 0) { - found = true; - break; - } - } - assertTrue("Failed to find " + cell, found); + walCells.add(cell); } } - } finally { - if (reader != null) reader.close(); } + + // Read all cells from region + List regionCells = new ArrayList<>(); + try (RegionScanner scanner = region.getScanner(new Scan())) { + List tmpCells; + do { + tmpCells = new ArrayList<>(); + scanner.nextRaw(tmpCells); + regionCells.addAll(tmpCells); + } while (!tmpCells.isEmpty()); + } + + Collections.sort(walCells, CellComparatorImpl.COMPARATOR); + int found = 0; + for (int i = 0, j = 0; i < walCells.size() && j < regionCells.size(); ) { + int compareResult = PrivateCellUtil + .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, walCells.get(i), + regionCells.get(j)); + if (compareResult == 0) { + i++; + j++; + found++; + } else if (compareResult > 0) { + j++; + } else { + i++; + } + } + assertEquals("Only found " + found + " cells in region, but there are " + walCells.size() + + " cells in recover edits", found, walCells.size()); return count; } } -- 2.7.4