From 786da0c21d7d3eeae28b3ea2b5fe87d1794b4d13 Mon Sep 17 00:00:00 2001 From: Sandeep Pal Date: Sun, 17 Nov 2019 16:08:52 -0800 Subject: [PATCH] HBASE-23309: Entries get replicated even though all cells get filtered through WalCellFilter --- .../replication/ChainWALEntryFilter.java | 17 +++++-- .../TestReplicationWALEntryFilters.java | 47 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java index 2bb981119a..036cf53a1c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ChainWALEntryFilter.java @@ -37,6 +37,10 @@ public class ChainWALEntryFilter implements WALEntryFilter { private final WALEntryFilter[] filters; private WALCellFilter[] cellFilters; + // To allow the empty entries to get filtered, we want to this optional flag to decide + // if we want to filter the entries which have no cells or all cells got filtered though WALCellFilter + private boolean filterEmptyEntry = false; + public ChainWALEntryFilter(WALEntryFilter...filters) { this.filters = filters; initCellFilters(); @@ -68,13 +72,16 @@ public class ChainWALEntryFilter implements WALEntryFilter { @Override public Entry filter(Entry entry) { + if (entry == null) { + return null; + } for (WALEntryFilter filter : filters) { - if (entry == null) { - return null; - } entry = filter.filter(entry); } filterCells(entry); + if (filterEmptyEntry && entry.getEdit().isEmpty()) { + return null; + } return entry; } @@ -94,4 +101,8 @@ public class ChainWALEntryFilter implements WALEntryFilter { } return cell; } + + public void setFilterEmptyEntry(boolean filterEmptyEntry) { + this.filterEmptyEntry = filterEmptyEntry; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java index 7faaefb8f3..c98c9e5ff1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWALEntryFilters.java @@ -155,6 +155,53 @@ public class TestReplicationWALEntryFilters { } }; + public static class FilterSomeCellsWALCellFilter implements WALEntryFilter, WALCellFilter { + @Override + public Entry filter(Entry entry) { + return entry; + } + + @Override + public Cell filterCell(Entry entry, Cell cell) { + if (Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()).equals("a")) { + return null; + } else { + return cell; + } + } + } + + public static class FilterAllCellsWALCellFilter implements WALEntryFilter, WALCellFilter { + @Override + public Entry filter(Entry entry) { + return entry; + } + + @Override + public Cell filterCell(Entry entry, Cell cell) { + return null; + } + } + + @Test + public void testChainWALEntryWithCellFilter() { + Entry userEntry = createEntry(null, a, b, c); + ChainWALEntryFilter filterSomeCells = new ChainWALEntryFilter(new FilterSomeCellsWALCellFilter()); + // since WALCellFilter filter cells with rowkey 'a' + assertEquals(createEntry(null, b,c), filterSomeCells.filter(userEntry)); + + Entry userEntry2 = createEntry(null, b, c, d); + // since there is no cell to get filtered, nothing should get filtered + assertEquals(userEntry2, filterSomeCells.filter(userEntry2)); + + ChainWALEntryFilter filterAllCells = new ChainWALEntryFilter(new FilterAllCellsWALCellFilter()); + assertEquals(createEntry(null), filterAllCells.filter(userEntry)); + // let's set the filter empty entry flag to true now for the above case + filterAllCells.setFilterEmptyEntry(true); + // since WALCellFilter filter all cells, whole entry should be filtered + assertEquals(null, filterAllCells.filter(userEntry)); + } + @Test public void testChainWALEntryFilter() { Entry userEntry = createEntry(null, a, b, c); -- 2.23.0