From 360b910cc7927835ac42874ce00256ed258a9bc9 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Sat, 23 Jun 2018 23:02:17 -0700 Subject: [PATCH] HBASE-20781 Save recalculating families in a WALEdit batch of Cells Pass the Set of families through to the WAL rather than recalculate a Set already known. --- .../apache/hadoop/hbase/regionserver/HRegion.java | 15 ++++++------ .../hadoop/hbase/regionserver/wal/FSWALEntry.java | 13 ++++++---- .../java/org/apache/hadoop/hbase/wal/WALEdit.java | 28 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index d71e988b53..fd05d2eaf8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -3416,18 +3416,19 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * @param familyMap map of family->edits * @param walEdit the destination entry to append into */ - private void addFamilyMapToWALEdit(Map> familyMap, - WALEdit walEdit) { - for (List edits : familyMap.values()) { + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { + for (Map.Entry> e: familyMap.entrySet()) { // Optimization: 'foreach' loop is not used. See: // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects - assert edits instanceof RandomAccess; - int listSize = edits.size(); - for (int i=0; i < listSize; i++) { - Cell cell = edits.get(i); + int listSize = e.getValue().size(); + for (int i = 0; i < listSize; i++) { + Cell cell = e.getValue().get(i); walEdit.add(cell); } } + // Add all families mentioned by Cells as a Set. Needed later in processing pipeline. This + // saves our having to recalculate it. + walEdit.setFamilies(familyMap.keySet()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java index ac5d3ed37b..aff686137c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java @@ -63,11 +63,16 @@ class FSWALEntry extends Entry { this.inMemstore = inMemstore; this.regionInfo = regionInfo; this.txid = txid; - if (inMemstore) { - // construct familyNames here to reduce the work of log sinker. - this.familyNames = collectFamilies(edit.getCells()); + Set families = edit.getFamilies(); + if (families != null) { + this.familyNames = families; } else { - this.familyNames = Collections. emptySet(); + if (inMemstore) { + // construct familyNames here to reduce the work of log sinker. + this.familyNames = collectFamilies(edit.getCells()); + } else { + this.familyNames = Collections.emptySet(); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java index 1d4dc1be1d..f31d259479 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALEdit.java @@ -20,6 +20,10 @@ package org.apache.hadoop.hbase.wal; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeSet; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -73,6 +77,9 @@ public class WALEdit implements HeapSize { private ArrayList cells = null; + // All the Cell families in cells or null. + private NavigableSet families = null; + public WALEdit() { this(false); } @@ -90,6 +97,27 @@ public class WALEdit implements HeapSize { cells = new ArrayList<>(cellCount); } + /** + * Make sure you pass ALL families here; all families mentioned in this WALEdit, in + * {@link #getCells()}. + * Idea is to save re-reading the Cells to figure the set of column families; if we know all + * families, pass them in here. If non-null, we'll presume we have the complete Set and will + * NOT try to recalculate. + */ + public void setFamilies(Set families) { + if (this.families == null) { + this.families = new TreeSet(Bytes.BYTES_COMPARATOR); + } + this.families.addAll(families); + } + + /** + * @return All families in {@link #getCells()} + */ + public Set getFamilies() { + return this.families; + } + /** * @param f * @return True is f is {@link #METAFAMILY} -- 2.16.3