From b9698254b226fcf1c0ae74caa21f3d637e4e4bfb Mon Sep 17 00:00:00 2001 From: eshcar Date: Sun, 28 Feb 2016 22:44:04 +0200 Subject: [PATCH] simplifying Segment hierarchy --- .../hbase/regionserver/AbstractMemStore.java | 34 +-- .../hadoop/hbase/regionserver/DefaultMemStore.java | 14 -- .../hbase/regionserver/ImmutableSegment.java | 39 +-- .../regionserver/ImmutableSegmentAdapter.java | 107 -------- .../hbase/regionserver/MutableCellSetSegment.java | 153 ------------ .../regionserver/MutableCellSetSegmentScanner.java | 258 -------------------- .../hadoop/hbase/regionserver/MutableSegment.java | 39 +-- .../apache/hadoop/hbase/regionserver/Segment.java | 118 +++++++-- .../hadoop/hbase/regionserver/SegmentFactory.java | 9 +- .../hadoop/hbase/regionserver/SegmentScanner.java | 230 ++++++++++++++++- 10 files changed, 371 insertions(+), 630 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 18d2f8a..08e3f5e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -226,6 +226,23 @@ public abstract class AbstractMemStore implements MemStore { return getSnapshot().getSize(); } + /** + * Remove n key from the memstore. Only cells that have the same key and the + * same memstoreTS are removed. It is ok to not update timeRangeTracker + * in this call. It is possible that we can optimize this method by using + * tailMap/iterator, but since this method is called rarely (only for + * error recovery), we can leave those optimization for the future. + * @param cell + */ + @Override + public void rollback(Cell cell) { + // If the key is in the active, delete it. Update this.size. + long sz = active.rollback(cell); + if (sz != 0) { + setOldestEditTimeToNow(); + } + } + @Override public String toString() { StringBuffer buf = new StringBuffer(); @@ -241,23 +258,6 @@ public abstract class AbstractMemStore implements MemStore { return buf.toString(); } - protected void rollbackInSnapshot(Cell cell) { - // If the key is in the snapshot, delete it. We should not update - // this.size, because that tracks the size of only the memstore and - // not the snapshot. The flush of this snapshot to disk has not - // yet started because Store.flush() waits for all rwcc transactions to - // commit before starting the flush to disk. - snapshot.rollback(cell); - } - - protected void rollbackInActive(Cell cell) { - // If the key is in the memstore, delete it. Update this.size. - long sz = active.rollback(cell); - if (sz != 0) { - setOldestEditTimeToNow(); - } - } - protected Configuration getConfiguration() { return conf; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 92bb7b6..1bb9343 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -118,20 +118,6 @@ public class DefaultMemStore extends AbstractMemStore { } /** - * Remove n key from the memstore. Only cells that have the same key and the - * same memstoreTS are removed. It is ok to not update timeRangeTracker - * in this call. It is possible that we can optimize this method by using - * tailMap/iterator, but since this method is called rarely (only for - * error recovery), we can leave those optimization for the future. - * @param cell - */ - @Override - public void rollback(Cell cell) { - rollbackInSnapshot(cell); - rollbackInActive(cell); - } - - /** * @param cell Find the row that comes after this one. If null, we return the * first. * @return Next row or null if none found. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index cfcd81e..86f77da 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -18,55 +18,30 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.CollectionBackedScanner; /** * ImmutableSegment is an abstract class that extends the API supported by a {@link Segment}, * and is not needed for a {@link MutableSegment}. Specifically, the method * {@link ImmutableSegment#getKeyValueScanner()} builds a special scanner for the * {@link MemStoreSnapshot} object. - * In addition, this class overrides methods that are not likely to be supported by an immutable - * segment, e.g. {@link Segment#rollback(Cell)} and {@link Segment#getCellSet()}, which - * can be very inefficient. */ @InterfaceAudience.Private -public abstract class ImmutableSegment extends Segment { +public class ImmutableSegment extends Segment { - public ImmutableSegment(Segment segment) { + protected ImmutableSegment(Segment segment) { super(segment); } /** - * Removes the given cell from this segment. - * By default immutable store segment can not rollback - * It may be invoked by tests in specific cases where it is known to be supported {@link - * ImmutableSegmentAdapter} - */ - @Override - public long rollback(Cell cell) { - return 0; - } - - /** - * Returns a set of all the cells in the segment. - * The implementation of this method might be very inefficient for some immutable segments - * that do not maintain a cell set. Therefore by default this method is not supported. - * It may be invoked by tests in specific cases where it is known to be supported {@link - * ImmutableSegmentAdapter} - */ - @Override - public CellSet getCellSet() { - throw new NotImplementedException("Immutable Segment does not support this operation by " + - "default"); - } - - /** - * Builds a special scanner for the MemStoreSnapshot object that may be different than the + * Builds a special scanner for the MemStoreSnapshot object that is different than the * general segment scanner. * @return a special scanner for the MemStoreSnapshot object */ - public abstract KeyValueScanner getKeyValueScanner(); + public KeyValueScanner getKeyValueScanner() { + return new CollectionBackedScanner(getCellSet(), getComparator()); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java deleted file mode 100644 index 058865a..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegmentAdapter.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.commons.logging.Log; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.CollectionBackedScanner; - -/** - * This segment is adapting a mutable segment making it into an immutable segment. - * This is used when a mutable segment is moved to being a snapshot or pushed into a compaction - * pipeline, that consists only of immutable segments. - * The compaction may generate different type of immutable segment - */ -@InterfaceAudience.Private -public class ImmutableSegmentAdapter extends ImmutableSegment { - - final private MutableSegment adaptee; - - public ImmutableSegmentAdapter(MutableSegment segment) { - super(segment); - this.adaptee = segment; - } - - @Override - public KeyValueScanner getKeyValueScanner() { - return new CollectionBackedScanner(adaptee.getCellSet(), adaptee.getComparator()); - } - - @Override - public SegmentScanner getSegmentScanner(long readPoint) { - return adaptee.getSegmentScanner(readPoint); - } - - @Override - public boolean isEmpty() { - return adaptee.isEmpty(); - } - - @Override - public int getCellsCount() { - return adaptee.getCellsCount(); - } - - @Override - public long add(Cell cell) { - return adaptee.add(cell); - } - - @Override - public Cell getFirstAfter(Cell cell) { - return adaptee.getFirstAfter(cell); - } - - @Override - public void close() { - adaptee.close(); - } - - @Override - public Cell maybeCloneWithAllocator(Cell cell) { - return adaptee.maybeCloneWithAllocator(cell); - } - - @Override - public Segment setSize(long size) { - adaptee.setSize(size); - return this; - } - - @Override - public long getSize() { - return adaptee.getSize(); - } - - @Override - public long rollback(Cell cell) { - return adaptee.rollback(cell); - } - - @Override - public CellSet getCellSet() { - return adaptee.getCellSet(); - } - - @Override - public void dump(Log log) { - adaptee.dump(log); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java deleted file mode 100644 index 743416c..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegment.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.util.Iterator; -import java.util.SortedSet; - -import org.apache.commons.logging.Log; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * This mutable store segment encapsulates a mutable cell set and its respective memory allocation - * buffers (MSLAB). - */ -@InterfaceAudience.Private -final class MutableCellSetSegment extends MutableSegment { - - private volatile CellSet cellSet; - private final CellComparator comparator; - - // Instantiate objects only using factory - MutableCellSetSegment(CellSet cellSet, MemStoreLAB memStoreLAB, long size, - CellComparator comparator) { - super(memStoreLAB, size); - this.cellSet = cellSet; - this.comparator = comparator; - } - - @Override - public SegmentScanner getSegmentScanner(long readPoint) { - return new MutableCellSetSegmentScanner(this, readPoint); - } - - @Override - public boolean isEmpty() { - return getCellSet().isEmpty(); - } - - @Override - public int getCellsCount() { - return getCellSet().size(); - } - - @Override - public long add(Cell cell) { - boolean succ = getCellSet().add(cell); - long s = AbstractMemStore.heapSizeChange(cell, succ); - updateMetaInfo(cell, s); - // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. - // When we use ACL CP or Visibility CP which deals with Tags during - // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not - // parse the byte[] to identify the tags length. - if(cell.getTagsLength() > 0) { - tagsPresent = true; - } - return s; - } - - @Override - public long rollback(Cell cell) { - Cell found = get(cell); - if (found != null && found.getSequenceId() == cell.getSequenceId()) { - long sz = AbstractMemStore.heapSizeChange(cell, true); - remove(cell); - incSize(-sz); - return sz; - } - return 0; - } - - @Override - public Cell getFirstAfter(Cell cell) { - SortedSet snTailSet = tailSet(cell); - if (!snTailSet.isEmpty()) { - return snTailSet.first(); - } - return null; - } - - @Override - public void dump(Log log) { - for (Cell cell: getCellSet()) { - log.debug(cell); - } - } - - @Override - public SortedSet tailSet(Cell firstCell) { - return getCellSet().tailSet(firstCell); - } - @Override - public CellSet getCellSet() { - return cellSet; - } - @Override - public CellComparator getComparator() { - return comparator; - } - - //*** Methods for MemStoreSegmentsScanner - public Cell last() { - return getCellSet().last(); - } - - public Iterator iterator() { - return getCellSet().iterator(); - } - - public SortedSet headSet(Cell firstKeyOnRow) { - return getCellSet().headSet(firstKeyOnRow); - } - - public int compare(Cell left, Cell right) { - return getComparator().compare(left, right); - } - - public int compareRows(Cell left, Cell right) { - return getComparator().compareRows(left, right); - } - - private Cell get(Cell cell) { - return getCellSet().get(cell); - } - - private boolean remove(Cell e) { - return getCellSet().remove(e); - } - - // methods for tests - @Override - Cell first() { - return this.getCellSet().first(); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java deleted file mode 100644 index 17791ff..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableCellSetSegmentScanner.java +++ /dev/null @@ -1,258 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.io.IOException; -import java.util.Iterator; -import java.util.SortedSet; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * A scanner of a single cells segment {@link MutableCellSetSegment}. - */ -@InterfaceAudience.Private -class MutableCellSetSegmentScanner extends SegmentScanner { - - // the observed structure - private final MutableCellSetSegment segment; - // the highest relevant MVCC - private long readPoint; - // the current iterator that can be reinitialized by - // seek(), backwardSeek(), or reseek() - private Iterator iter; - // the pre-calculated cell to be returned by peek() - private Cell current = null; - // or next() - // A flag represents whether could stop skipping KeyValues for MVCC - // if have encountered the next row. Only used for reversed scan - private boolean stopSkippingKVsIfNextRow = false; - // last iterated KVs by seek (to restore the iterator state after reseek) - private Cell last = null; - - public MutableCellSetSegmentScanner(MutableCellSetSegment segment, long readPoint) { - super(); - this.segment = segment; - this.readPoint = readPoint; - iter = segment.iterator(); - // the initialization of the current is required for working with heap of SegmentScanners - current = getNext(); - //increase the reference count so the underlying structure will not be de-allocated - this.segment.incScannerCount(); - } - - /** - * Look at the next Cell in this scanner, but do not iterate the scanner - * @return the currently observed Cell - */ - @Override - public Cell peek() { // sanity check, the current should be always valid - if (current!=null && current.getSequenceId() > readPoint) { - throw new RuntimeException("current is invalid: read point is "+readPoint+", " + - "while current sequence id is " +current.getSequenceId()); - } - - return current; - } - - /** - * Return the next Cell in this scanner, iterating the scanner - * @return the next Cell or null if end of scanner - */ - @Override - public Cell next() throws IOException { - Cell oldCurrent = current; - current = getNext(); // update the currently observed Cell - return oldCurrent; - } - - /** - * Seek the scanner at or after the specified Cell. - * @param cell seek value - * @return true if scanner has values left, false if end of scanner - */ - @Override - public boolean seek(Cell cell) throws IOException { - if(cell == null) { - close(); - return false; - } - // restart the iterator from new key - iter = segment.tailSet(cell).iterator(); - // last is going to be reinitialized in the next getNext() call - last = null; - current = getNext(); - return (current != null); - } - - /** - * Reseek the scanner at or after the specified KeyValue. - * This method is guaranteed to seek at or after the required key only if the - * key comes after the current position of the scanner. Should not be used - * to seek to a key which may come before the current position. - * - * @param cell seek value (should be non-null) - * @return true if scanner has values left, false if end of scanner - */ - @Override - public boolean reseek(Cell cell) throws IOException { - - /* - See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. - This code is executed concurrently with flush and puts, without locks. - The ideal implementation for performance would use the sub skip list implicitly - pointed by the iterator. Unfortunately the Java API does not offer a method to - get it. So we remember the last keys we iterated to and restore - the reseeked set to at least that point. - */ - iter = segment.tailSet(getHighest(cell, last)).iterator(); - current = getNext(); - return (current != null); - } - - /** - * Seek the scanner at or before the row of specified Cell, it firstly - * tries to seek the scanner at or after the specified Cell, return if - * peek KeyValue of scanner has the same row with specified Cell, - * otherwise seek the scanner at the first Cell of the row which is the - * previous row of specified KeyValue - * - * @param key seek Cell - * @return true if the scanner is at the valid KeyValue, false if such Cell does not exist - */ - @Override - public boolean backwardSeek(Cell key) throws IOException { - seek(key); // seek forward then go backward - if (peek() == null || segment.compareRows(peek(), key) > 0) { - return seekToPreviousRow(key); - } - return true; - } - - /** - * Seek the scanner at the first Cell of the row which is the previous row - * of specified key - * - * @param cell seek value - * @return true if the scanner at the first valid Cell of previous row, - * false if not existing such Cell - */ - @Override - public boolean seekToPreviousRow(Cell cell) throws IOException { - boolean keepSeeking = false; - Cell key = cell; - - do { - Cell firstKeyOnRow = CellUtil.createFirstOnRow(key); - SortedSet cellHead = segment.headSet(firstKeyOnRow); - Cell lastCellBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); - if (lastCellBeforeRow == null) { - current = null; - return false; - } - Cell firstKeyOnPreviousRow = CellUtil.createFirstOnRow(lastCellBeforeRow); - this.stopSkippingKVsIfNextRow = true; - seek(firstKeyOnPreviousRow); - this.stopSkippingKVsIfNextRow = false; - if (peek() == null - || segment.getComparator().compareRows(peek(), firstKeyOnPreviousRow) > 0) { - keepSeeking = true; - key = firstKeyOnPreviousRow; - continue; - } else { - keepSeeking = false; - } - } while (keepSeeking); - return true; - } - - /** - * Seek the scanner at the first KeyValue of last row - * - * @return true if scanner has values left, false if the underlying data is empty - */ - @Override - public boolean seekToLastRow() throws IOException { - Cell higherCell = segment.isEmpty() ? null : segment.last(); - if (higherCell == null) { - return false; - } - - Cell firstCellOnLastRow = CellUtil.createFirstOnRow(higherCell); - - if (seek(firstCellOnLastRow)) { - return true; - } else { - return seekToPreviousRow(higherCell); - } - } - - @Override protected Segment getSegment() { - return segment; - } - - /********************* Private Methods **********************/ - - /** - * Private internal method for iterating over the segment, - * skipping the cells with irrelevant MVCC - */ - private Cell getNext() { - Cell startKV = current; - Cell next = null; - - try { - while (iter.hasNext()) { - next = iter.next(); - if (next.getSequenceId() <= this.readPoint) { - return next; // skip irrelevant versions - } - if (stopSkippingKVsIfNextRow && // for backwardSeek() stay in the - startKV != null && // boundaries of a single row - segment.compareRows(next, startKV) > 0) { - return null; - } - } // end of while - - return null; // nothing found - } finally { - if (next != null) { - // in all cases, remember the last KV we iterated to, needed for reseek() - last = next; - } - } - } - - /** - * Private internal method that returns the higher of the two key values, or null - * if they are both null - */ - private Cell getHighest(Cell first, Cell second) { - if (first == null && second == null) { - return null; - } - if (first != null && second != null) { - int compare = segment.compare(first, second); - return (compare > 0 ? first : second); - } - return (first != null ? first : second); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index fcaddb0..aef70e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -18,34 +18,43 @@ */ package org.apache.hadoop.hbase.regionserver; -import java.util.SortedSet; - import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; /** - * An abstraction of a mutable segment in memstore, specifically the active segment. + * A mutable segment in memstore, specifically the active segment. */ @InterfaceAudience.Private -public abstract class MutableSegment extends Segment { +public class MutableSegment extends Segment { - protected MutableSegment(MemStoreLAB memStoreLAB, long size) { - super(memStoreLAB, size); + protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, + long size) { + super(cellSet, comparator, memStoreLAB, size); } /** - * Returns a subset of the segment cell set, which starts with the given cell - * @param firstCell a cell in the segment - * @return a subset of the segment cell set, which starts with the given cell + * Adds the given cell into the segment + * @return the change in the heap size */ - public abstract SortedSet tailSet(Cell firstCell); + public long add(Cell cell) { + return internalAdd(cell); + } /** - * Returns the Cell comparator used by this segment - * @return the Cell comparator used by this segment + * Removes the given cell from the segment + * @return the change in the heap size */ - public abstract CellComparator getComparator(); + public long rollback(Cell cell) { + Cell found = getCellSet().get(cell); + if (found != null && found.getSequenceId() == cell.getSequenceId()) { + long sz = AbstractMemStore.heapSizeChange(cell, true); + getCellSet().remove(cell); + incSize(-sz); + return sz; + } + return 0; + } //methods for test @@ -53,5 +62,7 @@ public abstract class MutableSegment extends Segment { * Returns the first cell in the segment * @return the first cell in the segment */ - abstract Cell first(); + Cell first() { + return this.getCellSet().first(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 7891809..77cf02a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -18,10 +18,13 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.util.Iterator; +import java.util.SortedSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -38,12 +41,17 @@ import org.apache.hadoop.hbase.util.ByteRange; @InterfaceAudience.Private public abstract class Segment { + private volatile CellSet cellSet; + private final CellComparator comparator; private volatile MemStoreLAB memStoreLAB; private final AtomicLong size; private final TimeRangeTracker timeRangeTracker; protected volatile boolean tagsPresent; - protected Segment(MemStoreLAB memStoreLAB, long size) { + protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long + size) { + this.cellSet = cellSet; + this.comparator = comparator; this.memStoreLAB = memStoreLAB; this.size = new AtomicLong(size); this.timeRangeTracker = new TimeRangeTracker(); @@ -51,6 +59,8 @@ public abstract class Segment { } protected Segment(Segment segment) { + this.cellSet = segment.getCellSet(); + this.comparator = segment.getComparator(); this.memStoreLAB = segment.getMemStoreLAB(); this.size = new AtomicLong(segment.getSize()); this.timeRangeTracker = segment.getTimeRangeTracker(); @@ -58,46 +68,40 @@ public abstract class Segment { } /** - * Creates the scanner that is able to scan the concrete segment + * Creates the scanner for the given read point * @return a scanner for the given read point */ - public abstract SegmentScanner getSegmentScanner(long readPoint); + public SegmentScanner getSegmentScanner(long readPoint) { + return new SegmentScanner(this, readPoint); + } /** * Returns whether the segment has any cells * @return whether the segment has any cells */ - public abstract boolean isEmpty(); + public boolean isEmpty() { + return getCellSet().isEmpty(); + } /** * Returns number of cells in segment * @return number of cells in segment */ - public abstract int getCellsCount(); - - /** - * Adds the given cell into the segment - * @return the change in the heap size - */ - public abstract long add(Cell cell); - - /** - * Removes the given cell from the segment - * @return the change in the heap size - */ - public abstract long rollback(Cell cell); + public int getCellsCount() { + return getCellSet().size(); + } /** * Returns the first cell in the segment that has equal or greater key than the given cell * @return the first cell in the segment that has equal or greater key than the given cell */ - public abstract Cell getFirstAfter(Cell cell); - - /** - * Returns a set of all cells in the segment - * @return a set of all cells in the segment - */ - public abstract CellSet getCellSet(); + public Cell getFirstAfter(Cell cell) { + SortedSet snTailSet = tailSet(cell); + if (!snTailSet.isEmpty()) { + return snTailSet.first(); + } + return null; + } /** * Closing a segment before it is being discarded @@ -190,11 +194,71 @@ public abstract class Segment { return timeRangeTracker; } + //*** Methods for SegmentsScanner + public Cell last() { + return getCellSet().last(); + } + + public Iterator iterator() { + return getCellSet().iterator(); + } + + public SortedSet headSet(Cell firstKeyOnRow) { + return getCellSet().headSet(firstKeyOnRow); + } + + public int compare(Cell left, Cell right) { + return getComparator().compare(left, right); + } + + public int compareRows(Cell left, Cell right) { + return getComparator().compareRows(left, right); + } + + /** + * Returns a set of all cells in the segment + * @return a set of all cells in the segment + */ + protected CellSet getCellSet() { + return cellSet; + } + + /** + * Returns the Cell comparator used by this segment + * @return the Cell comparator used by this segment + */ + protected CellComparator getComparator() { + return comparator; + } + + protected long internalAdd(Cell cell) { + boolean succ = getCellSet().add(cell); + long s = AbstractMemStore.heapSizeChange(cell, succ); + updateMetaInfo(cell, s); + // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. + // When we use ACL CP or Visibility CP which deals with Tags during + // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not + // parse the byte[] to identify the tags length. + if(cell.getTagsLength() > 0) { + tagsPresent = true; + } + return s; + } + protected void updateMetaInfo(Cell toAdd, long s) { getTimeRangeTracker().includeTimestamp(toAdd); size.addAndGet(s); } + /** + * Returns a subset of the segment cell set, which starts with the given cell + * @param firstCell a cell in the segment + * @return a subset of the segment cell set, which starts with the given cell + */ + protected SortedSet tailSet(Cell firstCell) { + return getCellSet().tailSet(firstCell); + } + private MemStoreLAB getMemStoreLAB() { return memStoreLAB; } @@ -203,7 +267,11 @@ public abstract class Segment { /** * Dumps all cells of the segment into the given log */ - public abstract void dump(Log log); + void dump(Log log) { + for (Cell cell: getCellSet()) { + log.debug(cell); + } + } @Override public String toString() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java index ccb11df..394ffa1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java @@ -54,7 +54,7 @@ public final class SegmentFactory { } public ImmutableSegment createImmutableSegment(final Configuration conf, MutableSegment segment) { - return generateImmutableSegment(conf, segment); + return new ImmutableSegment(segment); } public MutableSegment createMutableSegment(final Configuration conf, CellComparator comparator, long size) { @@ -64,16 +64,11 @@ public final class SegmentFactory { //****** private methods to instantiate concrete store segments **********// - private ImmutableSegment generateImmutableSegment(final Configuration conf, - MutableSegment segment) { - // TBD use configuration to set type of segment - return new ImmutableSegmentAdapter(segment); - } private MutableSegment generateMutableSegment( final Configuration conf, CellComparator comparator, MemStoreLAB memStoreLAB, long size) { // TBD use configuration to set type of segment CellSet set = new CellSet(comparator); - return new MutableCellSetSegment(set, memStoreLAB, size, comparator); + return new MutableSegment(set, comparator, memStoreLAB, size); } private MemStoreLAB getMemStoreLAB(Configuration conf) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java index 8852d5c..b5aabb8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java @@ -19,20 +19,193 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.util.Iterator; +import java.util.SortedSet; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; /** - * An abstraction for store segment scanner. + * A scanner of a single memstore segment. */ @InterfaceAudience.Private -public abstract class SegmentScanner implements KeyValueScanner { +public class SegmentScanner implements KeyValueScanner { private long sequenceID = Long.MAX_VALUE; - protected abstract Segment getSegment(); + // the observed structure + private final Segment segment; + // the highest relevant MVCC + private long readPoint; + // the current iterator that can be reinitialized by + // seek(), backwardSeek(), or reseek() + private Iterator iter; + // the pre-calculated cell to be returned by peek() + private Cell current = null; + // or next() + // A flag represents whether could stop skipping KeyValues for MVCC + // if have encountered the next row. Only used for reversed scan + private boolean stopSkippingKVsIfNextRow = false; + // last iterated KVs by seek (to restore the iterator state after reseek) + private Cell last = null; + + protected SegmentScanner(Segment segment, long readPoint) { + this.segment = segment; + this.readPoint = readPoint; + iter = segment.iterator(); + // the initialization of the current is required for working with heap of SegmentScanners + current = getNext(); + //increase the reference count so the underlying structure will not be de-allocated + this.segment.incScannerCount(); + } + + /** + * Look at the next Cell in this scanner, but do not iterate the scanner + * @return the currently observed Cell + */ + @Override + public Cell peek() { // sanity check, the current should be always valid + if (current!=null && current.getSequenceId() > readPoint) { + throw new RuntimeException("current is invalid: read point is "+readPoint+", " + + "while current sequence id is " +current.getSequenceId()); + } + + return current; + } + + /** + * Return the next Cell in this scanner, iterating the scanner + * @return the next Cell or null if end of scanner + */ + @Override + public Cell next() throws IOException { + Cell oldCurrent = current; + current = getNext(); // update the currently observed Cell + return oldCurrent; + } + + /** + * Seek the scanner at or after the specified Cell. + * @param cell seek value + * @return true if scanner has values left, false if end of scanner + */ + @Override + public boolean seek(Cell cell) throws IOException { + if(cell == null) { + close(); + return false; + } + // restart the iterator from new key + iter = segment.tailSet(cell).iterator(); + // last is going to be reinitialized in the next getNext() call + last = null; + current = getNext(); + return (current != null); + } + + /** + * Reseek the scanner at or after the specified KeyValue. + * This method is guaranteed to seek at or after the required key only if the + * key comes after the current position of the scanner. Should not be used + * to seek to a key which may come before the current position. + * + * @param cell seek value (should be non-null) + * @return true if scanner has values left, false if end of scanner + */ + @Override + public boolean reseek(Cell cell) throws IOException { + + /* + See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. + This code is executed concurrently with flush and puts, without locks. + The ideal implementation for performance would use the sub skip list implicitly + pointed by the iterator. Unfortunately the Java API does not offer a method to + get it. So we remember the last keys we iterated to and restore + the reseeked set to at least that point. + */ + iter = segment.tailSet(getHighest(cell, last)).iterator(); + current = getNext(); + return (current != null); + } + + /** + * Seek the scanner at or before the row of specified Cell, it firstly + * tries to seek the scanner at or after the specified Cell, return if + * peek KeyValue of scanner has the same row with specified Cell, + * otherwise seek the scanner at the first Cell of the row which is the + * previous row of specified KeyValue + * + * @param key seek Cell + * @return true if the scanner is at the valid KeyValue, false if such Cell does not exist + */ + @Override + public boolean backwardSeek(Cell key) throws IOException { + seek(key); // seek forward then go backward + if (peek() == null || segment.compareRows(peek(), key) > 0) { + return seekToPreviousRow(key); + } + return true; + } + + /** + * Seek the scanner at the first Cell of the row which is the previous row + * of specified key + * + * @param cell seek value + * @return true if the scanner at the first valid Cell of previous row, + * false if not existing such Cell + */ + @Override + public boolean seekToPreviousRow(Cell cell) throws IOException { + boolean keepSeeking = false; + Cell key = cell; + + do { + Cell firstKeyOnRow = CellUtil.createFirstOnRow(key); + SortedSet cellHead = segment.headSet(firstKeyOnRow); + Cell lastCellBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); + if (lastCellBeforeRow == null) { + current = null; + return false; + } + Cell firstKeyOnPreviousRow = CellUtil.createFirstOnRow(lastCellBeforeRow); + this.stopSkippingKVsIfNextRow = true; + seek(firstKeyOnPreviousRow); + this.stopSkippingKVsIfNextRow = false; + if (peek() == null + || segment.getComparator().compareRows(peek(), firstKeyOnPreviousRow) > 0) { + keepSeeking = true; + key = firstKeyOnPreviousRow; + continue; + } else { + keepSeeking = false; + } + } while (keepSeeking); + return true; + } + + /** + * Seek the scanner at the first KeyValue of last row + * + * @return true if scanner has values left, false if the underlying data is empty + */ + @Override + public boolean seekToLastRow() throws IOException { + Cell higherCell = segment.isEmpty() ? null : segment.last(); + if (higherCell == null) { + return false; + } + + Cell firstCellOnLastRow = CellUtil.createFirstOnRow(higherCell); + + if (seek(firstCellOnLastRow)) { + return true; + } else { + return seekToPreviousRow(higherCell); + } + } /** * Get the sequence id associated with this KeyValueScanner. This is required @@ -140,6 +313,10 @@ public abstract class SegmentScanner implements KeyValueScanner { return getSegment().shouldSeek(scan,oldestUnexpiredTS); } + protected Segment getSegment(){ + return segment; + } + //debug method @Override public String toString() { @@ -149,4 +326,51 @@ public abstract class SegmentScanner implements KeyValueScanner { return res; } + /********************* Private Methods **********************/ + + /** + * Private internal method for iterating over the segment, + * skipping the cells with irrelevant MVCC + */ + private Cell getNext() { + Cell startKV = current; + Cell next = null; + + try { + while (iter.hasNext()) { + next = iter.next(); + if (next.getSequenceId() <= this.readPoint) { + return next; // skip irrelevant versions + } + if (stopSkippingKVsIfNextRow && // for backwardSeek() stay in the + startKV != null && // boundaries of a single row + segment.compareRows(next, startKV) > 0) { + return null; + } + } // end of while + + return null; // nothing found + } finally { + if (next != null) { + // in all cases, remember the last KV we iterated to, needed for reseek() + last = next; + } + } + } + + /** + * Private internal method that returns the higher of the two key values, or null + * if they are both null + */ + private Cell getHighest(Cell first, Cell second) { + if (first == null && second == null) { + return null; + } + if (first != null && second != null) { + int compare = segment.compare(first, second); + return (compare > 0 ? first : second); + } + return (first != null ? first : second); + } + } -- 1.7.10.2 (Apple Git-33)