From 067ab9b41c77a63ce982c29895a527624c67c424 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 29 Mar 2018 15:48:29 -0700 Subject: [PATCH] W-4850564 Apply HBASE-20312 (CCSMap: A faster, GC-friendly, less memory Concurrent Map for memstore) Moved some memstore code back to versions on branch-1.1 Applied https://issues.apache.org/jira/secure/attachment/12916795/ccsmap-branch-1.1.patch Reason: Experimental, performance improvement Test Plan: Unit tests, manual verification --- hbase-client/src/test/resources/hbase-site.xml | 4 + .../java/org/apache/hadoop/hbase/KeyValue.java | 110 +- .../org/apache/hadoop/hbase/util/ClassSize.java | 2 +- .../hbase/util/CompactedConcurrentSkipListMap.java | 3308 ++++++++++++++++++++ ...ctedConcurrentSkipListMapPerformanceTester.java | 849 +++++ .../hbase/util/CompactedConcurrentSkipListSet.java | 138 + .../hadoop/hbase/util/CompactedTypeHelper.java | 91 + .../java/org/apache/hadoop/hbase/TestKeyValue.java | 57 + .../hadoop/hbase/regionserver/CellSkipListSet.java | 139 + .../hadoop/hbase/regionserver/DefaultMemStore.java | 460 ++- .../apache/hadoop/hbase/regionserver/HRegion.java | 10 +- .../apache/hadoop/hbase/regionserver/HStore.java | 16 +- .../hadoop/hbase/regionserver/HeapMemStoreLAB.java | 16 +- .../apache/hadoop/hbase/regionserver/MemStore.java | 31 +- .../hbase/regionserver/MemStoreChunkPool.java | 16 +- .../apache/hadoop/hbase/regionserver/Store.java | 23 +- .../org/apache/hadoop/hbase/io/TestHeapSize.java | 6 +- .../hbase/regionserver/TestDefaultMemStore.java | 241 +- .../hadoop/hbase/regionserver/TestHRegion.java | 61 +- .../hbase/regionserver/TestMemStoreChunkPool.java | 24 +- .../hadoop/hbase/regionserver/TestStore.java | 12 +- .../util/TestCompactedConcurrentSkipList.java | 1891 +++++++++++ hbase-server/src/test/resources/hbase-site.xml | 4 + 23 files changed, 7124 insertions(+), 385 deletions(-) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMap.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMapPerformanceTester.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListSet.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedTypeHelper.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompactedConcurrentSkipList.java diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml index 5788238045..147f4f244c 100644 --- a/hbase-client/src/test/resources/hbase-site.xml +++ b/hbase-client/src/test/resources/hbase-site.xml @@ -33,4 +33,8 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.hregion.memstore.ccsmap.enabled + true + diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 193d5b185e..d9d58d65be 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hadoop.hbase.util.CompactedTypeHelper; import org.apache.hadoop.io.RawComparator; import com.google.common.annotations.VisibleForTesting; @@ -1181,7 +1182,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, return "empty"; } return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) + "/vlen=" - + getValueLength() + "/seqid=" + seqId; + + getValueLength() + "/seqid=" + getSequenceId(); } /** @@ -2876,4 +2877,111 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, return super.equals(other); } } + + static public class CompactedCellTypeHelper implements + CompactedTypeHelper { + private KVComparator comparator; + + final static void writeSeqId(byte[] data, int offset, int length, long seqId) { + int pos = offset + lengthWithoutMemstore(length); + data[pos] = (byte) (seqId >> 56); + data[pos + 1] = (byte) (seqId >> 48); + data[pos + 2] = (byte) (seqId >> 40); + data[pos + 3] = (byte) (seqId >> 32); + data[pos + 4] = (byte) (seqId >> 24); + data[pos + 5] = (byte) (seqId >> 16); + data[pos + 6] = (byte) (seqId >> 8); + data[pos + 7] = (byte) (seqId); + } + + final static long readSeqId(byte[] data, int offset, int len) { + int mo = offset + lengthWithoutMemstore(len); + return (data[mo] & 0xFFL) << 56 | (data[mo + 1] & 0xFFL) << 48 + | (data[mo + 2] & 0xFFL) << 40 | (data[mo + 3] & 0xFFL) << 32 + | (data[mo + 4] & 0xFFL) << 24 | (data[mo + 5] & 0xFFL) << 16 + | (data[mo + 6] & 0xFFL) << 8 | (data[mo + 7] & 0xFFL); + } + + + @InterfaceAudience.Private + static class OnPageKeyValue extends KeyValue { + byte[] data; + int offset; + int length; + + public OnPageKeyValue(byte[] data, int offset, int length) { + super(data, offset, lengthWithoutMemstore(length)); + this.data = data; + this.offset = offset; + this.length = length; + } + + @Override + public void setSequenceId(long seqId) { + writeSeqId(data, offset, length, seqId); + } + + @Override + public long getSequenceId() { + return readSeqId(data, offset, length); + } + } + + public CompactedCellTypeHelper(KVComparator c) { + this.comparator = c; + } + + @Override + public int getCompactedSize(Cell key, Cell value) { + long sz = KeyValue.getKeyValueDataStructureSize(key.getRowLength(), + key.getFamilyLength(), key.getQualifierLength(), + key.getValueLength(), key.getTagsLength()) + Bytes.SIZEOF_LONG; // mvcc + if (sz > Integer.MAX_VALUE) { + throw new RuntimeException("Too big cell"); + } + return (int) sz; + } + + @Override + public void compact(Cell key, Cell value, byte[] data, int offset, int len) { + assert key == value; + KeyValue kv = new KeyValue(key); + assert len == kv.getLength() + Bytes.SIZEOF_LONG; + System.arraycopy(kv.bytes, kv.offset, data, offset, kv.length); + writeSeqId(data, offset, len, key.getSequenceId()); + } + + final static int lengthWithoutMemstore(int len) { + return len - Bytes.SIZEOF_LONG; + } + + @Override + public org.apache.hadoop.hbase.util.CompactedTypeHelper.KVPair decomposte( + byte[] data, int offset, int len) { + KeyValue kv = new OnPageKeyValue(data, offset, len); + return new KVPair(kv, kv); + } + + @Override + public int compare(byte[] ldata, int loffset, int llen, byte[] rdata, + int roffset, int rlen) { + CompactedTypeHelper.KVPair lc = this.decomposte(ldata, + loffset, llen); + CompactedTypeHelper.KVPair rc = this.decomposte(rdata, + roffset, rlen); + return this.comparator.compare(lc.key, rc.key); + } + + @Override + public int compare(Cell key, byte[] data, int offset, int len) { + CompactedTypeHelper.KVPair rc = this.decomposte(data, offset, + len); + return this.comparator.compare(key, rc.key); + } + + @Override + public int compare(Cell lkey, Cell rkey) { + return comparator.compare(lkey, rkey); + } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index b27eac9683..fa0778228e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -178,7 +178,7 @@ public class ClassSize { TIMERANGE_TRACKER = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2); - CELL_SKIPLIST_SET = align(OBJECT + REFERENCE); + CELL_SKIPLIST_SET = align(OBJECT + 3 * REFERENCE + Bytes.SIZEOF_BOOLEAN); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMap.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMap.java new file mode 100644 index 0000000000..c03e7f386f --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMap.java @@ -0,0 +1,3308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.Serializable; +import java.lang.reflect.Field; +import java.security.InvalidParameterException; +import java.util.AbstractCollection; +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.NoSuchElementException; +import java.util.Random; +import java.util.Set; +import java.util.SortedSet; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.CompactedTypeHelper.KVPair; + +import sun.misc.Unsafe; + + +@SuppressWarnings("restriction") +@InterfaceAudience.Private +public class CompactedConcurrentSkipListMap extends AbstractMap + implements ConcurrentNavigableMap, Cloneable, Serializable { + + public static class PutAndFetch { + private V1 old; + private V1 cur; + + PutAndFetch(final V1 old, final V1 cur) { + this.old = old; + this.cur = cur; + } + + public V1 old() { + return old; + } + + public V1 current() { + return cur; + } + } + + private static class Utils { + static Unsafe getUnsafe() { + Unsafe unsafe = null; + try { + Class clazz = Unsafe.class; + Field f; + f = clazz.getDeclaredField("theUnsafe"); + + f.setAccessible(true); + unsafe = (Unsafe) f.get(clazz); + } catch (Exception e) { + throw new RuntimeException("Initialize Unsafe instance failed"); + } + return unsafe; + } + + final static Unsafe UNSAFE = Utils.getUnsafe(); + + final static private int BASE = getUnsafe().arrayBaseOffset(byte[].class); + + final static private int OBJECT_BASE = getUnsafe().arrayBaseOffset(Object[].class); + final static private int OBJECT_SCALE = getUnsafe().arrayIndexScale(Object[].class); + + final static void clear(Unsafe unsafe, byte[] data, int localoffset, int len) { + Arrays.fill(data, localoffset, localoffset + len, (byte)0); + } + + final static void putLongVolatile(Unsafe unsafe, byte[] data, int localoffset, + long value) { + assert localoffset < data.length; + unsafe.putLongVolatile(data, BASE + localoffset, value); + } + + final static boolean compareAndSetLong(Unsafe unsafe, byte[] data, int localoffset, + long expect, long update) { + assert localoffset < data.length; + return unsafe.compareAndSwapLong(data, BASE + localoffset, expect, update); + } + + final static boolean compareAndSetObject(Unsafe unsafe, Object[] objects, int offset, Object expect, Object update) { + return unsafe.compareAndSwapObject(objects, OBJECT_BASE + offset * OBJECT_SCALE, expect, update); + } + + final static long getLongVolotile(Unsafe unsafe, byte[] data, int localoffset) { + assert localoffset < data.length; + return unsafe.getLongVolatile(data, BASE + localoffset); + } + + final static Object getObjectVolatile(Unsafe unsafe, Object[] objects, int offset) { + return unsafe.getObjectVolatile(objects, OBJECT_BASE + offset * OBJECT_SCALE); + } + + @SuppressWarnings("deprecation") + final static long getLong(Unsafe unsafe, byte[] data, int localoffset) { + assert localoffset < data.length; + return unsafe.getLong(data, BASE + localoffset); + } + + final static int getShiftFromX(int x) { + if ((x & ~x) != 0) { + throw new IllegalArgumentException("Page size must be power of 2"); + } + int ret = 0; + while ((x >>= 1) > 0) ret++; + return ret; + } + } + + @InterfaceAudience.Private + static public interface PageAllocator { + public abstract Object[] allocateHeapKVPage(int sz); + + public abstract byte[] allocatePages(int sz); + } + + static private final long serialVersionUID = 1L; + + @InterfaceAudience.Private + static public final class MemoryUsage { + public long totalSize; + + public long dataSpace; + public long trashDataSpace; + public long maxDataSpae; + + public long heapKVCapacity; + public long maxHeapKVCapacity; + public long trashHeapKVCount; + public long trashHeapKVSize; + public long aliveHeapKVCount; + public long aliveHeapKVSize; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("CompactedConcurrentSkipListMapMemoryUsage:[TotalSize:") + .append(aliveHeapKVSize + trashHeapKVSize + dataSpace); + sb.append(", DataSpace:").append(dataSpace); + sb.append(", TrashData:").append(trashDataSpace); + sb.append(", HeapKVCapacity:").append(heapKVCapacity); + sb.append(", MaxHeapKVCapacity:").append(maxHeapKVCapacity); + sb.append(", AliveHeapKVSize:").append(aliveHeapKVSize); + sb.append(", AliveHeapKVCount:").append(aliveHeapKVCount); + sb.append(", TrashHeapKVSize:").append(trashHeapKVSize); + sb.append(", TrashHeapKVCount:").append(trashHeapKVCount); + sb.append("]"); + return sb.toString(); + }; + } + + /** + * Parameters about pages in map. + */ + @InterfaceAudience.Private + static public class PageSetting { + /** + * Single data page's size, must be power of 2 + */ + public int pageSize = DEFAULT_PAGE_SIZE; + /** + * The max data page this map can support; + */ + public int pages = DEFAULT_PAGES; + /* + * Single heapKV page's size, must be power of 2 + */ + public int heapKVPageSize = DEFAULT_HEAPKV_PAGE_SIZE; + /** + * The max heapKV page this map can support + */ + public int heapKVPages = DEFAULT_HEAPKV_PAGES; + /** + * KV's size beyond heapKVThreshold will be placed on heap instead of on + * page + */ + public int heapKVThreshold = DEFAULT_HEAPKV_THRESHOLD; + PageAllocator pageAllocator = null; + + static public PageSetting create() { + return new PageSetting(); + } + + public PageSetting setDataPageSize(int pageSize) { + this.pageSize = pageSize; + return this; + } + + public PageSetting setPages(int maxPages) { + this.pages = maxPages; + return this; + } + + public PageSetting setHeapKVPageSize(int heapKVPageSize) { + this.heapKVPageSize = heapKVPageSize; + return this; + } + + public PageSetting setHeapKVPages(int heapKVPages) { + this.heapKVPages = heapKVPages; + return this; + } + + public PageSetting setHeapKVThreshold(int heapKVThreshold) { + this.heapKVThreshold = heapKVThreshold; + return this; + } + + public PageSetting setPageAllocator(PageAllocator allocator) { + this.pageAllocator = allocator; + return this; + } + } + + /** + * A simple page manager can support data/heapkv pages. + * + * @param

+ */ + static abstract class PageManager

{ + final String name; + final int maxPage; + final int pageSize; + final int pageLocationShift; + final int alignedPageOffsetMask; + final int maxAllocatingPages; + final int alignShift; + + final long maxSize; + final long maxAlignedSize; + + Object[] pages; + final AtomicLong alignedTail = new AtomicLong(0); + + final Unsafe unsafe = Utils.getUnsafe(); + + /** + * how many pages we can use + */ + final AtomicInteger usingPages = new AtomicInteger(0); + + /** + * total pages including which are being created. + */ + final AtomicInteger totalPages = new AtomicInteger(0); + + protected abstract P allocatePage(int size); + + protected abstract P allocateFirstPage(); + + /** + * A simple manger for allocating space on page, positioning, allocating + * pages. + * + * PageManager won't concern what is stored in side the page. + * + * @param name + * The page manager's name + * @param maxPage + * How many pages the manager can support. + * @param pageSize + * Single page's size + * @param maxAllocatingPages + * How many new pages could be allocating at the same time + * @param alignSize + * How much the offset passed-in and return shild be left shfit to real byte address + */ + PageManager(String name, int maxPage, int pageSize, int maxAllocatingPages, int alignShift) { + this.name = name; + this.maxPage = maxPage; + this.alignShift = alignShift; + if ((pageSize & (pageSize - 1)) != 0) { + throw new InvalidParameterException(""); + } + if (pageSize <= 0) { + throw new InvalidParameterException("Invalid page size: " + pageSize); + } + this.pageSize = pageSize; + this.pageLocationShift = Utils.getShiftFromX((this.pageSize >> alignShift)); + this.alignedPageOffsetMask = (this.pageSize >> alignShift) - 1; + + this.maxAllocatingPages = maxAllocatingPages; + this.maxSize = (long) maxPage * (long) pageSize; + this.maxAlignedSize = this.maxSize >> alignShift; + this.pages = new Object[this.maxPage]; + + // To avoid waste of pages for empty map, we allocate a small page for + // skiplist's head + doAllocatePage(0); + } + + /** + * Single page's size + * @return + */ + public final int pageSize() { + return this.pageSize; + } + + /** + * The max page count manager could support + * @return + */ + public final int maxPage() { + return this.maxPage; + } + + /** + * The max space this page manager could support + * @return + */ + public final long maxSize() { + return this.maxSize; + } + + /** + * How much space is already used + * @return + */ + public final long used() { + return (this.alignedTail.get() << alignShift); + } + + /** + * How much space is allocated + * @return + */ + public final long capacity() { + long ignoreFirstPage = this.usingPages.get() - 1; + return (long) ignoreFirstPage * this.pageSize; + } + + /** + * How many pages have been used + * @return + */ + public final int getAllocatedPages() { + return this.usingPages.get() - 1; + } + + /** + * Get the page for a given offset; + * + * @param offset + * @return + */ + @SuppressWarnings("unchecked") + final public P getPage(long offset) { + int pageIndex = (int) (offset >> this.pageLocationShift); + Object ret = this.pages[pageIndex]; + if (ret == null) { + ret = Utils.getObjectVolatile(unsafe, pages, pageIndex); + } + return (P)ret; + } + + /** + * get real offset inside its page for given offset + */ + final public int getInPageOffset(long offset) { + return (int) (offset & alignedPageOffsetMask); + } + + /** + * @size The required space for a new node, in bytes + * + * @return + */ + final public long allocate(int size) { + if (size <= 0) { + throw new RuntimeException("Impossible, invalid size=" + size); + } + if (size > this.pageSize) { + throw new SpaceNotEnoughExcpetion(name + + " : allocating size is bigger than page size"); + } + int alignedSize = ((size - 1) >> alignShift) + 1; + for (;;) { + long current = alignedTail.get(); + long bound = ((current >> pageLocationShift) + 1) << pageLocationShift; + long ret = (bound - current >= alignedSize) ? current : bound; + // it is no need to check ret + size, since (ret) & (ret + size) will + // always on same page + if (ret >= maxAlignedSize) { + throw new SpaceNotEnoughExcpetion(name, used(), maxSize(), alignedSize << alignShift); + } + // page not enough + if ((ret >> pageLocationShift) + 1 > usingPages.get()) { + doAllocatePage((int) ((ret >> pageLocationShift) + 1)); + continue; // restart + } + // lost race for allocating memory; + if (!alignedTail.compareAndSet(current, ret + alignedSize)) { + continue; + } + return ret; + } + } + + /** + * Move the tail pointer for allocation to second page of the whole space. + * + * @return The tail pointer before moving. I.E, the space has been used in + * the first page + */ + final public long skipFirstPage() { + assert getAllocatedPages() == 0; + assert (alignedTail.get() >> pageLocationShift) == 0; + int nextPageIndex = 1; + long next = (nextPageIndex << pageLocationShift); + return this.alignedTail.getAndSet(next) << alignShift; + } + + /** + * Allocate a new page, and put it on to pages[] + * @param The new page's index + */ + private void doAllocatePage(int index) { + // Skip allocating new page if too many threads are creating new page. + if (this.totalPages.get() - index >= this.maxAllocatingPages) { + return; + } + P newPage = index == 0 ? allocateFirstPage() : allocatePage(this.pageSize); + totalPages.incrementAndGet(); + // Put newPage on page, restart if lost race + for (;;) { + int pos = usingPages.get(); + if (!Utils.compareAndSetObject(unsafe, pages, pos, null, newPage)) { + continue; + } + usingPages.incrementAndGet(); + break; + } + } + } + + static public class SpaceNotEnoughExcpetion extends RuntimeException { + public SpaceNotEnoughExcpetion(String name, long used, long limited, + long request) { + super(createMessage(name, used, limited, request)); + } + + public SpaceNotEnoughExcpetion(String message) { + super(message); + } + + private static String createMessage(String name, long used, long limited, + long request) { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(" space not enough, used:").append(used) + .append(" limimted").append(limited).append(" request:") + .append(request); + return sb.toString(); + } + + private static final long serialVersionUID = 3220889776327332235L; + } + + private static final Random seedGenerator = new Random(); + + public final static int DEFAULT_PAGE_SIZE = 2 * 1024 * 1024; // 2 mb + public final static int DEFAULT_PAGES = 4096; + + public final static int DEFAULT_HEAPKV_PAGE_SIZE = 1024; + public final static int DEFAULT_HEAPKV_PAGES = 1024; + public final static int DEFAULT_HEAPKV_THRESHOLD = DEFAULT_PAGE_SIZE; + + private final static int EQ = 1; + private final static int LT = 2; + private final static int GT = 0; + + private final static boolean ON_PAGE = true; + private final static boolean ON_HEAP = false; + private final static boolean NODE_DELETED = true; + private final static boolean NODE_PRESENT = false; + + + private final static long HEADER_M_DELETED = 0x8000000000000000L; + private final static long HEADER_M_PAGED = 0x4000000000000000L; + private final static long HEADER_M_LEVEL = 0x3E00000000000000L; + private final static long HEADER_M_LENGTH = 0x1FFFFFE00000000L; + private final static long HEADER_M_REFERENCE = 0x1FFFFFFFFL; + + private final static long HEADER_SHIFT_LEVEL = 57; + private final static long HEADER_SHIFT_LENGTH = 33; + + private final static long MASK_REAL_NODE = 0x7FFFFFFFFFFFFFFFL; + private final static long MASK_NODE_DELETE_MARKER = 0x8000000000000000L; + + private final static int MAX_ALLOCATING_PAGES = 2; + + /** + * The max level of index + */ + private static final int LEVEL_LIMIT = 31; + private static final long HEAD_NODE = 1; + private static final long NULL_NODE = 0; + + transient private final PageManager dataPageManager; + transient private final PageManager heapKVManager; + + private AtomicLong trashData = new AtomicLong(0); + private AtomicLong trashHeapKVCount = new AtomicLong(0); + private AtomicLong trashHeapKVSize = new AtomicLong(0); + private AtomicLong aliveHeapKVCount = new AtomicLong(0); + private AtomicLong aliveHeapKVSize = new AtomicLong(0); + + private final CompactedTypeHelper typeHelper; + private final PageSetting settings; + + + private transient int randomSeed = seedGenerator.nextInt() | 0x0100; // ensure + // nonzero + AtomicInteger maxLevel = new AtomicInteger(1); + + private transient KeySet keySet; + private transient Values values; + private transient EntrySet entrySet; + private transient SubMap descendingMap; + + /** + * Print skip list base&index structure for debug + */ + public void dump() { + StringBuilder sb = new StringBuilder(); + for (int level = getHeadHighestLevel(); level > 0; --level) { + sb.append("level ").append(level).append(":"); + long q = HEAD_NODE; + long r = NULL_NODE; + int cnt = 0; + + do { + r = indexGetNext(q, level); + sb.append('\t').append(cnt).append(":").append(q); + q = r; + cnt += 1; + } while (r != NULL_NODE); + sb.append('\n'); + } + + long n = HEAD_NODE; + long f = NULL_NODE; + int cnt = 0; + sb.append("base :"); + do { + f = nodeGetNext(n); + sb.append('\t').append(cnt).append(":").append(n).append(""); + n = f; + cnt += 1; + } while (f != NULL_NODE); + System.out.println(sb.toString()); + } + + public MemoryUsage getMemoryUsage() { + MemoryUsage mu = new MemoryUsage(); + mu.dataSpace = dataPageManager.capacity(); + mu.maxDataSpae = dataPageManager.maxSize(); + mu.trashDataSpace = trashData.get(); + mu.aliveHeapKVCount = aliveHeapKVCount.get(); + mu.aliveHeapKVSize = aliveHeapKVSize.get(); + mu.trashHeapKVCount = trashHeapKVCount.get(); + mu.trashHeapKVSize = trashHeapKVSize.get(); + mu.heapKVCapacity = heapKVManager.capacity(); + mu.maxHeapKVCapacity = heapKVManager.maxSize(); + mu.totalSize = mu.dataSpace + mu.aliveHeapKVSize + + mu.trashHeapKVSize; + return mu; + } + + public CompactedConcurrentSkipListMap(CompactedTypeHelper typeHelper) { + this(typeHelper, new PageSetting()); + } + + public CompactedConcurrentSkipListMap(CompactedTypeHelper typeHelper, + PageSetting ps) { + this.typeHelper = typeHelper; + this.settings = ps; + + if (settings.heapKVThreshold > getMaxOnHeapKVThreshold(ps.pageSize)) { + settings.heapKVThreshold = getMaxOnHeapKVThreshold(ps.pageSize); + } + + final CompactedConcurrentSkipListMap m = this; + final int dataPageAlignShift = Utils.getShiftFromX(Long.SIZE/Byte.SIZE); + final int firstPageSizeWithoutAligned = estimatedNodeSizeHeadNodeSize() + 1; + final int firstPageSizeForDataPageManager = (((firstPageSizeWithoutAligned - 1) >> dataPageAlignShift) + 1) << dataPageAlignShift; + this.dataPageManager = new PageManager("DataPage", ps.pages, + ps.pageSize, MAX_ALLOCATING_PAGES, dataPageAlignShift) { + @Override + protected byte[] allocatePage(int size) { + if (m.settings.pageAllocator != null) { + byte[] ret = m.settings.pageAllocator.allocatePages(size); + if (ret.length != size) { + throw new InvalidParameterException( + "Page size not consistent, expect: " + size + " , actual:" + + ret.length); + } + return ret; + } else { + return new byte[size]; + } + } + + @Override + protected byte[] allocateFirstPage() { + return new byte[firstPageSizeForDataPageManager]; + } + }; + + final int firstPageSizeForHeapKVManager = 1; + this.heapKVManager = new PageManager( + "HeapKV", ps.heapKVPages, ps.heapKVPageSize, MAX_ALLOCATING_PAGES, 0) { + @Override + protected Object[] allocatePage(int size) { + if (m.settings.pageAllocator != null) { + Object[] ret = m.settings.pageAllocator.allocateHeapKVPage(size); + if (ret.length != size) { + throw new InvalidParameterException( + "Page size not consistent, expect: " + size + " , actual:" + + ret.length); + } + return ret; + } else { + return new Object[size]; + } + } + + @Override + protected Object[] allocateFirstPage() { + return new Object[firstPageSizeForHeapKVManager]; + } + + }; + + // allocate 1 byte to skip address 0, since 0 is a special meaning NULL for node logic. + this.dataPageManager.allocate(1); + createHeadNode(); + long dpmFirstPageUsed = this.dataPageManager.skipFirstPage(); + assert dpmFirstPageUsed == firstPageSizeForDataPageManager; + assert this.dataPageManager.getAllocatedPages() == 0; + + this.heapKVManager.allocate(1); + long hkvmFirstPageUsed = this.heapKVManager.skipFirstPage(); + assert hkvmFirstPageUsed == firstPageSizeForHeapKVManager; + assert this.heapKVManager.getAllocatedPages() == 0; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + CompactedConcurrentSkipListMap ret = new CompactedConcurrentSkipListMap( + typeHelper, settings); + ret.putAll(this); + return ret; + } + + final private int getMaxOnHeapKVThreshold(int pageSize) { + int forHeader = 1; + int forNext = 1; + int forLevel = LEVEL_LIMIT; + int forData = pageSize - forHeader - forNext - forLevel; + return forData; + } + + final private int compareKeyAt(K key, long node) { + return compareKeyAt(key, node, nodeHeader(node)); + } + + final private int compareKeyAt(K key, long node, long header) { + assert !isNullNode(node); + assert node != HEAD_NODE; + long refNode = headerReference(header); + if (!isNullNode(refNode)) { + return compareKeyAt(key, refNode); + } + if (headerOnPage(header) == ON_PAGE) { + int length = headerDataLength(header); + long dataPos = dataPos(node, header); + byte[] data = getDataOnPage(dataPos); + int lo = getLocalOffsetOnPage(dataPos); + return typeHelper.compare(key, data, lo, length); + } else { + int heapKVOffset = headerDataLength(header); + KVPair kv = getOnHeapKV(heapKVOffset); + return typeHelper.compare(key, kv.key); + } + } + + final private int compare(K key1, K key2) { + return typeHelper.compare(key1, key2); + } + + @SuppressWarnings("unchecked") + public V putIfAbsent(K key, V value) { + if (value == null) { + throw new NullPointerException(); + } + return (V) doPut(key, value, true, false); + } + + public boolean remove(Object key, Object value) { + if (key == null) { + throw new NullPointerException(); + } + if (value == null) { + return false; + } + return doRemove(key, value) != null; + } + + @SuppressWarnings("unchecked") + private V doRemove(Object key, Object value) { + K k = (K) key; + if (k == null) { + throw new InvalidParameterException("Invalid key type."); + } + for (;;) { + long b = findPredecessor(k); + long n = nodeGetNext(b); + for (;;) { + if (isNullNode(n)) + return null; + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + break; + } + + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + int c = compareKeyAt(k, n, n_header); + if (c < 0) { + return null; + } + if (c > 0) { + b = n; + n = f; + continue; + } + KVPair kv = nodeGetKV(n, n_header); + if (value != null && !value.equals(kv.value)) { + return null; + } + + if (!nodeCasToDeleted(n, n_header)) { + break; + } + if (!nodeAppendMarker(n, f) || !nodeCasSetNext(b, n, f)) { + findNode(k); // Retry via findNode + } else { + findPredecessor(k); // Clean index + if (indexGetNext(HEAD_NODE, getHeadHighestLevel()) == NULL_NODE) { + tryReduceLevel(); + } + } + return kv.value; + } + } + } + + /** + * Try to reduce head node's max level + * + * @return true if succeed + */ + private boolean tryReduceLevel() { + int level = getHeadHighestLevel(); + if (level >= 3 && NULL_NODE == indexGetNext(HEAD_NODE, level)) { + return casHeadHighestLevel(level, level - 1); + } + return false; + } + + public boolean replace(K key, V oldValue, V newValue) { + if (oldValue == null || newValue == null) { + throw new NullPointerException(); + } + long z = 0; + for (;;) { + long n = findNode(key); + if (isNullNode(n)) { + return false; + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header)) { + KVPair oldKV = nodeGetKV(n, n_header); + if (!oldValue.equals(oldKV.value)) { + return false; + } + if (z == NULL_NODE) { + z = createNode(key, newValue, nodeGetNext(n)); + } else { + nodeSetNext(z, nodeGetNext(n)); + } + if (nodeCasToReference(n, n_header, z)) { + return true; + } + } + } + } + + public V replace(K key, V value) { + if (value == null) { + throw new NullPointerException(); + } + long z = NULL_NODE; + for (;;) { + long n = findNode(key); + if (isNullNode(n)) { + return null; + } + if (z == NULL_NODE) { + z = createNode(key, value, nodeGetNext(n)); + } else { + nodeSetNext(z, nodeGetNext(n)); + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header) && nodeCasToReference(n, n_header, z)) { + return nodeGetKV(n, n_header).value; + } + } + } + + /** + * Returns the number of key-value mappings in this map. Beware that, unlike + * in most collections, this method is NOT a constant-time operation. Because + * of the asynchronous nature of these maps, determining the current number of + * elements requires traversing them all to count them. Additionally, it is + * possible for the size to change during execution of this method, in which + * case the returned result will be inaccurate. Thus, this method is typically + * not very useful in concurrent applications. + * + * @return: the number of elements in this map + */ + public int size() { + int count = 0; + for (long n = findFirst(); !isNullNode(n); n = nodeGetNext(n)) { + if (!nodeDeleted(n)) { + count += 1; + } + } + return count; + } + + public boolean isEmpty() { + return isNullNode(findFirst()); + } + + public boolean containsKey(Object key) { + return doGet(key) != null; + } + + public boolean containsValue(Object value) { + if (value == null) { + throw new NullPointerException(); + } + for (long n = findFirst(); !isNullNode(n); n = nodeGetNext(n)) { + long n_header = nodeHeader(n); + if (!headerDeleted(n_header) + && value.equals(nodeGetKV(n, n_header).value)) { + return true; + } + } + return false; + } + + public V get(Object key) { + return doGet(key); + } + + /** + * Ensure the given node is NOT a marker + * + * @param node + */ + final private void nodeEnsureNotMarker(long node) { + if ((node & MASK_NODE_DELETE_MARKER) != 0) { + throw new RuntimeException( + "There may be a bug in CompatedConcurrentSkipListMap, node should not be a marker, node=" + + node); + } + } + + /** + * Get the given node's key-value pair's snapshot, the snapshot is determined + * by offset + * + * @param node + * @param offset + * offset snapshot of this node + * @return + */ + final private KVPair nodeGetKV(long node, long header) { + assert !isNullNode(node); + assert node != HEAD_NODE; + long refNode = headerReference(header); + if (!isNullNode(refNode)) { + return nodeGetKV2(refNode); + } + if (headerOnPage(header)) { + int length = headerDataLength(header); + long dataPos = dataPos(node, header); + byte[] data = getDataOnPage(dataPos); + int lo = getLocalOffsetOnPage(dataPos); + return typeHelper.decomposte(data, lo, length); + } else { + int heapKVOffset = headerDataLength(header); + KVPair kv = getOnHeapKV(heapKVOffset); + return kv; + } + } + + /** + * Get the given node's key-value pair, + * + * @param node + * @return + */ + final private KVPair nodeGetKV2(long node) { + return nodeGetKV(node, nodeHeader(node)); + } + + /** + * Specialized variant of findNode to perform Map.get. Does a weak traversal, + * not bothering to fix any deleted index nodes, returning early if it happens + * to see key in index, and passing over any deleted base nodes, falling back + * to getUsingFindNode only if it would otherwise return value from an ongoing + * deletion. Also uses "bound" to eliminate need for some comparisons (see + * Pugh Cookbook). Also folds uses of null checks and node-skipping because + * markers have null keys. + * + * @param key + * @return the value related to the key. + */ + @SuppressWarnings("unchecked") + private V doGet(Object key) { + K k = (K) key; + if (k == null) { + throw new InvalidParameterException("Invalid key type"); + } + long bound = NULL_NODE; + int level = getHeadHighestLevel(); + long q = HEAD_NODE; + long r = indexGetNext(q, level); + long n = NULL_NODE; + int c; + for (;;) { + // Traverse right + if (r != NULL_NODE && (n = r) != bound) { + long n_header = nodeHeader(n); + if ((c = compareKeyAt(k, n, n_header)) > 0) { + q = r; + r = indexGetNext(r, level); + continue; + } else if (c == 0) { + if (headerDeleted(n_header)) { + return getUsingFindNode(k); // maybe lost race when n is deleting + } else { + return nodeGetKV(n, n_header).value; + } + } else { + bound = n; + } + } + + // Traverse down + if ((level = level - 1) <= 0) { + break; + } else { + r = indexGetNext(q, level); + } + } + + // Traverse on base + for (n = nodeGetNext(q); !isNullNode(n); n = nodeGetNext(n)) { + long n_header = nodeHeader(n); + if ((c = compareKeyAt(k, n, n_header)) == 0) { + if (headerDeleted(n_header)) { + return getUsingFindNode(k); // maybe lost race when n is deleting + } else { + return nodeGetKV(n, n_header).value; + } + } else if (c < 0) { + break; + } + } + return null; + } + + /** + * Performs map.get via findNode. Used as a backup if doGet encounters an + * in-progress deletion. + * + * @param key + * @return + */ + private V getUsingFindNode(K key) { + /** + * Loop needed here and elsewhere in case value field goes null just as it + * is about to be returned, in which case we lost a race with a deletion, so + * must retry. + */ + for (;;) { + long n = findNode(key); + if (isNullNode(n)) { + return null; + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header)) { + return nodeGetKV(n, n_header).value; + } + } + } + + public V put(K key, V value) { + if (value == null) { + throw new NullPointerException(); + } + return (V) doPut(key, value, false, false); + } + + public PutAndFetch putAndFetch(K key, V value) { + return (PutAndFetch) doPut(key, value, false, true); + } + + /** + * Main insertion method. Adds element if not present, or replaces value if + * present and onlyIfAbsent is false. + * + * @param key + * @param value + * @param onlyIfAbsent + * if should not insert if already present + * @return the old value, or null if newly inserted + */ + private Object doPut(K key, V value, boolean onlyIfAbsent, boolean fetch) { + long z = NULL_NODE; + boolean succ = false; + try { + for (;;) { + long b = findPredecessor(key); + long n = nodeGetNext(b); + long f = NULL_NODE; + int c = 0; + + for (;;) { + if (n != 0) { // b has a succ node + f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read, restart + break; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + c = compareKeyAt(key, n, n_header); + if (c > 0) { + b = n; + n = f; + continue; + } + + // update kv on node-n + if (c == 0) { + if (z == NULL_NODE) { // create new node if needed + z = createNode(key, value, f); + } + if (onlyIfAbsent || (succ = nodeCasToReference(n, n_header, z))) { + if (fetch) { + KVPair old = nodeGetKV(n, n_header); + KVPair current = nodeGetKV2(z); + assert old != null; + assert current != null; + return new PutAndFetch(old.value, current.value); + } else { + KVPair kv = nodeGetKV(n, n_header); + return kv.value; + } + } else { + break; // restart if lost race to replace value + } + } + } + + if (z == 0) { + // new node hasn't been created, create a new node and set its next + // to n + z = createNode(key, value, n); + } else { + // new node is already created, set its next to n + nodeSetNext(z, n); + } + + // append z after b + if (!(succ = nodeCasSetNext(b, n, z))) { + break; // restart if lost race to append z after b + } + + int level = nodeLevel(z); + if (level > 0) { + insertIndex(z, level, key); + } + + if (fetch) { + return new PutAndFetch(null, nodeGetKV2(z).value); + } else { + return null; + } + } // end for + } // end for + } finally { + // New node is created but failed to insert into map, + // do some cleaning work + if (!succ && !isNullNode(z)) { + trackTrashNode(z, nodeHeader(z)); + } + } + } + + /** + * insert node z into index + * @param z + * insert node + * @param level + * the node's level is supposed + * @param key + * the key of the insert node + */ + private void insertIndex(long z, int level, K key) { + int max = getHeadHighestLevel(); + if (level <= max) { + addIndex(z, level, max, key); + } else { // Add a new level + int newLevel = level = max + 1; + casHeadHighestLevel(max, newLevel); + addIndex(z, level, level, key); + } + } + + /** + * The main method to insert a node into index + * @param node + * @param indexLevel The level the node + * @param maxLevel The map's max level, + * @param key + */ + private void addIndex(long node, int indexLevel, int maxLevel, K key) { + // Track next level to insert in case of retries + int insertionLevel = indexLevel; + if (key == null) { + throw new NullPointerException(); + } + // Similar to findPredecessor, but adding index nodes along path to key. + for (;;) { + int level = maxLevel; + long q = HEAD_NODE; + long r = indexGetNext(q, level); + long t = node; + for (;;) { + if (r != NULL_NODE) { + // compare before deletion check avoids needing recheck + long r_header = nodeHeader(r); + int c = compareKeyAt(key, r, r_header); + if (headerDeleted(r_header)) { + if (!indexUnlink(q, r, level)) { + break; + } + r = indexGetNext(r, level); + continue; + } + if (c > 0) { + q = r; + r = indexGetNext(r, level); + continue; + } + } + + if (level == insertionLevel) { + // Don't insert index if node already deleted + if (nodeDeleted(t)) { + findNode(key); // cleans up + return; + } + if (!indexLink(q, r, t, level)) { + break; // restart + } + if (--insertionLevel == 0) { + // need final deletion check before return + if (nodeDeleted(t)) { + findNode(key); + } + return; + } + } + + level -= 1; + r = indexGetNext(q, level); + } + } + } + + /** + * Find the node of a given key, fix any deleted node on way + * @param key + * @return + */ + private long findNode(K key) { + for (;;) { + long b = findPredecessor(key); + long n = nodeGetNext(b); + for (;;) { + if (isNullNode(n)) { + return NULL_NODE; + } + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + break; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + int c = compareKeyAt(key, n, n_header); + if (c == 0) { + return n; + } + if (c < 0) { + return NULL_NODE; + } + b = n; + n = f; + } + } + } + + /** + * Get head node's max level + */ + final private int getHeadHighestLevel() { + return maxLevel.get(); + } + + /** + * Compare and set the head node's level + * @param cmp The head node's expected level + * @param val The new level will be set to the head node + * @return true if succeed + */ + final private boolean casHeadHighestLevel(int cmp, int val) { + return maxLevel.compareAndSet(cmp, val); + } + + /** + * Find a base-level node with key strictly less than given key. + * Unlink indexes to deleted nodes found along the way. + * @param key + * @return + */ + private long findPredecessor(K key) { + if (key == null) { + throw new NullPointerException(); + } + for (;;) { + long q = HEAD_NODE; + int level = getHeadHighestLevel(); + long r = indexGetNext(q, level); + for (;;) { + if (!isNullNode(r)) { + long r_header = nodeHeader(r); + if (headerDeleted(r_header)) { + if (!indexUnlink(q, r, level)) { + break; // restart + } + r = indexGetNext(q, level); // reread + continue; + } + if (compareKeyAt(key, r, r_header) > 0) { + q = r; + r = indexGetNext(r, level); + continue; + } + } + level -= 1; // Go down + if (level > 0) { + r = indexGetNext(q, level); + } else { + return q; + } + } + } + } + + /** + * Link index at level from q->right to q->newRight->Right + * @param q + * @param right + * @param newRight + * @param level + * @return True if succeed + */ + private boolean indexLink(long q, long right, long newRight, int level) { + pageSet(indexPos(newRight, level), right); + return !nodeDeleted(q) + && pageCompareAndSet(indexPos(q, level), right, newRight); + } + + /** + * Unlink rightNode at level + * + * @param node + * @param rightNode + * @param level + * @return True if succeed + */ + final private boolean indexUnlink(long node, long rightNode, int level) { + assert !isNullNode(node); + assert rightNode != HEAD_NODE; + long newRight = indexGetNext(rightNode, level); + return !nodeDeleted(node) + && pageCompareAndSet(indexPos(node, level), rightNode, newRight); + } + + /** + * Get the node's next node at index level + * @param node + * @param level + * @return + */ + final private long indexGetNext(long node, int level) { + assert level > 0; + return getAt(indexPos(node, level)); + } + + /** + * Whether a node is NULL + * @param node + * @return True if the node is NULL + */ + final private boolean isNullNode(long node) { + return (node & ~MASK_NODE_DELETE_MARKER) == 0; + } + + + /** + * Whether a node is a marker + * @param next + * @return True if the node is a makrer + */ + final private boolean isMarker(long next) { + return (next & MASK_NODE_DELETE_MARKER) != 0; + } + + /** + * Get real node of a coded marker. + * @param marker + * @return The real node + */ + final private long markerDecode(long marker) { + return MASK_REAL_NODE & marker; + } + + /** + * Encode a node as marker + * @param node + * @return The marked node + */ + final private long markerEncode(long node) { + return (node | MASK_NODE_DELETE_MARKER); + } + + + /** + * Validate the reference + * @param ref + */ + final private void ensureReferenceValid(long ref) { + if (ref <= HEAD_NODE || ref > HEADER_M_REFERENCE) { + throw new RuntimeException("Bug! refNode=" + ref); + } + } + + + final private long headerReference(long header) { + return header & HEADER_M_REFERENCE; + } + + final private long headerEstimateNodeSize(long header) { + return estimatedNodeSize(headerDataLength(header), headerLevel(header), + headerOnPage(header)); + } + + final private boolean isNodeReference(long header) { + return headerReference(header) != NULL_NODE; + } + + final private long dataPosWithLevel(long node, int level) { + return indexPos(node, level) + 1; + } + + final private long dataPos(long node, long header) { + return dataPosWithLevel(node, headerLevel(header)); + } + + /** + * Get position of node's header + * @param node + * @return + */ + final private long nodePosHeader(long node) { + return node & MASK_REAL_NODE; + } + + /** + * Get position of node's base next + * @param node + * @return + */ + final private long nodePosNext(long node) { + return (node & MASK_REAL_NODE) + 1; + } + + /** + * Get position of node's index area + * @param node + * @return + */ + final private long nodePosIndex(long node) { + return (node & MASK_REAL_NODE) + 2; + } + + /** + * @param node + * @param level + * @return + */ + final private long indexPos(long node, int level) { + return nodePosIndex(node) + level - 1; + } + + /** + * Compare and set the node's next from n to z + * @param node + * @param n + * @param z + * @return + */ + final private boolean nodeCasSetNext(long node, long n, long z) { + assert !isNullNode(node); + return pageCompareAndSet(nodePosNext(node), n, z); + } + + /** + * Set the node's base next + * @param node + * @param next + */ + final private void nodeSetNext(long node, long next) { + pageSet(nodePosNext(node), next); + } + + /** + * Set the node's header + * @param node + * @param header + */ + final private void nodeSetHeader(long node, long header) { + pageSet(nodePosHeader(node), header); + } + + /** + * Accumulate trash info into memory usage summary + * @param node + * @param header + * @param seenOffset + */ + final private void trackTrashNode(long node, long header) { + if (isNodeReference(header)) { + header = nodeHeader(headerReference(header)); + } + + this.trashData.addAndGet(headerEstimateNodeSize(header)); + if (!headerOnPage(header)) { + long pos = dataPos(node, header); + Utils.getLong(Utils.UNSAFE, getDataOnPage(pos), getLocalOffsetOnPage(pos)); + } + } + + /** + * Try to change a node's offset to delete state + * @param node + * @param seenOffset + * @return + */ + final private boolean nodeCasToDeleted(long node, long header) { + if (headerDeleted(header) == NODE_DELETED) { + throw new RuntimeException("Bug:Impossible."); + } + long newHeader = header | HEADER_M_DELETED; + long pos = nodePosHeader(node); + boolean ret = pageCompareAndSet(pos, header, newHeader); + if (ret) { + trackTrashNode(node, header); + } + return ret; + } + + /** + * Try to change a node's offset to a reference to another node + * + * @param node + * @param seenOffset + * @param refNode + * @return + */ + private boolean nodeCasToReference(long node, long oldHeader, long refNode) { + if (headerDeleted(oldHeader)) { + throw new RuntimeException("Bug: Impossible."); + } + ensureReferenceValid(refNode); + long newHeader = (refNode & HEADER_M_REFERENCE) | (oldHeader & (~HEADER_M_REFERENCE)); + boolean ret = pageCompareAndSet(nodePosHeader(node), oldHeader, newHeader); + if (ret) { + trackTrashNode(node, oldHeader); + } + return ret; + } + + + /** + * Try to set the node's base next to a marker + * @param node + * @param f + * @return + */ + final private boolean nodeAppendMarker(long node, long f) { + nodeEnsureNotMarker(node); + long fWithMarker = markerEncode(f); + return nodeCasSetNext(node, f, fWithMarker); + } + + /** + * Get a node's index level + * @param node + * @return + */ + final private int nodeLevel(long node) { + return headerLevel(nodeHeader(node)); + } + + /** + * Get the node's header + * @param node + * @return + */ + final private long nodeHeader(long node) { + long ret = getAt(nodePosHeader(node)); + return ret; + } + + /** + * If a deleted is observed by other threads, they will call this method to + * help for this node's deletion + * + * @param node + * @param b + * @param f + */ + final private void nodeHelpDelete(long node, long b, long f) { + long n_next = nodeGetNext(node); + long b_next = nodeGetNext(b); + if (f == n_next && node == b_next) { + if (!isMarker(f)) { // not already marked + nodeAppendMarker(node, f); + } else { // already marked + long new_b_next = isMarker(node) ? f : markerDecode(f); + nodeCasSetNext(b, node, new_b_next); + } + } + } + + /** + * Get node's base next + * @param node + * @return + */ + final private long nodeGetNext(long node) { + assert !isNullNode(node); + long next = getAt(nodePosNext(node)); + return next; + } + + /** + * Is node deleted + * @param node + * @return + */ + final private boolean nodeDeleted(long node) { + assert !isNullNode(node); + long header = nodeHeader(node); + return headerDeleted(header); + } + + // ////////////////////////////////////////// + // manipulating methods for header + // ////////////////////////////////////////// + + /** + * Is the header indicate that node is on data page + * @param header + * @return + */ + final private boolean headerOnPage(long header) { + return (header & HEADER_M_PAGED) != 0 ? ON_HEAP : ON_PAGE; + } + + final private boolean headerDeleted(long header) { + return (header & HEADER_M_DELETED) != 0 ? NODE_DELETED : NODE_PRESENT; + } + + /** + * Get the node's index level from header + * @param header + * @return + */ + final private int headerLevel(long header) { + return (int)((header & HEADER_M_LEVEL) >> HEADER_SHIFT_LEVEL); + } + + final private int headerDataLength(long header) { + return (int)((header & HEADER_M_LENGTH) >> HEADER_SHIFT_LENGTH); + } + + /** + * Encode a header by parameters below + */ + final private long headerMake(boolean deleted, boolean onPage, + long heapKVOffset, long dataLength, int level) { + long ret = 0; + ret = (deleted == NODE_DELETED) ? (ret | HEADER_M_DELETED) : ret; + ret = (onPage == ON_HEAP) ? (ret | HEADER_M_PAGED) : ret; + ret = ret | (((long)level) << HEADER_SHIFT_LEVEL); + ret = ret | (((onPage == ON_PAGE) ? dataLength : heapKVOffset) << HEADER_SHIFT_LENGTH); + return ret; + } + + final private int alignOffsetToLocalOffset(int i) { + return i << 3; + } + + final private long getAt(long globalOffset) { + byte[] data = getDataOnPage(globalOffset); + int pos = getLocalOffsetOnPage(globalOffset); + return Utils.getLongVolotile(Utils.UNSAFE, data, pos); + } + + /** + * CAS data at position i + * @param i + * @param expect The data expected to be + * @param update + * @return + */ + final private boolean pageCompareAndSet(long i, long expect, long update) { + byte[] data = getDataOnPage(i); + int pos = getLocalOffsetOnPage(i); + return Utils.compareAndSetLong(Utils.UNSAFE, data, pos, expect, update); + } + + /** + * Set value at position i directly + * @param i + * @param value + */ + final private void pageSet(long i, long value) { + byte[] data = this.getDataOnPage(i); + int pos = this.getLocalOffsetOnPage(i); + Utils.putLongVolatile(Utils.UNSAFE, data, pos, value); + } + + /** + * Get how many byte would the node use + * + */ + private int estimatedNodeSize(int kvSize, int level, boolean onPage) { + int longCount = 2; // for header & next; + longCount += level; // for level + if (onPage == ON_PAGE) { + if (kvSize <= 0) { + throw new RuntimeException("Impossible: bug"); + } + longCount += ((kvSize - 1) >> 3) + 1; + } else { + longCount += 1; // for data length of on heap kv + } + return longCount << 3; + } + + /** + * Create a node and store key value into map + * @param key + * @param value + * @param next the node's next node at base-level + * @return + */ + private long createNode(K key, V value, long next) { + int level = randomLevel(); + boolean onPage; + int kvSize = typeHelper.getCompactedSize(key, value); + long node = NULL_NODE; + + onPage = kvSize > settings.heapKVThreshold ? ON_HEAP : ON_PAGE; + int nodeSize = estimatedNodeSize(kvSize, level, onPage); + node = allocatePageSpace(nodeSize, level); + long heapKVOffset = NULL_NODE; + if (onPage == ON_PAGE) { + putDataOnPage(key, value, node, level, kvSize); + } else { + heapKVOffset = putDataOnHeap(key, value, node, level, kvSize); + } + long header = headerMake(NODE_PRESENT, onPage, heapKVOffset, kvSize, level); + nodeSetHeader(node, header); + nodeSetNext(node, next); + + if (onPage == ON_HEAP) { + aliveHeapKVCount.incrementAndGet(); + aliveHeapKVSize.addAndGet(kvSize); + } + return node; + } + + private int estimatedNodeSizeHeadNodeSize() { + int dataLength = 1; + return estimatedNodeSize(dataLength, LEVEL_LIMIT, ON_PAGE); + } + + private long createHeadNode() { + int dataLength = 1; + long header = headerMake(NODE_PRESENT, ON_PAGE, 0, dataLength, LEVEL_LIMIT); + int nodeSize = estimatedNodeSizeHeadNodeSize(); + long node = allocatePageSpace(nodeSize, LEVEL_LIMIT); + assert node == HEAD_NODE; + nodeSetHeader(node, header); + return node; + } + + /** + * Allocate space for a new node + * @param size How many integers this node needs + * @return + */ + private long allocatePageSpace(int size, int level) { + long node = dataPageManager.allocate(size); + int inclusiveStart = getLocalOffsetOnPage(nodePosHeader(node)); + int exclusiveEnd = getLocalOffsetOnPage(indexPos(node, level) + 1); + byte[] data = getDataOnPage(node); + Utils.clear(Utils.UNSAFE, data, inclusiveStart, (exclusiveEnd - inclusiveStart)); + return node; + } + + /** + * Generate a random level, the random level won't be larger than the map's + * highest level + 1 + * + * @return a random level + */ + private int randomLevel() { + int x = randomSeed; + x ^= x << 13; + x ^= x >>> 17; + randomSeed = x ^= x << 5; + if ((x & 0x8001) != 0) // test highest and lowest bits + return 0; + int level = 1; + while (((x >>>= 1) & 1) != 0) + ++level; + int maxLevel = getHeadHighestLevel(); + return (level > maxLevel + 1) ? maxLevel + 1 : level; + } + + /** + * Get data page of a given offset + * @param offset + * @return + */ + final private byte[] getDataOnPage(long globalOffset) { + return dataPageManager.getPage(globalOffset); + } + + final private int getLocalOffsetOnPage(long globalOffset) { + int inPageOffset = dataPageManager.getInPageOffset(globalOffset); + return alignOffsetToLocalOffset(inPageOffset); + } + + + /** + * Place key and value on to the page, return a global offset to indicate + * where the data is. + * + * @param key + * @param value + * @param sz The size of compacted key-value + * @return Global offset indicating where the data is. + */ + final private void putDataOnPage(K key, V value, long node, int level, int kvsize) { + long dataPos = dataPosWithLevel(node, level); + byte[] data = getDataOnPage(dataPos); + int localOffset = getLocalOffsetOnPage(dataPos); + typeHelper.compact(key, value, data, + localOffset, kvsize); + } + + /** + * Place key-value on heap, return a global offset to indicate where the data + * is + * + * @param key + * @param value + * @return Global offset indicating where the data is. + */ + final private long putDataOnHeap(K key, V value, long node, int level, int kvSize) { + long offset = heapKVManager.allocate(1); + Object[] a = heapKVManager.getPage(offset); + KVPair newKV = new KVPair(key, value); + a[heapKVManager.getInPageOffset(offset)] = newKV; + // put on heap kv's size on the first data's position of page + pageSet(dataPosWithLevel(node, level), kvSize); + return offset; + } + + /** + * Get key-value by offset + * @param offset + * @return + */ + @SuppressWarnings("unchecked") + final private KVPair getOnHeapKV(long offset) { + Object[] a = heapKVManager.getPage(offset); + int lo = heapKVManager.getInPageOffset(offset); + return (KVPair)a[lo]; + } + + public V remove(Object key) { + return doRemove(key, null); + } + + public void clear() { + while (!isEmpty()) { + doRemoveFirstEntry(); + } + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public Collection values() { + Values vs = values; + return (vs != null) ? vs : (values = new Values(this)); + } + + public Set> entrySet() { + EntrySet es = entrySet; + return (es != null) ? es : (entrySet = new EntrySet(this)); + } + + public Map.Entry lowerEntry(K key) { + return getNear(key, LT); + } + + public K lowerKey(K key) { + long n = findNear(key, LT); + return (isNullNode(n)) ? null : nodeGetKV2(n).key; + } + + public Map.Entry floorEntry(K key) { + return getNear(key, LT | EQ); + } + + public K floorKey(K key) { + Map.Entry entry = floorEntry(key); + return entry == null ? null : entry.getKey(); + } + + public Map.Entry ceilingEntry(K key) { + return getNear(key, GT | EQ); + } + + public K ceilingKey(K key) { + Map.Entry entry = ceilingEntry(key); + return entry == null ? null : entry.getKey(); + } + + public Map.Entry higherEntry(K key) { + return getNear(key, GT); + } + + public K higherKey(K key) { + Map.Entry entry = getNear(key, GT); + return (entry == null) ? null : entry.getKey(); + } + + /** + * Find first node of the map. If no node in this map, return NULL_NODE + * @return + */ + private long findFirst() { + for (;;) { + long b = HEAD_NODE; + long n = nodeGetNext(b); + if (isNullNode(n)) { + return NULL_NODE; + } + if (!nodeDeleted(n)) { + return n; + } + nodeHelpDelete(n, b, nodeGetNext(n)); + } + } + + /** + * Find last node of the map. If no node in this map, return NULL_NODE + * + * @return + */ + private long findLast() { + long q = HEAD_NODE; + int level = getHeadHighestLevel(); + for (;;) { + long r; + if ((r = indexGetNext(q, level)) != NULL_NODE) { + if (nodeDeleted(r)) { + indexUnlink(q, r, level); + q = HEAD_NODE; // restart + level = getHeadHighestLevel(); + } else { + q = r; + } + } else if ((level = level - 1) > 0) { + continue; + } else { + long b = q; + long n = nodeGetNext(b); + for (;;) { + if (isNullNode(n)) { + return (b == HEAD_NODE) ? NULL_NODE : b; + } + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + break; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + b = n; + n = f; + } + q = HEAD_NODE; + level = getHeadHighestLevel(); + } + } + } + + /** + * Get the entry near the key. The node's relation to the key is specified by + * the rel + * + * @param key + * @param rel LT, EQ, GT, LT|EQ or GT|EQ + * @return + */ + private AbstractMap.SimpleImmutableEntry getNear(K key, int rel) { + for (;;) { + long n = findNear(key, rel); + if (isNullNode(n)) { + return null; + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header)) { + KVPair kv = nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + /** + * Get the node near the key. The node's relation to the key is specified by + * the rel + * + * @param kkey + * @param rel + * @return + */ + private long findNear(K kkey, int rel) { + for (;;) { + long b = findPredecessor(kkey); + long n = nodeGetNext(b); + for (;;) { + if (isNullNode(n)) { + return ((rel & LT) == 0 || b == HEAD_NODE) ? NULL_NODE : b; + } + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + break; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + int c = compareKeyAt(kkey, n, n_header); + if ((c == 0 && (rel & EQ) != 0) || (c < 0 && (rel & LT) == 0)) { + return n; + } + if (c <= 0 && (rel & LT) != 0) { + return (b == HEAD_NODE) ? NULL_NODE : b; + } + b = n; + n = f; + } + } + } + + public Entry firstEntry() { + for (;;) { + long n = findFirst(); + if (isNullNode(n)) { + return null; + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header)) { + KVPair kv = nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + public Map.Entry lastEntry() { + for (;;) { + long n = findLast(); + if (isNullNode(n)) { + return null; + } + long n_header = nodeHeader(n); + if (!headerDeleted(n_header)) { + KVPair kv = nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + /** + * Remove first entry in the map + * @return Removed entry. If no entry to remove, return null + */ + private Map.Entry doRemoveFirstEntry() { + for (;;) { + long b = HEAD_NODE; + long n = nodeGetNext(b); + if (isNullNode(n)) { + return null; + } + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + continue; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { + nodeHelpDelete(n, b, f); + continue; + } + if (!nodeCasToDeleted(n, n_header)) { + continue; + } + if (!nodeAppendMarker(n, f) || !nodeCasSetNext(b, n, f)) { + findFirst(); // retry + } + clearIndexToFirst(); + KVPair kv = nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + + /** + * Clears out index nodes associated with deleted first entry. + */ + private void clearIndexToFirst() { + for (;;) { + long q = HEAD_NODE; + int level = getHeadHighestLevel(); + for (;;) { + long r = indexGetNext(q, level); + if (r != NULL_NODE && nodeDeleted(r) && !indexUnlink(q, r, level)) { + break; // retry + } + if ((level = level - 1) <= 0) { + if (indexGetNext(HEAD_NODE, getHeadHighestLevel()) == NULL_NODE) { + tryReduceLevel(); + } + return; + } + } + } + } + + /** + * Get the node before last node by traversing index + * @return The node before the last node. + */ + private long findPredecessorOfLast() { + for (;;) { + long q = HEAD_NODE; + int level = getHeadHighestLevel(); + for (;;) { + long r; + if ((r = indexGetNext(q, level)) != NULL_NODE) { + if (nodeDeleted(r)) { + indexUnlink(q, r, level); + break; // lost race, restart + } + // proceed as far across as possible without overshooting + if (!isNullNode(nodeGetNext(r))) { + q = r; + continue; + } + } + if ((level = level - 1) <= 0) { + return q; + } + } + } + } + + /** + * Remove last entry in the map + * @return removed entry + */ + private Map.Entry doRemoveLastEntry() { + for (;;) { + long b = findPredecessorOfLast(); + long n = nodeGetNext(b); + if (isNullNode(n)) { + if (b == HEAD_NODE) { + return null; + } else { + continue; // all b's successors are deleted; retry + } + } + // Traverse next + for (;;) { + long f = nodeGetNext(n); + if (n != nodeGetNext(b)) { // inconsistent read + break; + } + long n_header = nodeHeader(n); + if (headerDeleted(n_header)) { // n is deleted + nodeHelpDelete(n, b, f); + break; + } + if (isMarker(n) || nodeDeleted(b)) { // b is deleted + break; + } + if (!isNullNode(f)) { + b = n; + n = f; + continue; + } + if (!nodeCasToDeleted(n, n_header)) { + break; + } + KVPair kv = nodeGetKV(n, n_header); + if (!nodeAppendMarker(n, f) || !nodeCasSetNext(b, n, f)) { + findNode(kv.key); // Retry via findNode + } else { + findPredecessor(kv.key); // clean index + if (indexGetNext(HEAD_NODE, getHeadHighestLevel()) == NULL_NODE) { + tryReduceLevel(); + } + } + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + + public Map.Entry pollFirstEntry() { + return doRemoveFirstEntry(); + } + + public java.util.Map.Entry pollLastEntry() { + return doRemoveLastEntry(); + } + + /** + * Wrapper for key comparator + * @param + */ + private class KeyComparator implements Comparator { + @SuppressWarnings("unchecked") + public int compare(T o1, T o2) { + K k1 = (K) o1; + K k2 = (K) o2; + return typeHelper.compare(k1, k2); + } + } + + public Comparator comparator() { + return new KeyComparator(); + } + + public K firstKey() { + long n = findFirst(); + if (isNullNode(n)) { + throw new NoSuchElementException(); + } + return nodeGetKV2(n).key; + } + + public K lastKey() { + long n = findLast(); + if (isNullNode(n)) { + throw new NoSuchElementException(); + } + return nodeGetKV2(n).key; + } + + public ConcurrentNavigableMap subMap(K fromKey, boolean fromInclusive, + K toKey, boolean toInclusive) { + if (fromKey == null || toKey == null) { + throw new NullPointerException(); + } + return new SubMap(this, fromKey, fromInclusive, toKey, toInclusive, + false); + } + + public ConcurrentNavigableMap headMap(K toKey, boolean inclusive) { + if (toKey == null) { + throw new NullPointerException(); + } + return new SubMap(this, null, false, toKey, inclusive, false); + } + + public ConcurrentNavigableMap tailMap(K fromKey, boolean inclusive) { + if (fromKey == null) { + throw new NullPointerException(); + } + return new SubMap(this, fromKey, inclusive, null, false, false); + } + + public ConcurrentNavigableMap subMap(K fromKey, K toKey) { + return subMap(fromKey, true, toKey, false); + } + + public ConcurrentNavigableMap headMap(K toKey) { + return headMap(toKey, false); + } + + public ConcurrentNavigableMap tailMap(K fromKey) { + return tailMap(fromKey, true); + } + + public ConcurrentNavigableMap descendingMap() { + ConcurrentNavigableMap dm = descendingMap; + return (dm != null) ? dm : (descendingMap = new SubMap(this, null, + false, null, false, true)); + } + + public NavigableSet navigableKeySet() { + return keySet(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public NavigableSet keySet() { + KeySet ks = keySet; + return (ks != null) ? ks : (keySet = new KeySet(this)); + } + + public NavigableSet descendingKeySet() { + return descendingMap().navigableKeySet(); + } + + // Base of iterator classes: + abstract class Iter implements Iterator { + // the last node returned by next() + long lastReturned; + + // the next node to return from next(); + long next; + // the next node's offset + long next_header; + + // Initializes ascending iterator for entire range. + Iter() { + for (;;) { + next = findFirst(); + if (isNullNode(next)) { + break; + } + long x = nodeHeader(next); + if (!headerDeleted(x)) { + next_header = x; + break; + } + } + } + + public final boolean hasNext() { + return !isNullNode(next); + } + + final void advance() { + if (isNullNode(next)) { + throw new NoSuchElementException(); + } + lastReturned = next; + for (;;) { + next = nodeGetNext(next); + if (isNullNode(next)) { + break; + } + long x = nodeHeader(next); + if (!headerDeleted(x)) { + next_header = x; + break; + } + } + } + + public void remove() { + long l = lastReturned; + if (isNullNode(l)) { + throw new IllegalStateException(); + } + // It would not be worth all of the overhead to directly + // unlink from here. Using remove is fast enough. + CompactedConcurrentSkipListMap.this.remove(nodeGetKV2(l).key); + lastReturned = NULL_NODE; + } + } + + final class ValueIterator extends Iter { + public V next() { + KVPair kv = nodeGetKV(next, next_header); + advance(); + return kv.value; + } + } + + final class KeyIterator extends Iter { + public K next() { + KVPair kv = nodeGetKV(next, next_header); + advance(); + return kv.key; + } + } + + final class EntryIterator extends Iter> { + public Entry next() { + KVPair kv = nodeGetKV(next, next_header); + advance(); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + + Iterator keyIterator() { + return new KeyIterator(); + } + + Iterator valueIterator() { + return new ValueIterator(); + } + + Iterator> entryIterator() { + return new EntryIterator(); + } + + static final class SubMap extends AbstractMap implements + ConcurrentNavigableMap, Cloneable, Serializable { + private static final long serialVersionUID = 1L; + // Underlying map + private final CompactedConcurrentSkipListMap m; + // lower bound key, or null if from start + private final K lo; + // upper bound key, or null if to end + private final K hi; + // inclusion flag for lo + private final boolean loInclusive; + // inclusion flag for hi + private final boolean hiInclusive; + // direction + private final boolean isDescending; + // Lazily initialized view holders + private transient KeySet keySetView; + private transient Collection valuesView; + private transient Set> entrySetView; + + public SubMap(CompactedConcurrentSkipListMap map, K fromKey, + boolean fromInclusive, K toKey, boolean toInclusive, + boolean isDescending) { + if (fromKey != null && toKey != null + && map.typeHelper.compare(fromKey, toKey) > 0) { + throw new IllegalArgumentException("inconsistent range"); + } + this.m = map; + this.lo = fromKey; + this.hi = toKey; + this.loInclusive = fromInclusive; + this.hiInclusive = toInclusive; + this.isDescending = isDescending; + } + + private boolean tooLow(K key) { + if (lo != null) { + int c = m.compare(key, lo); + if (c < 0 || (c == 0 && !loInclusive)) { + return true; + } + } + return false; + } + + private boolean tooLowNode(long node) { + assert !m.isNullNode(node); + if (lo != null) { + int c = m.compareKeyAt(lo, node); + if (c > 0 || (c == 0 && !loInclusive)) { + return true; + } + } + return false; + } + + private boolean tooHigh(K key) { + if (hi != null) { + int c = m.compare(key, hi); + if (c > 0 || (c == 0 && !hiInclusive)) { + return true; + } + } + return false; + } + + private boolean tooHighNode(long node) { + if (hi != null) { + int c = m.compareKeyAt(hi, node); + if (c < 0 || (c == 0 && !hiInclusive)) { + return true; + } + } + return false; + } + + private boolean keyInBounds(K key) { + return !tooLow(key) && !tooHigh(key); + } + + private boolean nodeInBounds(long node) { + return !tooLowNode(node) && !tooHighNode(node); + } + + private void checkKeyBounds(K key) throws IllegalArgumentException { + if (key == null) { + throw new NullPointerException(); + } + if (!keyInBounds(key)) { + throw new IllegalArgumentException("key out of range"); + } + } + + // Returns true if node key is less than upper bound of range + private boolean isBeforeEnd(long node) { + if (m.isNullNode(node)) { + return false; + } + if (hi == null) { + return true; + } + return !tooHighNode(node); + } + + private Map.Entry getNearEntry(K key, int rel) { + if (isDescending) { // adjust relation for direction + if ((rel & LT) == 0) { + rel |= LT; + } else { + rel &= ~LT; + } + } + if (tooLow(key)) { + return ((rel & LT) != 0) ? null : lowestEntry(); + } + if (tooHigh(key)) { + return ((rel & LT) != 0) ? highestEntry() : null; + } + for (;;) { + long n = m.findNear(key, rel); + if (m.isNullNode(n) || !nodeInBounds(n)) { + return null; + } + long n_header = m.nodeHeader(n); + if (!m.headerDeleted(n_header)) { + KVPair kv = m.nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + private K getNearKey(K key, int rel) { + Map.Entry entry = getNearEntry(key, rel); + return entry == null ? null : entry.getKey(); + } + + private long loNode() { + if (lo == null) { + return m.findFirst(); + } else if (loInclusive) { + return m.findNear(lo, GT | EQ); + } else { + return m.findNear(lo, GT); + } + } + + private long hiNode() { + if (hi == null) { + return m.findLast(); + } else if (hiInclusive) { + return m.findNear(hi, LT | EQ); + } else { + return m.findNear(hi, LT); + } + } + + private Map.Entry lowestEntry() { + for (;;) { + long n = loNode(); + if (!isBeforeEnd(n)) { + return null; + } + long n_header = m.nodeHeader(n); + if (m.headerDeleted(n_header)) { + return null; + } + KVPair kv = m.nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + + private K lowestKey() { + Map.Entry entry = lowestEntry(); + if (entry != null) { + return entry.getKey(); + } + throw new NoSuchElementException(); + } + + private Map.Entry highestEntry() { + for (;;) { + long n = hiNode(); + if (m.isNullNode(n) || !nodeInBounds(n)) { + return null; + } + long n_header = m.nodeHeader(n); + if (!m.headerDeleted(n_header)) { + KVPair kv = m.nodeGetKV(n, n_header); + return new AbstractMap.SimpleImmutableEntry(kv.key, kv.value); + } + } + } + + private K highestKey() { + Map.Entry entry = highestEntry(); + if (entry != null) { + return entry.getKey(); + } + throw new NoSuchElementException(); + } + + @Override + public V put(K key, V value) { + checkKeyBounds(key); + return m.put(key, value); + } + + @SuppressWarnings("unchecked") + public V get(Object key) { + if (key == null) { + throw new NullPointerException(); + } + K k = (K) key; + return ((!keyInBounds(k)) ? null : m.get(k)); + } + + public V putIfAbsent(K key, V value) { + checkKeyBounds(key); + return m.putIfAbsent(key, value); + } + + @SuppressWarnings("unchecked") + public V remove(Object key) { + K k = (K) key; + return (!keyInBounds(k)) ? null : m.remove(k); + } + + @SuppressWarnings("unchecked") + public boolean remove(Object key, Object value) { + K k = (K) key; + return keyInBounds(k) && m.remove(k, value); + } + + private Map.Entry removeHighest() { + for (;;) { + long n = hiNode(); + if (m.isNullNode(n)) { + return null; + } + if (!nodeInBounds(n)) { + return null; + } + KVPair kv = m.nodeGetKV2(n); + V v = m.doRemove(kv.key, null); + if (v != null) { + return new AbstractMap.SimpleImmutableEntry(kv.key, v); + } + } + } + + private Map.Entry removeLowest() { + for (;;) { + long n = loNode(); + if (m.isNullNode(n)) { + return null; + } + if (!nodeInBounds(n)) { + return null; + } + KVPair kv = m.nodeGetKV2(n); + V v = m.doRemove(kv.key, null); + if (v != null) { + return new AbstractMap.SimpleImmutableEntry(kv.key, v); + } + } + } + + public boolean replace(K key, V oldValue, V newValue) { + checkKeyBounds(key); + ; + return m.replace(key, oldValue, newValue); + } + + public V replace(K key, V value) { + checkKeyBounds(key); + return m.replace(key, value); + } + + public Map.Entry lowerEntry(K key) { + return getNearEntry(key, LT); + } + + public K lowerKey(K key) { + return getNearKey(key, LT); + } + + public Map.Entry floorEntry(K key) { + return getNearEntry(key, (LT | EQ)); + } + + public K floorKey(K key) { + return getNearKey(key, (LT | EQ)); + } + + public Map.Entry ceilingEntry(K key) { + return getNearEntry(key, (GT | EQ)); + } + + public K ceilingKey(K key) { + return getNearKey(key, (GT | EQ)); + } + + public Map.Entry higherEntry(K key) { + return getNearEntry(key, (GT)); + } + + public K higherKey(K key) { + return getNearKey(key, (GT)); + } + + public java.util.Map.Entry firstEntry() { + return isDescending ? highestEntry() : lowestEntry(); + } + + public java.util.Map.Entry lastEntry() { + return isDescending ? lowestEntry() : highestEntry(); + } + + public Map.Entry pollFirstEntry() { + return isDescending ? removeHighest() : removeLowest(); + } + + public java.util.Map.Entry pollLastEntry() { + return isDescending ? removeLowest() : removeHighest(); + } + + public Comparator comparator() { + Comparator cmp = m.comparator(); + if (isDescending) { + return Collections.reverseOrder(cmp); + } else { + return cmp; + } + } + + public K firstKey() { + return isDescending ? highestKey() : lowestKey(); + } + + public K lastKey() { + return isDescending ? lowestKey() : highestKey(); + } + + private SubMap newSubMap(K fromKey, boolean fromInclusive, K toKey, + boolean toInclusive) { + if (isDescending) { + K tk = fromKey; + fromKey = toKey; + toKey = tk; + boolean ti = fromInclusive; + fromInclusive = toInclusive; + toInclusive = ti; + } + if (lo != null) { + if (fromKey == null) { + fromKey = lo; + fromInclusive = loInclusive; + } else { + int c = m.compare(fromKey, lo); + if (c < 0 || (c == 0 && !loInclusive && fromInclusive)) { + throw new IllegalArgumentException("key out of range"); + } + } + } + if (hi != null) { + if (toKey == null) { + toKey = hi; + toInclusive = hiInclusive; + } else { + int c = m.compare(toKey, hi); + if (c > 0 || (c == 0 && !hiInclusive && toInclusive)) { + throw new IllegalArgumentException("Key out of range"); + } + } + } + return new SubMap(m, fromKey, fromInclusive, toKey, toInclusive, + isDescending); + } + + public ConcurrentNavigableMap subMap(K fromKey, + boolean fromInclusive, K toKey, boolean toInclusive) { + if (fromKey == null || toKey == null) { + throw new NullPointerException(); + } + return newSubMap(fromKey, fromInclusive, toKey, toInclusive); + } + + public ConcurrentNavigableMap headMap(K toKey, boolean inclusive) { + if (toKey == null) { + throw new NullPointerException(); + } + return newSubMap(null, false, toKey, inclusive); + } + + public ConcurrentNavigableMap tailMap(K fromKey, boolean inclusive) { + if (fromKey == null) { + throw new NullPointerException(); + } + return newSubMap(fromKey, inclusive, null, false); + } + + public ConcurrentNavigableMap subMap(K fromKey, K toKey) { + return subMap(fromKey, true, toKey, false); + } + + public ConcurrentNavigableMap headMap(K toKey) { + return headMap(toKey, false); + } + + public ConcurrentNavigableMap tailMap(K fromKey) { + return tailMap(fromKey, true); + } + + public ConcurrentNavigableMap descendingMap() { + return new SubMap(m, lo, loInclusive, hi, hiInclusive, + !isDescending); + } + + public NavigableSet navigableKeySet() { + return keySet(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public NavigableSet keySet() { + KeySet ks = keySetView; + return (ks != null) ? ks : (keySetView = new KeySet(this)); + } + + public NavigableSet descendingKeySet() { + return descendingMap().keySet(); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override + public Set> entrySet() { + Set> es = entrySetView; + return (es != null) ? es : (entrySetView = new EntrySet(this)); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override + public Collection values() { + Collection vs = valuesView; + return (vs != null) ? vs : (valuesView = new Values(this)); + } + + @Override + public int size() { + int count = 0; + for (long n = loNode(); isBeforeEnd(n); n = m.nodeGetNext(n)) { + if (!m.nodeDeleted(n)) { + ++count; + } + } + return count; + } + + @Override + public void clear() { + for (long n = loNode(); isBeforeEnd(n); n = m.nodeGetNext(n)) { + if (!m.nodeDeleted(n)) { + KVPair kv = m.nodeGetKV2(n); + m.remove(kv.key); + } + } + } + + @Override + public boolean containsValue(Object value) { + if (value == null) { + throw new NullPointerException(); + } + for (long n = loNode(); isBeforeEnd(n); n = m.nodeGetNext(n)) { + long n_header = m.nodeHeader(n); + if (!m.headerDeleted(n_header)) { + KVPair kv = m.nodeGetKV2(n); + if (value.equals(kv.value)) { + return true; + } + } + } + return false; + } + + @SuppressWarnings("unchecked") + @Override + public boolean containsKey(Object key) { + K k = (K) key; + return keyInBounds(k) && m.containsKey(key); + } + + public Iterator> entryIterator() { + return new SubMapEntryIterator(); + } + + @Override + public boolean isEmpty() { + return !isBeforeEnd(loNode()); + } + + public Iterator keyIterator() { + return new SubMapKeyIterator(); + } + + public Iterator valueIterator() { + return new SubMapValueIterator(); + } + + abstract class SubMapIter implements Iterator { + long lastReturned; + KVPair lastReturned_KV; // just like cache + long next; + long next_header; + + public SubMapIter() { + for (;;) { + next = isDescending ? hiNode() : loNode(); + if (m.isNullNode(next)) { + break; + } + long x = m.nodeHeader(next); + if (!m.headerDeleted(next_header)) { + if (!nodeInBounds(next)) { + next = NULL_NODE; + } else { + next_header = x; + } + break; + } + } + } + + public final boolean hasNext() { + return !m.isNullNode(next); + } + + final void advance() { + if (m.isNullNode(next)) { + throw new NoSuchElementException(); + } + lastReturned_KV = m.nodeGetKV(next, next_header); + lastReturned = next; + if (isDescending) { + descend(); + } else { + ascend(); + } + } + + private void ascend() { + for (;;) { + next = m.nodeGetNext(next); + if (m.isNullNode(next)) { + break; + } + long x = m.nodeHeader(next); + if (!m.headerDeleted(x)) { + if (tooHighNode(next)) { + next = NULL_NODE; + } else { + next_header = x; + } + break; + } + } + } + + private void descend() { + for (;;) { + next = m.findNear(lastReturned_KV.key, LT); + if (m.isNullNode(next)) { + break; + } + long x = m.nodeHeader(next); + if (!m.headerDeleted(x)) { + if (tooLowNode(next)) { + next = NULL_NODE; + } else { + next_header = x; + } + break; + } + } + } + + public void remove() { + if (m.isNullNode(lastReturned)) { + throw new IllegalStateException(); + } + m.remove(lastReturned_KV.key); + lastReturned = NULL_NODE; + } + } + + final class SubMapEntryIterator extends SubMapIter> { + public Map.Entry next() { + if (m.isNullNode(next)) { + throw new IllegalStateException(); + } + advance(); + return new AbstractMap.SimpleImmutableEntry(lastReturned_KV.key, + lastReturned_KV.value); + } + } + + final class SubMapKeyIterator extends SubMapIter { + public K next() { + if (m.isNullNode(next)) { + throw new IllegalStateException(); + } + advance(); + return lastReturned_KV.key; + } + } + + final class SubMapValueIterator extends SubMapIter { + public V next() { + if (m.isNullNode(next)) { + throw new IllegalStateException(); + } + advance(); + return lastReturned_KV.value; + } + } + } + + static final List toList(Collection c) { + List list = new ArrayList(); + for (X e : c) { + list.add(e); + } + return list; + } + + static final class KeySet extends AbstractSet implements + NavigableSet { + private final ConcurrentNavigableMap m; + + KeySet(ConcurrentNavigableMap map) { + m = map; + } + + public Comparator comparator() { + return m.comparator(); + } + + public E first() { + return m.firstKey(); + } + + public E last() { + return m.lastKey(); + } + + public E lower(E e) { + return m.lowerKey(e); + } + + public E floor(E e) { + return m.floorKey(e); + } + + public E ceiling(E e) { + return m.ceilingKey(e); + } + + public E higher(E e) { + return m.higherKey(e); + } + + public E pollFirst() { + Map.Entry e = m.pollFirstEntry(); + return e == null ? null : e.getKey(); + } + + public E pollLast() { + Map.Entry e = m.pollLastEntry(); + return e == null ? null : e.getKey(); + } + + public NavigableSet descendingSet() { + return new CompactedConcurrentSkipListSet(m.descendingMap()); + } + + public Iterator descendingIterator() { + return descendingSet().iterator(); + } + + public NavigableSet subSet(E fromElement, boolean fromInclusive, + E toElement, boolean toInclusive) { + return new CompactedConcurrentSkipListSet(m.subMap(fromElement, + fromInclusive, toElement, toInclusive)); + } + + public NavigableSet headSet(E toElement, boolean inclusive) { + return new CompactedConcurrentSkipListSet(m.headMap(toElement, + inclusive)); + } + + public NavigableSet tailSet(E fromElement, boolean inclusive) { + return new CompactedConcurrentSkipListSet(m.tailMap(fromElement, + inclusive)); + } + + public SortedSet subSet(E fromElement, E toElement) { + return new CompactedConcurrentSkipListSet(m.subMap(fromElement, + toElement)); + } + + public SortedSet headSet(E toElement) { + return new CompactedConcurrentSkipListSet(m.headMap(toElement)); + } + + public SortedSet tailSet(E fromElement) { + return new CompactedConcurrentSkipListSet(m.tailMap(fromElement)); + } + + @Override + public Iterator iterator() { + if (m instanceof CompactedConcurrentSkipListMap) { + return ((CompactedConcurrentSkipListMap) m).keyIterator(); + } else { + return ((CompactedConcurrentSkipListMap.SubMap) m) + .keyIterator(); + } + } + + @Override + public int size() { + return m.size(); + } + + public boolean isEmpty() { + return m.isEmpty(); + } + + public boolean contains(Object o) { + return m.containsKey(o); + } + + public boolean remove(Object o) { + return m.remove(o) != null; + } + + public void clear() { + m.clear(); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Set)) { + return false; + } + Collection c = (Collection) o; + try { + return containsAll(c) && c.containsAll(this); + } catch (ClassCastException e) { + return false; + } catch (NullPointerException e) { + return false; + } + } + + public Object[] toArray() { + return toList(this).toArray(); + } + + public T[] toArray(T[] a) { + return toList(this).toArray(a); + } + } + + static final class EntrySet extends AbstractSet> { + private final ConcurrentNavigableMap m; + + EntrySet(ConcurrentNavigableMap map) { + m = map; + } + + public Iterator> iterator() { + if (m instanceof CompactedConcurrentSkipListMap) { + return ((CompactedConcurrentSkipListMap) m).entryIterator(); + } else { + return ((SubMap) m).entryIterator(); + } + } + + @SuppressWarnings("unchecked") + public boolean contains(Object o) { + if (!(o instanceof Map.Entry)) { + return false; + } + Map.Entry e = (Map.Entry) o; + V1 v = m.get(e.getKey()); + return v != null && v.equals(e.getValue()); + } + + @SuppressWarnings("unchecked") + public boolean remove(Object o) { + if (!(o instanceof Map.Entry)) { + return false; + } + Map.Entry e = (Map.Entry) o; + return m.remove(e.getKey(), e.getValue()); + } + + public boolean isEmpty() { + return m.isEmpty(); + } + + public int size() { + return m.size(); + } + + public void clear() { + m.clear(); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Set)) { + return false; + } + Collection c = (Collection) o; + try { + return containsAll(c) && c.containsAll(this); + } catch (ClassCastException e) { + return false; + } catch (NullPointerException e) { + return false; + } + } + + public Object[] toArray() { + return toList(this).toArray(); + } + + public T[] toArray(T[] a) { + return toList(this).toArray(a); + } + } + + static final class Values extends AbstractCollection { + private final ConcurrentNavigableMap m; + + Values(ConcurrentNavigableMap map) { + m = map; + } + + public Iterator iterator() { + if (m instanceof CompactedConcurrentSkipListMap) { + return ((CompactedConcurrentSkipListMap) m).valueIterator(); + } else { + return ((SubMap) m).valueIterator(); + } + } + + public boolean isEmpty() { + return m.isEmpty(); + } + + public int size() { + return m.size(); + } + + public boolean contains(Object o) { + return m.containsValue(o); + } + + public void clear() { + m.clear(); + } + + public Object[] toArray() { + return toList(this).toArray(); + } + + public T[] toArray(T[] a) { + return toList(this).toArray(a); + } + } +} \ No newline at end of file diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMapPerformanceTester.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMapPerformanceTester.java new file mode 100644 index 0000000000..5f21926fd0 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListMapPerformanceTester.java @@ -0,0 +1,849 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import com.google.common.io.Files; + +@InterfaceAudience.Private +public class CompactedConcurrentSkipListMapPerformanceTester { + static class IntegerTypeHelper implements CompactedTypeHelper { + @Override + public int getCompactedSize(Integer key, String value) { + return 4 + value.length(); + } + + private void copyIntToArray(int val, byte[] data, int offset) { + for (int pos = offset; pos < offset + 4; ++pos) { + data[pos] = (byte) (val & BYTE); + val = val >> 8; + } + } + + static int BYTE = 0xFF; + + private final int getIntFromArray(byte[] data, int offset) { + int ret = 0; + ret = ((data[offset + 3] & BYTE) << 24) | ((data[offset + 2] & BYTE)) << 16 + | ((data[offset + 1] & BYTE) << 8) | (data[offset] & BYTE); + return ret; + } + + @Override + public void compact(Integer key, String value, byte[] data, int offset, + int len) { + copyIntToArray(key.intValue(), data, offset); + //byte[] src = value.getBytes(); + //System.arraycopy(src, 0, data, offset + 4, src.length); + } + + static private String value = new String("emptyvalue"); + + @Override + public CompactedTypeHelper.KVPair decomposte( + byte[] data, int offset, int len) { + int intkey = getIntFromArray(data, offset); + // we don't care which value it is + return new KVPair(Integer.valueOf(intkey), value); + } + + private int compare(int key1, int key2) { + if (key1 < key2) { + return -1; + } else if (key2 < key1) { + return 1; + } else { + return 0; + } + } + + @Override + public int compare(byte[] data1, int offset1, int len1, byte[] data2, + int offset2, int len2) { + int key1 = getIntFromArray(data1, offset1); + int key2 = getIntFromArray(data2, offset2); + return compare(key1, key2); + } + + @Override + public int compare(Integer key, byte[] data, int offset, int len) { + return compare(key.intValue(), getIntFromArray(data, offset)); + } + + @Override + public int compare(Integer key1, Integer key2) { + return compare(key1.intValue(), key2.intValue()); + } + + } + + static abstract class AbstractMap { + protected Config conf; + AbstractMap(Config conf) { + this.conf = conf; + } + abstract boolean put(Random random); + abstract boolean put(Random random, AbstractMap anothor); + abstract boolean get(Random random); + abstract public long size(); + } + + static class IntegerMap extends AbstractMap { + final Map embededMap; + final int keyRange; + Map createMapForTest() { + if (conf.isCompcatedMap) { + return new CompactedConcurrentSkipListMap( + new IntegerTypeHelper(), CompactedConcurrentSkipListMap.PageSetting.create() + .setDataPageSize(conf.dataPageSize)); + } else { + return new ConcurrentSkipListMap(); + } + } + + static ThreadLocal staticValue = new ThreadLocal(); + static String createValue(int len) { + String ret = staticValue.get(); + if (ret != null && ret.length() == len) { + return new String(ret); + } else { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < len; ++i) { + sb.append('0'); + } + staticValue.set(sb.toString()); + return new String(staticValue.get()); + } + } + + IntegerMap(Config conf) { + super(conf); + this.embededMap = createMapForTest(); + this.keyRange = conf.getKeyDistirbuteRange(); + } + + @Override + boolean put(Random random) { + return (embededMap.put(random.nextInt(), createValue(conf.getValueLength())) == null); + } + + @Override + boolean get(Random random) { + int key = random.nextInt(this.keyRange); + if (random.nextBoolean()) { + key = -key; + } + String ret = embededMap.get(key); + return (ret != null); + } + + @Override + boolean put(Random random, AbstractMap anothor) { + IntegerMap a = (IntegerMap) anothor; + int key = random.nextInt(); + String value = createValue(conf.getValueLength()); + this.embededMap.put(key, value); + return (a.embededMap.put(key, value) == null); + } + + @Override + public long size() { + return embededMap.size(); + } + } + + static class KeyValueMap extends AbstractMap { + final ConcurrentNavigableMap embededMap; + final int keyRange; + KeyValueMap(Config conf) { + super(conf); + this.embededMap = createMapForTest(conf); + this.keyRange = conf.getKeyDistirbuteRange(); + } + + + static ConcurrentNavigableMap createMapForTest(Config config) { + if (config.isCompcatedMap) { + CompactedKeyValueTypeHelper typeHelper = new CompactedKeyValueTypeHelper( + KeyValue.COMPARATOR); + return new CompactedConcurrentSkipListMap( + typeHelper, CompactedConcurrentSkipListMap.PageSetting.create() + .setDataPageSize(config.dataPageSize)); + } else { + return new ConcurrentSkipListMap(KeyValue.COMPARATOR); + } + } + + static ThreadLocal kvgs = new ThreadLocal(); + + private KeyValueGenerator getKeyValueGenerator() { + KeyValueGenerator ret = kvgs.get(); + if (ret == null) { + ret = new KeyValueGenerator(conf); + kvgs.set(ret); + } + return ret; + } + + @Override + boolean put(Random random) { + KeyValueGenerator kvg = getKeyValueGenerator(); + KeyValue kv = kvg.next(random); + return (embededMap.put(kv, kv) == null); + } + + + @Override + boolean get(Random random) { + KeyValueGenerator kvg = getKeyValueGenerator(); + KeyValue kv = kvg.nextForRead(random); + return (embededMap.get(kv) != null); + } + + + @Override + boolean put(Random random, AbstractMap anothor) { + KeyValueMap a = (KeyValueMap) anothor; + KeyValueGenerator kvg = getKeyValueGenerator(); + KeyValue kv = kvg.next(random); + this.embededMap.put(kv, kv); + return (a.embededMap.put(kv, kv) == null); + } + + + @Override + public long size() { + return embededMap.size(); + } + } + + + static class Config { + int dataPageSize = 256 * 1024; + int metaPageSize = 128 * 1024; + int rowKeylength = 10; + int qualifier = 3; + int familyLength = 3; + int vlen = 30; // total length of key-value + int threads = 8; + + boolean isCompcatedMap = true; + int mapSize = 200 * 1024 * 1024; + int presentMap = 32; + int testingTime = 0; // seconds + boolean saveOutput = false; + + final static String READ = "read"; + final static String WRITE = "write"; + final static String MEMORY = "memory"; + + String type = "write"; + + final static String SCENE_KV = "kv"; + final static String SCENE_INT = "int"; + + String scene = "kv"; + + + + int valueLength = 0; + + + + public Config() { + } + + public Config(Config conf) { + this.dataPageSize = conf.dataPageSize; + this.metaPageSize = conf.metaPageSize; + this.vlen = conf.vlen; + this.threads = conf.threads; + this.isCompcatedMap = conf.isCompcatedMap; + this.mapSize = conf.mapSize; + this.presentMap = conf.presentMap; + this.testingTime = conf.testingTime; + this.type = conf.type; + this.rowKeylength = conf.rowKeylength; + this.familyLength = conf.familyLength; + this.scene = conf.scene; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("test=").append(type).append('\n'); + sb.append("vlen=").append(vlen).append('\n'); + sb.append("threads=").append(threads).append('\n'); + sb.append("dataPageSize=").append(dataPageSize).append('\n'); + sb.append("mataPageSize=").append(metaPageSize).append('\n'); + sb.append("qualifierLen=").append(qualifier).append('\n'); + sb.append("familyLen=").append(familyLength).append('\n'); + sb.append("mapSize=").append(mapSize).append('\n'); + sb.append("scene=").append("scene").append('\n'); + sb.append("saveoutput=").append(saveOutput).append('\n'); + if (this.type.equals(WRITE)) { + sb.append("isCompcatedMap=").append(isCompcatedMap).append('\n'); + } + sb.append("testingtime=").append(testingTime).append('\n'); + return sb.toString(); + } + + public int getEntryCount() { + return mapSize / vlen; + } + + public String getMapType() { + return (isCompcatedMap ? "COMPACTED" : "STAND"); + } + + public int getKeyDistirbuteRange() { + int factor = scene.equals(SCENE_KV) ? 5 : 2; + int range = (mapSize / vlen) * factor; + range = range < 1024 ? 1024 : range; + return range; + } + + public int getValueLength() { + if (valueLength == 0) { + int v = 0; + if (this.scene.equals(SCENE_KV)) { + v = vlen - (Long.SIZE >> 3) - rowKeylength - qualifier - familyLength; + } else { + v = vlen - (Integer.SIZE >> 3); + } + valueLength = v < 0 ? 1 : v; + } + + return valueLength; + } + + public void loadConfigFromProperty() { + isCompcatedMap = Boolean.valueOf(System.getProperty("test.compact", "true")); + threads = Integer.valueOf(System.getProperty("test.threads", Integer.toString(threads))); + presentMap = Integer.valueOf(System.getProperty("test.maps", Integer.toString(presentMap))); + vlen = Integer.valueOf(System.getProperty("test.vlen", Integer.toString(vlen))); + dataPageSize = metaPageSize = Integer.valueOf(System.getProperty("test.pagesize", Integer.toString(dataPageSize))); + mapSize = Integer.valueOf(System.getProperty("test.mapsize", Integer.toString(this.mapSize))); + type = System.getProperty("test.type", "write"); + rowKeylength = Integer.valueOf(System.getProperty("test.rowkey", Integer.toString(rowKeylength))); + qualifier = Integer.valueOf(System.getProperty("test.qualifier", Integer.toString(qualifier))); + familyLength = Integer.valueOf(System.getProperty("test.familiy", Integer.toString(familyLength))); + scene = System.getProperty("test.scene", SCENE_KV); + saveOutput = Boolean.valueOf(System.getProperty("test.sav", Boolean.toString(saveOutput))); + if (WRITE.equals(type)) { + testingTime = Integer.valueOf(System.getProperty("test.time", Integer.toString(300))); + } else if (READ.equals(type)){ + testingTime = Integer.valueOf(System.getProperty("test.time", Integer.toString(120))); + } + } + + public String getCaseName() { + StringBuilder sb = new StringBuilder(); + sb.append(scene).append('_').append(type).append("_vlen").append(vlen); + if (!READ.equals(this.type)) { + sb.append("_").append(getMapType()); + } + return sb.toString(); + } + } + + static AbstractMap createSceneMap(Config config) { + if (Config.SCENE_KV.equals(config.scene)) { + return new KeyValueMap(config); + } else if (Config.SCENE_INT.equals(config.scene)){ + return new IntegerMap(config); + } else { + throw new IllegalArgumentException("Invalid scene: " + config.scene); + } + } + + public static int DEFAULT_DATA_PAGE_SIZE = 256 * 1024; // 256k + public static int DEFAULT_DATA_PAGE = 16 * 1024; // 16k + public static int DEFAULT_META_PAGE_SIZE = 128 * 1024; // 128k + public static int DEFAULT_META_PAGE = 8 * 1024; // 8k + public static int DEFAULT_OUT_PAGE_PAGE_SIZE = 1024; + public static int DEFAULT_OUT_PAGE = 1024; + public static int DEFAULT_BIG_KV_THRESHOLD = 16 * 1024; // 16k + static ThreadLocal random = new ThreadLocal(); + static class MapContainer { + AtomicReferenceArray maps; + List mapSize = new ArrayList(); + Config conf; + + public MapContainer(Config config) { + this.conf = config; + maps = new AtomicReferenceArray(conf.presentMap); + for (int i = 0; i < conf.presentMap; ++i) { + maps.set(i, createSceneMap(config)); + mapSize.add(new AtomicInteger(0)); + } + } + + int putOneRecord() { + // init thread local random object + Random r = null; + r = getThreadLocalRandom(); + + for (;;) { + int mapIdx = r.nextInt(conf.presentMap); + int curSize = mapSize.get(mapIdx).get(); + if (curSize < 0 || curSize > conf.mapSize) { + // map needs updating, leave + continue; + } + AbstractMap curMap = maps.get(mapIdx); + if (curMap == null) { + // map is swapping, leave + continue; + } + curMap.put(r); + curSize = mapSize.get(mapIdx).addAndGet(4 + conf.vlen); + if (curSize > conf.mapSize) { + // try new map; acquire lock by null pointer + if (maps.compareAndSet(mapIdx, curMap, null)) { + System.out.println("Swaping map at slot " + mapIdx); + maps.set(mapIdx, createSceneMap(conf)); + mapSize.get(mapIdx).set(0); + } + } + return 4 + conf.vlen; + } + + } + } + + static class KeyValueGenerator { + + final Config conf; + final int range; + + byte[] rowKeyBuffer; + byte[] qualifierBuffer; + byte[] familyBuffer; + byte[] valueBuffer; + public KeyValueGenerator(Config config) { + this.conf = config; + range = config.getKeyDistirbuteRange(); + rowKeyBuffer = new byte[config.rowKeylength]; + qualifierBuffer = new byte[config.qualifier]; + familyBuffer = new byte[config.familyLength]; + valueBuffer = new byte[config.getValueLength()]; + + // set value + for (int i = 0; i < valueBuffer.length; ++i) { + valueBuffer[i] = '0'; + } + // set family + for (int i = 0; i < familyBuffer.length; ++i) { + familyBuffer[i] = 'a'; + } + } + + void fillBufferFromInt(int v, byte[] buf) { + for (int i = 0; i < buf.length; ++i) { + buf[i] = (byte)('A' + (v & 0x1F)); + v = v >> 5; + } + } + + public KeyValue next(Random random) { + int k = random.nextInt(range); + fillBufferFromInt(k, rowKeyBuffer); + fillBufferFromInt(k, qualifierBuffer); + KeyValue ret = new KeyValue(rowKeyBuffer, familyBuffer, qualifierBuffer, + k, Type.Put, valueBuffer); + return ret; + } + + public KeyValue nextForRead(Random random) { + int k = random.nextInt(range); + fillBufferFromInt(k, rowKeyBuffer); + fillBufferFromInt(k, qualifierBuffer); + KeyValue ret = new KeyValue(rowKeyBuffer, familyBuffer, qualifierBuffer, k, + Type.Put); + return ret; + } + } + + static ThreadLocal staticValue = new ThreadLocal(); + + + + static void doReadTest(Config conf) { + Config conf1 = new Config(conf); conf1.isCompcatedMap = true; + Config conf2 = new Config(conf); conf2.isCompcatedMap = false; + AbstractMap mCompcated = createSceneMap(conf1); + AbstractMap mStand = createSceneMap(conf2); + + System.out.println("Building map"); + Random r = new Random(); + int elementCount = 0; + + for (int i = 0; i < conf.getEntryCount(); ++i) { + if (mCompcated.put(r, mStand)) { + elementCount++; + } + if (i % 100000 == 0) { + System.out.println("Building map i=" + i); + } + } + System.out.println("There are " + elementCount + " element in map"); + double spd1 = testMapRead(conf1, mCompcated); + double spd2 = testMapRead(conf2, mStand); + + System.out.println("\nCompcatd map read speed: " + spd1); + System.out.println("\nStand map read speed: " + spd2); + } + + static double testMapRead(Config conf, AbstractMap map) { + AtomicLong counter = new AtomicLong(0); + System.out.println("\n\nStart doing read test for " + conf.getMapType() + " map"); + List threads = new ArrayList(); + for (int i = 0; i < conf.threads; ++i) { + Thread t = new Thread(runningReadThread(map, conf, counter), "Get-Thread-" + i); + threads.add(t); + } + + for (Thread t : threads) { + t.start(); + } + + long start = System.currentTimeMillis(); + long now = start + 1; + try { + do { + long last = counter.get(); + long startLine = System.currentTimeMillis(); + Thread.sleep(1000); + now = System.currentTimeMillis(); + long time = now - startLine; + long qps = counter.get() - last; + System.out.println("Current qps: " + qps * 1000 / (double)time); + } while (now - start < conf.testingTime * 1000); + } catch (InterruptedException ie) { + } + + for (Thread t : threads) { + t.interrupt(); + t.stop(); + } + + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + //ignore + } + } + + double spd = counter.get() * 1000 / (double)(now - start); + System.out.println("Average qps for " + + (conf.isCompcatedMap ? "compcated" : "stand") + " map: " + + spd); + return spd; + } + + static Runnable runningReadThread(final AbstractMap m, final Config conf, final AtomicLong counter) { + return new Runnable() { + + @Override + public void run() { + Random r = getThreadLocalRandom(); + + + for (;;) { + m.get(r); + counter.incrementAndGet(); + } + } + }; + } + + static Runnable runningWriteThread(final MapContainer mc, + final AtomicLong counter) { + return new Runnable() { + @Override + public void run() { + while (true) { + int res = mc.putOneRecord(); + counter.addAndGet(res); + } + } + }; + } + + static void redirectOutput(Config conf) throws FileNotFoundException { + String outputFileName = conf.getCaseName() + ".out"; + System.out.println("Redirecing output to file " + outputFileName); + PrintStream ps = new PrintStream(outputFileName); + System.setOut(ps); + } + + static void copyGCLog(Config conf) throws IOException { + RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); + List arguments = runtimeMxBean.getInputArguments(); + String src = null; + for (String s : arguments) { + if (s.startsWith("-Xloggc:")) { + src = s.replaceAll("-Xloggc:", ""); + break; + } + } + if (src != null) { + String dst = conf.getCaseName() + "_gc.log"; + System.out.println("Copy gc log from " + src + " to " + dst); + Files.copy(new File(src), new File(dst)); + } else { + System.out.println("GC log not found in vm arguments!"); + } + } + + public static void main(String[] args) throws IOException { + System.out.println("Starting performace test for CompactdConcurrentSkipListMap"); + Config conf = new Config(); + conf.loadConfigFromProperty(); + if (conf.saveOutput) { + redirectOutput(conf); + } + + + System.out.println("Config:"); + System.out.println(conf.toString()); + + if (Config.WRITE.equals(conf.type)) { + doWriteTest(conf); + } else if (Config.READ.equals(conf.type)) { + doReadTest(conf); + } else if (Config.MEMORY.equals(conf.type)) { + doMemoryTest(conf); + } + + copyGCLog(conf); + } + + static AbstractMap[] maps; + + @SuppressWarnings("DM_GC") + private static void doMemoryTest(Config conf) { + + + Random r = new Random() { + int last = 0; + + @Override + public int nextInt() { + last += 1; + return last; + } + + @Override + public int nextInt(int range) { + return nextInt() % range; + } + }; + + maps = new AbstractMap[conf.presentMap]; + System.out.println("Going to put " + conf.getEntryCount() + + " elements into " + maps.length +" " + conf.getMapType() + " map"); + + int entryCount = conf.getEntryCount(); + for (int i = 0; i < maps.length; ++i) { + AbstractMap m = createSceneMap(conf); + maps[i] = m; + for (int j = 0; j < entryCount; ++j) { + m.put(r); + } + } + + long elementCount = 0; + for (int i = 0; i < maps.length; ++i) { + elementCount += maps[i].size(); + } + System.out.println("There are " + elementCount + " entries in " + maps.length + " map"); + System.out.println("gc starting.."); + System.gc(); + System.out.println("gc finished.."); + System.out.println("Free memory: " + Runtime.getRuntime().freeMemory()); + System.out.println("Total memory: " + Runtime.getRuntime().totalMemory()); + System.out.println("Used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 + " KB"); + } + + private static void doWriteTest(Config conf) { + MapContainer mc = new MapContainer(conf); + AtomicLong counter = new AtomicLong(); + List threads = new ArrayList(); + for (int i = 0; i < conf.threads; ++i) { + threads.add(new Thread(runningWriteThread(mc, counter), "Put-Thread-" + i)); + } + for (Thread t : threads) { + t.start(); + } + + long start = System.currentTimeMillis(); + long now = start + 1; + try { + do { + long startLine = System.currentTimeMillis(); + long last = counter.get(); + Thread.sleep(1000); + now = System.currentTimeMillis(); + long time = now - startLine; + long data = counter.get() - last; + System.out.println("Current through put: " + (data * 1000 / 1024) / (double) time); + } while ((now - start) < conf.testingTime * 1000); + } catch (InterruptedException e) { + } + + + for (Thread t : threads) { + t.interrupt(); + t.stop(); + } + + try { + for (Thread t : threads) { + t.join(); + } + } catch (InterruptedException e) { + } + + System.out.println("Average write through put: " + + (counter.get() * 1000 / 1024) / (double) (now - start) + " kb/s"); + } + + private static Random getThreadLocalRandom() { + Random r; + if ((r = random.get()) == null) { + random.set(new Random()); + r = random.get(); + } + return r; + } + + @InterfaceAudience.Private + static public class CompactedKeyValueTypeHelper + implements CompactedTypeHelper { + + private KeyValue.KVComparator comparator; + + public CompactedKeyValueTypeHelper(KeyValue.KVComparator kvCompare) { + this.comparator = kvCompare; + } + + public final static int lengthWithoutMemstore(int len) { + return len - Bytes.SIZEOF_LONG; + } + + @Override + public int getCompactedSize(KeyValue key, KeyValue value) { + return key.getLength() + Bytes.SIZEOF_LONG; // mvcc's size + } + + @Override + public void compact(KeyValue key, KeyValue value, byte[] data, int offset, + int len) { + assert len == key.getLength() + Bytes.SIZEOF_LONG; + System.arraycopy(key.getBuffer(), key.getOffset(), data, offset, + key.getLength()); + long mts = key.getMvccVersion(); + int mo = offset + lengthWithoutMemstore(len); + data[mo] = (byte) (mts >> 56); + data[mo + 1] = (byte) (mts >> 48); + data[mo + 2] = (byte) (mts >> 40); + data[mo + 3] = (byte) (mts >> 32); + data[mo + 4] = (byte) (mts >> 24); + data[mo + 5] = (byte) (mts >> 16); + data[mo + 6] = (byte) (mts >> 8); + data[mo + 7] = (byte) (mts); + } + + @Override + public KVPair decomposte(byte[] data, int offset, + int len) { + KeyValue kv = new KeyValue(data, offset, lengthWithoutMemstore(len)); + kv.setSequenceId(getMvccFromRaw(data, offset, len)); + return new KVPair(kv, kv); + } + + public final static long getMvccFromRaw(byte[] data, int offset, int len) { + int mo = offset + lengthWithoutMemstore(len); + return (data[mo] & 0xFFL) << 56 | (data[mo + 1] & 0xFFL) << 48 + | (data[mo + 2] & 0xFFL) << 40 | (data[mo + 3] & 0xFFL) << 32 + | (data[mo + 4] & 0xFFL) << 24 | (data[mo + 5] & 0xFFL) << 16 + | (data[mo + 6] & 0xFFL) << 8 | (data[mo + 7] & 0xFFL); + } + + private final static int keyLength(byte[] data, int offset) { + return Bytes.toInt(data, offset); + } + + private final static int rawCompare(KeyValue.KVComparator comparator, + byte[] ldata, int loffset, int llen, byte[] rdata, int roffset, + int rlen) { + int ret = comparator.compare(ldata, loffset + KeyValue.ROW_OFFSET, + keyLength(ldata, loffset), rdata, roffset + KeyValue.ROW_OFFSET, + keyLength(rdata, roffset)); + if (ret != 0) + return ret; + // Negate this comparison so later edits show up first + return -Long.compare(getMvccFromRaw(ldata, loffset, llen), + getMvccFromRaw(rdata, roffset, rlen)); + } + + @Override + public int compare(byte[] ldata, int loffset, int llen, byte[] rdata, + int roffset, int rlen) { + return rawCompare(comparator, ldata, loffset, llen, rdata, roffset, rlen); + } + + @Override + public int compare(KeyValue key, byte[] rdata, int roffset, int rlen) { + int ret = comparator.compare(key.getBuffer(), + key.getOffset() + KeyValue.ROW_OFFSET, key.getKeyLength(), rdata, + roffset + KeyValue.ROW_OFFSET, keyLength(rdata, roffset)); + if (ret != 0) + return ret; + // Negate this comparison so later edits show up first + return -Long.compare(key.getMvccVersion(), + getMvccFromRaw(rdata, roffset, rlen)); + } + + @Override + public int compare(KeyValue lkey, KeyValue rkey) { + return comparator.compare(lkey, rkey); + } + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListSet.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListSet.java new file mode 100644 index 0000000000..3837eb4de3 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedConcurrentSkipListSet.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.Serializable; +import java.util.AbstractSet; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.NavigableSet; +import java.util.SortedSet; +import java.util.concurrent.ConcurrentNavigableMap; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +@InterfaceAudience.Private +public class CompactedConcurrentSkipListSet extends AbstractSet implements + NavigableSet, Cloneable, Serializable { + + ConcurrentNavigableMap m; + + CompactedConcurrentSkipListSet(ConcurrentNavigableMap m) { + this.m = m; + } + + @Override + public Comparator comparator() { + return m.comparator(); + } + + @Override + public E first() { + return m.firstKey(); + } + + @Override + public E last() { + return m.lastKey(); + } + + @Override + public E ceiling(E e) { + return m.ceilingKey(e); + } + + @Override + public Iterator descendingIterator() { + return m.descendingKeySet().iterator(); + } + + @Override + public NavigableSet descendingSet() { + return m.descendingKeySet(); + } + + @Override + public E floor(E e) { + return m.floorKey(e); + } + + @Override + public SortedSet headSet(E e) { + return new CompactedConcurrentSkipListSet(m.headMap(e)); + } + + @Override + public NavigableSet headSet(E e, boolean inclusive) { + return new CompactedConcurrentSkipListSet(m.headMap(e, inclusive)); + } + + @Override + public E higher(E e) { + return m.higherKey(e); + } + + @Override + public E lower(E e) { + return m.lowerKey(e); + } + + @Override + public E pollFirst() { + Entry e = m.pollFirstEntry(); + return e == null ? null : e.getKey(); + } + + @Override + public E pollLast() { + Entry e = m.pollLastEntry(); + return e == null ? null : e.getKey(); + } + + @Override + public SortedSet subSet(E from, E to) { + return new CompactedConcurrentSkipListSet(m.subMap(from, to)); + } + + @Override + public NavigableSet subSet(E from, boolean fromInclusive, E to, boolean toInclusive) { + return new CompactedConcurrentSkipListSet(m.subMap(from, fromInclusive, to, toInclusive)); + } + + @Override + public SortedSet tailSet(E e) { + return new CompactedConcurrentSkipListSet(m.tailMap(e)); + } + + @Override + public NavigableSet tailSet(E e, boolean inclusive) { + return new CompactedConcurrentSkipListSet(m.tailMap(e, inclusive)); + } + + @Override + public Iterator iterator() { + return m.navigableKeySet().iterator(); + } + + @Override + public int size() { + return m.size(); + } + +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedTypeHelper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedTypeHelper.java new file mode 100644 index 0000000000..8f6dbf2458 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CompactedTypeHelper.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +@InterfaceAudience.Private +public interface CompactedTypeHelper { + + @InterfaceAudience.Private + static public class KVPair { + public KVPair() { + } + + public KVPair(K k, V v) { + this.key = k; + this.value = v; + } + + public K key; + public V value; + } + + /** + * Get estimated size of compacted k-v + * @param key + * @param value + * @return estimated size + */ + int getCompactedSize(K key, V value); + + /** + * Compact the key&value into data. + * ONLY the area from data[offset] to data[offset+len-1] can be used for this key&value + * If you write out the area, the result is unpredictable + * + * @param key + * @param value + * @param data + * @param offset + * @param len + */ + void compact(K key, V value, byte[] data, int offset, int len); + + /** + * Deserialize the key&value from data + * ONLY the area from data[offset] to data[offset+len-1] can be used for this key&value + * If you write out the area, the result is unpredictable + * + * @param data + * @param offset + * @param len + * @return + */ + KVPair decomposte(byte[] data, int offset, int len); + + /** + * Compare kv1 at data1[offset1, offset1+len1) with kv2 at data2[offset2, offset2+len2) + * if key1 == key2: + * return 0 + * if key1 > key2: + * return >0 + * if key1 < key2: + * return <0 + * @param data1 kv1's data fragment + * @param offset1 kv1's data offset in the fragment + * @param len1 kv1's data length in the fragment + * @param data2 kv2's data fragment + * @param offset TODO + * @param len2 + * @return + */ + int compare(byte[] data1, int offset1, int len1, byte[] data2, int offset, int len2); + int compare(K key, byte[] data, int offset, int len); + int compare(K key1, K key2); +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java index 3baf729ded..f9e24cefda 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.TreeSet; @@ -38,11 +39,15 @@ import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.MetaComparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CompactedTypeHelper; +import org.apache.hadoop.hbase.util.CompactedTypeHelper.KVPair; +import org.junit.Assert; import static org.junit.Assert.assertNotEquals; public class TestKeyValue extends TestCase { private static final Log LOG = LogFactory.getLog(TestKeyValue.class); + static private final Random random = new Random(); public void testColumnCompare() throws Exception { final byte [] a = Bytes.toBytes("aaa"); @@ -848,5 +853,57 @@ public class TestKeyValue extends TestCase { assertEquals(kvA1.hashCode(), kvA2.hashCode()); assertNotEquals(kvA1.hashCode(), kvB.hashCode()); } + + private Type[] typeArray = new Type[]{Type.Delete, Type.DeleteColumn, Type.DeleteFamily, Type.Put, Type.Maximum, Type.Minimum}; + private KeyValue randomKeyValue() { + String key = Integer.toString(random.nextInt()); + String qualifier = Integer.toString(random.nextInt()); + qualifier = qualifier.substring(random.nextInt(qualifier.length() - 1)); + String family = Integer.toString(random.nextInt()); + family = family.substring(random.nextInt(family.length() - 1)); + String value = Integer.toBinaryString(random.nextInt()); + value = value.substring(random.nextInt(value.length() - 1)); + long ts = random.nextLong(); + Type type = typeArray[random.nextInt(typeArray.length)]; + return new KeyValue(key.getBytes(), family.getBytes(), qualifier.getBytes(), ts, type, value.getBytes()); + } + public void testKeyValueCompactedTypeHelper() { + final int RANDOM_TEST_TIMES = 100000; + KVComparator standardCompacrator = KeyValue.COMPARATOR; + CompactedTypeHelper compactedHelper = new KeyValue.CompactedCellTypeHelper(standardCompacrator); + for (int i = 0; i < RANDOM_TEST_TIMES; ++i) { + KeyValue kv1 = randomKeyValue(); + KeyValue kv2 = randomKeyValue(); + // Test comparator + assertEquals( + standardCompacrator.compare(kv1, kv2), + compactedHelper.compare(kv1, kv2.getBuffer(), kv2.getOffset(), + kv2.getLength())); + assertEquals( + standardCompacrator.compare(kv1, kv2), + compactedHelper.compare(kv1.getBuffer(), kv1.getOffset(), + kv1.getLength(), kv2.getBuffer(), kv2.getOffset(), + kv2.getLength())); + assertEquals(standardCompacrator.compare(kv1, kv2), compactedHelper.compare(kv1, kv2)); + + + // Test compact + int csz = compactedHelper.getCompactedSize(kv1, kv1); + int bufferLength = 1 + csz + random.nextInt(csz); + byte[] buffer = new byte[bufferLength]; + int offset = random.nextInt(buffer.length - csz); + compactedHelper.compact(kv1, kv1, buffer, offset, csz); + + assertEquals(compactedHelper.compare(kv1, buffer, offset, csz), 0); + + // Test decomposte + KVPair kvpair = compactedHelper.decomposte(buffer, offset, csz); + assertEquals(standardCompacrator.compare(kv1, kvpair.key), 0); + assertEquals(standardCompacrator.compare(kvpair.value, kv1), 0); + assertEquals(kv1.getMvccVersion(), kvpair.key.getMvccVersion()); + assertEquals(kv1.getMvccVersion(), kvpair.value.getMvccVersion()); + + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java index d1eb8763bf..ef6c632ed5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSkipListSet.java @@ -26,9 +26,17 @@ import java.util.SortedSet; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.CompactedConcurrentSkipListMap; +import org.apache.hadoop.hbase.util.CompactedConcurrentSkipListMap.PutAndFetch; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.ReflectionUtils; /** * A {@link java.util.Set} of {@link Cell}s implemented on top of a @@ -46,14 +54,92 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; */ @InterfaceAudience.Private public class CellSkipListSet implements NavigableSet { + private final static Log LOG = LogFactory.getLog(CellSkipListSet.class); private final ConcurrentNavigableMap delegatee; + private final MemStoreChunkPool chunkPool; + private final MemStoreLAB allocator; + private boolean isEmbededCCSMap = false; + + final private class PageAllocatorForCompactedMap implements + CompactedConcurrentSkipListMap.PageAllocator { + + @Override + public Object[] allocateHeapKVPage(int sz) { + return new Object[sz]; + } + + @Override + public byte[] allocatePages(int sz) { + if (allocator != null) { + ByteRange a = allocator.allocateBytes(sz); + // The chunk is only used for pages, always get a clean new page once + assert a.getOffset() == 0; + assert a.getBytes().length >= sz; + return a.getBytes(); + } else { + return new byte[sz]; + } + } + } + + CellSkipListSet(Configuration conf, final KeyValue.KVComparator c) { + if (conf.getBoolean(DefaultMemStore.USEMSLAB_KEY, + DefaultMemStore.USEMSLAB_DEFAULT)) { + this.chunkPool = MemStoreChunkPool.getPool(conf); + } else { + this.chunkPool = null; + } + isEmbededCCSMap = conf.getBoolean(DefaultMemStore.USECCSMAP_KEY, + DefaultMemStore.USECCSMAP_DEFAULT); + if (this.chunkPool != null) { + if (isEmbededCCSMap) { + this.allocator = new HeapMemStoreLAB(conf, + this.chunkPool.getChunkSize()); + } else { + String className = conf.get(DefaultMemStore.MSLAB_CLASS_NAME, + HeapMemStoreLAB.class.getName()); + this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, + new Class[] { Configuration.class }, new Object[] { conf }); + } + } else { + this.allocator = null; + } + + if (isEmbededCCSMap) { + delegatee = createCompcatedSkipList(conf, c); + } else { + delegatee = new ConcurrentSkipListMap(c); + } + } + + private ConcurrentNavigableMap createCompcatedSkipList( + Configuration conf, KeyValue.KVComparator c) { + CompactedConcurrentSkipListMap.PageSetting ps = new CompactedConcurrentSkipListMap.PageSetting(); + + int pageSize = chunkPool == null ? conf.getInt( + HeapMemStoreLAB.CHUNK_SIZE_KEY, HeapMemStoreLAB.CHUNK_SIZE_DEFAULT) + : this.chunkPool.getChunkSize(); + int threshold = conf.getInt(HeapMemStoreLAB.MAX_ALLOC_KEY, + HeapMemStoreLAB.MAX_ALLOC_DEFAULT); + + ps.setDataPageSize(pageSize); + ps.setHeapKVThreshold(threshold); + ps.setPageAllocator(new PageAllocatorForCompactedMap()); + + return new CompactedConcurrentSkipListMap( + new KeyValue.CompactedCellTypeHelper(c), ps); + } CellSkipListSet(final KeyValue.KVComparator c) { this.delegatee = new ConcurrentSkipListMap(c); + this.chunkPool = null; + this.allocator = null; } CellSkipListSet(final ConcurrentNavigableMap m) { this.delegatee = m; + this.chunkPool = null; + this.allocator = null; } @Override @@ -132,6 +218,22 @@ public class CellSkipListSet implements NavigableSet { public NavigableSet tailSet(Cell fromElement, boolean inclusive) { return new CellSkipListSet(this.delegatee.tailMap(fromElement, inclusive)); } + + /** + * Add cell into set, and return the just added cell + * @param e + * @return A Pair + * Boolean: Whether the added cell is not present + * Cell: The cell stored in the map + */ + public Pair addAndFetch(Cell e) { + if (isEmbededCCSMap) { + PutAndFetch result = ((CompactedConcurrentSkipListMap) delegatee) + .putAndFetch(e, e); + return new Pair(result.old() == null, result.current()); + } + return new Pair(this.delegatee.put(e, e) == null, e); + } @Override public Comparator comparator() { @@ -212,4 +314,41 @@ public class CellSkipListSet implements NavigableSet { public T[] toArray(T[] a) { throw new UnsupportedOperationException("Not implemented"); } + + void incrScannerReference() { + if (allocator != null) { + allocator.incScannerCount(); + } + } + + void decrScannerReference() { + if (allocator != null) { + allocator.decScannerCount(); + } + } + + void close() { + if (isEmbededCCSMap) { + CompactedConcurrentSkipListMap m = (CompactedConcurrentSkipListMap) delegatee; + if (m != null && LOG.isTraceEnabled()) { + LOG.trace("Sealed ccsmap, memory summary: " + + m.getMemoryUsage().toString()); + } + } + if (allocator != null) { + allocator.close(); + } + } + + MemStoreLAB getAllocator() { + return this.allocator; + } + + boolean isEntryUncleanable() { + return this.isEmbededCCSMap; + } + + boolean isCCSMap() { + return this.isEmbededCCSMap; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 884ef29a0c..ad083cbdae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionserver; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -40,13 +41,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.CollectionBackedScanner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.ReflectionUtils; +import org.apache.hadoop.hbase.util.Pair; import org.apache.htrace.Trace; import com.google.common.annotations.VisibleForTesting; @@ -70,26 +70,38 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceAudience.Private public class DefaultMemStore implements MemStore { private static final Log LOG = LogFactory.getLog(DefaultMemStore.class); - @VisibleForTesting static final String USEMSLAB_KEY = "hbase.hregion.memstore.mslab.enabled"; - private static final boolean USEMSLAB_DEFAULT = true; - private static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; + static final boolean USEMSLAB_DEFAULT = true; + static final String USECCSMAP_KEY = "hbase.hregion.memstore.ccsmap.enabled"; + static final boolean USECCSMAP_DEFAULT = false; + static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; private Configuration conf; - @VisibleForTesting + // MemStore. Use a CellSkipListSet rather than SkipListSet because of the + // better semantics. The Map will overwrite if passed a key it already had + // whereas the Set will not add new Cell if key is same though value might be + // different. Value is not important -- just make sure always same + // reference passed. + volatile CellSkipListSet cellSet; + + // Snapshot of memstore. Made for flusher. + volatile CellSkipListSet snapshot; + final KeyValue.KVComparator comparator; + // Used to track own heapSize + final AtomicLong size; + private volatile long snapshotSize; + private volatile boolean tagsPresent; + // Used to track when to flush - private volatile long timeOfOldestEdit = Long.MAX_VALUE; + volatile long timeOfOldestEdit = Long.MAX_VALUE; - private volatile long snapshotId; - private volatile boolean tagsPresent; + TimeRangeTracker timeRangeTracker; + TimeRangeTracker snapshotTimeRangeTracker; - @VisibleForTesting - volatile Section activeSection; - @VisibleForTesting - volatile Section snapshotSection; + volatile long snapshotId; /** * Default constructor. Used for tests. @@ -106,8 +118,31 @@ public class DefaultMemStore implements MemStore { final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; - this.activeSection = Section.newActiveSection(comparator, conf); - this.snapshotSection = Section.newSnapshotSection(comparator); + this.cellSet = createCellSkipListSet(conf, comparator); + this.snapshot = createCellSkipListSet(conf, comparator); + timeRangeTracker = new TimeRangeTracker(); + snapshotTimeRangeTracker = new TimeRangeTracker(); + this.size = new AtomicLong(DEEP_OVERHEAD); + this.snapshotSize = 0; + } + + private CellSkipListSet createCellSkipListSet(Configuration conf, + KeyValue.KVComparator c) { + return new CellSkipListSet(conf, c); + } + + @VisibleForTesting + CellSkipListSet getCellSkipListSet() { + return this.cellSet; + } + + void dump() { + for (Cell cell: this.cellSet) { + LOG.info(cell); + } + for (Cell cell: this.snapshot) { + LOG.info(cell); + } } /** @@ -118,23 +153,25 @@ public class DefaultMemStore implements MemStore { public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. - if (!snapshotSection.getCellSkipListSet().isEmpty()) { + if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); - if (!activeSection.getCellSkipListSet().isEmpty()) { - snapshotSection = activeSection; - activeSection = Section.newActiveSection(comparator, conf); - snapshotSection.getHeapSize().addAndGet(-DEEP_OVERHEAD); + this.snapshotSize = keySize(); + if (!this.cellSet.isEmpty()) { + this.snapshot = this.cellSet; + this.cellSet = createCellSkipListSet(conf, comparator); + this.snapshotTimeRangeTracker = this.timeRangeTracker; + this.timeRangeTracker = new TimeRangeTracker(); + // Reset heap to not include any keys + this.size.set(DEEP_OVERHEAD); timeOfOldestEdit = Long.MAX_VALUE; } } - MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, - snapshotSection.getCellSkipListSet().size(), snapshotSection.getHeapSize().get(), - snapshotSection.getTimeRangeTracker(), - new CollectionBackedScanner(snapshotSection.getCellSkipListSet(), this.comparator), - this.tagsPresent); + MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), + this.snapshotSize, this.snapshotTimeRangeTracker, + new CollectionBackedScanner(snapshot, this.comparator), this.tagsPresent); this.tagsPresent = false; return memStoreSnapshot; } @@ -147,41 +184,47 @@ public class DefaultMemStore implements MemStore { */ @Override public void clearSnapshot(long id) throws UnexpectedStateException { + CellSkipListSet discardedSnapshot = null; if (this.snapshotId == -1) return; // already cleared if (this.snapshotId != id) { throw new UnexpectedStateException("Current snapshot id is " + this.snapshotId + ",passed " + id); } - // OK. Passed in snapshot is same as current snapshot. - MemStoreLAB tmpAllocator = snapshotSection.getMemStoreLAB(); - snapshotSection = Section.newSnapshotSection(comparator); - if (tmpAllocator != null) { - tmpAllocator.close(); + // OK. Passed in snapshot is same as current snapshot. If not-empty, + // create a new snapshot and let the old one go. + if (!this.snapshot.isEmpty()) { + discardedSnapshot = this.snapshot; + this.snapshot = createCellSkipListSet(conf, comparator); + this.snapshotTimeRangeTracker = new TimeRangeTracker(); } + this.snapshotSize = 0; this.snapshotId = -1; + if (discardedSnapshot != null) { + discardedSnapshot.close(); + } } @Override public long getFlushableSize() { - long snapshotSize = snapshotSection.getHeapSize().get(); - return snapshotSize > 0 ? snapshotSize : keySize(); + return this.snapshotSize > 0 ? this.snapshotSize : keySize(); } @Override public long getSnapshotSize() { - return snapshotSection.getHeapSize().get(); + return this.snapshotSize; } /** * Write an update * @param cell - * @return approximate size of the passed cell. + * @return approximate size of the passed KV & newly added KV which maybe different than the + * passed-in KV */ @Override public long add(Cell cell) { - Cell toAdd = maybeCloneWithAllocator(cell); + Cell toAdd = maybeCloneWithAllocator(cell, cellSet); boolean mslabUsed = (toAdd != cell); - return internalAdd(toAdd, mslabUsed); + return internalAdd(toAdd, mslabUsed).getFirst(); } @Override @@ -189,21 +232,21 @@ public class DefaultMemStore implements MemStore { return timeOfOldestEdit; } - private boolean addToCellSet(Cell e) { - boolean b = this.activeSection.getCellSkipListSet().add(e); + private Pair addToCellSet(Cell e) { + Pair result = this.cellSet.addAndFetch(e); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. // When we use ACL CP or Visibility CP which deals with Tags during // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not // parse the byte[] to identify the tags length. - if(e.getTagsLength() > 0) { + if (e.getTagsLength() > 0) { tagsPresent = true; } setOldestEditTimeToNow(); - return b; + return result; } private boolean removeFromCellSet(Cell e) { - boolean b = this.activeSection.getCellSkipListSet().remove(e); + boolean b = this.cellSet.remove(e); setOldestEditTimeToNow(); return b; } @@ -221,20 +264,24 @@ public class DefaultMemStore implements MemStore { * Callers should ensure they already have the read lock taken * @param toAdd the cell to add * @param mslabUsed whether using MSLAB - * @return the heap size change in bytes + * @return A Pair + * Long: The size changed + * Cell: The cell in the embedded map */ - private long internalAdd(final Cell toAdd, boolean mslabUsed) { - boolean notPresent = addToCellSet(toAdd); - long s = heapSizeChange(toAdd, notPresent); + private Pair internalAdd(final Cell toAdd, boolean mslabUsed) { + Pair ret = addToCellSet(toAdd); + boolean notPresent = ret.getFirst(); + Cell added = ret.getSecond(); + long s = heapSizeChange(toAdd, notPresent || cellSet.isEntryUncleanable()); // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // than the counted number) if (!notPresent && mslabUsed) { s += getCellLength(toAdd); } - activeSection.getTimeRangeTracker().includeTimestamp(toAdd); - activeSection.getHeapSize().addAndGet(s); - return s; + timeRangeTracker.includeTimestamp(toAdd); + this.size.addAndGet(s); + return new Pair(s, added); } /** @@ -245,13 +292,15 @@ public class DefaultMemStore implements MemStore { return KeyValueUtil.length(cell); } - private Cell maybeCloneWithAllocator(Cell cell) { - if (activeSection.getMemStoreLAB() == null) { + private Cell maybeCloneWithAllocator(Cell cell, CellSkipListSet cellSet) { + // CCSMap has its own logic to do entry copy&compact, we should not handle + // for it + if (cellSet == null || cellSet.getAllocator() == null || cellSet.isCCSMap()) { return cell; } int len = getCellLength(cell); - ByteRange alloc = activeSection.getMemStoreLAB().allocateBytes(len); + ByteRange alloc = cellSet.getAllocator().allocateBytes(len); if (alloc == null) { // The allocation was too large, allocator decided // not to do anything with it. @@ -271,27 +320,35 @@ public class DefaultMemStore implements MemStore { * tailMap/iterator, but since this method is called rarely (only for * error recovery), we can leave those optimization for the future. * @param cell + * @Return the size of bytes which are rolled back */ @Override - public void rollback(Cell cell) { + public long rollback(Cell cell) { // If the key is in the snapshot, delete it. We should not update // this.size, because that tracks the size of only the memstore and // not the snapshot. The flush of this snapshot to disk has not // yet started because Store.flush() waits for all rwcc transactions to // commit before starting the flush to disk. - Cell found = snapshotSection.getCellSkipListSet().get(cell); + long rollbackSize = 0; + Cell found = this.snapshot.get(cell); if (found != null && found.getSequenceId() == cell.getSequenceId()) { - snapshotSection.getCellSkipListSet().remove(cell); + this.snapshot.remove(cell); long sz = heapSizeChange(cell, true); - snapshotSection.getHeapSize().addAndGet(-sz); + if (!snapshot.isEntryUncleanable()) { + this.snapshotSize -= sz; + } } // If the key is in the memstore, delete it. Update this.size. - found = activeSection.getCellSkipListSet().get(cell); + found = this.cellSet.get(cell); if (found != null && found.getSequenceId() == cell.getSequenceId()) { removeFromCellSet(cell); - long sz = heapSizeChange(found, true); - activeSection.getHeapSize().addAndGet(-sz); + long s = heapSizeChange(cell, true); + if (!cellSet.isEntryUncleanable()) { + rollbackSize = s; + this.size.addAndGet(-s); + } } + return rollbackSize; } /** @@ -301,9 +358,9 @@ public class DefaultMemStore implements MemStore { */ @Override public long delete(Cell deleteCell) { - Cell toAdd = maybeCloneWithAllocator(deleteCell); + Cell toAdd = maybeCloneWithAllocator(deleteCell, cellSet); boolean mslabUsed = (toAdd != deleteCell); - return internalAdd(toAdd, mslabUsed); + return internalAdd(toAdd, mslabUsed).getFirst(); } /** @@ -312,8 +369,7 @@ public class DefaultMemStore implements MemStore { * @return Next row or null if none found. */ Cell getNextRow(final Cell cell) { - return getLowest(getNextRow(cell, activeSection.getCellSkipListSet()), - getNextRow(cell, snapshotSection.getCellSkipListSet())); + return getLowest(getNextRow(cell, this.cellSet), getNextRow(cell, this.snapshot)); } /* @@ -358,8 +414,8 @@ public class DefaultMemStore implements MemStore { */ @Override public void getRowKeyAtOrBefore(final GetClosestRowBeforeTracker state) { - getRowKeyAtOrBefore(activeSection.getCellSkipListSet(), state); - getRowKeyAtOrBefore(snapshotSection.getCellSkipListSet(), state); + getRowKeyAtOrBefore(cellSet, state); + getRowKeyAtOrBefore(snapshot, state); } /* @@ -457,7 +513,7 @@ public class DefaultMemStore implements MemStore { long now) { Cell firstCell = KeyValueUtil.createFirstOnRow(row, family, qualifier); // Is there a Cell in 'snapshot' with the same TS? If so, upgrade the timestamp a bit. - SortedSet snSs = snapshotSection.getCellSkipListSet().tailSet(firstCell); + SortedSet snSs = snapshot.tailSet(firstCell); if (!snSs.isEmpty()) { Cell snc = snSs.first(); // is there a matching Cell in the snapshot? @@ -475,7 +531,7 @@ public class DefaultMemStore implements MemStore { // so we cant add the new Cell w/o knowing what's there already, but we also // want to take this chance to delete some cells. So two loops (sad) - SortedSet ss = activeSection.getCellSkipListSet().tailSet(firstCell); + SortedSet ss = cellSet.tailSet(firstCell); for (Cell cell : ss) { // if this isnt the row we are interested in, then bail: if (!CellUtil.matchingColumn(cell, family, qualifier) @@ -494,7 +550,7 @@ public class DefaultMemStore implements MemStore { // 'now' and a 0 memstoreTS == immediately visible List cells = new ArrayList(1); cells.add(new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue))); - return upsert(cells, 1L); + return upsertAndFetch(cells, 1L).getFirst(); } /** @@ -517,11 +573,21 @@ public class DefaultMemStore implements MemStore { */ @Override public long upsert(Iterable cells, long readpoint) { + throw new RuntimeException("Bug: deprecated methods, use upsertAndFetch instead"); + } + + @Override + public Pair> upsertAndFetch(Collection cells, + long readpoint) { long size = 0; + ArrayList retCells = new ArrayList(cells.size()); for (Cell cell : cells) { - size += upsert(cell, readpoint); + //size += upsert(cell, readpoint); + Pair res = upsert(cell, readpoint); + size += res.getFirst(); + retCells.add(res.getSecond()); } - return size; + return new Pair>(size, retCells); } /** @@ -538,14 +604,15 @@ public class DefaultMemStore implements MemStore { * @param cell * @return change in size of MemStore */ - private long upsert(Cell cell, long readpoint) { + private Pair upsert(Cell cell, long readpoint) { // Add the Cell to the MemStore // Use the internalAdd method here since we (a) already have a lock // and (b) cannot safely use the MSLAB here without potentially // hitting OOME - see TestMemStore.testUpsertMSLAB for a // test that triggers the pathological case if we don't avoid MSLAB // here. - long addedSize = internalAdd(cell, false); + Pair added = internalAdd(cell, false); + long addedSize = added.getFirst(); // Get the Cells for the row/family/qualifier regardless of timestamp. // For this case we want to clean up any other puts @@ -553,14 +620,22 @@ public class DefaultMemStore implements MemStore { cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); - SortedSet ss = activeSection.getCellSkipListSet().tailSet(firstCell); + SortedSet ss = cellSet.tailSet(firstCell); Iterator it = ss.iterator(); // versions visible to oldest scanner int versionsVisible = 0; while ( it.hasNext() ) { Cell cur = it.next(); - if (cell == cur) { + /** + * We must ensure the read out cell is totally all the same with the one just put in, + * so we compare the whole contents(key/value/mvcc) between them + * + * Note that there is no sure that the cell just put in is the same instance with the + * just read-out. Since underlying memstore data structure may compact key-value to avoid + * small objects allocation. + */ + if (comparator.compare(cell, cur) == 0) { // ignore the one just put in continue; } @@ -575,8 +650,10 @@ public class DefaultMemStore implements MemStore { // false means there was a change, so give us the size. long delta = heapSizeChange(cur, true); - addedSize -= delta; - activeSection.getHeapSize().addAndGet(-delta); + if (!cellSet.isEntryUncleanable()) { + addedSize -= delta; + this.size.addAndGet(-delta); + } it.remove(); setOldestEditTimeToNow(); } else { @@ -588,7 +665,8 @@ public class DefaultMemStore implements MemStore { break; } } - return addedSize; + added.setFirst(addedSize); + return added; } /* @@ -632,41 +710,19 @@ public class DefaultMemStore implements MemStore { */ @Override public List getScanners(long readPt) { - return Collections. singletonList( - new MemStoreScanner(activeSection, snapshotSection, readPt, comparator)); + return Collections. singletonList(new MemStoreScanner(readPt)); } /** * Check if this memstore may contain the required keys - * @param scan scan - * @param store holds reference to cf - * @param oldestUnexpiredTS + * @param scan * @return False if the key definitely does not exist in this Memstore */ - public boolean shouldSeek(Scan scan, Store store, long oldestUnexpiredTS) { - return shouldSeek(activeSection.getTimeRangeTracker(), - snapshotSection.getTimeRangeTracker(), scan, store, oldestUnexpiredTS); - } - - /** - * Check if this memstore may contain the required keys - * @param activeTimeRangeTracker the tracker of active data - * @param snapshotTimeRangeTracker the tracker of snapshot data - * @param scan scan - * @param store holds reference to cf - * @param oldestUnexpiredTS - * @return False if the key definitely does not exist in this Memstore - */ - private static boolean shouldSeek(TimeRangeTracker activeTimeRangeTracker, - TimeRangeTracker snapshotTimeRangeTracker, Scan scan, Store store, long oldestUnexpiredTS) { - byte[] cf = store.getFamily().getName(); - TimeRange timeRange = scan.getColumnFamilyTimeRange().get(cf); - if (timeRange == null) { - timeRange = scan.getTimeRange(); - } - return (activeTimeRangeTracker.includesTimeRange(timeRange) || - snapshotTimeRangeTracker.includesTimeRange(timeRange)) && - (Math.max(activeTimeRangeTracker.getMax(), snapshotTimeRangeTracker.getMax()) >= oldestUnexpiredTS); + public boolean shouldSeek(Scan scan, long oldestUnexpiredTS) { + return (timeRangeTracker.includesTimeRange(scan.getTimeRange()) || + snapshotTimeRangeTracker.includesTimeRange(scan.getTimeRange())) + && (Math.max(timeRangeTracker.getMax(), snapshotTimeRangeTracker.getMax()) >= + oldestUnexpiredTS); } /* @@ -675,7 +731,7 @@ public class DefaultMemStore implements MemStore { * map and snapshot. * This behaves as if it were a real scanner but does not maintain position. */ - protected static class MemStoreScanner extends NonLazyKeyValueScanner { + protected class MemStoreScanner extends NonLazyKeyValueScanner { // Next row information for either cellSet or snapshot private Cell cellSetNextRow = null; private Cell snapshotNextRow = null; @@ -689,18 +745,20 @@ public class DefaultMemStore implements MemStore { private Iterator snapshotIt; // The cellSet and snapshot at the time of creating this scanner - private final Section activeAtCreation; - private final Section snapshotAtCreation; + private CellSkipListSet cellSetAtCreation; + private CellSkipListSet snapshotAtCreation; // the pre-calculated Cell to be returned by peek() or next() private Cell theNext; - + // A flag represents whether could stop skipping Cells for MVCC // if have encountered the next row. Only used for reversed scan private boolean stopSkippingCellsIfNextRow = false; - private final long readPoint; - private final KeyValue.KVComparator comparator; + private long readPoint; + + private boolean closed = false; + /* Some notes... @@ -722,17 +780,14 @@ public class DefaultMemStore implements MemStore { the adds to kvset in the MemStoreScanner. */ - MemStoreScanner(Section activeSection, Section snapshotSection, long readPoint, final KeyValue.KVComparator c) { + MemStoreScanner(long readPoint) { + super(); + this.readPoint = readPoint; - this.comparator = c; - activeAtCreation = activeSection; - snapshotAtCreation = snapshotSection; - if (activeAtCreation.getMemStoreLAB() != null) { - activeAtCreation.getMemStoreLAB().incScannerCount(); - } - if (snapshotAtCreation.getMemStoreLAB() != null) { - snapshotAtCreation.getMemStoreLAB().incScannerCount(); - } + cellSetAtCreation = cellSet; + snapshotAtCreation = snapshot; + this.cellSetAtCreation.incrScannerReference(); + this.snapshotAtCreation.incrScannerReference(); if (Trace.isTracing() && Trace.currentSpan() != null) { Trace.currentSpan().addTimelineAnnotation("Creating MemStoreScanner"); } @@ -786,8 +841,8 @@ public class DefaultMemStore implements MemStore { } // kvset and snapshot will never be null. // if tailSet can't find anything, SortedSet is empty (not null). - cellSetIt = activeAtCreation.getCellSkipListSet().tailSet(key).iterator(); - snapshotIt = snapshotAtCreation.getCellSkipListSet().tailSet(key).iterator(); + cellSetIt = cellSetAtCreation.tailSet(key).iterator(); + snapshotIt = snapshotAtCreation.tailSet(key).iterator(); cellSetItRow = null; snapshotItRow = null; @@ -829,8 +884,8 @@ public class DefaultMemStore implements MemStore { get it. So we remember the last keys we iterated to and restore the reseeked set to at least that point. */ - cellSetIt = activeAtCreation.getCellSkipListSet().tailSet(getHighest(key, cellSetItRow)).iterator(); - snapshotIt = snapshotAtCreation.getCellSkipListSet().tailSet(getHighest(key, snapshotItRow)).iterator(); + cellSetIt = cellSetAtCreation.tailSet(getHighest(key, cellSetItRow)).iterator(); + snapshotIt = snapshotAtCreation.tailSet(getHighest(key, snapshotItRow)).iterator(); return seekInSubLists(key); } @@ -898,23 +953,27 @@ public class DefaultMemStore implements MemStore { return (first != null ? first : second); } - @Override public synchronized void close() { + if (this.closed) { + return; + } this.cellSetNextRow = null; this.snapshotNextRow = null; this.cellSetIt = null; this.snapshotIt = null; - - if (activeAtCreation != null && activeAtCreation.getMemStoreLAB() != null) { - activeAtCreation.getMemStoreLAB().decScannerCount(); + + if (this.cellSetAtCreation != null && !this.closed) { + this.cellSetAtCreation.decrScannerReference(); } - if (snapshotAtCreation != null && snapshotAtCreation.getMemStoreLAB() != null) { - snapshotAtCreation.getMemStoreLAB().decScannerCount(); + if (this.snapshotAtCreation != null && !this.closed) { + this.snapshotAtCreation.decrScannerReference(); } this.cellSetItRow = null; this.snapshotItRow = null; + + this.closed = true; } /** @@ -926,12 +985,6 @@ public class DefaultMemStore implements MemStore { return Long.MAX_VALUE; } - @Override - public boolean shouldUseScanner(Scan scan, Store store, long oldestUnexpiredTS) { - return shouldSeek(activeAtCreation.getTimeRangeTracker(), - snapshotAtCreation.getTimeRangeTracker(), scan, store, oldestUnexpiredTS); - } - /** * Seek scanner to the given key first. If it returns false(means * peek()==null) or scanner's peek row is bigger than row of given key, seek @@ -952,46 +1005,38 @@ public class DefaultMemStore implements MemStore { * specified key, then seek to the first KeyValue of previous row */ @Override - public synchronized boolean seekToPreviousRow(Cell originalKey) { - boolean keepSeeking = false; - Cell key = originalKey; - do { - Cell firstKeyOnRow = KeyValueUtil.createFirstOnRow(key.getRowArray(), key.getRowOffset(), - key.getRowLength()); - SortedSet cellHead = activeAtCreation.getCellSkipListSet().headSet(firstKeyOnRow); - Cell cellSetBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); - SortedSet snapshotHead = snapshotAtCreation.getCellSkipListSet() - .headSet(firstKeyOnRow); - Cell snapshotBeforeRow = snapshotHead.isEmpty() ? null : snapshotHead - .last(); - Cell lastCellBeforeRow = getHighest(cellSetBeforeRow, snapshotBeforeRow); - if (lastCellBeforeRow == null) { - theNext = null; - return false; - } - Cell firstKeyOnPreviousRow = KeyValueUtil.createFirstOnRow(lastCellBeforeRow.getRowArray(), - lastCellBeforeRow.getRowOffset(), lastCellBeforeRow.getRowLength()); - this.stopSkippingCellsIfNextRow = true; - seek(firstKeyOnPreviousRow); - this.stopSkippingCellsIfNextRow = false; - if (peek() == null - || comparator.compareRows(peek(), firstKeyOnPreviousRow) > 0) { - keepSeeking = true; - key = firstKeyOnPreviousRow; - continue; - } else { - keepSeeking = false; - } - } while (keepSeeking); + public synchronized boolean seekToPreviousRow(Cell key) { + Cell firstKeyOnRow = KeyValueUtil.createFirstOnRow(key.getRowArray(), key.getRowOffset(), + key.getRowLength()); + SortedSet cellHead = cellSetAtCreation.headSet(firstKeyOnRow); + Cell cellSetBeforeRow = cellHead.isEmpty() ? null : cellHead.last(); + SortedSet snapshotHead = snapshotAtCreation + .headSet(firstKeyOnRow); + Cell snapshotBeforeRow = snapshotHead.isEmpty() ? null : snapshotHead + .last(); + Cell lastCellBeforeRow = getHighest(cellSetBeforeRow, snapshotBeforeRow); + if (lastCellBeforeRow == null) { + theNext = null; + return false; + } + Cell firstKeyOnPreviousRow = KeyValueUtil.createFirstOnRow(lastCellBeforeRow.getRowArray(), + lastCellBeforeRow.getRowOffset(), lastCellBeforeRow.getRowLength()); + this.stopSkippingCellsIfNextRow = true; + seek(firstKeyOnPreviousRow); + this.stopSkippingCellsIfNextRow = false; + if (peek() == null + || comparator.compareRows(peek(), firstKeyOnPreviousRow) > 0) { + return seekToPreviousRow(lastCellBeforeRow); + } return true; } @Override public synchronized boolean seekToLastRow() { - Cell first = activeAtCreation.getCellSkipListSet().isEmpty() ? null - : activeAtCreation.getCellSkipListSet().last(); - Cell second = snapshotAtCreation.getCellSkipListSet().isEmpty() ? null - : snapshotAtCreation.getCellSkipListSet().last(); + Cell first = cellSetAtCreation.isEmpty() ? null : cellSetAtCreation + .last(); + Cell second = snapshotAtCreation.isEmpty() ? null + : snapshotAtCreation.last(); Cell higherCell = getHighest(first, second); if (higherCell == null) { return false; @@ -1007,12 +1052,13 @@ public class DefaultMemStore implements MemStore { } } - public final static long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT - + (4 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN); + public final static long FIXED_OVERHEAD = ClassSize.align( + ClassSize.OBJECT + (7 * ClassSize.REFERENCE) + (3 * Bytes.SIZEOF_LONG) + + Bytes.SIZEOF_BOOLEAN); public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + - (2 * ClassSize.ATOMIC_LONG) + (2 * ClassSize.TIMERANGE_TRACKER) + - (2 * ClassSize.CELL_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP)); + ClassSize.ATOMIC_LONG + (2 * ClassSize.TIMERANGE_TRACKER) + + (2 * ClassSize.CELL_SKIPLIST_SET)); /* * Calculate how the MemStore size has changed. Includes overhead of the @@ -1036,7 +1082,7 @@ public class DefaultMemStore implements MemStore { */ @Override public long heapSize() { - return activeSection.getHeapSize().get(); + return size.get(); } @Override @@ -1085,64 +1131,4 @@ public class DefaultMemStore implements MemStore { } LOG.info("Exiting."); } - - /** - * Contains the fields which are useful to MemStoreScanner. - */ - @InterfaceAudience.Private - @VisibleForTesting - static class Section { - /** - * MemStore. Use a CellSkipListSet rather than SkipListSet because of the - * better semantics. The Map will overwrite if passed a key it already had - * whereas the Set will not add new Cell if key is same though value might be - * different. Value is not important -- just make sure always same reference passed. - */ - private final CellSkipListSet cellSet; - private final TimeRangeTracker tracker = new TimeRangeTracker(); - /** - * Used to track own heapSize. - */ - private final AtomicLong heapSize; - private final MemStoreLAB allocator; - - static Section newSnapshotSection(final KeyValue.KVComparator c) { - return new Section(c, null, 0); - } - - static Section newActiveSection(final KeyValue.KVComparator c, - final Configuration conf) { - return new Section(c, conf, DEEP_OVERHEAD); - } - - private Section(final KeyValue.KVComparator c, - final Configuration conf, long initHeapSize) { - this.cellSet = new CellSkipListSet(c); - this.heapSize = new AtomicLong(initHeapSize); - if (conf != null && conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { - String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); - this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, - new Class[]{Configuration.class}, new Object[]{conf}); - } else { - this.allocator = null; - } - } - - CellSkipListSet getCellSkipListSet() { - return cellSet; - } - - TimeRangeTracker getTimeRangeTracker() { - return tracker; - } - - AtomicLong getHeapSize() { - return heapSize; - } - - MemStoreLAB getMemStoreLAB() { - return allocator; - } - - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 8867b625f9..f9ce97b62f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -3231,7 +3231,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi byte[] byteNow = Bytes.toBytes(now); // Nothing to put/delete -- an exception in the above such as NoSuchColumnFamily? - if (numReadyToWrite <= 0) return 0L; + if (numReadyToWrite <= 0) return 0; // We've now grabbed as many mutations off the list as we can @@ -3965,7 +3965,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi store.rollback(cell); kvsRolledback++; } - LOG.debug("rollbackMemstore rolled back " + kvsRolledback); + LOG.debug("rollbackMemstore rolled back " + kvsRolledback + ", region = " + this.getRegionInfo().getEncodedName()); } @Override @@ -7708,7 +7708,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi if (store.getFamily().getMaxVersions() == 1) { // upsert if VERSIONS for this CF == 1 // Is this right? It immediately becomes visible? St.Ack 20150907 - size += store.upsert(entry.getValue(), getSmallestReadPoint()); + Pair> res = store.upsertAndFetch(entry.getValue(), getSmallestReadPoint()); + size += res.getFirst(); } else { // otherwise keep older versions around for (Cell cell: entry.getValue()) { @@ -7942,7 +7943,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi List results = entry.getValue(); if (store.getFamily().getMaxVersions() == 1) { // Upsert if VERSIONS for this CF == 1 - accumulatedResultSize += store.upsert(results, getSmallestReadPoint()); + Pair> res = store.upsertAndFetch(results, getSmallestReadPoint()); + accumulatedResultSize += res.getFirst(); // TODO: St.Ack 20151222 Why no rollback in this case? } else { // Otherwise keep older versions around diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 812ee969bd..2cf332838f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; @@ -720,10 +721,13 @@ public class HStore implements Store { } @Override - public void rollback(final Cell cell) { + /** + * @Return the size of bytes which are rolled back + */ + public long rollback(final Cell cell) { lock.readLock().lock(); try { - this.memstore.rollback(cell); + return this.memstore.rollback(cell); } finally { lock.readLock().unlock(); } @@ -2396,9 +2400,15 @@ public class HStore implements Store { @Override public long upsert(Iterable cells, long readpoint) throws IOException { + throw new RuntimeException("Bug: deprecated method, use upsertAndFetch instead"); + } + + @Override + public Pair> upsertAndFetch(Collection cells, + long readpoint) throws IOException { this.lock.readLock().lock(); try { - return this.memstore.upsert(cells, readpoint); + return this.memstore.upsertAndFetch(cells, readpoint); } finally { this.lock.readLock().unlock(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java index d8fa5c3934..b0616d913b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java @@ -87,9 +87,9 @@ public class HeapMemStoreLAB implements MemStoreLAB { this(new Configuration()); } - public HeapMemStoreLAB(Configuration conf) { + public HeapMemStoreLAB(Configuration conf, int maxAllocation) { chunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT); - maxAlloc = conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT); + maxAlloc = maxAllocation; this.chunkPool = MemStoreChunkPool.getPool(conf); // currently chunkQueue is only used for chunkPool if (this.chunkPool != null) { @@ -103,6 +103,10 @@ public class HeapMemStoreLAB implements MemStoreLAB { maxAlloc <= chunkSize, MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); } + + public HeapMemStoreLAB(Configuration conf) { + this(conf, conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT)); + } /** * Allocate a slice of the given length. @@ -144,6 +148,10 @@ public class HeapMemStoreLAB implements MemStoreLAB { @Override public void close() { this.closed = true; + if (LOG.isTraceEnabled()) { + LOG.trace("Still have " + openScannerCount.get() + + " active scanner when close HeapMemStoreLab"); + } // We could put back the chunks to pool for reusing only when there is no // opening scanner which will read their data if (chunkPool != null && openScannerCount.get() == 0 @@ -166,6 +174,10 @@ public class HeapMemStoreLAB implements MemStoreLAB { @Override public void decScannerCount() { int count = this.openScannerCount.decrementAndGet(); + if (count < 0) { + LOG.warn("Detected scanner count of MemStoreLab is a negative number: " + + count); + } if (chunkPool != null && count == 0 && this.closed && reclaimed.compareAndSet(false, true)) { chunkPool.putbackChunks(this.chunkQueue); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java index 658ba488ea..7b4b9b61fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java @@ -17,11 +17,14 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; +import java.util.Collection; import java.util.List; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.util.Pair; /** * The MemStore holds in-memory modifications to the Store. Modifications are {@link Cell}s. @@ -67,9 +70,9 @@ public interface MemStore extends HeapSize { /** * Write an update * @param cell - * @return approximate size of the passed cell. + * @return memstore size delta */ - long add(final Cell cell); + public long add(Cell cell); /** * @return Oldest timestamp of all the Cells in the MemStore @@ -80,8 +83,9 @@ public interface MemStore extends HeapSize { * Remove n key from the memstore. Only kvs that have the same key and the same memstoreTS are * removed. It is ok to not update timeRangeTracker in this call. * @param cell + * @Return the size of bytes which are rolled back */ - void rollback(final Cell cell); + long rollback(final Cell cell); /** * Write a delete @@ -129,7 +133,28 @@ public interface MemStore extends HeapSize { * @param readpoint readpoint below which we can safely remove duplicate Cells. * @return change in memstore size */ + @Deprecated long upsert(Iterable cells, long readpoint); + + /** + * Update or insert the specified cells. + *

+ * For each Cell, insert into MemStore. This will atomically upsert the value for that + * row/family/qualifier. If a Cell did already exist, it will then be removed. + *

+ * Currently the memstoreTS is kept at 0 so as each insert happens, it will be immediately + * visible. May want to change this so it is atomic across all KeyValues. + *

+ * This is called under row lock, so Get operations will still see updates atomically. Scans will + * only see each KeyValue update as atomic. + * @param cells + * @param readpoint readpoint below which we can safely remove duplicate Cells. + * @return A Pair> + * Long: memstore size delta + * Collection: The added cells stored in the embedded store. + */ + Pair> upsertAndFetch(Collection cells, + long readpoint); /** * @return scanner over the memstore. This might include scanner over the snapshot when one is diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index eb43b43383..807038409a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -38,7 +38,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** - * A pool of {@link HeapMemStoreLAB.Chunk} instances. + * A pool of {@link HeapMemStoreLAB$Chunk} instances. * * MemStoreChunkPool caches a number of retired chunks for reusing, it could * decrease allocating bytes when writing, thereby optimizing the garbage @@ -77,6 +77,7 @@ public class MemStoreChunkPool { private static final int statThreadPeriod = 60 * 5; private AtomicLong createdChunkCount = new AtomicLong(); private AtomicLong reusedChunkCount = new AtomicLong(); + private AtomicLong managedChunkCount = new AtomicLong(); MemStoreChunkPool(Configuration conf, int chunkSize, int maxCount, int initialCount) { @@ -85,6 +86,7 @@ public class MemStoreChunkPool { this.reclaimedChunks = new LinkedBlockingQueue(); for (int i = 0; i < initialCount; i++) { Chunk chunk = new Chunk(chunkSize); + managedChunkCount.incrementAndGet(); chunk.init(); reclaimedChunks.add(chunk); } @@ -106,6 +108,7 @@ public class MemStoreChunkPool { if (chunk == null) { chunk = new Chunk(chunkSize); createdChunkCount.incrementAndGet(); + managedChunkCount.incrementAndGet(); } else { chunk.reset(); reusedChunkCount.incrementAndGet(); @@ -129,6 +132,7 @@ public class MemStoreChunkPool { if (LOG.isTraceEnabled()) { LOG.trace("Left " + chunks.size() + " unreclaimable chunks, removing them from queue"); } + managedChunkCount.addAndGet(-chunks.size()); chunks.clear(); } } @@ -140,6 +144,7 @@ public class MemStoreChunkPool { */ void putbackChunk(Chunk chunk) { if (reclaimedChunks.size() >= this.maxCount) { + managedChunkCount.decrementAndGet(); return; } reclaimedChunks.add(chunk); @@ -153,6 +158,7 @@ public class MemStoreChunkPool { * Only used in testing */ void clearChunks() { + managedChunkCount.addAndGet(-this.reclaimedChunks.size()); this.reclaimedChunks.clear(); } @@ -175,10 +181,12 @@ public class MemStoreChunkPool { if (!LOG.isDebugEnabled()) return; long created = createdChunkCount.get(); long reused = reusedChunkCount.get(); + long managed = managedChunkCount.get(); long total = created + reused; LOG.debug("Stats: current pool size=" + reclaimedChunks.size() + ",created chunk count=" + created + ",reused chunk count=" + reused + + ",managed chunk count=" + managed + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent( (float) reused / (float) total, 2))); } @@ -187,8 +195,6 @@ public class MemStoreChunkPool { * @param conf * @return the global MemStoreChunkPool instance */ - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="DC_DOUBLECHECK", - justification="Intentional") static MemStoreChunkPool getPool(Configuration conf) { if (GLOBAL_INSTANCE != null) return GLOBAL_INSTANCE; @@ -232,6 +238,10 @@ public class MemStoreChunkPool { int getMaxCount() { return this.maxCount; } + + int getChunkSize() { + return chunkSize; + } @VisibleForTesting static void clearDisableFlag() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index e7a4de5a86..a51f5d1ad0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.util.Pair; /** * Interface for objects that hold a column family in a Region. Its a memstore and a set of zero or @@ -140,7 +141,26 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf * @return memstore size delta * @throws IOException */ + @Deprecated long upsert(Iterable cells, long readpoint) throws IOException; + + /** + * Adds or replaces the specified KeyValues. + *

+ * For each KeyValue specified, if a cell with the same row, family, and qualifier exists in + * MemStore, it will be replaced. Otherwise, it will just be inserted to MemStore. + *

+ * This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic + * across all of them. + * @param cells + * @param readpoint readpoint below which we can safely remove duplicate KVs + * @return A Pair> + * Long: memstore size delta + * Collection: The added cells stored in the embedded store. + * @throws IOException + */ + Pair> upsertAndFetch(Collection cells, + long readpoint) throws IOException; /** * Adds a value to the memstore @@ -159,8 +179,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf * & memstoreTS match the key & memstoreTS value of the cell * parameter. * @param cell + * @Return the size of bytes which are rolled back */ - void rollback(final Cell cell); + long rollback(final Cell cell); /** * Find the key that matches row exactly, or the one that immediately precedes it. WARNING: diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index aa1febbb61..6a92a710b6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -303,18 +303,14 @@ public class TestHeapSize { // DefaultMemStore Deep Overhead actual = DefaultMemStore.DEEP_OVERHEAD; expected = ClassSize.estimateBase(cl, false); - expected += (2 * ClassSize.estimateBase(AtomicLong.class, false)); + expected += ClassSize.estimateBase(AtomicLong.class, false); expected += (2 * ClassSize.estimateBase(CellSkipListSet.class, false)); - expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false)); expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false)); if(expected != actual) { ClassSize.estimateBase(cl, true); ClassSize.estimateBase(AtomicLong.class, true); - ClassSize.estimateBase(AtomicLong.class, true); ClassSize.estimateBase(CellSkipListSet.class, true); ClassSize.estimateBase(CellSkipListSet.class, true); - ClassSize.estimateBase(ConcurrentSkipListMap.class, true); - ClassSize.estimateBase(ConcurrentSkipListMap.class, true); ClassSize.estimateBase(TimeRangeTracker.class, true); ClassSize.estimateBase(TimeRangeTracker.class, true); assertEquals(expected, actual); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index f11f7cfcfe..3e6def24b0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -52,19 +52,17 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.wal.WALFactory; +import org.junit.Assert; import org.junit.experimental.categories.Category; import com.google.common.base.Joiner; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** memstore test case */ @Category(MediumTests.class) public class TestDefaultMemStore extends TestCase { - private static final Log LOG = LogFactory.getLog(TestDefaultMemStore.class); + private final Log LOG = LogFactory.getLog(this.getClass()); private DefaultMemStore memstore; private static final int ROW_COUNT = 10; private static final int QUALIFIER_COUNT = ROW_COUNT; @@ -86,25 +84,30 @@ public class TestDefaultMemStore extends TestCase { byte [] other = Bytes.toBytes("somethingelse"); KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); this.memstore.add(samekey); - Cell found = this.memstore.activeSection.getCellSkipListSet().first(); - assertEquals(1, this.memstore.activeSection.getCellSkipListSet().size()); + Cell found = this.memstore.cellSet.first(); + assertEquals(1, this.memstore.cellSet.size()); assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found)); } public void testPutSameCell() { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); + memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR); + byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); long sizeChangeForFirstCell = this.memstore.add(kv); long sizeChangeForSecondCell = this.memstore.add(kv); // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(DefaultMemStore.heapSizeChange(kv, true), sizeChangeForFirstCell); - if (this.memstore.activeSection.getMemStoreLAB() != null) { + if (this.memstore.cellSet.getAllocator() != null) { // make sure memstore size increased when using MSLAB assertEquals(memstore.getCellLength(kv), sizeChangeForSecondCell); // make sure chunk size increased even when writing the same cell, if using MSLAB - if (this.memstore.activeSection.getMemStoreLAB() instanceof HeapMemStoreLAB) { + if (this.memstore.cellSet.getAllocator() instanceof HeapMemStoreLAB) { assertEquals(2 * memstore.getCellLength(kv), - ((HeapMemStoreLAB) this.memstore.activeSection.getMemStoreLAB()).getCurrentChunk().getNextFreeOffset()); + ((HeapMemStoreLAB) this.memstore.cellSet.getAllocator()).getCurrentChunk().getNextFreeOffset()); } } else { // make sure no memstore size change w/o MSLAB @@ -117,14 +120,13 @@ public class TestDefaultMemStore extends TestCase { * @throws IOException */ public void testScanAcrossSnapshot() throws IOException { + Configuration conf = HBaseConfiguration.create(); int rowCount = addRows(this.memstore); List memstorescanners = this.memstore.getScanners(0); Scan scan = new Scan(); List result = new ArrayList(); - Configuration conf = HBaseConfiguration.create(); - ScanInfo scanInfo = - new ScanInfo(conf, null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0, - this.memstore.comparator); + ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, + KeepDeletedCells.FALSE, 0, this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); int count = 0; @@ -268,8 +270,7 @@ public class TestDefaultMemStore extends TestCase { final byte[] q2 = Bytes.toBytes("q2"); final byte[] v = Bytes.toBytes("value"); - MultiVersionConcurrencyControl.WriteEntry w = - mvcc.begin(); + MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); KeyValue kv1 = new KeyValue(row, f, q1, v); kv1.setSequenceId(w.getWriteNumber()); @@ -278,7 +279,7 @@ public class TestDefaultMemStore extends TestCase { KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); assertScannerResults(s, new KeyValue[]{}); - mvcc.completeAndWait(w); + mvcc.complete(w); s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); assertScannerResults(s, new KeyValue[]{kv1}); @@ -291,7 +292,7 @@ public class TestDefaultMemStore extends TestCase { s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); assertScannerResults(s, new KeyValue[]{kv1}); - mvcc.completeAndWait(w); + mvcc.complete(w); s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); assertScannerResults(s, new KeyValue[]{kv1, kv2}); @@ -312,8 +313,7 @@ public class TestDefaultMemStore extends TestCase { final byte[] v2 = Bytes.toBytes("value2"); // INSERT 1: Write both columns val1 - MultiVersionConcurrencyControl.WriteEntry w = - mvcc.begin(); + MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); KeyValue kv11 = new KeyValue(row, f, q1, v1); kv11.setSequenceId(w.getWriteNumber()); @@ -322,7 +322,7 @@ public class TestDefaultMemStore extends TestCase { KeyValue kv12 = new KeyValue(row, f, q2, v1); kv12.setSequenceId(w.getWriteNumber()); memstore.add(kv12); - mvcc.completeAndWait(w); + mvcc.complete(w); // BEFORE STARTING INSERT 2, SEE FIRST KVS KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); @@ -343,7 +343,7 @@ public class TestDefaultMemStore extends TestCase { assertScannerResults(s, new KeyValue[]{kv11, kv12}); // COMPLETE INSERT 2 - mvcc.completeAndWait(w); + mvcc.complete(w); // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS. // See HBASE-1485 for discussion about what we should do with @@ -364,8 +364,7 @@ public class TestDefaultMemStore extends TestCase { final byte[] q2 = Bytes.toBytes("q2"); final byte[] v1 = Bytes.toBytes("value1"); // INSERT 1: Write both columns val1 - MultiVersionConcurrencyControl.WriteEntry w = - mvcc.begin(); + MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); KeyValue kv11 = new KeyValue(row, f, q1, v1); kv11.setSequenceId(w.getWriteNumber()); @@ -374,7 +373,7 @@ public class TestDefaultMemStore extends TestCase { KeyValue kv12 = new KeyValue(row, f, q2, v1); kv12.setSequenceId(w.getWriteNumber()); memstore.add(kv12); - mvcc.completeAndWait(w); + mvcc.complete(w); // BEFORE STARTING INSERT 2, SEE FIRST KVS KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); @@ -392,7 +391,7 @@ public class TestDefaultMemStore extends TestCase { assertScannerResults(s, new KeyValue[]{kv11, kv12}); // COMPLETE DELETE - mvcc.completeAndWait(w); + mvcc.complete(w); // NOW WE SHOULD SEE DELETE s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); @@ -438,8 +437,7 @@ public class TestDefaultMemStore extends TestCase { private void internalRun() throws IOException { for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) { - MultiVersionConcurrencyControl.WriteEntry w = - mvcc.begin(); + MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); // Insert the sequence value (i) byte[] v = Bytes.toBytes(i); @@ -447,7 +445,7 @@ public class TestDefaultMemStore extends TestCase { KeyValue kv = new KeyValue(row, f, q1, i, v); kv.setSequenceId(w.getWriteNumber()); memstore.add(kv); - mvcc.completeAndWait(w); + mvcc.complete(w); // Assert that we can read back KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); @@ -492,7 +490,7 @@ public class TestDefaultMemStore extends TestCase { for (int i = 0; i < snapshotCount; i++) { addRows(this.memstore); runSnapshot(this.memstore); - assertEquals("History not being cleared", 0, this.memstore.snapshotSection.getCellSkipListSet().size()); + assertEquals("History not being cleared", 0, this.memstore.snapshot.size()); } } @@ -513,7 +511,7 @@ public class TestDefaultMemStore extends TestCase { m.add(key2); assertTrue("Expected memstore to hold 3 values, actually has " + - m.activeSection.getCellSkipListSet().size(), m.activeSection.getCellSkipListSet().size() == 3); + m.cellSet.size(), m.cellSet.size() == 3); } ////////////////////////////////////////////////////////////////////////////// @@ -524,6 +522,7 @@ public class TestDefaultMemStore extends TestCase { * @throws InterruptedException */ public void testGetNextRow() throws Exception { + Configuration conf = HBaseConfiguration.create(); addRows(this.memstore); // Add more versions to make it a little more interesting. Thread.sleep(1); @@ -542,10 +541,9 @@ public class TestDefaultMemStore extends TestCase { } } //starting from each row, validate results should contain the starting row - Configuration conf = HBaseConfiguration.create(); for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { - ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, - KeepDeletedCells.FALSE, 0, this.memstore.comparator); + ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, + 0, this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; InternalScanner scanner = new StoreScanner(new Scan( Bytes.toBytes(startRowId)), scanInfo, scanType, null, @@ -587,12 +585,12 @@ public class TestDefaultMemStore extends TestCase { memstore.add(new KeyValue(row, fam ,qf3, val)); //Creating a snapshot memstore.snapshot(); - assertEquals(3, memstore.snapshotSection.getCellSkipListSet().size()); + assertEquals(3, memstore.snapshot.size()); //Adding value to "new" memstore - assertEquals(0, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam ,qf4, val)); memstore.add(new KeyValue(row, fam ,qf5, val)); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(2, memstore.cellSet.size()); } ////////////////////////////////////////////////////////////////////////////// @@ -614,7 +612,7 @@ public class TestDefaultMemStore extends TestCase { memstore.add(put2); memstore.add(put3); - assertEquals(3, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(3, memstore.cellSet.size()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); memstore.delete(del2); @@ -625,9 +623,9 @@ public class TestDefaultMemStore extends TestCase { expected.add(put2); expected.add(put1); - assertEquals(4, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(4, memstore.cellSet.size()); int i = 0; - for(Cell cell : memstore.activeSection.getCellSkipListSet()) { + for(Cell cell : memstore.cellSet) { assertEquals(expected.get(i++), cell); } } @@ -648,7 +646,7 @@ public class TestDefaultMemStore extends TestCase { memstore.add(put2); memstore.add(put3); - assertEquals(3, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(3, memstore.cellSet.size()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val); @@ -661,9 +659,9 @@ public class TestDefaultMemStore extends TestCase { expected.add(put1); - assertEquals(4, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(4, memstore.cellSet.size()); int i = 0; - for (Cell cell: memstore.activeSection.getCellSkipListSet()) { + for (Cell cell: memstore.cellSet) { assertEquals(expected.get(i++), cell); } } @@ -701,9 +699,9 @@ public class TestDefaultMemStore extends TestCase { - assertEquals(5, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(5, memstore.cellSet.size()); int i = 0; - for (Cell cell: memstore.activeSection.getCellSkipListSet()) { + for (Cell cell: memstore.cellSet) { assertEquals(expected.get(i++), cell); } } @@ -717,8 +715,8 @@ public class TestDefaultMemStore extends TestCase { memstore.add(new KeyValue(row, fam, qf, ts, val)); KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); memstore.delete(delete); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); - assertEquals(delete, memstore.activeSection.getCellSkipListSet().first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteVersion() throws IOException { @@ -730,8 +728,8 @@ public class TestDefaultMemStore extends TestCase { "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); - assertEquals(delete, memstore.activeSection.getCellSkipListSet().first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteColumn() throws IOException { // add a put to memstore @@ -742,8 +740,8 @@ public class TestDefaultMemStore extends TestCase { KeyValue.Type.DeleteColumn, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); - assertEquals(delete, memstore.activeSection.getCellSkipListSet().first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } public void testRetainsDeleteFamily() throws IOException { // add a put to memstore @@ -754,8 +752,8 @@ public class TestDefaultMemStore extends TestCase { KeyValue.Type.DeleteFamily, "dont-care"); memstore.delete(delete); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); - assertEquals(delete, memstore.activeSection.getCellSkipListSet().first()); + assertEquals(2, memstore.cellSet.size()); + assertEquals(delete, memstore.cellSet.first()); } //////////////////////////////////// @@ -765,32 +763,29 @@ public class TestDefaultMemStore extends TestCase { /** * Test to ensure correctness when using Memstore with multiple timestamps */ - public void testMultipleTimestamps() throws Exception { + public void testMultipleTimestamps() throws IOException { long[] timestamps = new long[] {20,10,5,1}; Scan scan = new Scan(); for (long timestamp: timestamps) addRows(memstore,timestamp); - byte[] fam = Bytes.toBytes("fam"); - HColumnDescriptor hcd = mock(HColumnDescriptor.class); - when(hcd.getName()).thenReturn(fam); - Store store = mock(Store.class); - when(store.getFamily()).thenReturn(hcd); - scan.setColumnFamilyTimeRange(fam, 0, 2); - assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE)); + scan.setTimeRange(0, 2); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); - scan.setColumnFamilyTimeRange(fam, 20, 82); - assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE)); + scan.setTimeRange(20, 82); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); - scan.setColumnFamilyTimeRange(fam, 10, 20); - assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE)); + scan.setTimeRange(10, 20); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); - scan.setColumnFamilyTimeRange(fam, 8, 12); - assertTrue(memstore.shouldSeek(scan, store, Long.MIN_VALUE)); + scan.setTimeRange(8, 12); + assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE)); - scan.setColumnFamilyTimeRange(fam, 28, 42); - assertTrue(!memstore.shouldSeek(scan, store, Long.MIN_VALUE)); + /*This test is not required for correctness but it should pass when + * timestamp range optimization is on*/ + //scan.setTimeRange(28, 42); + //assertTrue(!memstore.shouldSeek(scan)); } //////////////////////////////////// @@ -813,6 +808,7 @@ public class TestDefaultMemStore extends TestCase { public void testUpsertMSLAB() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR); int ROW_SIZE = 2048; @@ -838,6 +834,78 @@ public class TestDefaultMemStore extends TestCase { + " (heapsize: " + memstore.heapSize() + " size: " + size + ")"); } + + public void testUpsertKeepOnlyOneCell() throws Exception { + for (int i = 0; i < 4; ++i) { + boolean useccsmap = (i & 0x1F) == 0; + boolean usemslab = (i & 0x2F) == 0; + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, usemslab); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, useccsmap); + memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR); + + byte[] row = Bytes.toBytes(0); + byte[] qualifier = new byte[4]; + + KeyValue kv1 = new KeyValue(row, FAMILY, qualifier, 1, Bytes.toBytes(1)); + kv1.setSequenceId(1); + KeyValue kv2 = new KeyValue(row, FAMILY, qualifier, 1, Bytes.toBytes(1)); + kv2.setSequenceId(2); + memstore.add(kv1); + memstore.add(kv2); + + memstore.updateColumnValue(row, FAMILY, qualifier, 3, 3); + + List scanners = memstore.getScanners(0); + for (KeyValueScanner s : scanners) { + s.seek(kv1); + Assert.assertEquals(null, s.next()); + } + } + } + + /** + * Test when we use CCSMAP in memstore, heap size will increase when upsert the same column + * @throws Exception + */ + public void testCCSMapHeapSizeChanged() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, true); + memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR); + if (!memstore.cellSet.isCCSMap()) { + LOG.info("The ccsmap has been hook closed, skip ..."); + return; + } + + int ROW_SIZE = 50; + byte[] qualifier = new byte[ROW_SIZE - 4]; + + // Open ccsmap, the heap size will increase when upsert the same column + long ts=0; + byte[] rowBytes = Bytes.toBytes(0); + memstore.updateColumnValue(rowBytes, FAMILY, qualifier, 0, ++ts); + for (int newValue = 0; newValue < 100; newValue++) { + long ret = memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts); + Assert.assertTrue(ret > 0); + } + // Delete the same column twice, the heap size still changed + KeyValue deletedKV = new KeyValue(rowBytes, FAMILY, qualifier, 123, KeyValue.Type.Delete); + memstore.delete(deletedKV); + Assert.assertTrue(memstore.delete(deletedKV) > 0); + // Put the same cell twice, the heap size still changed + KeyValue newKV = new KeyValue(rowBytes, FAMILY, qualifier, 456, KeyValue.Type.Put, rowBytes); + { + long beforeSize = memstore.size(); + memstore.add(newKV); + long afterSize = memstore.size(); + Assert.assertTrue(afterSize > beforeSize); + } + // Roll back cell, the heap size won't changed + long sz = memstore.size(); + memstore.rollback(newKV); + Assert.assertTrue(memstore.size() == sz); + } ////////////////////////////////////////////////////////////////////////////// // Helpers @@ -854,8 +922,9 @@ public class TestDefaultMemStore extends TestCase { */ public void testUpsertMemstoreSize() throws Exception { Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR); - long oldSize = memstore.activeSection.getHeapSize().get(); + long oldSize = memstore.size.get(); List l = new ArrayList(); KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); @@ -865,19 +934,19 @@ public class TestDefaultMemStore extends TestCase { kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1); l.add(kv1); l.add(kv2); l.add(kv3); - this.memstore.upsert(l, 2);// readpoint is 2 - long newSize = this.memstore.activeSection.getHeapSize().get(); + this.memstore.upsertAndFetch(l, 2);// readpoint is 2 + long newSize = this.memstore.size.get(); assert(newSize > oldSize); //The kv1 should be removed. - assert(memstore.activeSection.getCellSkipListSet().size() == 2); + assert(memstore.cellSet.size() == 2); KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); kv4.setSequenceId(1); l.clear(); l.add(kv4); - this.memstore.upsert(l, 3); - assertEquals(newSize, this.memstore.activeSection.getHeapSize().get()); + this.memstore.upsertAndFetch(l, 2); + assertEquals(newSize, this.memstore.size.get()); //The kv2 should be removed. - assert(memstore.activeSection.getCellSkipListSet().size() == 2); + assert(memstore.cellSet.size() == 2); //this.memstore = null; } @@ -918,7 +987,7 @@ public class TestDefaultMemStore extends TestCase { KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); kv1.setSequenceId(100); l.add(kv1); - memstore.upsert(l, 1000); + memstore.upsertAndFetch(l, 1000); t = memstore.timeOfOldestEdit(); assertTrue(t == 1234); } finally { @@ -956,10 +1025,10 @@ public class TestDefaultMemStore extends TestCase { edge.setCurrentTimeMillis(1234); s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v")); edge.setCurrentTimeMillis(1234 + 100); - StringBuffer sb = new StringBuffer(); - assertTrue(region.shouldFlush(sb) == false); + StringBuffer why = new StringBuffer(); + assertTrue(region.shouldFlush(why) == false); edge.setCurrentTimeMillis(1234 + 10000); - assertTrue(region.shouldFlush(sb) == expected); + assertTrue(region.shouldFlush(why) == expected); } finally { EnvironmentEdgeManager.reset(); } @@ -967,7 +1036,7 @@ public class TestDefaultMemStore extends TestCase { public void testShouldFlushMeta() throws Exception { // write an edit in the META and ensure the shouldFlush (that the periodic memstore - // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though + // flusher invokes) returns true after META_CACHE_FLUSH_INTERVAL (even though // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value) Configuration conf = new Configuration(); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10); @@ -987,13 +1056,13 @@ public class TestDefaultMemStore extends TestCase { desc.addFamily(new HColumnDescriptor("foo".getBytes())); HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, - wFactory.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace())); + wFactory.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace())); HRegion.addRegionToMETA(meta, r); edge.setCurrentTimeMillis(1234 + 100); - StringBuffer sb = new StringBuffer(); - assertTrue(meta.shouldFlush(sb) == false); + StringBuffer why = new StringBuffer(); + assertTrue(meta.shouldFlush(why) == false); edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1); - assertTrue(meta.shouldFlush(sb) == true); + assertTrue(meta.shouldFlush(why) == true); } private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { @@ -1038,10 +1107,10 @@ public class TestDefaultMemStore extends TestCase { private long runSnapshot(final DefaultMemStore hmc) throws UnexpectedStateException { // Save off old state. - int oldHistorySize = hmc.snapshotSection.getCellSkipListSet().size(); + int oldHistorySize = hmc.snapshot.size(); MemStoreSnapshot snapshot = hmc.snapshot(); // Make some assertions about what just happened. - assertTrue("History size has not increased", oldHistorySize < hmc.snapshotSection.getCellSkipListSet().size()); + assertTrue("History size has not increased", oldHistorySize < hmc.snapshot.size()); long t = memstore.timeOfOldestEdit(); assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE); hmc.clearSnapshot(snapshot.getId()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 529fc51bca..efc1283ea0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -328,33 +328,42 @@ public class TestHRegion { super.sync(txid, forceSync); } } - - FileSystem fs = FileSystem.get(CONF); - Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); - MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); - HRegion region = initHRegion(tableName, null, null, name.getMethodName(), - CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES); - - Store store = region.getStore(COLUMN_FAMILY_BYTES); - // Get some random bytes. - byte [] value = Bytes.toBytes(name.getMethodName()); - faultyLog.setStoreFlushCtx(store.createFlushContext(12345)); - - Put put = new Put(value); - put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); - faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); - - boolean threwIOE = false; + + //disable ccsmap + boolean old = TEST_UTIL.getConfiguration().getBoolean(DefaultMemStore.USECCSMAP_KEY, DefaultMemStore.USECCSMAP_DEFAULT); + TEST_UTIL.getConfiguration().setBoolean(DefaultMemStore.USECCSMAP_KEY, false); try { - region.put(put); - } catch (IOException ioe) { - threwIOE = true; + FileSystem fs = FileSystem.get(CONF); + Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); + MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); + HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES); + + Store store = region.getStore(COLUMN_FAMILY_BYTES); + // Get some random bytes. + byte [] value = Bytes.toBytes(name.getMethodName()); + StoreFlushContext flushContext = store.createFlushContext(12345); + faultyLog.setStoreFlushCtx(flushContext); + + Put put = new Put(value); + put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); + faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); + + boolean threwIOE = false; + try { + region.put(put); + } catch (IOException ioe) { + threwIOE = true; + } finally { + assertTrue("The regionserver should have thrown an exception", threwIOE); + } + long sz = store.getFlushableSize(); + assertTrue("flushable size should be zero, but it is " + sz, sz == 0); + //close region will fail since FaultyFSLog + //HRegion.closeHRegion(region); } finally { - assertTrue("The regionserver should have thrown an exception", threwIOE); + TEST_UTIL.getConfiguration().setBoolean(DefaultMemStore.USECCSMAP_KEY, old); } - long sz = store.getFlushableSize(); - assertTrue("flushable size should be zero, but it is " + sz, sz == 0); - HRegion.closeHRegion(region); } /** @@ -2482,10 +2491,10 @@ public class TestHRegion { // This is kinda hacky, but better than nothing... long now = System.currentTimeMillis(); DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore; - Cell firstCell = memstore.activeSection.getCellSkipListSet().first(); + Cell firstCell = memstore.getCellSkipListSet().first(); assertTrue(firstCell.getTimestamp() <= now); now = firstCell.getTimestamp(); - for (Cell cell : memstore.activeSection.getCellSkipListSet()) { + for (Cell cell : memstore.getCellSkipListSet()) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java index 1cd8191c0d..2d16319bf5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.ByteRange; @@ -97,6 +98,7 @@ public class TestMemStoreChunkPool { @Test public void testPuttingBackChunksAfterFlushing() throws UnexpectedStateException { + //disable compact memory skip list byte[] row = Bytes.toBytes("testrow"); byte[] fam = Bytes.toBytes("testfamily"); byte[] qf1 = Bytes.toBytes("testqualifier1"); @@ -106,7 +108,10 @@ public class TestMemStoreChunkPool { byte[] qf5 = Bytes.toBytes("testqualifier5"); byte[] val = Bytes.toBytes("testval"); - DefaultMemStore memstore = new DefaultMemStore(); + // disable compacted memstore + Configuration newConf = HBaseConfiguration.create(); + newConf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); + DefaultMemStore memstore = new DefaultMemStore(newConf, KeyValue.COMPARATOR); // Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val)); @@ -115,13 +120,13 @@ public class TestMemStoreChunkPool { // Creating a snapshot MemStoreSnapshot snapshot = memstore.snapshot(); - assertEquals(3, memstore.snapshotSection.getCellSkipListSet().size()); + assertEquals(3, memstore.snapshot.size()); // Adding value to "new" memstore - assertEquals(0, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam, qf4, val)); memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(2, memstore.cellSet.size()); memstore.clearSnapshot(snapshot.getId()); int chunkCount = chunkPool.getPoolSize(); @@ -143,7 +148,10 @@ public class TestMemStoreChunkPool { byte[] qf7 = Bytes.toBytes("testqualifier7"); byte[] val = Bytes.toBytes("testval"); - DefaultMemStore memstore = new DefaultMemStore(); + // disable compacted memstore + Configuration newConf = HBaseConfiguration.create(); + newConf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); + DefaultMemStore memstore = new DefaultMemStore(newConf, KeyValue.COMPARATOR); // Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val)); @@ -152,13 +160,13 @@ public class TestMemStoreChunkPool { // Creating a snapshot MemStoreSnapshot snapshot = memstore.snapshot(); - assertEquals(3, memstore.snapshotSection.getCellSkipListSet().size()); + assertEquals(3, memstore.snapshot.size()); // Adding value to "new" memstore - assertEquals(0, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(0, memstore.cellSet.size()); memstore.add(new KeyValue(row, fam, qf4, val)); memstore.add(new KeyValue(row, fam, qf5, val)); - assertEquals(2, memstore.activeSection.getCellSkipListSet().size()); + assertEquals(2, memstore.cellSet.size()); // opening scanner before clear the snapshot List scanners = memstore.getScanners(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index e13c250864..9cc1a0c717 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -570,7 +570,7 @@ public class TestStore { this.store.snapshot(); flushStore(store, id++); Assert.assertEquals(storeFilessize, this.store.getStorefiles().size()); - Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).activeSection.getCellSkipListSet().size()); + Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).getCellSkipListSet().size()); } private void assertCheck() { @@ -615,7 +615,7 @@ public class TestStore { flushStore(store, id++); Assert.assertEquals(1, this.store.getStorefiles().size()); // from the one we inserted up there, and a new one - Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).activeSection.getCellSkipListSet().size()); + Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).getCellSkipListSet().size()); // how many key/values for this row are there? Get get = new Get(row); @@ -642,7 +642,9 @@ public class TestStore { @Test public void testICV_negMemstoreSize() throws IOException { - init(this.name.getMethodName()); + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(DefaultMemStore.USECCSMAP_KEY, false); // disable ccsmap in memstore + init(this.name.getMethodName(), conf); long time = 100; ManualEnvironmentEdge ee = new ManualEnvironmentEdge(); @@ -684,7 +686,7 @@ public class TestStore { } long computedSize=0; - for (Cell cell : ((DefaultMemStore)this.store.memstore).activeSection.getCellSkipListSet()) { + for (Cell cell : ((DefaultMemStore)this.store.memstore).getCellSkipListSet()) { long kvsize = DefaultMemStore.heapSizeChange(cell, true); //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize()); computedSize += kvsize; @@ -716,7 +718,7 @@ public class TestStore { // then flush. flushStore(store, id++); Assert.assertEquals(1, this.store.getStorefiles().size()); - Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).activeSection.getCellSkipListSet().size()); + Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).getCellSkipListSet().size()); // now increment again: newValue += 1; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompactedConcurrentSkipList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompactedConcurrentSkipList.java new file mode 100644 index 0000000000..a434bfe7d9 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompactedConcurrentSkipList.java @@ -0,0 +1,1891 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.security.InvalidParameterException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.TreeMap; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.CompactedConcurrentSkipListMap; + +import junit.framework.Assert; + +import org.apache.hadoop.hbase.util.CompactedConcurrentSkipListMap.MemoryUsage; +import org.apache.hadoop.hbase.util.CompactedTypeHelper.KVPair; +import org.apache.hadoop.hbase.util.TestCompactedConcurrentSkipList.BehaviorCollections.RandomBehavior; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestCompactedConcurrentSkipList { + static int BIG_KV_THRESHOLD = 1024; + static int META_PAGE_SIZE = 16 * 1024; + static int DATA_PAGE_SIZE = 16 * 1024; + static int DEFAULT_MIN_GROUP = 5; + static int DEFAULT_MAX_GROUP = 10; + + static class TypeHelper implements CompactedTypeHelper { + + @Override + public int getCompactedSize(Integer key, String value) { + return 4 + value.length(); + } + + private void copyIntToArray(int val, byte[] data, int offset) { + for (int pos = offset; pos < offset + 4; ++pos) { + data[pos] = (byte) (val & BYTE); + val = val >> 8; + } + } + + static int BYTE = 0xFF; + + private int getIntFromArray(byte[] data, int offset) { + int ret = 0; + for (int pos = offset + 3; pos >= offset; pos--) { + int b = data[pos]; + ret = (ret << 8) | (b & BYTE); + } + return ret; + } + + @Override + public void compact(Integer key, String value, byte[] data, int offset, + int len) { + Assert.assertEquals(4 + value.length(), len); + copyIntToArray(key.intValue(), data, offset); + byte[] src = value.getBytes(); + Assert.assertEquals(value.length(), src.length); + System.arraycopy(src, 0, data, offset + 4, src.length); + } + + @Override + public CompactedTypeHelper.KVPair decomposte(byte[] data, + int offset, int len) { + Assert.assertTrue(len > 4); + int intkey = getIntFromArray(data, offset); + String value = new String(data, offset + 4, len - 4); + return new KVPair(new Integer(intkey), value); + } + + private int compare(int key1, int key2) { + if (key1 < key2) { + return -1; + } else if (key2 < key1) { + return 1; + } else { + return 0; + } + } + + @Override + public int compare(byte[] data1, int offset1, int len1, byte[] data2, + int offset2, int len2) { + Assert.assertTrue(len1 > 4); + Assert.assertTrue(len2 > 4); + int key1 = getIntFromArray(data1, offset1); + int key2 = getIntFromArray(data2, offset2); + return compare(key1, key2); + } + + @Override + public int compare(Integer key, byte[] data, int offset, int len) { + Assert.assertTrue(len > 4); + return compare(key.intValue(), getIntFromArray(data, offset)); + } + + @Override + public int compare(Integer key1, Integer key2) { + return compare(key1.intValue(), key2.intValue()); + } + + } + + static interface KVGenerator { + abstract public KVPair[] getNextGroup(Random random); + + abstract public KVPair getNextKV(Random random); + + abstract public void validate(int key, String value); + + // whether this generator can provide a group of kv + public boolean groupKV(); + + public boolean singleKV(); + } + + static class KVGeneratorFactory { + static abstract class KVGeneratorBase implements KVGenerator { + private boolean single; + private boolean group; + + public KVGeneratorBase(boolean supportSingle, boolean supportGroup) { + this.single = supportSingle; + this.group = supportGroup; + } + + @Override + public KVPair[] getNextGroup(Random random) { + throw new UnsupportedOperationException(); + } + + @Override + public KVPair getNextKV(Random random) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean groupKV() { + return this.group; + } + + @Override + public boolean singleKV() { + return this.single; + } + } + + static private int getHashWithKeyRange(int keyRange, String value) { + return (int) ((long) value.hashCode() % keyRange); + } + + static private int getGroupSize(Random random, int minGroup, int maxGroup) { + if (minGroup == maxGroup) { + return minGroup; + } + int s = random.nextInt(maxGroup - minGroup); + return minGroup + s; + } + + static private char[] ALPHABELT = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0' }; + + static private String getValue(Random random, final double bigValueRate) { + int sz = (bigValueRate >= 0.0d && random.nextDouble() < bigValueRate) ? random + .nextInt(1024) + 1024 : random.nextInt(8) + 8; + StringBuilder sb = new StringBuilder(sz); + for (int i = 0; i < sz; ++i) { + sb.append(ALPHABELT[random.nextInt(ALPHABELT.length)]); + } + return sb.toString(); + } + + static public KVGenerator arbitaryKVGenerator(final int keyRange) { + return arbitaryKVGenerator(keyRange, -1.0f, DEFAULT_MIN_GROUP, + DEFAULT_MAX_GROUP); + } + + static public KVGenerator arbitaryKVGenerator(final int keyRange, + final double bigValueRate, final int minGroup, final int maxGroup) { + if (keyRange <= 0) { + throw new InvalidParameterException(); + } + return new KVGeneratorBase(true, true) { + @Override + public void validate(int key, String value) { + Assert.assertEquals(key, getHashWithKeyRange(keyRange, value)); + } + + @Override + public KVPair[] getNextGroup(Random random) { + int sz = getGroupSize(random, minGroup, maxGroup); + KVPair[] ret = new KVPair[sz]; + for (int i = 0; i < sz; ++i) { + ret[i] = getNextKV(random); + } + return ret; + } + + @Override + public KVPair getNextKV(Random random) { + String value = getValue(random, bigValueRate); + return new KVPair(getHashWithKeyRange(keyRange, + value), value); + } + }; + } + } + + class Operators { + final List functions = new ArrayList(); + final List threadNames = new ArrayList(); + final List threads = new ArrayList(); + Throwable[] exceptions; + String operatorsName = "Operator"; + final AtomicInteger runningThreads = new AtomicInteger(0); + + public void addFunctions(Runnable f) { + int idx = functions.size(); + String defaultname = operatorsName + "-Thread-" + idx; + addFunctions(f, defaultname); + } + + public void addFunctions(Runnable f, String threadName) { + functions.add(f); + threadNames.add(threadName); + } + + public void startAll() { + exceptions = new Throwable[functions.size()]; + + for (int i = 0; i < functions.size(); ++i) { + final int ii = i; + synchronized (runningThreads) { + runningThreads.incrementAndGet(); + } + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + functions.get(ii).run(); + } catch (Throwable e) { + exceptions[ii] = e; + } finally { + synchronized (runningThreads) { + runningThreads.decrementAndGet(); + runningThreads.notify(); + } + } + } + }, threadNames.get(i)); + threads.add(t); + } + for (Thread t : threads) { + t.start(); + } + } + + public void waitForAll() throws Throwable { + synchronized (runningThreads) { + while (runningThreads.get() > 0) { + try { + runningThreads.wait(); + } catch (InterruptedException e) { + break; + } + } + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // ignored + } + } + Throwable first = null; + for (int i = 0; i < exceptions.length; ++i) { + if (exceptions[i] != null) { + if (first != null) { + first = exceptions[i]; + } + exceptions[i].printStackTrace(); + } + } + if (first != null) { + throw first; + } + } + } + + private CompactedConcurrentSkipListMap createMapForTest( + boolean largeMap) { + if (!largeMap) { + return new CompactedConcurrentSkipListMap( + new TypeHelper(), CompactedConcurrentSkipListMap.PageSetting.create() + .setDataPageSize(DATA_PAGE_SIZE).setHeapKVThreshold(BIG_KV_THRESHOLD)); + } else { + CompactedConcurrentSkipListMap.PageSetting ps = CompactedConcurrentSkipListMap.PageSetting + .create(); + ps.setDataPageSize(256 * 1024).setHeapKVThreshold(4 * 1024); + return new CompactedConcurrentSkipListMap( + new TypeHelper(), ps); + } + } + + private ConcurrentSkipListMap createMapForStand() { + return new ConcurrentSkipListMap(); + } + + @Test + public void testPutAndGet() { + CompactedConcurrentSkipListMap m = createMapForTest(false); + m.put(100, "thanks"); + m.put(2, "hello"); + m.put(9, "year"); + Assert.assertEquals(m.get(-50), null); + Assert.assertEquals(m.get(99), null); + Assert.assertEquals(m.get(2), new String("hello")); + Assert.assertEquals(m.get(9), new String("year")); + Assert.assertEquals(m.get(100), new String("thanks")); + } + + @Test + public void randomWriteTest() { + CompactedConcurrentSkipListMap m = createMapForTest(false); + + // random put with overlap + final int testTimes = 128 * 1024; + final int keyRange = 32768; + Random random = new Random(); + KVGenerator kvg = KVGeneratorFactory.arbitaryKVGenerator(keyRange); + long start = System.currentTimeMillis(); + for (int i = 0; i < testTimes; ++i) { + KVPair kv = kvg.getNextKV(random); + m.put(kv.key, kv.value); + } + + // iterator map + int prev = Integer.MIN_VALUE; + int elementCount = 0; + for (Entry e : m.entrySet()) { + kvg.validate(e.getKey(), e.getValue()); + Assert.assertTrue(e.getKey() > prev); + prev = e.getKey(); + elementCount++; + } + System.out.println("There are " + elementCount + " in map"); + System.out.println("It takes " + (System.currentTimeMillis() - start) + + " ms do insertion " + testTimes + " times"); + System.out.println("MemoryUsage:"); + System.out.println(m.getMemoryUsage().toString()); + } + + @Test + public void testPutAndGetParallelly() throws Throwable { + final int THREADS = 10; + final int GROUPSIZE = 5; + final int COUNT = 10000; + + final CompactedConcurrentSkipListMap m = createMapForTest(false); + Operators ops = new Operators(); + for (int i = 0; i < THREADS; ++i) { + ops.addFunctions(new Runnable() { + @Override + public void run() { + KVGenerator kvg = KVGeneratorFactory.arbitaryKVGenerator(32767, 0.0f, + 1, GROUPSIZE); + for (int c = 0; c < COUNT; ++c) { + KVPair[] group = kvg.getNextGroup(new Random()); + for (int j = 0; j < group.length; ++j) { + m.put(group[j].key, group[j].value); + } + for (int j = 0; j < group.length; ++j) { + String value = m.get(group[j].key); + kvg.validate(group[j].key.intValue(), value); + } + } + } + }); + } + + ops.startAll(); + ops.waitForAll(); + + System.out.println("There are " + m.size() + " element in map"); + } + + static void putSimpleTestDataIntoMap(final Map map) { + map.put(1, "hello"); + map.put(6, "last"); + map.put(5, "quick"); + map.put(9, "tt"); + map.put(-30, "hbase"); + } + + static enum InputType { + MAIN_MAP, SUB_MAP, ITERATOR, ENTRYVIEW, KEYVIEW, VALUEVIEW, + } + + static class MapContext { + final int keyRange; + final int totalThread = 0; // 0 means single thread; + + int subRangeStart; + int subRangeEnd; + boolean descending; + final int multiThread; + + final ConcurrentNavigableMap mainMap1; + final ConcurrentNavigableMap mainMap2; + ConcurrentNavigableMap subMap1; + ConcurrentNavigableMap subMap2; + NavigableSet keyView1; + NavigableSet keyView2; + Collection valueView1; + Collection valueView2; + Set> entryView1; + Set> entryView2; + + InputType type; + + final int label; // different thread has different label + int viewOps; // how may behaviors has been done in this view + + public MapContext(ConcurrentNavigableMap map1, + ConcurrentNavigableMap map2, int range, int label, + int multiThread) { + this.keyRange = range; + this.mainMap1 = map1; + this.mainMap2 = map2; + this.label = label; + this.multiThread = multiThread; + reset(); + } + + public MapContext(ConcurrentNavigableMap map1, + ConcurrentNavigableMap map2, int range) { + this(map1, map2, range, -1, 0); + } + + public void reset() { + subRangeStart = Integer.MIN_VALUE; + subRangeEnd = Integer.MIN_VALUE; + descending = false; + subMap1 = null; + subMap2 = null; + keyView1 = null; + keyView2 = null; + valueView1 = null; + valueView2 = null; + entryView1 = null; + entryView2 = null; + type = InputType.MAIN_MAP; + viewOps = 0; + } + + public void reverse() { + this.descending = !descending; + } + } + + @Test + public void testEqual() throws Throwable { + final CompactedConcurrentSkipListMap mForTest = createMapForTest(false); + final ConcurrentSkipListMap mForStand = createMapForStand(); + putSimpleTestDataIntoMap(mForTest); + putSimpleTestDataIntoMap(mForStand); + Assert.assertTrue(mForTest.equals(mForStand)); + Assert.assertTrue(mForStand.equals(mForTest)); + } + + static interface Behavior { + InputType doBehavior(MapContext mc); + + String getName(); + } + + static class BehaviorCollections { + static ThreadLocal random = new ThreadLocal(); + static KVGenerator kvg; + + static Random getRandom() { + Random ret = random.get(); + if (ret == null) { + ret = new Random(); + random.set(ret); + } + return ret; + } + + public static abstract class AbstractBehavior implements Behavior { + String name; + int weight; + InputType type; + boolean supportMultiThread; + + public AbstractBehavior(String name, int w, InputType type, + boolean multiThread) { + this.name = name; + this.weight = w; + this.type = type; + this.supportMultiThread = multiThread; + } + + public String getName() { + return this.name + "(multithread=" + this.supportMultiThread + ",type=" + + this.type + ")"; + } + + public int getWeight() { + return this.weight; + } + + public boolean supportMultiThread() { + return this.supportMultiThread; + } + + public InputType getType() { + return this.type; + } + } + + static void initRandom(int range) { + kvg = KVGeneratorFactory.arbitaryKVGenerator(range); + } + + static int getRandomKey(MapContext mc) { + Random r = getRandom(); + if (mc.multiThread <= 1) { + // single thread + if (mc.subRangeStart == Integer.MIN_VALUE + || mc.subRangeEnd == Integer.MIN_VALUE) { + // get random key in whole range + return r.nextInt(mc.keyRange); + } else { + // get random key in sub range + int s = Math.max(Math.abs(mc.subRangeStart), Math.abs(mc.subRangeEnd)); + s = s == 0 ? 1 : s; + int key = r.nextInt(s); + return r.nextBoolean() ? key : -key; + } + } else { + // get random key in whole range + int threads = mc.multiThread; + if (mc.subRangeStart == Integer.MIN_VALUE || mc.subRangeEnd == Integer.MIN_VALUE) { + int base = mc.keyRange / threads; + base = base <= 0 ? 1 : base; + int k = r.nextInt(base); + int ret = k * threads + mc.label; + return ret; + } else { + // get random key in sub range + int s = Math.max(Math.abs(mc.subRangeStart), Math.abs(mc.subRangeEnd)); + int base = s / threads; + base = base == 0 ? 1 : 0; + int ss = r.nextInt(base); + int key = ss * base + mc.label; + int ret = r.nextBoolean() ? key : -key; + return ret; + } + } + } + + static KVPair getRandomKV(MapContext mc) { + if (mc.multiThread <= 1) { + if (mc.subRangeStart == Integer.MIN_VALUE + || mc.subRangeEnd == Integer.MIN_VALUE) { + // get kv in whole range + return kvg.getNextKV(getRandom()); + } else { + // get kv in sub range + KVPair kv = kvg.getNextKV(getRandom()); + int x = kv.value.hashCode(); + int s = Math.max(Math.abs(mc.subRangeStart), Math.abs(mc.subRangeEnd)); + kv.key = x % s; + return kv; + } + } else { + if (mc.subRangeStart == Integer.MIN_VALUE + || mc.subRangeEnd == Integer.MIN_VALUE) { + // get kv in whole range + KVPair kv = kvg.getNextKV(getRandom()); + kv.key = getRandomKey(mc); + return kv; + } else { + // get kv in sub range + KVPair kv = kvg.getNextKV(getRandom()); + kv.key = getRandomKey(mc); + return kv; + } + } + } + + static class MapRange { + int start; // always smaller than end + int end; // always bigger than start + boolean startInclusive; + boolean endInclusive; + + void reverse() { + int it = this.end; + this.end = this.start; + this.start = it; + + boolean bt = this.startInclusive; + this.startInclusive = this.endInclusive; + this.endInclusive = bt; + } + } + + private static int randomEdge(int start, int end, int keyRange) { + if (getRandom().nextInt(5) < 1) { // have 1/10 chance to get a edge + // outside keyrange + int ret = keyRange + getRandom().nextInt(keyRange); + return getRandom().nextBoolean() ? ret : -ret; + } + if (start == Integer.MIN_VALUE) { + start = -keyRange; + } + if (end == Integer.MIN_VALUE) { + end = keyRange; + } + if (end - start < 0) { + int tmp = start; + start = end; + end = tmp; + } + return getRandom().nextInt(end - start) + start; + } + + static MapRange getRandomMapRange(MapContext mc) { + MapRange mr = new MapRange(); + int start = randomEdge(mc.subRangeStart, mc.subRangeEnd, mc.keyRange); + int end = randomEdge(mc.subRangeStart, mc.subRangeEnd, mc.keyRange); + if (start > end) { + mr.start = end; + mr.end = start; + } else { + mr.start = start; + mr.end = start; + } + mr.startInclusive = getRandom().nextBoolean(); + mr.endInclusive = getRandom().nextBoolean(); + return mr; + } + + public static class Put extends AbstractBehavior { + public Put() { + super("put", 1000, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String ret1 = mc.mainMap1.put(kv.key, kv.value); + String ret2 = mc.mainMap2.put(kv.key, kv.value); + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapPut extends AbstractBehavior { + public SubMapPut() { + super("subMapPut", 1000, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + Throwable e1 = null; + Throwable e2 = null; + String ret1 = null; + String ret2 = null; + KVPair kv = getRandomKV(mc); + try { + ret1 = mc.mainMap1.put(kv.key, kv.value); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.mainMap2.put(kv.key, kv.value); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e2 != null || e1 != null) { + System.out.println(e1); + System.out.println(e2); + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.SUB_MAP; + } + } + + public static class PutIfAbsent extends AbstractBehavior { + public PutIfAbsent() { + super("putIfAbsent", 1000, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String ret1 = mc.mainMap1.putIfAbsent(kv.key, kv.value); + String ret2 = mc.mainMap2.putIfAbsent(kv.key, kv.value); + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapPutIfAbsent extends AbstractBehavior { + public SubMapPutIfAbsent() { + super("subMapPutIfAbsent", 1000, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + Throwable e1 = null; + Throwable e2 = null; + String ret1 = null; + String ret2 = null; + try { + ret1 = mc.mainMap1.putIfAbsent(kv.key, kv.value); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.mainMap2.putIfAbsent(kv.key, kv.value); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e2 != null || e1 != null) { + System.out.println(e1); + System.out.println(e2); + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.SUB_MAP; + } + } + + public static class PutAll extends AbstractBehavior { + public PutAll() { + super("putAll", 200, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + Map smallMap = new TreeMap(); + for (int i = 0; i < 5; ++i) { + KVPair kv = getRandomKV(mc); + smallMap.put(kv.key, kv.value); + } + mc.mainMap1.putAll(smallMap); + mc.mainMap2.putAll(smallMap); + return InputType.MAIN_MAP; + } + } + + public static class Remove extends AbstractBehavior { + public Remove() { + super("removeByKV", 300, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + int key = getRandomKey(mc); + String value = null; + if (getRandom().nextBoolean()) { + String v1 = mc.mainMap1.get(key); + String v2 = mc.mainMap2.get(key); + Assert.assertEquals(v1, v2); + value = v1; + } + + if (value == null) { + String ret1 = mc.mainMap1.remove(key); + String ret2 = mc.mainMap2.remove(key); + Assert.assertEquals(ret1, ret2); + } else { + boolean succ = getRandom().nextBoolean(); + if (!succ) { + value = kv.value; + } + boolean ret1 = mc.mainMap1.remove(key, value); + boolean ret2 = mc.mainMap2.remove(key, value); + Assert.assertEquals(ret1, ret2); + } + return InputType.MAIN_MAP; + } + } + + public static class SubMapRemove extends AbstractBehavior { + public SubMapRemove() { + super("subMapRemoveByKV", 300, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + int key = getRandomKey(mc); + String value = null; + if (getRandom().nextBoolean()) { + String v1 = mc.mainMap1.get(key); + String v2 = mc.mainMap2.get(key); + Assert.assertEquals(v1, v2); + value = v1; + } + + if (value == null) { + String ret1 = mc.subMap1.remove(key); + String ret2 = mc.subMap2.remove(key); + Assert.assertEquals(ret1, ret2); + } else { + boolean succ = getRandom().nextBoolean(); + if (!succ) { + value = kv.value; + } + boolean ret1 = mc.subMap1.remove(key, value); + boolean ret2 = mc.subMap2.remove(key, value); + Assert.assertEquals(ret1, ret2); + } + return InputType.SUB_MAP; + } + } + + public static class Replace extends AbstractBehavior { + public Replace() { + super("replace", 300, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String oldValue = null; + if (getRandom().nextBoolean()) { + oldValue = mc.mainMap1.get(kv.key); + if (getRandom().nextBoolean()) { + oldValue = oldValue + "a"; + } + } + + if (oldValue != null) { + boolean ret1 = mc.mainMap1.replace(kv.key, oldValue, kv.value); + boolean ret2 = mc.mainMap2.replace(kv.key, oldValue, kv.value); + Assert.assertEquals(ret1, ret2); + } else { + String ret1 = mc.mainMap1.replace(kv.key, kv.value); + String ret2 = mc.mainMap2.replace(kv.key, kv.value); + Assert.assertEquals(ret1, ret2); + } + return InputType.MAIN_MAP; + } + } + + public static class SubMapReplace extends AbstractBehavior { + public SubMapReplace() { + super("subMapReplace", 300, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String oldValue = null; + if (getRandom().nextBoolean()) { + oldValue = mc.mainMap1.get(kv.key); + if (getRandom().nextBoolean()) { + oldValue = oldValue + "a"; + } + } + + Throwable e1 = null; + Throwable e2 = null; + if (oldValue != null) { + boolean ret1 = false; + boolean ret2 = false; + try { + ret1 = mc.subMap1.replace(kv.key, oldValue, kv.value); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.subMap2.replace(kv.key, oldValue, kv.value); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + } else { + String ret1 = null; + String ret2 = null; + try { + ret1 = mc.subMap1.replace(kv.key, kv.value); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.subMap2.replace(kv.key, kv.value); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + } + return InputType.SUB_MAP; + } + } + + public static class Get extends AbstractBehavior { + public Get() { + super("get", 1000, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + String ret1 = mc.mainMap1.get(key); + String ret2 = mc.mainMap2.get(key); + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapGet extends AbstractBehavior { + public SubMapGet() { + super("subMapGet", 1000, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + String ret1 = mc.subMap1.get(key); + String ret2 = mc.subMap2.get(key); + Assert.assertEquals(ret1, ret2); + return InputType.SUB_MAP; + } + } + + public static class Near extends AbstractBehavior { + public Near() { + super("near", 1000, InputType.MAIN_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + int select = getRandom().nextInt(4); + if (getRandom().nextBoolean()) { + if (select == 0) { + Assert.assertEquals(mc.mainMap1.ceilingKey(key), + mc.mainMap2.ceilingKey(key)); + } else if (select == 1) { + Assert.assertEquals(mc.mainMap1.floorKey(key), + mc.mainMap2.floorKey(key)); + } else if (select == 2) { + Assert.assertEquals(mc.mainMap1.higherKey(key), + mc.mainMap2.higherKey(key)); + } else if (select == 3) { + Assert.assertEquals(mc.mainMap1.higherKey(key), + mc.mainMap2.higherKey(key)); + } + } else { + if (select == 0) { + Assert.assertEquals(mc.mainMap1.ceilingEntry(key), + mc.mainMap2.ceilingEntry(key)); + } else if (select == 1) { + Assert.assertEquals(mc.mainMap1.floorEntry(key), + mc.mainMap2.floorEntry(key)); + } else if (select == 2) { + Assert.assertEquals(mc.mainMap1.higherEntry(key), + mc.mainMap2.higherEntry(key)); + } else if (select == 3) { + Assert.assertEquals(mc.mainMap1.lowerEntry(key), + mc.mainMap2.lowerEntry(key)); + } + } + return InputType.MAIN_MAP; + } + } + + public static class SubMapNear extends AbstractBehavior { + public SubMapNear() { + super("subMapNear", 1000, InputType.SUB_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + int select = getRandom().nextInt(4); + if (getRandom().nextBoolean()) { + if (select == 0) { + Integer ret1 = mc.subMap1.ceilingKey(key); + Integer ret2 = mc.subMap2.ceilingKey(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 1) { + Integer ret1 = mc.subMap1.floorKey(key); + Integer ret2 = mc.subMap1.floorKey(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 2) { + Integer ret1 = mc.subMap1.higherKey(key); + Integer ret2 = mc.subMap2.higherKey(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 3) { + Integer ret1 = mc.subMap1.lowerKey(key); + Integer ret2 = mc.subMap2.lowerKey(key); + Assert.assertEquals(ret1, ret2); + } + } else { + + if (select == 0) { + Entry ret1 = mc.subMap1.ceilingEntry(key); + Entry ret2 = mc.subMap2.ceilingEntry(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 1) { + Entry ret1 = mc.subMap1.floorEntry(key); + Entry ret2 = mc.subMap2.floorEntry(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 2) { + Entry ret1 = mc.subMap1.higherEntry(key); + Entry ret2 = mc.subMap2.higherEntry(key); + Assert.assertEquals(ret1, ret2); + } else if (select == 3) { + Entry ret1 = mc.subMap1.lowerEntry(key); + Entry ret2 = mc.subMap2.lowerEntry(key); + Assert.assertEquals(ret1, ret2); + } + } + + return InputType.SUB_MAP; + } + } + + public static class ContainsKey extends AbstractBehavior { + public ContainsKey() { + super("containsKey", 1000, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + boolean ret1 = mc.mainMap1.containsKey(key); + boolean ret2 = mc.mainMap2.containsKey(key); + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapContainsKey extends AbstractBehavior { + public SubMapContainsKey() { + super("subMapContainsKey", 1000, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + boolean ret1 = mc.subMap1.containsKey(key); + boolean ret2 = mc.subMap2.containsKey(key); + Assert.assertEquals(ret1, ret2); + return InputType.SUB_MAP; + } + } + + public static Entry newEntry(int key, String value) { + return new AbstractMap.SimpleEntry(key, value); + } + + public static class EntrySetContains extends AbstractBehavior { + public EntrySetContains() { + super("entrySetContains", 500, InputType.ENTRYVIEW, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String value = mc.mainMap1.get(kv.key); + if (value != null) { + if (getRandom().nextBoolean()) { + value = value + 'x'; + } + } else { + value = kv.value; + } + Entry e = newEntry(kv.key, value); + boolean ret1 = mc.entryView1.contains(e); + boolean ret2 = mc.entryView2.contains(e); + Assert.assertEquals(ret1, ret2); + return InputType.ENTRYVIEW; + } + } + + public static class KeySetContains extends AbstractBehavior { + public KeySetContains() { + super("keySetContains", 1000, InputType.KEYVIEW, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + boolean ret1 = mc.keyView1.contains(key); + boolean ret2 = mc.keyView2.contains(key); + Assert.assertEquals(ret1, ret2); + return InputType.KEYVIEW; + } + } + + public static class EntrySetRemove extends AbstractBehavior { + public EntrySetRemove() { + super("entrySetRemove", 100, InputType.ENTRYVIEW, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + KVPair kv = getRandomKV(mc); + String value = mc.mainMap2.get(kv.key); + if (value != null) { + if (getRandom().nextBoolean()) { + value = value + 'x'; + } else { + value = kv.value; + } + } + Entry entry = newEntry(kv.key, value); + boolean ret1 = false; + boolean ret2 = false; + Throwable e1 = null; + Throwable e2 = null; + try { + ret1 = mc.entryView1.remove(entry); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.entryView2.remove(entry); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.ENTRYVIEW; + } + } + + public static class KeySetRemove extends AbstractBehavior { + public KeySetRemove() { + super("keySetRemove", 1000, InputType.KEYVIEW, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int key = getRandomKey(mc); + boolean ret1 = false; + boolean ret2 = false; + Throwable e1 = null; + Throwable e2 = null; + try { + ret1 = mc.keyView1.remove(key); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = mc.keyView2.remove(key); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.KEYVIEW; + } + } + + public static class EntryIterator extends AbstractBehavior { + public EntryIterator() { + super("entrySetIterator", 1000, InputType.ENTRYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Iterator> iter1 = mc.entryView1.iterator(); + Iterator> iter2 = mc.entryView2.iterator(); + int step = getRandom().nextInt(30); // max up to 30 step + while (iter2.hasNext()) { + Assert.assertTrue(iter1.hasNext()); + Entry ret1 = iter1.next(); + Entry ret2 = iter2.next(); + Assert.assertEquals(ret1, ret2); + if (getRandom().nextInt(100) < 1) { // 1/100 chance to remove this + // value + iter1.remove(); + iter2.remove(); + } + if (--step <= 0) { + break; + } + } + return InputType.ENTRYVIEW; + } + } + + public static class SubMap extends AbstractBehavior { + public SubMap() { + super("subMap", 400, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + MapRange mr = getRandomMapRange(mc); + int select = getRandom().nextInt(3); + if (select == 0) { // submap + mc.subMap1 = mc.mainMap1.subMap(mr.start, mr.startInclusive, mr.end, + mr.endInclusive); + mc.subMap2 = mc.mainMap2.subMap(mr.start, mr.startInclusive, mr.end, + mr.endInclusive); + } else if (select == 1) { // headmap + mc.subMap1 = mc.mainMap1.headMap(mr.end, mr.endInclusive); + mc.subMap2 = mc.mainMap2.headMap(mr.end, mr.endInclusive); + } else { // tailmap + mc.subMap1 = mc.mainMap1.tailMap(mr.start, mr.startInclusive); + mc.subMap2 = mc.mainMap2.tailMap(mr.start, mr.startInclusive); + } + mc.type = InputType.SUB_MAP; + return InputType.SUB_MAP; + } + } + + public static class KeySetIterator extends AbstractBehavior { + public KeySetIterator() { + super("keySetIterator", 1000, InputType.KEYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Iterator iter1 = null; + Iterator iter2 = null; + if (getRandom().nextBoolean()) { + iter1 = mc.keyView1.descendingIterator(); + iter2 = mc.keyView2.descendingIterator(); + } else { + iter1 = mc.keyView1.iterator(); + iter2 = mc.keyView2.iterator(); + } + int step = getRandom().nextInt(30); // max up to 30 step + while (iter2.hasNext()) { + Assert.assertTrue(iter2.hasNext()); + Integer ret1 = iter1.next(); + Integer ret2 = iter2.next(); + if (getRandom().nextInt(100) < 1) { // 1/100 chance to remove this key + iter1.remove(); + iter2.remove(); + } + if (--step <= 0) { + break; + } + } + return InputType.KEYVIEW; + } + } + + public static class EntrySetSize extends AbstractBehavior { + public EntrySetSize() { + super("entrySetSize", 1, InputType.ENTRYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int ret1 = mc.entryView1.size(); + int ret2 = mc.entryView2.size(); + Assert.assertEquals(ret1, ret2); + return InputType.ENTRYVIEW; + } + } + + public static class KeySetSize extends AbstractBehavior { + public KeySetSize() { + super("keySetSize", 1, InputType.KEYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int ret1 = mc.keyView1.size(); + int ret2 = mc.keyView2.size(); + Assert.assertEquals(ret1, ret2); + return InputType.KEYVIEW; + } + } + + public static class FirstOrLast extends AbstractBehavior { + public FirstOrLast() { + super("firstOrLast", 1000, InputType.MAIN_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + if (getRandom().nextBoolean()) { + Integer ret1 = null; + Integer ret2 = null; + if (getRandom().nextBoolean()) { + ret1 = mc.mainMap1.firstKey(); + ret2 = mc.mainMap2.firstKey(); + } else { + ret1 = mc.mainMap1.lastKey(); + ret2 = mc.mainMap2.lastKey(); + } + Assert.assertEquals(ret1, ret2); + } else { + Entry ret1 = null; + Entry ret2 = null; + if (getRandom().nextBoolean()) { + ret1 = mc.mainMap1.firstEntry(); + ret2 = mc.mainMap2.firstEntry(); + } else { + ret1 = mc.mainMap1.lastEntry(); + ret2 = mc.mainMap2.lastEntry(); + } + Assert.assertEquals(ret1, ret2); + } + return InputType.MAIN_MAP; + } + } + + public static class SubMapFirstOrLast extends AbstractBehavior { + public SubMapFirstOrLast() { + super("subMapFirstOrLast", 1000, InputType.SUB_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Assert.assertEquals(mc.subMap1.isEmpty(), mc.subMap2.isEmpty()); + Object ret1 = null; + Object ret2 = null; + Throwable e1 = null; + Throwable e2 = null; + if (getRandom().nextBoolean()) { + boolean first = getRandom().nextBoolean(); + try { + ret1 = first ? mc.subMap1.firstKey() : mc.subMap1.lastKey(); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = first ? mc.subMap2.firstKey() : mc.subMap2.lastKey(); + } catch (Throwable e) { + e2 = e; + } + } else { + boolean first = getRandom().nextBoolean(); + try { + ret1 = first ? mc.subMap1.firstEntry() : mc.subMap1.lastEntry(); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = first ? mc.subMap2.firstEntry() : mc.subMap2.lastEntry(); + } catch (Throwable e) { + e2 = e; + } + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + if (e2 == null || e1 == null) { + System.out.println(e1); + System.out.println(e2); + } + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.SUB_MAP; + } + } + + public static class Poll extends AbstractBehavior { + public Poll() { + super("poll", 250, InputType.MAIN_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Assert.assertEquals(mc.mainMap1.isEmpty(), mc.mainMap2.isEmpty()); + if (mc.mainMap1.isEmpty()) { + return InputType.MAIN_MAP; + } + Entry ret1 = null; + Entry ret2 = null; + if (getRandom().nextBoolean()) { + ret1 = mc.mainMap1.pollFirstEntry(); + ret2 = mc.mainMap2.pollFirstEntry(); + } else { + ret1 = mc.mainMap1.pollLastEntry(); + ret2 = mc.mainMap2.pollLastEntry(); + } + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapPoll extends AbstractBehavior { + public SubMapPoll() { + super("subMapPoll", 250, InputType.SUB_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Assert.assertEquals(mc.subMap1.isEmpty(), mc.subMap2.isEmpty()); + Entry ret1 = null; + Entry ret2 = null; + Throwable e1 = null; + Throwable e2 = null; + boolean first = getRandom().nextBoolean(); + try { + ret1 = first ? mc.subMap1.pollFirstEntry() : mc.subMap1 + .pollLastEntry(); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = first ? mc.subMap2.pollFirstEntry() : mc.subMap2 + .pollLastEntry(); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + ; + } + return InputType.SUB_MAP; + } + } + + public static class Size extends AbstractBehavior { + public Size() { + super("size", 1, InputType.MAIN_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int ret1 = mc.mainMap1.size(); + int ret2 = mc.mainMap2.size(); + Assert.assertEquals(ret1, ret2); + return InputType.MAIN_MAP; + } + } + + public static class SubMapSize extends AbstractBehavior { + public SubMapSize() { + super("subMapSize", 1, InputType.SUB_MAP, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Assert.assertEquals(mc.subMap1.size(), mc.subMap2.size()); + return InputType.SUB_MAP; + } + } + + public static class EntrySet extends AbstractBehavior { + public EntrySet() { + super("entrySet", 200, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + mc.entryView1 = mc.mainMap1.entrySet(); + mc.entryView2 = mc.mainMap2.entrySet(); + mc.viewOps = 0; + return (mc.type = InputType.ENTRYVIEW); + } + } + + public static class SubMapEntrySet extends AbstractBehavior { + public SubMapEntrySet() { + super("SubMapEntrySet", 200, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + mc.entryView1 = mc.subMap1.entrySet(); + mc.entryView2 = mc.subMap2.entrySet(); + return (mc.type = InputType.ENTRYVIEW); + } + } + + public static class KeySet extends AbstractBehavior { + public KeySet() { + super("keySet", 200, InputType.MAIN_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int select = getRandom().nextInt(3); + if (select == 0) { + mc.keyView1 = mc.mainMap1.keySet(); + mc.keyView2 = mc.mainMap2.keySet(); + } else if (select == 1) { + mc.keyView1 = mc.mainMap1.navigableKeySet(); + mc.keyView2 = mc.mainMap2.navigableKeySet(); + } else if (select == 2) { + mc.keyView1 = mc.mainMap1.descendingKeySet(); + mc.keyView2 = mc.mainMap2.descendingKeySet(); + mc.reverse(); + } + + return (mc.type = InputType.KEYVIEW); + } + } + + public static class SubMapKeySet extends AbstractBehavior { + public SubMapKeySet() { + super("subMapKeySet", 200, InputType.SUB_MAP, true); + } + + @Override + public InputType doBehavior(MapContext mc) { + int select = getRandom().nextInt(3); + if (select == 0) { + mc.keyView1 = mc.subMap1.keySet(); + mc.keyView2 = mc.subMap2.keySet(); + } else if (select == 1) { + mc.keyView1 = mc.subMap1.navigableKeySet(); + mc.keyView2 = mc.subMap2.navigableKeySet(); + } else if (select == 2) { + mc.keyView1 = mc.subMap1.descendingKeySet(); + mc.keyView2 = mc.subMap2.descendingKeySet(); + mc.reverse(); + } + return (mc.type = InputType.KEYVIEW); + } + } + + public static class KeySetNeer extends AbstractBehavior { + public KeySetNeer() { + super("keySetNeer", 1000, InputType.KEYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + int select = getRandom().nextInt(4); + int key = getRandomKey(mc); + Integer ret1 = null; + Integer ret2 = null; + if (select == 0) { + Assert.assertEquals(mc.keyView1.ceiling(key), + mc.keyView2.ceiling(key)); + } else if (select == 1) { + Assert.assertEquals(mc.keyView1.lower(key), mc.keyView1.lower(key)); + } else if (select == 2) { + Assert.assertEquals(mc.keyView1.higher(key), mc.keyView2.higher(key)); + } else { + Assert.assertEquals(mc.keyView1.floor(key), mc.keyView2.floor(key)); + } + + return (mc.type = InputType.KEYVIEW); + } + } + + public static class KeySetFirstOrLast extends AbstractBehavior { + public KeySetFirstOrLast() { + super("keySetFirstOrLast", 1000, InputType.KEYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Integer ret1 = null; + Integer ret2 = null; + Throwable e1 = null; + Throwable e2 = null; + boolean first = getRandom().nextBoolean(); + + try { + ret1 = first ? mc.keyView1.first() : mc.keyView1.last(); + } catch (Throwable e) { + e1 = e; + } + try { + ret2 = first ? mc.keyView2.first() : mc.keyView2.last(); + } catch (Throwable e) { + e2 = e; + } + Assert.assertEquals(ret1, ret2); + if (e1 != null || e2 != null) { + Assert.assertEquals(e1.getClass(), e2.getClass()); + } + return InputType.KEYVIEW; + } + } + + public static class KeySetPoll extends AbstractBehavior { + public KeySetPoll() { + super("keySetPool", 1000, InputType.KEYVIEW, false); + } + + @Override + public InputType doBehavior(MapContext mc) { + Integer ret1 = null; + Integer ret2 = null; + if (getRandom().nextBoolean()) { + ret1 = mc.keyView1.pollFirst(); + ret2 = mc.keyView2.pollFirst(); + } else { + ret1 = mc.keyView1.pollLast(); + ret2 = mc.keyView2.pollLast(); + } + Assert.assertEquals(ret1, ret2); + return InputType.KEYVIEW; + } + } + + public static class RandomBehavior { + Random r = getRandom(); + NavigableMap> behaviors = new TreeMap>(); + NavigableMap> weightedBehaviors = new TreeMap>(); + NavigableMap totalWeight = new TreeMap(); + + public RandomBehavior(boolean multiThread) { + + // get behaviors from Behavior collections + try { + Class[] classes = BehaviorCollections.class.getClasses(); + System.out.println("Get " + classes.length + + " classes from BehaviorCollections"); + for (int i = 0; i < classes.length; ++i) { + Class cls = classes[i]; + if (AbstractBehavior.class.isAssignableFrom(cls) + && !cls.equals(AbstractBehavior.class)) { + AbstractBehavior newBehavior; + + newBehavior = (AbstractBehavior) cls.newInstance(); + if (multiThread && !newBehavior.supportMultiThread()) { + continue; + } + + List list = behaviors + .get(newBehavior.getType()); + if (list == null) { + list = new ArrayList(); + behaviors.put(newBehavior.getType(), list); + } + list.add(newBehavior); + + System.out.println("Created new behavior " + + newBehavior.getName()); + } + } + } catch (InstantiationException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + + // create random map + for (Entry> entry : behaviors + .entrySet()) { + InputType type = entry.getKey(); + List bhs = entry.getValue(); + NavigableMap wm = new TreeMap(); + int totalw = 0; + for (AbstractBehavior bh : bhs) { + totalw += bh.getWeight(); + wm.put(totalw, bh); + System.out.println("Put " + bh.getName() + " into at " + type + + " of weight " + totalw); + } + totalWeight.put(type, totalw); + weightedBehaviors.put(type, wm); + } + } + + public Behavior nextBehavior(InputType inputType) { + int totalw = totalWeight.get(inputType).intValue(); + int l = getRandom().nextInt(totalw); + Entry entry = weightedBehaviors.get( + inputType).higherEntry(l); + return entry.getValue(); + } + } + + static public void initData(MapContext mc, int count) { + for (int i = 0; i < count; ++i) { + KVPair kv = getRandomKV(mc); + mc.mainMap1.put(kv.key, kv.value); + mc.mainMap2.put(kv.key, kv.value); + } + } + } // end of behavior collctions + + static class Performace { + Map behaviorToOps = new ConcurrentHashMap(); + Map behaviorToLatency = new ConcurrentHashMap(); + + void addOps(String behavior, long latencyInNs) { + Integer o = behaviorToOps.get(behavior); + o = o == null ? 1 : o.intValue() + 1; + behaviorToOps.put(behavior, o); + + Long l = behaviorToLatency.get(behavior); + l = l == null ? latencyInNs : l.longValue() + latencyInNs; + behaviorToLatency.put(behavior, l); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Entry e : behaviorToOps.entrySet()) { + int ops = e.getValue(); + long lat = behaviorToLatency.get(e.getKey()) / ops / 1000; + sb.append("Ops:").append(ops).append('\t').append("AvgLatency:") + .append(lat).append('\t').append("B:").append(e.getKey()) + .append('\n'); + } + return sb.toString(); + } + } + + @Test + public void testRandomBehavior() { + final int RANGE = 32768; + final int INIT_DATA = 50000; + final int COUNT = 10000000; + MapContext mc = new MapContext(createMapForTest(true), + createMapForStand(), RANGE); + BehaviorCollections.initRandom(RANGE); + BehaviorCollections.initData(mc, INIT_DATA); + System.out.println("There are " + mc.mainMap2.size() + + " element in this map"); + Performace performance = new Performace(); + RandomBehavior rb = new RandomBehavior(false); + for (int i = 0; i < COUNT; ++i) { + Behavior b = rb.nextBehavior(mc.type); + if (mc.type != InputType.MAIN_MAP) { + mc.viewOps += 1; + } + long start = System.nanoTime(); + b.doBehavior(mc); + long lat = System.nanoTime() - start; + performance.addOps(b.getName(), lat); + if (mc.viewOps > 20) { // up to 20 view operations + mc.reset(); + } + } + System.out.println("There are " + mc.mainMap2.size() + + " element in this map"); + System.out.println("Performace:"); + System.out.print(performance.toString()); + } + + @Test + public void testRandomBehaviorParallelly() throws InterruptedException { + final int RANGE = 32768; + final int INIT_DATA = 50000; + final int THREAD = 8; + final int COUNT = 1000000; // all threads will run COUNT times + final RandomBehavior rb = new RandomBehavior(true); + CompactedConcurrentSkipListMap mapForTest = createMapForTest(true); + ConcurrentSkipListMap mapForStand = createMapForStand(); + final MapContext[] mc = new MapContext[THREAD]; + for (int i = 0; i < THREAD; ++i) { + mc[i] = new MapContext(mapForTest, mapForStand, RANGE, i, THREAD); + } + + BehaviorCollections.initRandom(RANGE); + BehaviorCollections.initData( + new MapContext(mapForTest, mapForStand, RANGE), INIT_DATA); + System.out.println("There are " + mc[0].mainMap2.size() + + " element in this map"); + final Performace performance = new Performace(); + List threads = new ArrayList(); + for (int i = 0; i < THREAD; ++i) { + final MapContext context = mc[i]; + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + for (int c = 0; c < COUNT; ++c) { + Behavior b = rb.nextBehavior(context.type); + if (context.type != InputType.MAIN_MAP) { + context.viewOps += 1; + } + long start = System.nanoTime(); + b.doBehavior(context); + long lat = System.nanoTime() - start; + performance.addOps(b.getName(), lat); + if (context.viewOps > 20) { // up to 20 view operations + context.reset(); + } + } + } catch (Throwable e) { + e.printStackTrace(); + Assert.assertFalse(e.getMessage(), true); + } + + } + }); + threads.add(t); + } + + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + System.out.println("There are " + mc[0].mainMap2.size() + + " element in this map"); + System.out.println("Performace:"); + System.out.print(performance.toString()); + } + + @Test + public void testSingleKVLargerThanPage() { + int old = BIG_KV_THRESHOLD; + BIG_KV_THRESHOLD = DATA_PAGE_SIZE * 2; + try { + CompactedConcurrentSkipListMap m = createMapForTest(false); + int sz = DATA_PAGE_SIZE + 10; + StringBuilder sb = new StringBuilder(sz); + for (int i = 0; i < sz; ++i) { + sb.append('a'); + } + m.put(10, sb.toString()); + } finally { + BIG_KV_THRESHOLD = old; + } + } + + @Test + public void testSingleKVLargerThanDataLengthLimit() { + int len = 0x00FFFFFF + 1; + StringBuilder sb = new StringBuilder(len+1); + for (int i = 0; i < len; ++i) { + sb.append('a'); + } + CompactedConcurrentSkipListMap m = createMapForTest(false); + m.put(10, sb.toString()); + System.out.println(m.getMemoryUsage().toString()); + } + + @Test + public void testPageSize() { + CompactedConcurrentSkipListMap m = createMapForTest(false); + m.put(1, "a"); + MemoryUsage usage = m.getMemoryUsage(); + Assert.assertEquals(usage.dataSpace, DATA_PAGE_SIZE); + } + + @Test + public void testEmptyMapMemoryUsed() { + CompactedConcurrentSkipListMap m = createMapForTest(true); + MemoryUsage usage = m.getMemoryUsage(); + System.out.println(usage.toString()); + Assert.assertEquals(usage.dataSpace, 0); + Assert.assertEquals(usage.heapKVCapacity, 0); + } +} \ No newline at end of file diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index f005f05d9a..0db9264e62 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -170,4 +170,8 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.hregion.memstore.ccsmap.enabled + true + -- 2.16.3