From 2c6e81a2f3cc83777972de24372ef1f92462a6f9 Mon Sep 17 00:00:00 2001 From: anastas Date: Mon, 24 Apr 2017 11:59:51 +0300 Subject: [PATCH 1/3] My squashed commits --- .../hadoop/hbase/regionserver/ChunkCreator.java | 9 ++- .../hadoop/hbase/regionserver/TestCellFlatSet.java | 67 +++++++++++++++++++++- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index d550148..b043ec4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -35,6 +35,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.HeapMemoryTuneObserver; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; @@ -138,13 +139,17 @@ public class ChunkCreator { */ private Chunk createChunk(boolean pool) { int id = chunkID.getAndIncrement(); + Chunk c; assert id > 0; // do not create offheap chunk on demand if (pool && this.offheap) { - return new OffheapChunk(chunkSize, id, pool); + c = new OffheapChunk(chunkSize, id, pool); } else { - return new OnheapChunk(chunkSize, id, pool); + c = new OnheapChunk(chunkSize, id, pool); } + // write the chunk id as the first thing on the chunk's buffer + ByteBufferUtils.putInt(c.getData(),id); + return c; } @VisibleForTesting diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java index 09877b0..4e8fad3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java @@ -18,6 +18,10 @@ */ package org.apache.hadoop.hbase.regionserver; + +import java.lang.management.ManagementFactory; + +import java.nio.ByteBuffer; import java.util.Iterator; import java.util.NavigableMap; import java.util.NavigableSet; @@ -28,13 +32,19 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; + +import org.apache.hadoop.hbase.io.util.MemorySizeUtil; + + import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; - +import static org.junit.Assert.assertTrue; @Category({RegionServerTests.class, SmallTests.class}) public class TestCellFlatSet extends TestCase { @@ -47,6 +57,11 @@ public class TestCellFlatSet extends TestCase { private KeyValue lowerOuterCell; private KeyValue upperOuterCell; + + private CellChunkMap ccm; // for testing CellChunkMap + private static ChunkCreator chunkCreator; + + @Override protected void setUp() throws Exception { super.setUp(); @@ -71,17 +86,38 @@ public class TestCellFlatSet extends TestCase { ascCbOnHeap = new CellArrayMap(CellComparator.COMPARATOR,ascCells,0,NUM_OF_CELLS,false); descCells = new Cell[] {kv4,kv3,kv2,kv1}; descCbOnHeap = new CellArrayMap(CellComparator.COMPARATOR,descCells,0,NUM_OF_CELLS,true); + CONF.setBoolean(MemStoreLAB.USEMSLAB_KEY, true); CONF.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f); ChunkCreator.chunkPoolDisabled = false; + + + long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() + .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(CONF, false)); + chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, + globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); + assertTrue(chunkCreator != null); + + ccm = setUpCellChunkMap(); } - /* Create and test CellSet based on CellArrayMap */ - public void testCellBlocksOnHeap() throws Exception { + /* Create and test ascending CellSet based on CellArrayMap */ + @Test + public void testCellArrayMapAsc() throws Exception { CellSet cs = new CellSet(ascCbOnHeap); testCellBlocks(cs); testIterators(cs); } + + /* Create and test ascending CellSet based on CellChunkMap */ + @Test + public void testCellChunkMapAsc() throws Exception { + CellSet cs = new CellSet(ccm); + testCellBlocks(cs); + testIterators(cs); + testSubSet(cs); + } + @Test public void testAsc() throws Exception { CellSet ascCs = new CellSet(ascCbOnHeap); @@ -200,4 +236,29 @@ public class TestCellFlatSet extends TestCase { } assertEquals(NUM_OF_CELLS, count); } + + /* Create CellChunkMap with four cells inside the index chunk */ + private CellChunkMap setUpCellChunkMap() { + + Chunk idxChunk = chunkCreator.getChunk(); + ByteBuffer idxBuffer = idxChunk.getData(); // index chunk buffer for cell-representations + // allocate new chunk and use its buffer to hold the full data of the cells + Chunk dataChunk = chunkCreator.getChunk(); + + ByteBuffer dataBuffer = dataChunk.getData(); + int offset = Bytes.SIZEOF_INT; // skip the space for chunk ID + int pos = offset; + + for (Cell kv: ascCells) { + KeyValueUtil.appendTo(kv, dataBuffer, offset, false); // write deep cell data + + pos = ByteBufferUtils.putInt(idxBuffer, pos, dataChunk.getId()); // write data chunk index + pos = ByteBufferUtils.putInt(idxBuffer, pos, offset); // offset + pos = ByteBufferUtils.putInt(idxBuffer, pos, KeyValueUtil.length(kv)); // length + pos = ByteBufferUtils.putLong(idxBuffer, pos, kv.getSequenceId()); // length + offset += KeyValueUtil.length(kv); + } + Chunk [] chunkArray = {idxChunk}; + return new CellChunkMap(CellComparator.COMPARATOR,chunkArray,0,NUM_OF_CELLS,false); + } } -- 1.8.5.2 (Apple Git-48) From 86ea96ad6af100f47558e2ca87b3b6b4b0fe5690 Mon Sep 17 00:00:00 2001 From: anastas Date: Mon, 24 Apr 2017 12:20:02 +0300 Subject: [PATCH 2/3] continue with CellChunkMap debugging --- .../java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index b043ec4..d550148 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -35,7 +35,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.HeapMemoryTuneObserver; -import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; @@ -139,17 +138,13 @@ public class ChunkCreator { */ private Chunk createChunk(boolean pool) { int id = chunkID.getAndIncrement(); - Chunk c; assert id > 0; // do not create offheap chunk on demand if (pool && this.offheap) { - c = new OffheapChunk(chunkSize, id, pool); + return new OffheapChunk(chunkSize, id, pool); } else { - c = new OnheapChunk(chunkSize, id, pool); + return new OnheapChunk(chunkSize, id, pool); } - // write the chunk id as the first thing on the chunk's buffer - ByteBufferUtils.putInt(c.getData(),id); - return c; } @VisibleForTesting -- 1.8.5.2 (Apple Git-48) From 27921b97fe292d0c0275d9be0d4f831c232e4ed3 Mon Sep 17 00:00:00 2001 From: anastas Date: Mon, 24 Apr 2017 12:38:19 +0300 Subject: [PATCH 3/3] Continue with CellChunkMap debugging --- .../java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java | 7 ++++--- .../java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index a8ba512..68edd9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -70,9 +70,9 @@ public class CellChunkMap extends CellFlatMap { super(comparator, min, max, descending); this.chunkCreator = ChunkCreator.getInstance(); this.chunks = chunks; - + //TODO: change to int when the chunk ID size bug is fixed this.numOfCellsInsideChunk = // each chunk starts with its own ID following the cells data - (chunkCreator.getChunkSize() - Bytes.SIZEOF_INT) / SIZEOF_CELL_REF; + (chunkCreator.getChunkSize() - Bytes.SIZEOF_LONG) / SIZEOF_CELL_REF; } /* To be used by base (CellFlatMap) class only to create a sub-CellFlatMap @@ -91,7 +91,8 @@ public class CellChunkMap extends CellFlatMap { i = i - chunkIndex * numOfCellsInsideChunk; // get the index of the cell-representation // find inside the offset inside the chunk holding the index, skip bytes for chunk id - int offsetInBytes = Bytes.SIZEOF_INT + i*SIZEOF_CELL_REF; + //TODO: change to int when the chunk ID size bug is fixed + int offsetInBytes = Bytes.SIZEOF_LONG + i*SIZEOF_CELL_REF; // find the chunk holding the data of the cell, the chunkID is stored first int chunkId = ByteBufferUtils.toInt(block, offsetInBytes); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java index 4e8fad3..fd56881 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java @@ -246,7 +246,8 @@ public class TestCellFlatSet extends TestCase { Chunk dataChunk = chunkCreator.getChunk(); ByteBuffer dataBuffer = dataChunk.getData(); - int offset = Bytes.SIZEOF_INT; // skip the space for chunk ID + //TODO: change to int when the chunk ID size bug is fixed + int offset = Bytes.SIZEOF_LONG; // skip the space for chunk ID int pos = offset; for (Cell kv: ascCells) { -- 1.8.5.2 (Apple Git-48)