From 1489b4cd3696801a576e227e3d2eae1c2cf5f753 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 1 Jun 2018 15:36:51 -0700 Subject: [PATCH] current --- .../apache/hadoop/hbase/ByteBufferKeyValue.java | 30 +++++++++-- .../hadoop/hbase/NoTagsByteBufferKeyValue.java | 62 ---------------------- .../apache/hadoop/hbase/codec/KeyValueCodec.java | 27 ++++++++-- .../regionserver/ByteBufferChunkKeyValue.java | 48 ----------------- .../hadoop/hbase/regionserver/CellChunkMap.java | 4 +- .../hadoop/hbase/regionserver/MemStoreLABImpl.java | 37 ++++++++++--- .../regionserver/NoTagByteBufferChunkKeyValue.java | 48 ----------------- 7 files changed, 83 insertions(+), 173 deletions(-) delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsByteBufferKeyValue.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ByteBufferChunkKeyValue.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoTagByteBufferChunkKeyValue.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java index 963b4118ba..67ccc560cf 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java @@ -29,7 +29,12 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti /** * This Cell is an implementation of {@link ByteBufferExtendedCell} where the data resides in - * off heap/ on heap ByteBuffer + * off heap/ on heap ByteBuffer. + * + * This is our base type in hbase2. + * + * Avoid specialization so JIT sees one base type only ever (in the name of confusing it + * with myriad types; lets be monomorphic in our base type rather than megamorphic). */ @InterfaceAudience.Private public class ByteBufferKeyValue extends ByteBufferExtendedCell { @@ -317,9 +322,17 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { public ExtendedCell deepClone() { byte[] copy = new byte[this.length]; ByteBufferUtils.copyFromBufferToArray(copy, this.buf, this.offset, 0, this.length); - KeyValue kv = new KeyValue(copy, 0, copy.length); - kv.setSequenceId(this.getSequenceId()); - return kv; + ExtendedCell cell = deserialize(copy); + try { + cell.setSequenceId(this.getSequenceId()); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + return cell; + } + + protected ExtendedCell deserialize(byte [] bytes) { + return new KeyValue(bytes, 0, bytes.length); } /** @@ -355,4 +368,13 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { hash = 31 * hash + cell.getTypeByte(); return hash; } + + @Override + public int getChunkId() { + // The chunkId is embedded at the 0th offset of the bytebuffer + // Do not call this method unless you really need too... + return ByteBufferUtils.toInt(buf, 0); + } + + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsByteBufferKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsByteBufferKeyValue.java deleted file mode 100644 index f00961f666..0000000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsByteBufferKeyValue.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.util.ByteBufferUtils; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * An extension of the ByteBufferKeyValue where the tags length is always 0 - */ -@InterfaceAudience.Private -public class NoTagsByteBufferKeyValue extends ByteBufferKeyValue { - - public NoTagsByteBufferKeyValue(ByteBuffer buf, int offset, int length) { - super(buf, offset, length); - } - - public NoTagsByteBufferKeyValue(ByteBuffer buf, int offset, int length, long seqId) { - super(buf, offset, length, seqId); - } - - @Override - public byte[] getTagsArray() { - return HConstants.EMPTY_BYTE_ARRAY; - } - - @Override - public int getTagsLength() { - return 0; - } - - @Override - public int getSerializedSize(boolean withTags) { - return this.length; - } - - @Override - public ExtendedCell deepClone() { - byte[] copy = new byte[this.length]; - ByteBufferUtils.copyFromBufferToArray(copy, this.buf, this.offset, 0, this.length); - KeyValue kv = new NoTagsKeyValue(copy, 0, copy.length); - kv.setSequenceId(this.getSequenceId()); - return kv; - } -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java index 9a5db3c640..21acf03b84 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java @@ -22,10 +22,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.NoTagsByteBufferKeyValue; import org.apache.hadoop.hbase.NoTagsKeyValue; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -112,9 +114,28 @@ public class KeyValueCodec implements Codec { protected Cell createCell(ByteBuffer bb, int pos, int len) { // We know there is not going to be any tags. - return new NoTagsByteBufferKeyValue(bb, pos, len); + return new ByteBufferKeyValue(bb, pos, len) { + @Override + public byte[] getTagsArray() { + return HConstants.EMPTY_BYTE_ARRAY; + } + + @Override + public int getTagsLength() { + return 0; + } + + @Override + public int getSerializedSize(boolean withTags) { + return this.length; + } + + @Override + protected ExtendedCell deserialize(byte[] bytes) { + return new NoTagsKeyValue(bytes, 0, bytes.length); + } + }; } - } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ByteBufferChunkKeyValue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ByteBufferChunkKeyValue.java deleted file mode 100644 index 8278c420ff..0000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ByteBufferChunkKeyValue.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.ByteBufferKeyValue; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.util.ByteBufferUtils; - -/** - * ByteBuffer based cell which has the chunkid at the 0th offset - * @see MemStoreLAB - */ -//TODO : When moving this cell to CellChunkMap we will have the following things -// to be serialized -// chunkId (Integer) + offset (Integer) + length (Integer) + seqId (Long) = 20 bytes -@InterfaceAudience.Private -public class ByteBufferChunkKeyValue extends ByteBufferKeyValue { - public ByteBufferChunkKeyValue(ByteBuffer buf, int offset, int length) { - super(buf, offset, length); - } - - public ByteBufferChunkKeyValue(ByteBuffer buf, int offset, int length, long seqId) { - super(buf, offset, length, seqId); - } - - @Override - public int getChunkId() { - // The chunkId is embedded at the 0th offset of the bytebuffer - return ByteBufferUtils.toInt(buf, 0); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java index 0ef54a13e3..7ad1253d2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.regionserver; import java.nio.ByteBuffer; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @@ -128,6 +128,6 @@ public class CellChunkMap extends CellFlatMap { + chunk.isFromPool() + ". We were looking for a cell at index " + i); } - return new ByteBufferChunkKeyValue(buf, offsetOfCell, lengthOfCell, cellSeqID); + return new ByteBufferKeyValue(buf, offsetOfCell, lengthOfCell, cellSeqID); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index ac7223f2d8..75f590c963 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -29,9 +29,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.NoTagsKeyValue; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,13 +189,33 @@ public class MemStoreLABImpl implements MemStoreLAB { // TODO : write the seqid here. For writing seqId we should create a new cell type so // that seqId is not used as the state if (tagsLen == 0) { - // When tagsLen is 0, make a NoTagsByteBufferKeyValue version. This is an optimized class - // which directly return tagsLen as 0. So we avoid parsing many length components in - // reading the tagLength stored in the backing buffer. The Memstore addition of every Cell - // call getTagsLength(). - return new NoTagByteBufferChunkKeyValue(buf, offset, len, cell.getSequenceId()); + // When tagsLen is 0, return version of BBKV that short-circuits answers on tags. + // This is an optimized class which directly return tagsLen as 0. So we avoid + // parsing many length components in reading the tagLength stored in the backing + // buffer. The Memstore addition of every Cell calls getTagsLength(). + return new ByteBufferKeyValue(buf, offset, len, cell.getSequenceId()) { + @Override + public byte[] getTagsArray() { + return HConstants.EMPTY_BYTE_ARRAY; + } + + @Override + public int getTagsLength() { + return 0; + } + + @Override + public int getSerializedSize(boolean withTags) { + return this.length; + } + + @Override + protected ExtendedCell deserialize(byte[] bytes) { + return new NoTagsKeyValue(bytes, 0, bytes.length); + } + }; } else { - return new ByteBufferChunkKeyValue(buf, offset, len, cell.getSequenceId()); + return new ByteBufferKeyValue(buf, offset, len, cell.getSequenceId()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoTagByteBufferChunkKeyValue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoTagByteBufferChunkKeyValue.java deleted file mode 100644 index 1fb533a2c5..0000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NoTagByteBufferChunkKeyValue.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.NoTagsByteBufferKeyValue; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.util.ByteBufferUtils; - - -/** - * ByteBuffer based cell which has the chunkid at the 0th offset and with no tags - * @see MemStoreLAB - */ -@InterfaceAudience.Private -public class NoTagByteBufferChunkKeyValue extends NoTagsByteBufferKeyValue { - - public NoTagByteBufferChunkKeyValue(ByteBuffer buf, int offset, int length) { - super(buf, offset, length); - } - - public NoTagByteBufferChunkKeyValue(ByteBuffer buf, int offset, int length, long seqId) { - super(buf, offset, length, seqId); - } - - @Override - public int getChunkId() { - // The chunkId is embedded at the 0th offset of the bytebuffer - return ByteBufferUtils.toInt(buf, 0); - } - -} -- 2.16.3