diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 8835957..d014ecd 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -142,6 +142,8 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, Bytes.SIZEOF_INT /*keylength*/ + Bytes.SIZEOF_INT /*valuelength*/; + public static final int ROW_KEY_OFFSET = ROW_OFFSET + ROW_LENGTH_SIZE; + // Size of the length ints in a KeyValue datastructure. public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET; @@ -1323,7 +1325,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, */ @Override public int getRowOffset() { - return getKeyOffset() + Bytes.SIZEOF_SHORT; + return this.offset + ROW_KEY_OFFSET; } /** @@ -1354,7 +1356,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @return Family offset */ private int getFamilyOffset(int rlength) { - return this.offset + ROW_OFFSET + Bytes.SIZEOF_SHORT + rlength + Bytes.SIZEOF_BYTE; + return this.offset + ROW_KEY_OFFSET + rlength + Bytes.SIZEOF_BYTE; } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java deleted file mode 100644 index c4c8351..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/NoTagsKeyValue.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * An extension of the KeyValue where the tags length is always 0 - */ -@InterfaceAudience.Private -public class NoTagsKeyValue extends KeyValue { - public NoTagsKeyValue(byte[] bytes, int offset, int length) { - super(bytes, offset, length); - } - - @Override - public int getTagsLength() { - return 0; - } -} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedKeyValue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedKeyValue.java new file mode 100644 index 0000000..19b2233 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedKeyValue.java @@ -0,0 +1,67 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This class is an extension to KeyValue where rowLen, keyLen and valLen are cached. + * Parsing the backing byte[] every time to get these values will affect the performance. + * In read path, we tend to read these values many times in Comparator, SQM etc. + * Note: Please do not use these objects in write path and it will increase the heap space usage. + */ +@InterfaceAudience.Private +public class ContentSizeCachedKeyValue extends KeyValue { + + private static final int HEAP_SIZE_OVERHEAD = Bytes.SIZEOF_SHORT + (2 * Bytes.SIZEOF_INT); + + private short rowLen; + private int keyLen, valLen; + + public ContentSizeCachedKeyValue(byte[] bytes, int offset, int length) { + super(bytes, offset, length); + // We will read all these cached values at least once. Initialize now itself so that we can + // avoid uninitialized checks with every time call + rowLen = super.getRowLength(); + keyLen = super.getKeyLength(); + valLen = super.getValueLength(); + } + + @Override + public short getRowLength() { + return rowLen; + } + + @Override + public int getKeyLength() { + return this.keyLen; + } + + @Override + public int getValueLength() { + return this.valLen; + } + + @Override + public long heapSize() { + return super.heapSize() + HEAP_SIZE_OVERHEAD; + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedNoTagsKeyValue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedNoTagsKeyValue.java new file mode 100644 index 0000000..4e66c5d --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ContentSizeCachedNoTagsKeyValue.java @@ -0,0 +1,39 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * This class is an extension to ContentSizeCachedKeyValue where there are no tags in Cell. + * Note: Please do not use these objects in write path and it will increase the heap space usage. + */ +@InterfaceAudience.Private +public class ContentSizeCachedNoTagsKeyValue extends ContentSizeCachedKeyValue { + + public ContentSizeCachedNoTagsKeyValue(byte[] bytes, int offset, int length) { + super(bytes, offset, length); + } + + @Override + public int getTagsLength() { + return 0; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 4d1881d..13ab722 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -35,11 +35,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ContentSizeCachedKeyValue; +import org.apache.hadoop.hbase.ContentSizeCachedNoTagsKeyValue; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.NoTagsKeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.compress.Compression; @@ -137,7 +137,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { private List loadOnOpenBlocks = new ArrayList(); /** Minimum minor version supported by this HFile format */ - static final int MIN_MINOR_VERSION = 0; + static final int MIN_MINOR_VERSION = 3; /** Maximum minor version supported by this HFile format */ // We went to version 2 when we moved to pb'ing fileinfo and the trailer on @@ -825,22 +825,18 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { public Cell getKeyValue() { if (!isSeeked()) return null; - - if(currTagsLen > 0) { - KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset() + KeyValue ret; + if (currTagsLen > 0) { + ret = new ContentSizeCachedKeyValue(blockBuffer.array(), blockBuffer.arrayOffset() + blockBuffer.position(), getCellBufSize()); - if (this.reader.shouldIncludeMemstoreTS()) { - ret.setSequenceId(currMemstoreTS); - } - return ret; } else { - NoTagsKeyValue ret = new NoTagsKeyValue(blockBuffer.array(), - blockBuffer.arrayOffset() + blockBuffer.position(), getCellBufSize()); - if (this.reader.shouldIncludeMemstoreTS()) { - ret.setSequenceId(currMemstoreTS); - } - return ret; + ret = new ContentSizeCachedNoTagsKeyValue(blockBuffer.array(), blockBuffer.arrayOffset() + + blockBuffer.position(), getCellBufSize()); } + if (this.reader.shouldIncludeMemstoreTS()) { + ret.setSequenceId(currMemstoreTS); + } + return ret; } @Override @@ -1609,22 +1605,6 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { return true; // We load file info in constructor in version 2. } - /** - * Validates that the minor version is within acceptable limits. - * Otherwise throws an Runtime exception - */ - private void validateMinorVersion(Path path, int minorVersion) { - if (minorVersion < MIN_MINOR_VERSION || - minorVersion > MAX_MINOR_VERSION) { - String msg = "Minor version for path " + path + - " is expected to be between " + - MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION + - " but is found to be " + minorVersion; - LOG.error(msg); - throw new RuntimeException(msg); - } - } - @Override public HFileContext getFileContext() { return hfileContext;