Index: /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java =================================================================== --- /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java (revision 786666) +++ /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java (working copy) @@ -29,9 +29,12 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.io.hfile.HFile.BlockIndex; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.RawComparator; /** @@ -244,4 +247,31 @@ writer.append("1".getBytes(), "0".getBytes()); writer.close(); } + + /** + * Checks if the HeapSize calculator is within reason + */ + public void testHeapSizeForBlockIndex() { + ClassSize cs = null; + Class cl = null; + long expected = 0L; + long actual = 0L; + try { + cs = new ClassSize(); + } catch(Exception e) {} + + //KeyValue + cl = BlockIndex.class; + expected = cs.estimateBase(cl, false); + BlockIndex bi = new BlockIndex(Bytes.BYTES_RAWCOMPARATOR); + actual = bi.heapSize(); + //Since we have a [[]] in BlockIndex and the checker only sees the [] we + // miss a MULTI_ARRAY which is 4*Reference = 32 B + actual -= 32; + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + } + } \ No newline at end of file Index: /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java =================================================================== --- /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java (revision 0) +++ /home/erik/workspace/hbase/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java (revision 0) @@ -0,0 +1,72 @@ +package org.apache.hadoop.hbase.io; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.io.hfile.LruBlockCache; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; + +import junit.framework.TestCase; + +/** + * Testing the sizing that HeapSize offers and compares to the size given by + * ClassSize. + */ +public class TestHeapSize extends TestCase { + static final Log LOG = LogFactory.getLog(TestHeapSize.class); + // List of classes implementing HeapSize + // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry, HStoreKey + // KeyValue, LruBlockCache, LruHashMap, Put, HLogKey + + /** + * Testing the classes that implements HeapSize and are a part of 0.20. + * Some are not tested here for example BlockIndex which is tested in + * TestHFile since it is a non public class + */ + public void testSizes() { + ClassSize cs = null; + Class cl = null; + long expected = 0L; + long actual = 0L; + try { + cs = new ClassSize(); + } catch(Exception e) {} + + //KeyValue + cl = KeyValue.class; + expected = cs.estimateBase(cl, false); + KeyValue kv = new KeyValue(); + actual = kv.heapSize(); + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //LruBlockCache + cl = LruBlockCache.class; + expected = cs.estimateBase(cl, false); + LruBlockCache c = new LruBlockCache(1,1,200); + //Since minimum size for the for a LruBlockCache is 1 + //we need to remove one reference from the heapsize + actual = c.heapSize() - HeapSize.REFERENCE; + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + + //Put + cl = Put.class; + expected = cs.estimateBase(cl, false); + //The actual TreeMap is not included in the above calculation + expected += HeapSize.TREEMAP_SIZE; + Put put = new Put(Bytes.toBytes("")); + actual = put.heapSize(); + if(expected != actual) { + cs.estimateBase(cl, true); + assertEquals(expected, actual); + } + } + +} Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java (working copy) @@ -43,7 +43,7 @@ // Time at which this edit was written. private long writeTime; private int HEAP_TAX = HeapSize.OBJECT + (2 * HeapSize.BYTE_ARRAY) + - (2 * HeapSize.LONG); + (2 * Bytes.SIZEOF_LONG); /** Writable Consructor -- Do not use. */ public HLogKey() { Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (working copy) @@ -33,7 +33,6 @@ * A query matcher that is specifically designed for the scan case. */ public class ScanQueryMatcher extends QueryMatcher { - private Filter filter; // have to support old style filter for now. private RowFilterInterface oldFilter; Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/regionserver/LruHashMap.java (working copy) @@ -19,11 +19,6 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.hbase.io.HeapSize; - import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -31,6 +26,15 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.util.Bytes; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + + + + /** * The LruHashMap is a memory-aware HashMap with a configurable maximum * memory footprint. @@ -62,8 +66,9 @@ private static final float DEFAULT_LOAD_FACTOR = 0.75f; /** Memory overhead of this Object (for HeapSize) */ - private static final int OVERHEAD = 5 * HeapSize.LONG + 2 * HeapSize.INT + - 2 * HeapSize.FLOAT + 3 * HeapSize.REFERENCE + 1 * HeapSize.ARRAY; + private static final int OVERHEAD = 5 * Bytes.SIZEOF_LONG + + 2 * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_FLOAT + 3 * HeapSize.REFERENCE + + 1 * HeapSize.ARRAY; /** Load factor allowed (usually 75%) */ private final float loadFactor; @@ -922,8 +927,8 @@ protected static class Entry implements Map.Entry, HeapSize { /** The baseline overhead memory usage of this class */ - static final int OVERHEAD = 1 * HeapSize.LONG + 5 * HeapSize.REFERENCE + - 2 * HeapSize.INT; + static final int OVERHEAD = 1 * Bytes.SIZEOF_LONG + 5 * HeapSize.REFERENCE + + 2 * Bytes.SIZEOF_INT; /** The key */ protected final K key; Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/KeyValue.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; @@ -1783,9 +1784,8 @@ // HeapSize public long heapSize() { - int dataLen = bytes.length + (bytes.length % 8); - return HeapSize.OBJECT + HeapSize.BYTE_ARRAY + dataLen + - (2 * HeapSize.INT); + return ClassSize.alignSize(HeapSize.OBJECT + HeapSize.REFERENCE + + HeapSize.BYTE_ARRAY + length + (2 * Bytes.SIZEOF_INT)); } // Writable Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (working copy) @@ -24,6 +24,8 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; import java.util.ArrayList; import java.util.Collection; @@ -63,10 +65,6 @@ /** The default load factor to use */ public static final float DEFAULT_LOAD_FACTOR = 0.75f; - /** Memory overhead of this Object (for HeapSize) */ - private static final int OVERHEAD = 5 * HeapSize.LONG + 2 * HeapSize.INT + - 2 * HeapSize.FLOAT + 3 * HeapSize.REFERENCE + 1 * HeapSize.ARRAY; - /** Load factor allowed (usually 75%) */ private final float loadFactor; /** Number of key/vals in the map */ @@ -91,6 +89,11 @@ /** Number of unsuccessful (not found) get() calls */ private long missCount = 0; + /** Memory overhead of this Object (for HeapSize) */ + private static final int OVERHEAD = (int)ClassSize.alignSize(HeapSize.OBJECT + + 1 * Bytes.SIZEOF_FLOAT + 2 * Bytes.SIZEOF_INT + 1 * HeapSize.ARRAY + + 3 * HeapSize.REFERENCE + 4 * Bytes.SIZEOF_LONG); + /** * Constructs a new, empty map with the specified initial capacity, * load factor, and maximum memory usage. @@ -266,8 +269,7 @@ * @return hit ratio (double between 0 and 1) */ public double getHitRatio() { - return (double)((double)hitCount/ - ((double)(hitCount+missCount))); + return ((double)hitCount) / ((double)(hitCount+missCount)); } /** @@ -955,10 +957,6 @@ */ protected static class Entry implements Map.Entry, HeapSize { - /** The baseline overhead memory usage of this class */ - static final int OVERHEAD = 1 * HeapSize.LONG + 5 * HeapSize.REFERENCE + - 2 * HeapSize.INT; - /** The key */ protected final String key; /** The value */ @@ -976,6 +974,10 @@ /** The precomputed heap size of this entry */ protected long heapSize; + /** The baseline overhead memory usage of this class */ + static final int OVERHEAD = HeapSize.OBJECT + 5 * HeapSize.REFERENCE + + 1 * Bytes.SIZEOF_INT + 1 * Bytes.SIZEOF_LONG; + /** * Create a new entry. * @@ -1137,7 +1139,8 @@ * @return size of String in bytes */ private long heapSize(String s) { - return HeapSize.OBJECT + alignSize(s.length()*2); + return HeapSize.STRING_SIZE + + ClassSize.alignSize(s.length() * Bytes.SIZEOF_CHAR); } /** @@ -1145,18 +1148,9 @@ * @return size of ByteBuffer in bytes */ private long heapSize(ByteBuffer b) { - return HeapSize.BYTE_BUFFER + alignSize(b.capacity()); + return HeapSize.BYTE_BUFFER + ClassSize.alignSize(b.capacity()); } - /** - * Aligns a number to 8. - * @param num number to align to 8 - * @return smallest number >= input that is a multiple of 8 - */ - private long alignSize(long num) { - if(num % 8 == 0) return num; - return (num + (8 - (num % 8))); - } } } Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java (working copy) @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; @@ -1333,11 +1334,15 @@ long [] blockOffsets; int [] blockDataSizes; int size = 0; - + /* Needed doing lookup on blocks. */ final RawComparator comparator; + static final int OVERHEAD = (int)ClassSize.alignSize(HeapSize.OBJECT + + 2 * Bytes.SIZEOF_INT + 1 * HeapSize.MULTI_ARRAY + 2 * HeapSize.ARRAY + + 4 * HeapSize.REFERENCE); + /* * Shutdown default constructor */ @@ -1493,8 +1498,25 @@ } public long heapSize() { - return this.size; + long size = OVERHEAD; + + //Calculating the size of blockKeys + if(blockKeys != null) { + for(byte [] bs : blockKeys) { + size += HeapSize.MULTI_ARRAY; + size += ClassSize.alignSize(bs.length); + } + } + if(blockOffsets != null) { + size += blockOffsets.length * Bytes.SIZEOF_LONG; + } + if(blockDataSizes != null) { + size += blockDataSizes.length * Bytes.SIZEOF_INT; + } + + return size; } + } /* Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/HeapSize.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/HeapSize.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/io/HeapSize.java (working copy) @@ -45,36 +45,35 @@ /** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */ static final int OBJECT = 2 * REFERENCE; - /** - * The following types are always allocated in blocks of 8 bytes (on 64bit) - * For example, if you have two ints in a class, it will use 8 bytes. - * If you have three ints in a class, it will use 16 bytes. - */ - static final int SHORT = 4; - static final int INT = 4; - static final int FLOAT = 4; - static final int BOOLEAN = 4; - static final int CHAR = 4; - static final int BYTE = 1; - - /** These types are always 8 bytes */ - static final int DOUBLE = 8; - static final int LONG = 8; - /** Array overhead */ static final int ARRAY = 3 * REFERENCE; + + /** OverHead for nested arrays */ static final int MULTI_ARRAY = (4 * REFERENCE) + ARRAY; /** Byte arrays are fixed size below plus its length, 8 byte aligned */ static final int BYTE_ARRAY = 3 * REFERENCE; - static final int BLOCK_SIZE_TAX = 8; - + /** Overhead for ByteBuffer */ static final int BYTE_BUFFER = 56; - + + /** String overhead */ + static final int STRING_SIZE = 64; + + /** Overhead for ArrayList(0) */ + static final int ARRAYLIST_SIZE = 64; + + /** Overhead for TreeMap */ + static final int TREEMAP_SIZE = 80; + + /** Overhead for entry in map */ + static final int MAP_ENTRY_SIZE = 64; + + /** * @return Approximate 'exclusive deep size' of implementing object. Includes * count of payload and hosting object sizings. */ public long heapSize(); + } Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/ClassSize.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/ClassSize.java (revision 0) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/ClassSize.java (revision 0) @@ -0,0 +1,195 @@ +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.io.HeapSize; + +/** + * Class for determining the "size" of a class, an attempt to calculate the + * actual bytes that an object of this class will occupy in memory + * + * The core of this class is taken from the Derby project + */ +public class ClassSize { + static final Log LOG = LogFactory.getLog(ClassSize.class); + + private int refSize; +// private static final int objectOverhead = 2; // references, not bytes! + private int minObjectSize; + + /** + * Constructor + * @throws Exception + */ + public ClassSize() throws Exception{ + // Figure out whether this is a 32 or 64 bit machine. + Runtime runtime = Runtime.getRuntime(); + int loops = 10; + int sz = 0; + for(int i = 0; i < loops; i++) { + cleaner(runtime, i); + long memBase = runtime.totalMemory() - runtime.freeMemory(); + Object[] junk = new Object[10000]; + cleaner(runtime, i); + long memUsed = runtime.totalMemory() - runtime.freeMemory() - memBase; + sz = (int)((memUsed + junk.length/2)/junk.length); + if(sz > 0 ) { + break; + } + } + + refSize = ( 4 > sz) ? 4 : sz; + minObjectSize = 4*refSize; + } + + /** + * The estimate of the size of a class instance depends on whether the JVM + * uses 32 or 64 bit addresses, that is it depends on the size of an object + * reference. It is a linear function of the size of a reference, e.g. + * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes). + * + * This method returns the coefficients of the linear function, e.g. {24, 5} + * in the above example. + * + * @param cl A class whose instance size is to be estimated + * @return an array of 3 integers. The first integer is the size of the + * primitives, the second the number of arrays and the third the number of + * references. + */ + private int [] getSizeCoefficients(Class cl, boolean debug) { + int primitives = 0; + int arrays = 0; + int references = HeapSize.OBJECT / HeapSize.REFERENCE; + + for( ; null != cl; cl = cl.getSuperclass()) { + Field[] field = cl.getDeclaredFields(); + if( null != field) { + for( int i = 0; i < field.length; i++) { + if( ! Modifier.isStatic( field[i].getModifiers())) { + Class fieldClass = field[i].getType(); + if( fieldClass.isArray()){ + arrays++; + } + else if(! fieldClass.isPrimitive()){ + references++; + } + else {// Is simple primitive + String name = fieldClass.getName(); + + if(name.equals("int") || name.equals( "I")) + primitives += Bytes.SIZEOF_INT; + else if(name.equals("long") || name.equals( "J")) + primitives += Bytes.SIZEOF_LONG; + else if(name.equals("boolean") || name.equals( "Z")) + primitives += Bytes.SIZEOF_BOOLEAN; + else if(name.equals("short") || name.equals( "S")) + primitives += Bytes.SIZEOF_SHORT; + else if(name.equals("byte") || name.equals( "B")) + primitives += Bytes.SIZEOF_BYTE; + else if(name.equals("char") || name.equals( "C")) + primitives += Bytes.SIZEOF_CHAR; + else if(name.equals("float") || name.equals( "F")) + primitives += Bytes.SIZEOF_FLOAT; + else if(name.equals("double") || name.equals( "D")) + primitives += Bytes.SIZEOF_DOUBLE; + } + if(debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug(field[i].getName()+ "\n\t" +field[i].getType()); + } + } + } + } + } + } + return new int [] {primitives, arrays, references}; + } + + /** + * Estimate the static space taken up by a class instance given the + * coefficients returned by getSizeCoefficients. + * + * @param coeff the coefficients + * + * @return the size estimate, in bytes + */ + private long estimateBaseFromCoefficients(int [] coeff, boolean debug) { + int size = coeff[0] + (coeff[1]*4 + coeff[2])*refSize; + + // Round up to a multiple of 8 + size = (int)alignSize(size); + if(debug) { + if (LOG.isDebugEnabled()) { + // Write out region name as string and its encoded name. + LOG.debug("Primitives " + coeff[0] + ", arrays " + coeff[1] + + ", references(inlcuding " + HeapSize.OBJECT + + ", for object overhead) " + coeff[2] + ", refSize " + refSize + + ", size " + size); + } + } + return (size < minObjectSize) ? minObjectSize : size; + } + + /** + * Estimate the static space taken up by the fields of a class. This includes + * the space taken up by by references (the pointer) but not by the referenced + * object. So the estimated size of an array field does not depend on the size + * of the array. Similarly the size of an object (reference) field does not + * depend on the object. + * + * @return the size estimate in bytes. + */ + public long estimateBase(Class cl, boolean debug) { + return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug); + } + + /** + * Tries to clear all the memory used to estimate the reference size for the + * current JVM + * @param runtime + * @param i + * @throws Exception + */ + private void cleaner(Runtime runtime, int i) throws Exception{ + Thread.sleep(i*1000); + runtime.gc();runtime.gc(); runtime.gc();runtime.gc();runtime.gc(); + runtime.runFinalization(); + } + + + /** + * Aligns a number to 8. + * @param num number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public static long alignSize(int num) { + int aligned = (num + 7)/8; + aligned *= 8; + return aligned; + } + +} Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/Bytes.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/Bytes.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/util/Bytes.java (working copy) @@ -41,34 +41,45 @@ public class Bytes { /** - * Size of long in bytes + * Size of boolean in bytes */ - public static final int SIZEOF_LONG = Long.SIZE/Byte.SIZE; - + public static final int SIZEOF_BOOLEAN = Byte.SIZE/Byte.SIZE; + /** - * Size of int in bytes + * Size of byte in bytes */ - public static final int SIZEOF_INT = Integer.SIZE/Byte.SIZE; + public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; /** - * Size of short in bytes + * Size of char in bytes */ - public static final int SIZEOF_SHORT = Short.SIZE/Byte.SIZE; - + public static final int SIZEOF_CHAR = Character.SIZE/Byte.SIZE; + /** + * Size of double in bytes + */ + public static final int SIZEOF_DOUBLE = Double.SIZE/Byte.SIZE; + + /** * Size of float in bytes */ public static final int SIZEOF_FLOAT = Float.SIZE/Byte.SIZE; - + /** - * Size of double in bytes + * Size of int in bytes */ - public static final int SIZEOF_DOUBLE = Double.SIZE/Byte.SIZE; + public static final int SIZEOF_INT = Integer.SIZE/Byte.SIZE; + + /** + * Size of long in bytes + */ + public static final int SIZEOF_LONG = Long.SIZE/Byte.SIZE; /** - * Size of byte in bytes + * Size of short in bytes */ - public static final int SIZEOF_BYTE = 1; + public static final int SIZEOF_SHORT = Short.SIZE/Byte.SIZE; + /** * Estimate of size cost to pay beyond payload in jvm for instance of byte []. @@ -265,11 +276,11 @@ return result; } - public static String toStringBinary(final byte []b) { + public static String toStringBinary(final byte [] b) { return toStringBinary(b, 0, b.length); } - public static String toStringBinary(final byte []b, int off, int len) { + public static String toStringBinary(final byte [] b, int off, int len) { String result = null; try { String first = new String(b, off, len, "ISO-8859-1"); @@ -1126,4 +1137,5 @@ } return value; } + } Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Put.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Put.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Put.java (working copy) @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; /** @@ -47,10 +48,15 @@ private long timestamp = HConstants.LATEST_TIMESTAMP; private long lockId = -1L; private boolean writeToWAL = true; + private Map> familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); - /** Constructor for Writable. DO NOT USE */ + private static final long OVERHEAD = ClassSize.alignSize(HeapSize.OBJECT + + 1 * HeapSize.REFERENCE + 1 * HeapSize.ARRAY + 2 * Bytes.SIZEOF_LONG + + 1 * Bytes.SIZEOF_BOOLEAN + 1 * HeapSize.REFERENCE + HeapSize.TREEMAP_SIZE); + + /** Constructor for Writable. DO NOT USE */ public Put() {} /** @@ -124,7 +130,7 @@ public void add(byte [] family, byte [] qualifier, long timestamp, byte [] value) { List list = familyMap.get(family); if(list == null) { - list = new ArrayList(); + list = new ArrayList(0); } KeyValue kv = new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.Put, value); @@ -263,13 +269,29 @@ //HeapSize public long heapSize() { - long totalSize = 0; - for(Map.Entry> entry : this.familyMap.entrySet()) { - for(KeyValue kv : entry.getValue()) { - totalSize += kv.heapSize(); - } - } - return totalSize; + long heapsize = OVERHEAD; + heapsize += ClassSize.alignSize(this.row.length); + + + for(Map.Entry> entry : this.familyMap.entrySet()) { + //Adding entry overhead + heapsize += HeapSize.MAP_ENTRY_SIZE; + + //Adding key overhead + heapsize += HeapSize.REFERENCE + HeapSize.ARRAY + + ClassSize.alignSize(entry.getKey().length); + + //This part is kinds tricky since the JVM can reuse references if you + //store the same value, but have a good match with SizeOf at the moment + //Adding value overhead + heapsize += HeapSize.REFERENCE + HeapSize.ARRAYLIST_SIZE; + int size = entry.getValue().size(); + heapsize += size * HeapSize.REFERENCE; + for(KeyValue kv : entry.getValue()) { + heapsize += kv.heapSize(); + } + } + return heapsize; } //Writable Index: /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Scan.java =================================================================== --- /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Scan.java (revision 786666) +++ /home/erik/workspace/hbase/src/java/org/apache/hadoop/hbase/client/Scan.java (working copy) @@ -85,7 +85,6 @@ public Scan(byte [] startRow, Filter filter) { this(startRow); this.filter = filter; - } /** @@ -118,7 +117,6 @@ public Scan addFamily(byte [] family) { familyMap.remove(family); familyMap.put(family, null); - return this; } @@ -204,7 +202,7 @@ * Get all available versions. */ public Scan setMaxVersions() { - this.maxVersions = Integer.MAX_VALUE; + this.maxVersions = Integer.MAX_VALUE; return this; } @@ -236,7 +234,6 @@ */ public Scan setOldFilter(RowFilterInterface filter) { oldFilter = filter; - return this; } @@ -246,7 +243,6 @@ */ public Scan setFamilyMap(Map> familyMap) { this.familyMap = familyMap; - return this; }