diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java index d878f65..6242daf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java @@ -99,8 +99,8 @@ public void assignSlot(int slot, byte[] keyBytes, int keyStart, int keyLength, } public VectorMapJoinFastBytesHashMap( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); valueStore = new VectorMapJoinFastValueStore(writeBuffersSize); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java index b328efd..1a41961 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java @@ -92,8 +92,8 @@ public void assignSlot(int slot, byte[] keyBytes, int keyStart, int keyLength, } public VectorMapJoinFastBytesHashMultiSet( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java index c9b23bf..331867c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java @@ -79,8 +79,8 @@ public void assignSlot(int slot, byte[] keyBytes, int keyStart, int keyLength, } public VectorMapJoinFastBytesHashSet( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java index dc0476b..4b372ae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java @@ -105,6 +105,10 @@ public void add(byte[] keyBytes, int keyStart, int keyLength, BytesWritable curr private void expandAndRehash() { + // We allocate triples, so we cannot go above Integer.MAX_VALUE / 6. + if (logicalHashBucketCount > ONE_SIXTH_LIMIT) { + throwExpandError(ONE_QUARTER_LIMIT, "Bytes"); + } int newLogicalHashBucketCount = logicalHashBucketCount * 2; int newLogicalHashBucketMask = newLogicalHashBucketCount - 1; int newMetricPutConflict = 0; @@ -210,8 +214,8 @@ private void allocateBucketArray() { } public VectorMapJoinFastBytesHashTable( - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); allocateBucketArray(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java index 262b619..c32964b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java @@ -32,7 +32,7 @@ public VectorMapJoinHashMapResult createHashMapResult() { public VectorMapJoinFastHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java index 5f7c6a7..2b4a2b8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java @@ -42,7 +42,7 @@ public void set(long count) { public VectorMapJoinFastHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java index 8509971..9339250 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java @@ -38,7 +38,7 @@ public VectorMapJoinHashSetResult createHashSetResult() { public VectorMapJoinFastHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java index 7df9eed..2eb1521 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java @@ -31,12 +31,25 @@ protected float loadFactor; protected final int writeBuffersSize; + protected long estimatedKeyCount; + protected int metricPutConflict; protected int largestNumberOfSteps; protected int keysAssigned; protected int resizeThreshold; protected int metricExpands; + public static int ONE_QUARTER_LIMIT = (Integer.MAX_VALUE / 4) + 1; + public static int ONE_SIXTH_LIMIT = (Integer.MAX_VALUE / 6) + 1; + + public void throwExpandError(int limit, String dataTypeName) { + throw new RuntimeException( + "Vector MapJoin " + dataTypeName + " Hash Table cannot grow any more. " + + "Current logical size is " + logicalHashBucketCount + ". " + + "Doubling limit is " + limit + ". " + + "Estimated key count was " + (estimatedKeyCount == -1 ? "not available" : estimatedKeyCount) + "."); + } + private static void validateCapacity(long capacity) { if (Long.bitCount(capacity) != 1) { throw new AssertionError("Capacity must be a power of two"); @@ -51,13 +64,15 @@ private static int nextHighestPowerOfTwo(int v) { } public VectorMapJoinFastHashTable( - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { initialCapacity = (Long.bitCount(initialCapacity) == 1) ? initialCapacity : nextHighestPowerOfTwo(initialCapacity); validateCapacity(initialCapacity); + this.estimatedKeyCount = estimatedKeyCount; + logicalHashBucketCount = initialCapacity; logicalHashBucketMask = logicalHashBucketCount - 1; resizeThreshold = (int)(logicalHashBucketCount * loadFactor); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java index cd51d0d..6fe98f9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java @@ -107,9 +107,9 @@ public void assignSlot(int slot, long key, boolean isNewKey, BytesWritable curre public VectorMapJoinFastLongHashMap( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); valueStore = new VectorMapJoinFastValueStore(writeBuffersSize); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java index 032233a..9140aee 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java @@ -96,8 +96,8 @@ public void assignSlot(int slot, long key, boolean isNewKey, BytesWritable curre public VectorMapJoinFastLongHashMultiSet( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java index 21701d4..d3efb11 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java @@ -92,8 +92,8 @@ public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { public VectorMapJoinFastLongHashSet( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { super(minMaxEnabled, isOuterJoin, hashTableKeyType, - initialCapacity, loadFactor, writeBuffersSize); + initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java index bc892ba..a12c36e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java @@ -156,6 +156,10 @@ public void add(long key, BytesWritable currentValue) { private void expandAndRehash() { + // We allocate pairs, so we cannot go above Integer.MAX_VALUE / 4. + if (logicalHashBucketCount > ONE_QUARTER_LIMIT) { + throwExpandError(ONE_QUARTER_LIMIT, "Long"); + } int newLogicalHashBucketCount = logicalHashBucketCount * 2; int newLogicalHashBucketMask = newLogicalHashBucketCount - 1; int newMetricPutConflict = 0; @@ -262,8 +266,8 @@ private void allocateBucketArray() { public VectorMapJoinFastLongHashTable( boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); this.isOuterJoin = isOuterJoin; this.hashTableKeyType = hashTableKeyType; PrimitiveTypeInfo[] primitiveTypeInfos = { hashTableKeyType.getPrimitiveTypeInfo() }; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java index cee3b3b..add4788 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java @@ -50,7 +50,7 @@ public void testPutRow(byte[] currentKey, byte[] currentValue) throws HiveExcept public VectorMapJoinFastMultiKeyHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java index ff82ac4..faefdbb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java @@ -48,8 +48,8 @@ public void testPutRow(byte[] currentKey) throws HiveException, IOException { public VectorMapJoinFastMultiKeyHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java index de0666d..5328910 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java @@ -48,8 +48,8 @@ public void testPutRow(byte[] currentKey) throws HiveException, IOException { public VectorMapJoinFastMultiKeyHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java index 35af1d1..f13034f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java @@ -39,8 +39,8 @@ public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws public VectorMapJoinFastStringHashMap( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java index 36120b7..53ad7b4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java @@ -39,8 +39,8 @@ public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws public VectorMapJoinFastStringHashMultiSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java index 2ed6ab3..723c729 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java @@ -39,8 +39,8 @@ public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws public VectorMapJoinFastStringHashSet( boolean isOuterJoin, - int initialCapacity, float loadFactor, int writeBuffersSize) { - super(initialCapacity, loadFactor, writeBuffersSize); + int initialCapacity, float loadFactor, int writeBuffersSize, long estimatedKeyCount) { + super(initialCapacity, loadFactor, writeBuffersSize, estimatedKeyCount); stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java index 069cc9a..05f1cf1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java @@ -55,13 +55,14 @@ private final int threshold; private final float loadFactor; private final int wbSize; - private final long keyCount; + + private final long estimatedKeyCount; private final VectorMapJoinFastHashTable vectorMapJoinFastHashTable; public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, - long keyCount) throws SerDeException { + long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; @@ -71,7 +72,7 @@ public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); - this.keyCount = keyCount; + this.estimatedKeyCount = estimatedKeyCount; // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj); // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold); @@ -79,7 +80,7 @@ public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize); int newThreshold = HashMapWrapper.calculateTableSize( - keyCountAdj, threshold, loadFactor, keyCount); + keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); @@ -114,17 +115,17 @@ private VectorMapJoinFastHashTable createHashTable(int newThreshold) { case HASH_MAP: hashTable = new VectorMapJoinFastLongHashMap( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastLongHashMultiSet( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastLongHashSet( minMaxEnabled, isOuterJoin, hashTableKeyType, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; @@ -134,17 +135,17 @@ private VectorMapJoinFastHashTable createHashTable(int newThreshold) { case HASH_MAP: hashTable = new VectorMapJoinFastStringHashMap( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastStringHashMultiSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastStringHashSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; @@ -154,17 +155,17 @@ private VectorMapJoinFastHashTable createHashTable(int newThreshold) { case HASH_MAP: hashTable = new VectorMapJoinFastMultiKeyHashMap( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_MULTISET: hashTable = new VectorMapJoinFastMultiKeyHashMultiSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; case HASH_SET: hashTable = new VectorMapJoinFastMultiKeyHashSet( isOuterJoin, - newThreshold, loadFactor, writeBufferSize); + newThreshold, loadFactor, writeBufferSize, estimatedKeyCount); break; } break; diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java index 8525e99..c9c767f 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java @@ -42,7 +42,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -76,7 +76,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -103,7 +103,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -141,7 +141,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashMap map = - new VectorMapJoinFastMultiKeyHashMap(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMap(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -185,7 +185,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashMap map = - new VectorMapJoinFastMultiKeyHashMap(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMap(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -247,7 +247,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -262,7 +262,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); @@ -277,7 +277,7 @@ public void testReallyBig() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( - false,LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java index 449a8b2..de30b31 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMultiSet.java @@ -37,7 +37,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -65,7 +65,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -91,7 +91,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -126,7 +126,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashMultiSet map = - new VectorMapJoinFastMultiKeyHashMultiSet(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMultiSet(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -168,7 +168,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashMultiSet map = - new VectorMapJoinFastMultiKeyHashMultiSet(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashMultiSet(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -228,7 +228,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); @@ -243,7 +243,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java index ef7c91c..969c7d1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashSet.java @@ -37,7 +37,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -65,7 +65,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -91,7 +91,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,CAPACITY, LOAD_FACTOR, WB_SIZE); + false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -125,7 +125,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastMultiKeyHashSet map = - new VectorMapJoinFastMultiKeyHashSet(false,CAPACITY, 1f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashSet(false,CAPACITY, 1f, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -167,7 +167,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashSet map = - new VectorMapJoinFastMultiKeyHashSet(false,1, 0.0000001f, WB_SIZE); + new VectorMapJoinFastMultiKeyHashSet(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -227,7 +227,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); @@ -242,7 +242,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashSet map = new VectorMapJoinFastMultiKeyHashSet( - false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashSet verifyTable = new VerifyFastBytesHashSet(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java index e8bbee3..8659766 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java @@ -39,7 +39,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -72,7 +72,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -102,7 +102,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -138,7 +138,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -181,7 +181,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -241,7 +241,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); @@ -256,7 +256,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashMap verifyTable = new VerifyFastLongHashMap(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java index 9e94611..beae727 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMultiSet.java @@ -39,7 +39,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -66,7 +66,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -94,7 +94,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -128,7 +128,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -169,7 +169,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -227,7 +227,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); @@ -242,7 +242,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java index 698bcdc..2e77abf 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashSet.java @@ -39,7 +39,7 @@ public void testOneKey() throws Exception { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -66,7 +66,7 @@ public void testMultipleKeysSingleValue() throws Exception { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -94,7 +94,7 @@ public void testGetNonExistent() throws Exception { VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -126,7 +126,7 @@ public void testFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE); + false, false, HashTableKeyType.LONG, CAPACITY, 1f, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -167,7 +167,7 @@ public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE); + false, false, HashTableKeyType.LONG, 1, 0.0000001f, WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -225,7 +225,7 @@ public void testMultipleKeysMultipleValue() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + false, false, HashTableKeyType.LONG, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); @@ -240,7 +240,7 @@ public void testLargeAndExpand() throws Exception { // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastLongHashSet map = new VectorMapJoinFastLongHashSet( - false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE); + false, false, HashTableKeyType.LONG, MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastLongHashSet verifyTable = new VerifyFastLongHashSet(); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java index 3f02eb3..ebb243e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java @@ -147,7 +147,7 @@ public void testBigIntRows() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -171,7 +171,7 @@ public void testIntRows() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -195,7 +195,7 @@ public void testStringRows() throws Exception { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -219,7 +219,7 @@ public void testMultiKeyRows1() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -243,7 +243,7 @@ public void testMultiKeyRows2() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -267,7 +267,7 @@ public void testMultiKeyRows3() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -291,7 +291,7 @@ public void testBigIntRowsClipped() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -315,7 +315,7 @@ public void testIntRowsClipped() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -339,7 +339,7 @@ public void testStringRowsClipped() throws Exception { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -363,7 +363,7 @@ public void testMultiKeyRowsClipped1() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -387,7 +387,7 @@ public void testMultiKeyRowsClipped2() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -411,7 +411,7 @@ public void testMultiKeyRowsClipped3() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -436,7 +436,7 @@ public void testBigIntRowsExact() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -460,7 +460,7 @@ public void testIntRowsExact() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -484,7 +484,7 @@ public void testStringRowsExact() throws Exception { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -508,7 +508,7 @@ public void testMultiKeyRowsExact1() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -532,7 +532,7 @@ public void testMultiKeyRowsExact2() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -556,7 +556,7 @@ public void testMultiKeyRowsExact3() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -580,7 +580,7 @@ public void testBigIntRowsClippedExact() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.LONG, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -604,7 +604,7 @@ public void testIntRowsClippedExact() throws Exception { VectorMapJoinFastLongHashMap map = new VectorMapJoinFastLongHashMap( false, false, HashTableKeyType.INT, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -628,7 +628,7 @@ public void testStringRowsClippedExact() throws Exception { VectorMapJoinFastStringHashMap map = new VectorMapJoinFastStringHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -652,7 +652,7 @@ public void testMultiKeyRowsClippedExact1() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -676,7 +676,7 @@ public void testMultiKeyRowsClippedExact2() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); @@ -700,7 +700,7 @@ public void testMultiKeyRowsClippedExact3() throws Exception { VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, - LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE); + LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap();