diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java index 89087e1450..e401b21545 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java @@ -208,6 +208,10 @@ protected final long findReadSlot( protected long[] slotTriples; private void allocateBucketArray() { + // We allocate triples, so we cannot go above highest Integer power of 2 / 6. + if (logicalHashBucketCount > ONE_SIXTH_LIMIT) { + throwExpandError(ONE_SIXTH_LIMIT, "Bytes"); + } int slotTripleArraySize = 3 * logicalHashBucketCount; slotTriples = new long[slotTripleArraySize]; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java index f610653640..ad9af6ecbd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java @@ -259,6 +259,10 @@ protected long findReadSlot(long key, long hashCode) { protected long[] slotPairs; private void allocateBucketArray() { + // We allocate pairs, so we cannot go above highest Integer power of 2 / 4. + if (logicalHashBucketCount > ONE_QUARTER_LIMIT) { + throwExpandError(ONE_QUARTER_LIMIT, "Long"); + } int slotPairArraySize = 2 * logicalHashBucketCount; slotPairs = new long[slotPairArraySize]; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java index 905180431f..e816101ce3 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastBytesHashMap.java @@ -24,6 +24,7 @@ import java.util.Random; import org.apache.hadoop.hive.ql.exec.JoinUtil; +import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.CheckFastHashTable.VerifyFastBytesHashMap; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -298,6 +299,21 @@ public void testReallyBig() throws Exception { addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } + @Test + public void testOutOfBounds() throws Exception { + random = new Random(42662); + + int HIGHEST_INT_POWER_OF_2 = 1073741824; + // The c'tor should throw the error + try { + VectorMapJoinFastMultiKeyHashMap map = + new VectorMapJoinFastMultiKeyHashMap( + false, HIGHEST_INT_POWER_OF_2, LOAD_FACTOR, MODERATE_WB_SIZE, -1); + } catch (MapJoinMemoryExhaustionError e) { + System.out.println("Caught MapJoinMemoryExhaustionError =" + e.getMessage()); + } + } + /* // Can't seem to get mvn to give enough memory to run this successfully. @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java index 315a54dde5..ea6614ba26 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastLongHashMap.java @@ -22,6 +22,7 @@ import java.util.Random; import org.apache.hadoop.hive.ql.exec.JoinUtil; +import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.CheckFastHashTable.VerifyFastLongHashMap; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast.VectorMapJoinFastLongHashMap; @@ -275,6 +276,21 @@ public void testLargeAndExpand() throws Exception { addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } + @Test + public void testOutOfBounds() throws Exception { + random = new Random(42662); + + int HIGHEST_INT_POWER_OF_2 = 1073741824; + // The c'tor should throw the error + try { + VectorMapJoinFastMultiKeyHashMap map = + new VectorMapJoinFastMultiKeyHashMap( + false, HIGHEST_INT_POWER_OF_2, LOAD_FACTOR, MODERATE_WB_SIZE, -1); + } catch (MapJoinMemoryExhaustionError e) { + System.out.println("Caught MapJoinMemoryExhaustionError =" + e.getMessage()); + } + } + /* // Doesn't finish in a reasonable amount of time.... @Test