diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java index 5552dfb..14ecd25 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java @@ -177,6 +177,14 @@ public BytesBytesMultiHashMap getHashMapFromDisk(int rowCount) SerializationUtilities.releaseKryo(kryo); } + // In case the restored hashmap is empty (that can happen when it was spilled), + // we will instantiate a new BytesBytesMultiHashMap object to avoid having an empty refs array. + // Because this scenario is equivalent to the case of "spilled on creation". + if (restoredHashMap.size() == 0) { + LOG.warn("Restored an empty hash partition."); + restoredHashMap = new BytesBytesMultiHashMap(rowCount, loadFactor, wbSize, -1); + } + if (rowCount > 0) { restoredHashMap.expandAndRehashToTarget(rowCount); } @@ -385,6 +393,11 @@ private HybridHashTableContainer(float keyCountAdj, int threshold, float loadFac memoryUsed += hashPartitions[i].hashMap.memorySize(); } } + + if (writeBufferSize * (numPartitions - numPartitionsSpilledOnCreation) > memoryThreshold) { + LOG.error("There is not enough memory to allocate " + + (numPartitions - numPartitionsSpilledOnCreation) + " hash partitions."); + } assert numPartitionsSpilledOnCreation != numPartitions : "All partitions are directly spilled!" + " It is not supported now."; LOG.info("Number of partitions created: " + numPartitions); @@ -585,6 +598,10 @@ private int biggestPartition() { public long spillPartition(int partitionId) throws IOException { HashPartition partition = hashPartitions[partitionId]; int inMemRowCount = partition.hashMap.getNumValues(); + if (inMemRowCount == 0) { + LOG.warn("Trying to spill an empty hash partition! It may be due to " + + "hive.auto.convert.join.noconditionaltask.size being set too low."); + } File file = FileUtils.createLocalDirsTempFile( spillLocalDirs, "partition-" + partitionId + "-", null, false);