diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java index 5552dfb..53917cf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java @@ -177,6 +177,13 @@ public BytesBytesMultiHashMap getHashMapFromDisk(int rowCount) SerializationUtilities.releaseKryo(kryo); } + // In case the restored hashmap is empty (that can happen when it was spilled), + // we will instantiate a new BytesBytesMultiHashMap object to avoid having an empty refs array. + // Because this scenario is equivalent to the case of "spilled on creation". + if (restoredHashMap.size() == 0) { + restoredHashMap = new BytesBytesMultiHashMap(rowCount, loadFactor, wbSize, -1); + } + if (rowCount > 0) { restoredHashMap.expandAndRehashToTarget(rowCount); } @@ -585,6 +592,10 @@ private int biggestPartition() { public long spillPartition(int partitionId) throws IOException { HashPartition partition = hashPartitions[partitionId]; int inMemRowCount = partition.hashMap.getNumValues(); + if (inMemRowCount == 0) { + LOG.warn("Trying to spill an empty hash partition! It may be due to " + + "hive.auto.convert.join.noconditionaltask.size being set too low."); + } File file = FileUtils.createLocalDirsTempFile( spillLocalDirs, "partition-" + partitionId + "-", null, false);