diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 1ed9b9c..ce9a15a 100755 --- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.Map.Entry; import org.apache.commons.lang.StringUtils; @@ -80,6 +81,8 @@ private static final String CLASS_NAME = HiveInputFormat.class.getName(); private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); + private static final ReentrantLock tableDescLock = new ReentrantLock(); + /** * A cache of InputFormat instances. */ @@ -422,7 +425,6 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job for (Path dir : dirs) { PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, dir); Class inputFormatClass = part.getInputFileFormatClass(); - TableDesc table = part.getTableDesc(); TableScanOperator tableScan = null; List aliases = mrwork.getPathToAliases().get(dir.toString()); @@ -448,12 +450,19 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job } } - if (!currentDirs.isEmpty() && - inputFormatClass.equals(currentInputFormatClass) && - table.equals(currentTable) && - tableScan == currentTableScan) { - currentDirs.add(dir); - continue; + // HIVE-11749: Avoid race condition for getting table desc resource + TableDesc table = part.getTableDesc(); + tableDescLock.lock(); + try { + if (!currentDirs.isEmpty() && + inputFormatClass.equals(currentInputFormatClass) && + table.equals(currentTable) && + tableScan == currentTableScan) { + currentDirs.add(dir); + continue; + } + } finally { + tableDescLock.unlock(); } if (!currentDirs.isEmpty()) {