diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java index a8b9653411..8806d58202 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java @@ -99,7 +99,9 @@ LockState lock(LockRequest lock, String queryId, boolean isBlocking, List>> jumpTable; + private static Map> jumpTableFilters; + private void checkQFileTestHack() { boolean hackOn = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) || MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST); @@ -4420,6 +4411,7 @@ private LockResponse checkLock(Connection dbConn, long extLockId) // table locks for this db. boolean sawNull = false; strings.clear(); + Set partitions = new HashSet<>(strings); for (LockInfo info : locksBeingChecked) { if (info.table == null) { sawNull = true; @@ -4449,24 +4441,29 @@ private LockResponse checkLock(Connection dbConn, long extLockId) sawNull = true; break; } else { - strings.add(info.partition); - } - } - if (!sawNull) { - query.append(" AND (\"HL_PARTITION\" IS NULL OR \"HL_PARTITION\" IN("); - first = true; - for (String s : strings) { - if (first) first = false; - else query.append(", "); - query.append('\''); - query.append(s); - query.append('\''); + partitions.add(info.partition); } - query.append("))"); } } query.append(" AND \"HL_LOCK_EXT_ID\" < ").append(extLockId); + // Add additional filters + Set filters = new HashSet<>(); + locksBeingChecked.forEach(lockInfo -> filters.addAll(jumpTableFilters.get(lockInfo.type))); + + if (!filters.isEmpty()) { + query.append(" AND \"HL_LOCK_TYPE\" IN ("); + first = true; + for (LockType filter : filters) { + if (first) + first = false; + else + query.append(", "); + query.append('\'').append(getLockTypeChar(filter)).append('\''); + } + query.append(")"); + } + LOG.debug("Going to execute query <" + query.toString() + ">"); stmt = dbConn.createStatement(); rs = stmt.executeQuery(query.toString()); @@ -4474,6 +4471,12 @@ private LockResponse checkLock(Connection dbConn, long extLockId) while (rs.next()) { lockSet.add(new LockInfo(rs)); } + + if (!lockSet.isEmpty() && !sawNull) { + // filter out locks, whose partition name does not match. + lockSet.removeIf(info -> (info.partition != null && !partitions.contains(info.partition))); + } + // Turn the tree set into an array so we can move back and forth easily // in it. LockInfo[] locks = lockSet.toArray(new LockInfo[lockSet.size()]); @@ -4522,7 +4525,10 @@ private LockResponse checkLock(Connection dbConn, long extLockId) // We've found something that matches what we're trying to lock, // so figure out if we can lock it too. LockAction lockAction = jumpTable.get(info.type).get(locks[i].type).get(locks[i].state); - LOG.debug("desired Lock: " + info + " checked Lock: " + locks[i] + " action: " + lockAction); + if (LOG.isDebugEnabled()) { + LOG.debug("desired Lock: " + info + " checked Lock: " + locks[i] + + " action: " + lockAction); + } switch (lockAction) { case WAIT: if(!ignoreConflict(info, locks[i])) { @@ -4571,6 +4577,16 @@ private LockResponse checkLock(Connection dbConn, long extLockId) } return response; } + + private char getLockTypeChar(LockType lockType) { + switch (lockType) { + case EXCLUSIVE: return LOCK_EXCLUSIVE; + case SHARED_READ: return LOCK_SHARED; + case SHARED_WRITE: return LOCK_SEMI_SHARED; + default: return 'z'; + } + } + private void acquire(Connection dbConn, Statement stmt, List locksBeingChecked) throws SQLException, NoSuchLockException, MetaException { if(locksBeingChecked == null || locksBeingChecked.isEmpty()) { @@ -5241,7 +5257,22 @@ private static synchronized void buildJumpTable() { // No matter whether it has acquired or not, we cannot pass an exclusive. m2.put(LockState.ACQUIRED, LockAction.WAIT); m2.put(LockState.WAITING, LockAction.WAIT); + + // build filter conditions for locks + // SR=[E], SW=[E, SW], E=[SR, E, SW] + jumpTableFilters = new HashMap<>(3); + jumpTable.forEach((extLockType, val) -> { + Set filterSet = jumpTableFilters.computeIfAbsent(extLockType, v -> new HashSet<>()); + val.forEach((lockType, lockStateMap) -> { + lockStateMap.forEach((lockState, action) -> { + if (action == LockAction.WAIT) { + filterSet.add(lockType); + } + }); + }); + }); } + /** * Returns true if {@code ex} should be retried */