diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java index 1cc3074..e75a90a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java @@ -70,6 +70,9 @@ public HiveLockObjectData(String data) { lockTime = elem[1]; lockMode = elem[2]; queryStr = elem[3]; + if (elem.length >= 5) { + clientIp = elem[4]; + } } public String getQueryId() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java index 65b9136..1cfd525 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; @@ -296,45 +297,77 @@ private String getLockName(String parent, HiveLockMode mode) { private ZooKeeperHiveLock lock (HiveLockObject key, HiveLockMode mode, boolean keepAlive, boolean parentCreated) throws LockException { - int tryNum = 1; + int tryNum = 0; ZooKeeperHiveLock ret = null; + Set conflictingLocks = new HashSet(); do { + tryNum++; try { if (tryNum > 1) { Thread.sleep(sleepTime); prepareRetry(); } - ret = lockPrimitive(key, mode, keepAlive, parentCreated); + ret = lockPrimitive(key, mode, keepAlive, parentCreated, conflictingLocks); if (ret != null) { break; } - tryNum++; } catch (Exception e1) { - tryNum++; if (e1 instanceof KeeperException) { KeeperException e = (KeeperException) e1; switch (e.code()) { case CONNECTIONLOSS: case OPERATIONTIMEOUT: - LOG.warn("Possibly transient ZooKeeper exception: ", e); - break; + LOG.debug("Possibly transient ZooKeeper exception: ", e); + continue; default: LOG.error("Serious Zookeeper exception: ", e); break; } } if (tryNum >= numRetriesForLock) { + console.printError("Unable to acquire " + key.getData().getLockMode() + + ", " + mode + " lock " + key.getDisplayName() + " after " + + tryNum + " attempts."); + LOG.error("Exceeds maximum retries with errors: ", e1); + printConflictingLocks(key,mode,conflictingLocks); + conflictingLocks.clear(); throw new LockException(e1); } } } while (tryNum < numRetriesForLock); + if (ret == null) { + console.printError("Unable to acquire " + key.getData().getLockMode() + + ", " + mode + " lock " + key.getDisplayName() + " after " + + tryNum + " attempts."); + printConflictingLocks(key,mode,conflictingLocks); + } + conflictingLocks.clear(); return ret; } + private void printConflictingLocks(HiveLockObject key, HiveLockMode mode, + Set conflictingLocks) { + if (!conflictingLocks.isEmpty()) { + HiveLockObjectData requestedLock = new HiveLockObjectData(key.getData().toString()); + LOG.debug("Requested lock " + key.getDisplayName() + + ":: mode:" + requestedLock.getLockMode() + "," + mode + + "; query:" + requestedLock.getQueryStr()); + for (String conflictingLock : conflictingLocks) { + HiveLockObjectData conflictingLockData = new HiveLockObjectData(conflictingLock); + LOG.debug("Conflicting lock to " + key.getDisplayName() + + ":: mode:" + conflictingLockData.getLockMode() + + ";query:" + conflictingLockData.getQueryStr() + + ";queryId:" + conflictingLockData.getQueryId() + + ";clientIp:" + conflictingLockData.getClientIp()); + } + } + } + private ZooKeeperHiveLock lockPrimitive(HiveLockObject key, - HiveLockMode mode, boolean keepAlive, boolean parentCreated) + HiveLockMode mode, boolean keepAlive, boolean parentCreated, + Set conflictingLocks) throws KeeperException, InterruptedException { String res; @@ -394,9 +427,19 @@ private ZooKeeperHiveLock lockPrimitive(HiveLockObject key, } if ((childSeq >= 0) && (childSeq < seqNo)) { - zooKeeper.delete(res, -1); - console.printError("conflicting lock present for " - + key.getDisplayName() + " mode " + mode); + try{ + zooKeeper.delete(res, -1); + } finally { + if (LOG.isDebugEnabled()) { + Stat stat = new Stat(); + try { + String data = new String(zooKeeper.getData(child, false, stat)); + conflictingLocks.add(data); + } catch (Exception e) { + //ignored, + } + } + } return null; } } diff --git a/ql/src/test/results/clientnegative/insert_into1.q.out b/ql/src/test/results/clientnegative/insert_into1.q.out index a38b679..4bfbc98 100644 --- a/ql/src/test/results/clientnegative/insert_into1.q.out +++ b/ql/src/test/results/clientnegative/insert_into1.q.out @@ -13,8 +13,5 @@ PREHOOK: query: LOCK TABLE insert_into1_neg SHARED PREHOOK: type: LOCKTABLE POSTHOOK: query: LOCK TABLE insert_into1_neg SHARED POSTHOOK: type: LOCKTABLE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE +Unable to acquire IMPLICIT, EXCLUSIVE lock default@insert_into1_neg after 5 attempts. FAILED: Error in acquiring locks: Locks on the underlying objects cannot be acquired. retry after some time diff --git a/ql/src/test/results/clientnegative/insert_into2.q.out b/ql/src/test/results/clientnegative/insert_into2.q.out index f21823a..845d041 100644 --- a/ql/src/test/results/clientnegative/insert_into2.q.out +++ b/ql/src/test/results/clientnegative/insert_into2.q.out @@ -13,8 +13,5 @@ PREHOOK: query: LOCK TABLE insert_into1_neg EXCLUSIVE PREHOOK: type: LOCKTABLE POSTHOOK: query: LOCK TABLE insert_into1_neg EXCLUSIVE POSTHOOK: type: LOCKTABLE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE -conflicting lock present for default@insert_into1_neg mode EXCLUSIVE +Unable to acquire IMPLICIT, EXCLUSIVE lock default@insert_into1_neg after 5 attempts. FAILED: Error in acquiring locks: Locks on the underlying objects cannot be acquired. retry after some time diff --git a/ql/src/test/results/clientnegative/insert_into3.q.out b/ql/src/test/results/clientnegative/insert_into3.q.out index 5f1581e..baa6d5b 100644 --- a/ql/src/test/results/clientnegative/insert_into3.q.out +++ b/ql/src/test/results/clientnegative/insert_into3.q.out @@ -27,8 +27,5 @@ PREHOOK: query: LOCK TABLE insert_into3_neg PARTITION (ds='1') SHARED PREHOOK: type: LOCKTABLE POSTHOOK: query: LOCK TABLE insert_into3_neg PARTITION (ds='1') SHARED POSTHOOK: type: LOCKTABLE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE +Unable to acquire IMPLICIT, EXCLUSIVE lock default@insert_into3_neg@ds=1 after 5 attempts. FAILED: Error in acquiring locks: Locks on the underlying objects cannot be acquired. retry after some time diff --git a/ql/src/test/results/clientnegative/insert_into4.q.out b/ql/src/test/results/clientnegative/insert_into4.q.out index 5dcdd50..4e7c6fc 100644 --- a/ql/src/test/results/clientnegative/insert_into4.q.out +++ b/ql/src/test/results/clientnegative/insert_into4.q.out @@ -27,8 +27,5 @@ PREHOOK: query: LOCK TABLE insert_into3_neg PARTITION (ds='1') EXCLUSIVE PREHOOK: type: LOCKTABLE POSTHOOK: query: LOCK TABLE insert_into3_neg PARTITION (ds='1') EXCLUSIVE POSTHOOK: type: LOCKTABLE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE -conflicting lock present for default@insert_into3_neg@ds=1 mode EXCLUSIVE +Unable to acquire IMPLICIT, EXCLUSIVE lock default@insert_into3_neg@ds=1 after 5 attempts. FAILED: Error in acquiring locks: Locks on the underlying objects cannot be acquired. retry after some time diff --git a/ql/src/test/results/clientnegative/lockneg1.q.out b/ql/src/test/results/clientnegative/lockneg1.q.out index 6a76cd7..c8f514d 100644 --- a/ql/src/test/results/clientnegative/lockneg1.q.out +++ b/ql/src/test/results/clientnegative/lockneg1.q.out @@ -29,5 +29,5 @@ POSTHOOK: query: LOCK TABLE tstsrc SHARED POSTHOOK: type: LOCKTABLE PREHOOK: query: LOCK TABLE tstsrc EXCLUSIVE PREHOOK: type: LOCKTABLE -conflicting lock present for default@tstsrc mode EXCLUSIVE +Unable to acquire EXPLICIT, EXCLUSIVE lock default@tstsrc after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out index a9833a8..92d7447 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out @@ -8,5 +8,5 @@ POSTHOOK: query: lock database lockneg4 exclusive POSTHOOK: type: LOCKDATABASE PREHOOK: query: lock database lockneg4 shared PREHOOK: type: LOCKDATABASE -conflicting lock present for lockneg4 mode SHARED +Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out index d67365a..2fcff1b 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out @@ -10,5 +10,5 @@ PREHOOK: query: show locks database lockneg9 PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks database lockneg9 POSTHOOK: type: SHOWLOCKS -conflicting lock present for lockneg9 mode EXCLUSIVE +Unable to acquire IMPLICIT, EXCLUSIVE lock lockneg9 after 1 attempts. FAILED: Error in acquiring locks: Locks on the underlying objects cannot be acquired. retry after some time diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out index 89d3265..0242338 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out @@ -38,5 +38,5 @@ POSTHOOK: type: SHOWLOCKS lockneg2@tstsrcpart SHARED PREHOOK: query: lock database lockneg2 exclusive PREHOOK: type: LOCKDATABASE -conflicting lock present for lockneg2 mode EXCLUSIVE +Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask