Index: ql/src/test/results/clientpositive/lock1.q.out =================================================================== --- ql/src/test/results/clientpositive/lock1.q.out (revision 1042920) +++ ql/src/test/results/clientpositive/lock1.q.out (working copy) @@ -56,7 +56,7 @@ POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] default@tstsrc SHARED -QUERYID_LOCK:heyongqiang_20101203190404_a00af404-b094-4552-8ecc-0004930a878a TIME : 1291431894235 +LOCK_QUERYID:njain_20101206231818_eec284bc-9a38-4ba5-b59e-7bc770db8bd2 LOCK_TIME:1291706323232 LOCK_MODE:EXPLICIT PREHOOK: query: UNLOCK TABLE tstsrc PREHOOK: type: UNLOCKTABLE POSTHOOK: query: UNLOCK TABLE tstsrc @@ -101,7 +101,7 @@ POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] default@tstsrc SHARED -QUERYID_LOCK:heyongqiang_20101203190404_feadd884-14ab-4305-8078-605128a33c42 TIME : 1291431894956 +LOCK_QUERYID:njain_20101206231818_8eee6b8f-abba-41f5-8e70-e6c216d488cc LOCK_TIME:1291706324145 LOCK_MODE:EXPLICIT PREHOOK: query: SHOW LOCKS tstsrc PREHOOK: type: SHOWLOCKS POSTHOOK: query: SHOW LOCKS tstsrc @@ -130,9 +130,9 @@ POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] default@tstsrc SHARED -QUERYID_LOCK:heyongqiang_20101203190404_53022be2-f951-4b33-895a-d373a5791d16 TIME : 1291431895381 +LOCK_QUERYID:njain_20101206231818_d29191d9-ed78-4e6d-80df-2fc47d70adef LOCK_TIME:1291706324583 LOCK_MODE:EXPLICIT default@tstsrc SHARED -QUERYID_LOCK:heyongqiang_20101203190404_feadd884-14ab-4305-8078-605128a33c42 TIME : 1291431894956 +LOCK_QUERYID:njain_20101206231818_8eee6b8f-abba-41f5-8e70-e6c216d488cc LOCK_TIME:1291706324145 LOCK_MODE:EXPLICIT PREHOOK: query: SHOW LOCKS tstsrc PREHOOK: type: SHOWLOCKS POSTHOOK: query: SHOW LOCKS tstsrc Index: ql/src/test/results/clientpositive/lock2.q.out =================================================================== --- ql/src/test/results/clientpositive/lock2.q.out (revision 1042920) +++ ql/src/test/results/clientpositive/lock2.q.out (working copy) @@ -106,7 +106,7 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] default@tstsrcpart@ds=2008-04-08/hr=11 EXCLUSIVE -QUERYID_LOCK:njain_20101116183333_f625d313-c8ac-4ecf-87f8-ef6459651b9d TIME : 1289961212197 +LOCK_QUERYID:njain_20101206171111_df7566b7-88ed-482a-827f-6cc3fac55d63 LOCK_TIME:1291684308414 LOCK_MODE:EXPLICIT PREHOOK: query: UNLOCK TABLE tstsrc PREHOOK: type: UNLOCKTABLE POSTHOOK: query: UNLOCK TABLE tstsrc Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1042920) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -897,7 +897,7 @@ "-I", "at java", "-I", "at junit", "-I", "Caused by:", - "-I", "QUERYID_LOCK:", + "-I", "LOCK_QUERYID:", "-I", "[.][.][.] [0-9]* more", (new File(logDir, tname + ".out")).getPath(), outFileName }; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1042920) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -74,6 +74,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; import org.apache.hadoop.hive.ql.metadata.CheckResult; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -1347,11 +1348,12 @@ List locks = null; if (showLocks.getTableName() == null) { - locks = lockMgr.getLocks(isExt); + locks = lockMgr.getLocks(false, isExt); } else { locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(), - showLocks.getPartSpec()), isExt); + showLocks.getPartSpec()), + true, isExt); } Collections.sort(locks, new Comparator() { @@ -1383,15 +1385,11 @@ outStream.writeBytes(lock.getHiveLockMode().toString()); if (isExt) { outStream.write(terminator); - String lockData = lock.getHiveLockObject().getData(); + HiveLockObjectData lockData = lock.getHiveLockObject().getData(); if (lockData != null) { - String[] lockDataArr = lockData.split(":"); - if (lockDataArr.length == 1) { - outStream.writeBytes("QUERYID_LOCK:" + lockData); - } - else { - outStream.writeBytes("QUERYID_LOCK:" + lockDataArr[0] + " TIME : " + Long.parseLong(lockDataArr[1])); - } + outStream.writeBytes("LOCK_QUERYID:" + lockData.getQueryId() + " "); + outStream.writeBytes("LOCK_TIME:" + lockData.getLockTime() + " "); + outStream.writeBytes("LOCK_MODE:" + lockData.getLockMode() + " "); } } outStream.write(terminator); @@ -1433,7 +1431,10 @@ } Map partSpec = lockTbl.getPartSpec(); - String lockData = lockTbl.getQueryId() + ":" + String.valueOf(System.currentTimeMillis()); + HiveLockObjectData lockData = + new HiveLockObjectData(lockTbl.getQueryId(), + String.valueOf(System.currentTimeMillis()), + "EXPLICIT"); if (partSpec == null) { HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true, 0, 0); @@ -1495,7 +1496,7 @@ String tabName = unlockTbl.getTableName(); HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec()); - List locks = lockMgr.getLocks(obj, false); + List locks = lockMgr.getLocks(obj, false, false); if ((locks == null) || (locks.isEmpty())) { throw new HiveException("Table " + tabName + " is not locked "); } Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockManager.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockManager.java (revision 1042920) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockManager.java (working copy) @@ -28,14 +28,14 @@ * @param key object to be locked * @param mode mode of the lock (SHARED/EXCLUSIVE) * @param keepAlive if the lock needs to be persisted after the statement - * @param sleepTime - * @param numRetries + * @param sleepTime + * @param numRetries */ public HiveLock lock(HiveLockObject key, HiveLockMode mode, boolean keepAlive, int numRetries, int sleepTime) throws LockException; public void unlock(HiveLock hiveLock) throws LockException; - public List getLocks(boolean verifyTablePartitions) throws LockException; - public List getLocks(HiveLockObject key, boolean fetchData) throws LockException; + public List getLocks(boolean verifyTablePartitions, boolean fetchData) throws LockException; + public List getLocks(HiveLockObject key, boolean verifyTablePartitions, boolean fetchData) throws LockException; public void close() throws LockException; } Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (revision 1042920) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (working copy) @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockManagerCtx; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.conf.HiveConf; @@ -62,7 +63,7 @@ // All the locks are created under this parent private String parent; - + private int sessionTimeout; private String quorumServers; @@ -132,7 +133,7 @@ * @param keepAlive Whether the lock is to be persisted after the statement * @param numRetries number of retries when the lock can not be acquired * @param sleepTime sleep time between retries - * + * * Acuire the lock. Return null if a conflicting lock is present. **/ public ZooKeeperHiveLock lock(HiveLockObject key, HiveLockMode mode, @@ -147,10 +148,10 @@ String msg = null; try { if (keepAlive) { - res = zooKeeper.create(name, key.getData().getBytes(), + res = zooKeeper.create(name, key.getData().toString().getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); } else { - res = zooKeeper.create(name, key.getData().getBytes(), + res = zooKeeper.create(name, key.getData().toString().getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); } } catch (Exception e) { @@ -160,9 +161,9 @@ if (res != null) { break; } - + renewZookeeperInstance(sessionTimeout, quorumServers); - + if (tryNum == numRetries) { console.printError("Lock for " + key.getName() + " cannot be acquired in " + mode); @@ -170,7 +171,7 @@ } tryNum++; - + console.printInfo("Lock for " + key.getName() + " cannot be acquired in " + mode +", will retry again later..., more info: " + msg); @@ -259,13 +260,15 @@ } /* Get all locks */ - public List getLocks(boolean verifyTablePartition) throws LockException { - return getLocks(ctx.getConf(), zooKeeper, null, parent, verifyTablePartition, false); + public List getLocks(boolean verifyTablePartition, boolean fetchData) + throws LockException { + return getLocks(ctx.getConf(), zooKeeper, null, parent, verifyTablePartition, fetchData); } /* Get all locks for a particular object */ - public List getLocks(HiveLockObject key, boolean fetchData) throws LockException { - return getLocks(ctx.getConf(), zooKeeper, key, parent, false, fetchData); + public List getLocks(HiveLockObject key, boolean verifyTablePartitions, + boolean fetchData) throws LockException { + return getLocks(ctx.getConf(), zooKeeper, key, parent, verifyTablePartitions, fetchData); } /** @@ -285,27 +288,27 @@ // no locks present return locks; } - + for (String child : children) { child = "/" + parent + "/" + child; - HiveLockMode mode = getLockMode(conf, child); if (mode == null) { continue; } - - String data = "NULL"; + + HiveLockObjectData data = null; //set the lock object with a dummy data, and then do a set if needed. HiveLockObject obj = getLockObject(conf, child, mode, data, verifyTablePartition); if (obj == null) { continue; } - + if ((key == null) || (obj.getName().equals(key.getName()))) { + if (fetchData) { try { - data = new String(zkpClient.getData(child, new DummyWatcher(), null)); + data = new HiveLockObjectData(new String(zkpClient.getData(child, new DummyWatcher(), null))); } catch (Exception e) { LOG.error("Error in getting data for " + child + " " + e); // ignore error @@ -350,21 +353,21 @@ * The object may correspond to a table, a partition or a parent to a partition. * For eg: if Table T is partitioned by ds, hr and ds=1/hr=1 is a valid partition, * the lock may also correspond to T@ds=1, which is not a valid object - * @param verifyTablePartition + * @param verifyTablePartition **/ private static HiveLockObject getLockObject(HiveConf conf, String path, - HiveLockMode mode, String data, boolean verifyTablePartition) + HiveLockMode mode, HiveLockObjectData data, boolean verifyTablePartition) throws LockException { try { Hive db = Hive.get(conf); int indx = path.lastIndexOf(mode.toString()); String objName = path.substring(1, indx-1); String[] names = objName.split("/")[1].split("@"); - + if (names.length < 2) { return null; } - + if (!verifyTablePartition) { if (names.length == 2) { return new HiveLockObject(names, data); @@ -374,7 +377,7 @@ "/").split("@"), data); } } - + Table tab = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, //need to change to names[0] names[1], false); // do not throw exception if table does not exist if (tab == null) { Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java (revision 1042920) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java (working copy) @@ -22,26 +22,69 @@ import org.apache.hadoop.hive.ql.metadata.Table; public class HiveLockObject { - String [] pathNames = null; + public static class HiveLockObjectData { + + private String queryId; // queryId of the command + private String lockTime; // time at which lock was acquired + // mode of the lock: EXPLICIT(lock command)/IMPLICIT(query) + private String lockMode; + + public HiveLockObjectData(String queryId, + String lockTime, + String lockMode) { + this.queryId = queryId; + this.lockTime = lockTime; + this.lockMode = lockMode; + } + + + public HiveLockObjectData(String data) { + if (data == null) { + return; + } + + String[] elem = data.split(":"); + queryId = elem[0]; + lockTime = elem[1]; + lockMode = elem[2]; + } + + public String getQueryId() { + return queryId; + } + + public String getLockTime() { + return lockTime; + } + + public String getLockMode() { + return lockMode; + } + + public String toString() { + return queryId + ":" + lockTime + ":" + lockMode; + } + } + /* user supplied data for that object */ - private String data; + private HiveLockObjectData data; public HiveLockObject() { this.data = null; } - - public HiveLockObject(String[] paths, String lockData) { + + public HiveLockObject(String[] paths, HiveLockObjectData lockData) { this.pathNames = paths; this.data = lockData; } - - public HiveLockObject(Table tbl, String lockData) { + + public HiveLockObject(Table tbl, HiveLockObjectData lockData) { this(new String[] {tbl.getDbName(), tbl.getTableName()}, lockData); } - public HiveLockObject(Partition par, String lockData) { + public HiveLockObject(Partition par, HiveLockObjectData lockData) { this(new String[] { par.getTable().getDbName(), par.getTable().getTableName(), par.getName() }, lockData); } @@ -63,11 +106,11 @@ return ret; } - public String getData() { + public HiveLockObjectData getData() { return data; } - public void setData(String data) { + public void setData(HiveLockObjectData data) { this.data = data; } Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 1042920) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -67,6 +67,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockManagerCtx; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -459,13 +460,18 @@ throws SemanticException { List locks = new LinkedList(); + HiveLockObjectData lockData = + new HiveLockObjectData(plan.getQueryId(), + String.valueOf(System.currentTimeMillis()), + "IMPLICIT"); + if (t != null) { - locks.add(new LockObject(new HiveLockObject(t, plan.getQueryId()), mode)); + locks.add(new LockObject(new HiveLockObject(t, lockData), mode)); return locks; } if (p != null) { - locks.add(new LockObject(new HiveLockObject(p, plan.getQueryId()), mode)); + locks.add(new LockObject(new HiveLockObject(p, lockData), mode)); // All the parents are locked in shared mode mode = HiveLockMode.SHARED; @@ -485,14 +491,14 @@ try { locks.add(new LockObject(new HiveLockObject(new DummyPartition(p.getTable(), p.getTable() .getDbName() - + "@" + p.getTable().getTableName() + "@" + partialName), plan.getQueryId()), mode)); + + "@" + p.getTable().getTableName() + "@" + partialName), lockData), mode)); partialName += "/"; } catch (HiveException e) { throw new SemanticException(e.getMessage()); } } - locks.add(new LockObject(new HiveLockObject(p.getTable(), plan.getQueryId()), mode)); + locks.add(new LockObject(new HiveLockObject(p.getTable(), lockData), mode)); } return locks; } @@ -600,7 +606,7 @@ * object is not locked twice, and the list passed is sorted such that EXCLUSIVE locks * occur before SHARED locks. * @param sleepTime - * @param numRetries + * @param numRetries **/ private List acquireLocks(List lockObjects, LockObjectContainer notFound, int numRetries, int sleepTime)