diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index c9d18a64e6..abeb7fc9cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; @@ -809,13 +811,27 @@ private boolean isValidTxnListState() throws LockException { } Set nonSharedLocks = new HashSet<>(); for (HiveLock lock : ctx.getHiveLocks()) { - if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE || - lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) { - if (lock.getHiveLockObject().getPaths().length == 2) { - // Pos 0 of lock paths array contains dbname, pos 1 contains tblname - nonSharedLocks.add( - Warehouse.getQualifiedName( - lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1])); + if (lock.mayContainComponents()) { + // The lock may have multiple components, e.g., DbHiveLock, hence we need + // to check for each of them + for (LockComponent lckCmp : lock.getHiveLockComponents()) { + if (lckCmp.getType() == LockType.EXCLUSIVE || + lckCmp.getType() == LockType.SHARED_WRITE) { + nonSharedLocks.add( + Warehouse.getQualifiedName( + lckCmp.getDbname(), lckCmp.getTablename())); + } + } + } else { + // The lock has a single components, e.g., SimpleHiveLock or ZooKeeperHiveLock + if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE || + lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) { + if (lock.getHiveLockObject().getPaths().length == 2) { + // Pos 0 of lock paths array contains dbname, pos 1 contains tblname + nonSharedLocks.add( + Warehouse.getQualifiedName( + lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1])); + } } } } @@ -1938,6 +1954,7 @@ private void runInternal(String command, boolean alreadyCompiled) throws Command try { if (!isValidTxnListState()) { + LOG.info("Compiling after acquiring locks"); // Snapshot was outdated when locks were acquired, hence regenerate context, // txn list and retry // TODO: Lock acquisition should be moved before analyze, this is a bit hackish. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java index ab94475b64..1a042783b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.lockmgr; +import com.google.common.collect.ImmutableList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.DDLTask; import org.slf4j.Logger; @@ -114,7 +115,7 @@ LockState lock(LockRequest lock, String queryId, boolean isBlocking, List 0) { boolean logMsg = false; for(DbHiveLock l : locks) { @@ -310,14 +311,17 @@ public void refresh() { long lockId; String queryId; long txnId; + List components; DbHiveLock(long id) { lockId = id; } - DbHiveLock(long id, String queryId, long txnId) { + + DbHiveLock(long id, String queryId, long txnId, List components) { lockId = id; this.queryId = queryId; this.txnId = txnId; + this.components = ImmutableList.copyOf(components); } @Override @@ -330,6 +334,16 @@ public HiveLockMode getHiveLockMode() { throw new UnsupportedOperationException(); } + @Override + public boolean mayContainComponents() { + return true; + } + + @Override + public List getHiveLockComponents() { + return components; + } + @Override public boolean equals(Object other) { if (other instanceof DbHiveLock) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java index 5373137ad5..82392455b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java @@ -18,7 +18,33 @@ package org.apache.hadoop.hive.ql.lockmgr; +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.hive.metastore.api.LockComponent; + +import java.util.List; + public abstract class HiveLock { + public abstract HiveLockObject getHiveLockObject(); + public abstract HiveLockMode getHiveLockMode(); + + /** + * Returns true if for this lock implementation, a single lock can in turn + * lock multiple objects, e.g., multi-statement transaction. + */ + public boolean mayContainComponents() { + return false; + } + + /** + * Returns the lock components if a single lock can in turn + * lock multiple objects, e.g., multi-statement transaction. + * + * Returns an empty list if the lock does not have multiple + * components. + */ + public List getHiveLockComponents() { + return ImmutableList.of(); + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java index 43fc33c8bf..55f62dc3ac 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java @@ -163,7 +163,6 @@ public void testExistingOnly() throws Exception { msClient.close(); } - @Ignore @Test(timeout=80000) public void testQueueingWithThreads() throws Exception { final int PART_COUNT = 12;