diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 60ae06a49a..df6504e3cd 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2734,6 +2734,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_TXN_READ_LOCKS("hive.txn.read.locks", true, "Flag to turn off the read locks, when set to false. Although its not recommended, \n" + "but in performance critical scenarios this option may be exercised."), + HIVE_TXN_ENABLE_LOCKS("hive.txn.enable.locks", true, + "Flag to turn off all locks, when set to false. This is only intented to used in situations " + + "when a query is blocked on locks and it shouldn't be. A quick workaround can be to turn off locking " + + "entirely in the session until the problem is investigated and solved."), TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true, "Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire Exclusive locks for\n" + "transactional tables. This ensures that inserts (w/o overwrite) running concurrently\n" + diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index deaab89c1f..4bdf4883d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -405,6 +405,10 @@ LockState acquireLocks(QueryPlan plan, Context ctx, String username, boolean isB * mode of operation documented at {@link DbTxnManager#isExplicitTransaction}*/ return null; } + if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_ENABLE_LOCKS)) { + LOG.info("Locking is disabled in the session, not acquiring locks!"); + return null; + } LockRequestBuilder rqstBuilder = new LockRequestBuilder(queryId); //link queryId to txnId diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 2adabe7058..c2ac99ffd6 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -258,6 +258,30 @@ public void testBasicBlocking() throws Exception { Assert.assertEquals("Unexpected number of locks found", 0, locks.size()); } + /** + * Test for disabling the locks all together with {@link HiveConf.ConfVars.HIVE_TXN_ENABLE_LOCKS}. + * @throws Exception + */ + @Test + public void testBasicBlockingLocksDisabled() throws Exception { + dropTable(new String[] {"T6"}); + driver.run("create table if not exists T6(a int)"); + driver.compileAndRespond("select a from T6", true); + txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); //gets S lock on T6 + HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + swapTxnManager(txnMgr2); + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_ENABLE_LOCKS, false); + driver.run("drop table if exists T6"); + + List locks = getLocks(); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T6", null, locks); + txnMgr.rollbackTxn(); //release S on T6 + locks = getLocks(); + Assert.assertEquals("Unexpected number of locks found", 0, locks.size()); + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_ENABLE_LOCKS, true); + } + @Test public void testLockConflictDbTable() throws Exception { dropTable(new String[] {"temp.T7"}); @@ -724,6 +748,13 @@ private void testCheckExpectedLocks(boolean sharedWrite) throws Exception { */ @Test public void testCheckExpectedLocks2() throws Exception { + testCheckExpectedLocks2(true); + } + @Test + public void testCheckExpectedLocks2NoReadLock() throws Exception { + testCheckExpectedLocks2(false); + } + public void testCheckExpectedLocks2(boolean readLocks) throws Exception { dropTable(new String[] {"tab_acid", "tab_not_acid"}); driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); @@ -731,31 +762,44 @@ public void testCheckExpectedLocks2() throws Exception { "clustered by (na) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')"); driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')"); driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')"); + if (!readLocks) { + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_READ_LOCKS, false); + } txnMgr.openTxn(ctx, "T1"); driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List locks = getLocks(txnMgr); - Assert.assertEquals("Unexpected lock count", 6, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 6, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + } else { + Assert.assertEquals("Unexpected lock count", 0, locks.size()); + } HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr2.openTxn(ctx, "T2"); driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)", true); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false); locks = getLocks(txnMgr2); - Assert.assertEquals("Unexpected lock count", 7, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); - checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 7, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); + } else { + Assert.assertEquals("Unexpected lock count", 1, locks.size()); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + } + // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources. // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID semantics @@ -765,23 +809,39 @@ public void testCheckExpectedLocks2() throws Exception { driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true); ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false); locks = getLocks(txnMgr3); - Assert.assertEquals("Unexpected lock count", 8, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); - checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 8, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + } else { + Assert.assertEquals("Unexpected lock count", 2, locks.size()); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + } + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_READ_LOCKS, + HiveConf.ConfVars.HIVE_TXN_READ_LOCKS.defaultBoolVal); } /** * Check to make sure we acquire proper locks for queries involving non-strict locking */ @Test - public void testCheckExpectedReadLocksForNonAcidTables() throws Exception { + public void testCheckExpectedReadLocksForNonAcidTables() throws Exception{ + testCheckExpectedReadLocksForNonAcidTables(true); + } + @Test + public void testCheckExpectedReadLocksForNonAcidTablesNoReadLocks() throws Exception{ + testCheckExpectedReadLocksForNonAcidTables(false); + } + private void testCheckExpectedReadLocksForNonAcidTables(boolean readLocks) throws Exception { dropTable(new String[] {"tab_acid", "tab_not_acid"}); driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); @@ -792,41 +852,61 @@ public void testCheckExpectedReadLocksForNonAcidTables() throws Exception { // Test non-acid read-locking mode - the read locks are only obtained for the ACID side conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS, false); + if (!readLocks) { + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_READ_LOCKS, false); + } HiveTxnManager txnMgr1 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr1.openTxn(ctx, "T1"); driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na", true); txnMgr1.acquireLocks(driver.getPlan(), ctx, "T1"); List locks = getLocks(txnMgr1); - Assert.assertEquals("Unexpected lock count", 3, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 3, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + } else { + Assert.assertEquals("Unexpected lock count", 0, locks.size()); + } HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr2.openTxn(ctx, "T2"); driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)", true); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false); locks = getLocks(txnMgr2); - Assert.assertEquals("Unexpected lock count", 4, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); - checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 4, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + } else { + Assert.assertEquals("Unexpected lock count", 1, locks.size()); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + } + HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr3.openTxn(ctx, "T3"); driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true); ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false); locks = getLocks(txnMgr3); - Assert.assertEquals("Unexpected lock count", 5, locks.size()); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); - checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + if (readLocks) { + Assert.assertEquals("Unexpected lock count", 5, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + } else { + Assert.assertEquals("Unexpected lock count", 2, locks.size()); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + } conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS, HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS.defaultBoolVal); + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_READ_LOCKS, + HiveConf.ConfVars.HIVE_TXN_READ_LOCKS.defaultBoolVal); } @Test