diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 3c508ec6cf..8ec89c373b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -2819,7 +2819,7 @@ private static boolean isLockableTable(Table t) { List lockComponents = new ArrayList<>(); // For each source to read, get a shared lock for (ReadEntity input : inputs) { - if (!input.needsLock() || input.isUpdateOrDelete() || !AcidUtils.needsLock(input)) { + if (input.isDummy() || !input.needsLock() || input.isUpdateOrDelete() || !AcidUtils.needsLock(input)) { // We don't want to acquire read locks during update or delete as we'll be acquiring write // locks instead. Also, there's no need to lock temp tables since they're session wide continue; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index c033a94bfa..2d4683662b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -133,10 +133,9 @@ public void testMetadataOperationLocks() throws Exception { driver.compileAndRespond("insert into T values (1,2)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); //since LM is using non strict mode we get shared lock checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); //simulate concurrent session HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); @@ -144,9 +143,8 @@ public void testMetadataOperationLocks() throws Exception { driver.compileAndRespond("alter table T SET TBLPROPERTIES ('transactional'='true')", true); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 3, locks.size()); + Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T", null, locks); txnMgr2.rollbackTxn(); txnMgr.commitTxn(); @@ -616,33 +614,29 @@ public void checkExpectedLocks() throws Exception { driver.compileAndRespond("insert into nonAcidPart partition(p) values(1,2,3)", true); LockState lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false); List locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); txnMgr.rollbackTxn();; driver.compileAndRespond("insert into nonAcidPart partition(p=1) values(5,6)", true); lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "nonAcidPart", "p=1", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); txnMgr.rollbackTxn(); driver.compileAndRespond("insert into acidPart partition(p) values(1,2,3)", true); lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); txnMgr.rollbackTxn(); driver.compileAndRespond("insert into acidPart partition(p=1) values(5,6)", true); lockState = ((DbTxnManager) txnMgr).acquireLocks(driver.getPlan(), ctx, "Practical", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "acidPart", "p=1", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); txnMgr.rollbackTxn(); driver.compileAndRespond("update acidPart set b = 17 where a = 1", true); @@ -688,7 +682,7 @@ public void checkExpectedLocks2() throws Exception { driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)", true); LockState ls = ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false); locks = getLocks(txnMgr2); - Assert.assertEquals("Unexpected lock count", 8, locks.size()); + Assert.assertEquals("Unexpected lock count", 7, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); @@ -696,7 +690,6 @@ public void checkExpectedLocks2() throws Exception { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); - checkLock(LockType.SHARED_READ, LockState.WAITING, "_dummy_database", "_dummy_table", null, locks); // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources. // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID semantics @@ -706,7 +699,7 @@ public void checkExpectedLocks2() throws Exception { driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true); ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false); locks = getLocks(txnMgr3); - Assert.assertEquals("Unexpected lock count", 10, locks.size()); + Assert.assertEquals("Unexpected lock count", 8, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); @@ -714,9 +707,7 @@ public void checkExpectedLocks2() throws Exception { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks); - checkLock(LockType.SHARED_READ, LockState.WAITING, "_dummy_database", "_dummy_table", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true); } @@ -730,9 +721,8 @@ public void testLockingOnInsertIntoNonNativeTables() throws Exception { txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List locks = getLocks(txnMgr); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); } @Test @@ -744,9 +734,8 @@ public void testLockingOnInsertOverwriteNonNativeTables() throws Exception { driver.compileAndRespond("insert overwrite table tab_not_acid values(1,2)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List locks = getLocks(txnMgr); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); } /** The list is small, and the object is generated, so we don't use sets/equals/etc. */ @@ -823,13 +812,12 @@ public void testShowLocksFilterOptions() throws Exception { // SHOW LOCKS (no filter) List locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 7, locks.size()); + Assert.assertEquals("Unexpected lock count", 5, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "db1", "t14", "ds=today", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "db1", "t14", "ds=tomorrow", locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "db2", "t15", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "db2", "t16", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "db2", "t14", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); // SHOW LOCKS db2 locks = getLocksWithFilterOptions(txnMgr3, "db2", null, null); @@ -1864,9 +1852,8 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "target", null, locks); } else { - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); } DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); @@ -1880,16 +1867,12 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { txnMgr2.acquireLocks(driver.getPlan(), ctx, "T2", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", causeConflict ? 3 : 4, locks.size()); + Assert.assertEquals("Unexpected lock count", 3, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "target", null, locks); checkLock(LockType.SHARED_READ, causeConflict ? LockState.WAITING : LockState.ACQUIRED, "default", "source", null, locks); long extLockId = checkLock(LockType.SHARED_WRITE, causeConflict ? LockState.WAITING : LockState.ACQUIRED, "default", "target", null, locks).getLockid(); - if (!causeConflict) { - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); - } - txnMgr.commitTxn(); //commit T1 Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + @@ -1955,14 +1938,13 @@ public void testDynamicPartitionInsert() throws Exception { driver.compileAndRespond("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List locks = getLocks(txnMgr); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); //table is empty, so can only lock the table checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); Assert.assertEquals( "HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + TxnDbUtil.queryToString(conf, "select * from HIVE_LOCKS"), - 2, + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1)); txnMgr.rollbackTxn(); Assert.assertEquals( @@ -1988,10 +1970,9 @@ public void testDynamicPartitionInsert() throws Exception { driver.compileAndRespond("insert into target partition(p=1,q) values (10,2,2), (30,4,2), (50,6,3), (70,8,2)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); locks = getLocks(txnMgr); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); //Plan is using DummyPartition, so can only lock the table... unfortunately checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); long writeId = txnMgr.getTableWriteId("default", "target"); AddDynamicPartitions adp = new AddDynamicPartitions(txnid2, writeId, "default", "target", Arrays.asList("p=1/q=2","p=1/q=2")); adp.setOperationType(DataOperationType.INSERT); @@ -2167,9 +2148,8 @@ public void testShowTablesLock() throws Exception { driver.compileAndRespond("insert into T values(1,3)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr2.openTxn(ctx, "Fidler"); @@ -2177,9 +2157,8 @@ public void testShowTablesLock() throws Exception { driver.compileAndRespond("show tables", true); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Fidler"); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 3, locks.size()); + Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t", null, locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks); txnMgr.commitTxn(); txnMgr2.rollbackTxn(); @@ -2195,9 +2174,8 @@ public void testShowTablesLock() throws Exception { driver.compileAndRespond("insert into T2 partition(p=1) values(1,3)", true); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 2, locks.size()); + Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t2", "p=1", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr2.openTxn(ctx, "Fidler"); @@ -2205,9 +2183,8 @@ public void testShowTablesLock() throws Exception { driver.compileAndRespond("show tables", true); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Fidler", false); locks = getLocks(); - Assert.assertEquals("Unexpected lock count", 3, locks.size()); + Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "t2", "p=1", locks); - checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks); txnMgr.commitTxn(); txnMgr2.commitTxn();