diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b213609..dbc4fd2 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2561,6 +2561,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock, which\n" + "allows two concurrent writes to the same partition but still lets lock manager prevent\n" + "DROP TABLE etc. when the table is being written to"), + HIVE_TXN_STRICT_READ_LOCKS("hive.txn.strict.read.locks", true, "Read locks cannot protect\n" + + "non-ACID readers against concurrent inserts to the table. And the locks need to be explicitly\n" + + "unlocked partitition by partition, instead of being transactionally scoped.\n" + + "Allow these to be skipped in clusters where read locking is enabled globally to support ACID,\n." + + "but are being used for non-ACID tables unintentionally."), TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true, "Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire Exclusive locks for\n" + "transactional tables. This ensures that inserts (w/o overwrite) running concurrently\n" + diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index badcc55..09935ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -2338,6 +2338,7 @@ private static boolean isLockableTable(Table t) { public static List makeLockComponents(Set outputs, Set inputs, HiveConf conf) { List lockComponents = new ArrayList<>(); + final boolean strictReadLocks = conf.getBoolVar(ConfVars.HIVE_TXN_STRICT_READ_LOCKS); // For each source to read, get a shared lock for (ReadEntity input : inputs) { if (!input.needsLock() || input.isUpdateOrDelete() || !AcidUtils.needsLock(input)) { @@ -2373,11 +2374,17 @@ private static boolean isLockableTable(Table t) { // This is a file or something we don't hold locks for. continue; } + if (strictReadLocks == false && AcidUtils.isTransactionalTable(t) == false) { + // skip read-locks for non-transactional tables + // read-locks don't protect non-transactional tables data consistency + LOG.debug("Adding lock component to lock request on {} ", t); + continue; + } if (t != null) { compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t)); } LockComponent comp = compBuilder.build(); - LOG.debug("Adding lock component to lock request " + comp.toString()); + LOG.debug("Adding lock component to lock request {} ", comp); lockComponents.add(comp); } // For each source to write to, get the appropriate lock type. If it's diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 5e085f8..9c3de2e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -774,6 +774,57 @@ public void checkExpectedLocks2() throws Exception { conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true); } + /** + * Check to make sure we acquire proper locks for queries involving non-strict locking + */ + @Test + public void checkExpectedReadLocksNonStrict() throws Exception { + dropTable(new String[] {"tab_acid", "tab_not_acid"}); + checkCmdOnDriver(driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')")); + checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " + + "clustered by (na) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')")); + checkCmdOnDriver(driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')")); + checkCmdOnDriver(driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')")); + + final HiveTxnManager txnMgr = null; + // Test non-strict read-locking mode - the read locks are only obtained for the ACID side + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_READ_LOCKS, false); + HiveTxnManager txnMgr1 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr1.openTxn(ctx, "T1"); + checkCmdOnDriver(driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na", true)); + txnMgr1.acquireLocks(driver.getPlan(), ctx, "T1"); + List locks = getLocks(txnMgr1); + Assert.assertEquals("Unexpected lock count", 3, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + + HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr2.openTxn(ctx, "T2"); + checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)", true)); + LockState ls = ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false); + locks = getLocks(txnMgr2); + Assert.assertEquals("Unexpected lock count", 4, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + + HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr3.openTxn(ctx, "T3"); + checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true)); + ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false); + locks = getLocks(txnMgr3); + Assert.assertEquals("Unexpected lock count", 5, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_READ_LOCKS, + HiveConf.ConfVars.HIVE_TXN_STRICT_READ_LOCKS.defaultBoolVal); + } + @Test public void testLockingOnInsertIntoNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"});