diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b79515fcf0..d8aabd8c87 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2644,12 +2644,17 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock, which\n" + "allows two concurrent writes to the same partition but still lets lock manager prevent\n" + "DROP TABLE etc. when the table is being written to"), + HIVE_TXN_NONACID_READ_LOCKS("hive.txn.nonacid.read.locks", true, + "Flag to turn off the read locks for non-ACID tables, when set to false.\n" + + "Could be exercised to improve the performance of non-ACID tables in clusters where read locking " + + "is enabled globally to support ACID. Can cause issues with concurrent DDL operations, or slow S3 writes."), HIVE_TXN_READ_LOCKS("hive.txn.read.locks", true, - "flag to turn off the strict read lock when set to false"), + "Flag to turn off the read locks, when set to false. Although its not recommended, \n" + + "but in performance critical scenarios this option may be exercised."), TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true, "Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire Exclusive locks for\n" + - "transactional tables. This ensures that inserts (w/o overwrite) running concurrently\n" + - "are not hidden by the INSERT OVERWRITE."), + "transactional tables. This ensures that inserts (w/o overwrite) running concurrently\n" + + "are not hidden by the INSERT OVERWRITE."), HIVE_TXN_STATS_ENABLED("hive.txn.stats.enabled", true, "Whether Hive supports transactional stats (accurate stats for transactional tables)"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index ed68e0f4af..2f5ec5270c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -2845,8 +2845,10 @@ private static boolean isLockableTable(Table t) { public static List makeLockComponents(Set outputs, Set inputs, HiveConf conf) { List lockComponents = new ArrayList<>(); - // For each source to read, get a shared lock boolean skipReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_READ_LOCKS); + boolean skipNonAcidReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_NONACID_READ_LOCKS); + + // For each source to read, get a shared lock for (ReadEntity input : inputs) { if (input.isDummy() || !input.needsLock() @@ -2863,33 +2865,38 @@ private static boolean isLockableTable(Table t) { Table t = null; switch (input.getType()) { - case DATABASE: - compBuilder.setDbName(input.getDatabase().getName()); - break; - - case TABLE: - t = input.getTable(); - compBuilder.setDbName(t.getDbName()); - compBuilder.setTableName(t.getTableName()); - break; - - case PARTITION: - case DUMMYPARTITION: - compBuilder.setPartitionName(input.getPartition().getName()); - t = input.getPartition().getTable(); - compBuilder.setDbName(t.getDbName()); - compBuilder.setTableName(t.getTableName()); - break; - - default: - // This is a file or something we don't hold locks for. + case DATABASE: + compBuilder.setDbName(input.getDatabase().getName()); + break; + + case TABLE: + t = input.getTable(); + compBuilder.setDbName(t.getDbName()); + compBuilder.setTableName(t.getTableName()); + break; + + case PARTITION: + case DUMMYPARTITION: + compBuilder.setPartitionName(input.getPartition().getName()); + t = input.getPartition().getTable(); + compBuilder.setDbName(t.getDbName()); + compBuilder.setTableName(t.getTableName()); + break; + + default: + // This is a file or something we don't hold locks for. + continue; + } + if (skipNonAcidReadLock && !AcidUtils.isTransactionalTable(t)) { + // skip read-locks for non-transactional tables + // read-locks don't protect non-transactional tables data consistency continue; } if (t != null) { compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t)); } LockComponent comp = compBuilder.build(); - LOG.debug("Adding lock component to lock request " + comp.toString()); + LOG.debug("Adding lock component to lock request {} ", comp); lockComponents.add(comp); } // For each source to write to, get the appropriate lock type. If it's diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 50859d1128..48e9afc496 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -711,6 +711,58 @@ public void checkExpectedLocks2() throws Exception { conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true); } + /** + * Check to make sure we acquire proper locks for queries involving non-strict locking + */ + @Test + public void checkExpectedReadLocksForNonAcidTables() throws Exception { + dropTable(new String[] {"tab_acid", "tab_not_acid"}); + driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); + driver.run("create table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " + + "clustered by (na) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')"); + driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')"); + driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')"); + + // Test non-acid read-locking mode - the read locks are only obtained for the ACID side + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS, false); + + HiveTxnManager txnMgr1 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr1.openTxn(ctx, "T1"); + driver.compileAndRespond("select * from tab_acid inner join tab_not_acid on a = na", true); + txnMgr1.acquireLocks(driver.getPlan(), ctx, "T1"); + List locks = getLocks(txnMgr1); + Assert.assertEquals("Unexpected lock count", 3, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + + HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr2.openTxn(ctx, "T2"); + driver.compileAndRespond("insert into tab_not_acid partition(np='doh') values(5,6)", true); + LockState ls = ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "T2", false); + locks = getLocks(txnMgr2); + Assert.assertEquals("Unexpected lock count", 4, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks); + + HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + txnMgr3.openTxn(ctx, "T3"); + driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true); + ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false); + locks = getLocks(txnMgr3); + Assert.assertEquals("Unexpected lock count", 5, locks.size()); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks); + checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks); + checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks); + + conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS, + HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS.defaultBoolVal); + } + @Test public void testLockingOnInsertIntoNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"});