diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 08bc654..43e5dd9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -380,6 +380,8 @@ TXN_ABORTED(10263, "Transaction manager has aborted the transaction {0}.", true), DBTXNMGR_REQUIRES_CONCURRENCY(10264, "To use DbTxnManager you must set hive.support.concurrency=true"), + TXNMGR_CANNOT_CHANGE(10265, "Cannot change transaction manager within a session"), + TXNMGR_NOT_ACID(10266, "Cannot query an ACID table using a non-ACID transaction manager"), LOCK_NO_SUCH_LOCK(10270, "No record of lock {0} could be found, " + "may have timed out", true), diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8c880c3..b9ae494 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1626,6 +1626,8 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException // here, it means the table itself doesn't support it. throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, tab_name); } + // Disallow queries against ACID table with transaction managers other than DbTxnManager + checkAcidTxnManager(tab); if (tab.isView()) { if (qb.getParseInfo().isAnalyzeCommand()) { @@ -1744,6 +1746,9 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException .getMsg(ast, "The class is " + outputFormatClass.toString())); } + // Disallow INSERT against ACID table with transaction managers other than DbTxnManager + checkAcidTxnManager(ts.tableHandle); + // TableSpec ts is got from the query (user specified), // which means the user didn't specify partitions in their query, // but whether the table itself is partitioned is not know. @@ -12223,6 +12228,14 @@ protected boolean updating() { protected boolean deleting() { return false; } + + // Error out if the table is ACID but current session's transaction manager does not support ACID + protected void checkAcidTxnManager(Table tab) throws SemanticException { + if (AcidUtils.isAcidTable(tab) && !SessionState.get().getTxnMgr().supportsAcid()) { + throw new SemanticException("Table: " + tab + " - " + ErrorMsg.TXNMGR_NOT_ACID.getMsg()); + } + } + public static ASTNode genSelectDIAST(RowResolver rr) { LinkedHashMap> map = rr.getRslvMap(); ASTNode selectDI = new ASTNode(new CommonToken(HiveParser.TOK_SELECTDI, "TOK_SELECTDI")); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 9a3ba04..8b3e296 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; +import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; @@ -206,6 +207,13 @@ private static String setConf(String varname, String key, String varvalue, boole throw new IllegalArgumentException("hive configuration " + key + " does not exists."); } } + if (HiveConf.ConfVars.HIVE_TXN_MANAGER.varname.equals(key) && + SessionState.get().getTxnMgr() != null) { + StringBuilder message = new StringBuilder(); + message.append("'SET ").append(varname).append('=').append(varvalue); + message.append("' failed. "); + throw new IllegalArgumentException(message + ErrorMsg.TXNMGR_CANNOT_CHANGE.getMsg()); + } conf.verifyAndSet(key, value); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key)) { if (!"spark".equals(value)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index efeb70f..9bce822 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -410,6 +410,13 @@ public synchronized HiveTxnManager initTxnMgr(HiveConf conf) throws LockExceptio if (txnMgr == null) { txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); } + // Only allow changing transaction manager in tests + if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { + if (txnMgr != null) { + txnMgr.closeTxnManager(); + } + txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); + } return txnMgr; } diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 3bdcc21..84f2579 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.lockmgr; +import com.sun.xml.bind.v2.TODO; import junit.framework.Assert; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.LockState; @@ -241,6 +242,41 @@ public void testLockRetryLimit() throws Exception { otherTxnMgr.closeTxnManager(); } + @Test + public void testDummyTxnManagerOnAcidTable() throws Exception { + // Create an ACID table with DbTxnManager + CommandProcessorResponse cpr = driver.run("create table T10 (a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); + checkCmdOnDriver(cpr); + cpr = driver.run("create table T11 (a int, b int) clustered by(b) into 2 buckets stored as orc"); + checkCmdOnDriver(cpr); + + // Now switch to DummyTxnManager + conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager"); + txnMgr = SessionState.get().initTxnMgr(conf); + Assert.assertTrue(txnMgr instanceof DummyTxnManager); + + // All DML should fail with DummyTxnManager on ACID table + cpr = driver.compileAndRespond("select * from T10"); + Assert.assertEquals(40000, cpr.getResponseCode()); + Assert.assertTrue(cpr.getErrorMessage().contains("Cannot query an ACID table using a non-ACID transaction manager")); + + cpr = driver.compileAndRespond("insert into table T10 values (1, 2)"); + Assert.assertEquals(40000, cpr.getResponseCode()); + Assert.assertTrue(cpr.getErrorMessage().contains("Cannot query an ACID table using a non-ACID transaction manager")); + + cpr = driver.compileAndRespond("insert overwrite table T10 select a, b from T11"); + Assert.assertEquals(40000, cpr.getResponseCode()); + Assert.assertTrue(cpr.getErrorMessage().contains("Cannot query an ACID table using a non-ACID transaction manager")); + + cpr = driver.compileAndRespond("update T10 set a=0 where b=1"); + Assert.assertEquals(10294, cpr.getResponseCode()); + Assert.assertTrue(cpr.getErrorMessage().contains("Attempt to do update or delete using transaction manager that does not support these operations.")); + + cpr = driver.compileAndRespond("delete from T10"); + Assert.assertEquals(10294, cpr.getResponseCode()); + Assert.assertTrue(cpr.getErrorMessage().contains("Attempt to do update or delete using transaction manager that does not support these operations.")); + } + private void checkLock(LockType type, LockState state, String db, String table, String partition, ShowLocksResponseElement l) { Assert.assertEquals(l.toString(),l.getType(), type); Assert.assertEquals(l.toString(),l.getState(), state); diff --git ql/src/test/queries/clientnegative/acid_txn_manager.q ql/src/test/queries/clientnegative/acid_txn_manager.q new file mode 100644 index 0000000..ddf8f44 --- /dev/null +++ ql/src/test/queries/clientnegative/acid_txn_manager.q @@ -0,0 +1,3 @@ +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager; +select count(*) from src; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; diff --git ql/src/test/results/clientnegative/acid_txn_manager.q.out ql/src/test/results/clientnegative/acid_txn_manager.q.out new file mode 100644 index 0000000..9ad7ab2 --- /dev/null +++ ql/src/test/results/clientnegative/acid_txn_manager.q.out @@ -0,0 +1,10 @@ +PREHOOK: query: select count(*) from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +500 +Query returned non-zero code: 1, cause: 'SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager' failed. Cannot change transaction manager within a session