diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java index 372c709bed..9294c2b32c 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java @@ -1169,8 +1169,37 @@ public void testRecoverManyTimeouts() throws Exception { } finally { txnHandler.setTimeout(timeout); } + } - + @Test + public void testReplTimeouts() throws Exception { + long timeout = txnHandler.setTimeout(1); + try { + OpenTxnRequest request = new OpenTxnRequest(3, "me", "localhost"); + OpenTxnsResponse response = txnHandler.openTxns(request); + request.setReplPolicy("default.*"); + request.setReplSrcTxnIds(response.getTxn_ids()); + OpenTxnsResponse responseRepl = txnHandler.openTxns(request); + Thread.sleep(10); + txnHandler.performTimeOuts(); + GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo(); + int numAborted = 0; + int numOpen = 0; + for (TxnInfo txnInfo : rsp.getOpen_txns()) { + if (TxnState.ABORTED == txnInfo.getState()) { + assertTrue(response.getTxn_ids().contains(txnInfo.getId())); + numAborted++; + } + if (TxnState.OPEN == txnInfo.getState()) { + assertTrue(responseRepl.getTxn_ids().contains(txnInfo.getId())); + numOpen++; + } + } + assertEquals(3, numAborted); + assertEquals(3, numOpen); + } finally { + txnHandler.setTimeout(timeout); + } } @Test diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 4597166359..7b58cbe3e0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -79,6 +79,7 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_STARTED bigint NOT NULL," + " TXN_LAST_HEARTBEAT bigint NOT NULL," + " TXN_USER varchar(128) NOT NULL," + + " TXN_TYPE integer," + " TXN_HOST varchar(128) NOT NULL)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 21b98655e9..513d82efb6 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -215,6 +215,19 @@ //todo: make these like OperationType and remove above char constatns enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN} + public enum TxnType { + DEFAULT(0), REPL_CREATED(1), READ_ONLY(2); + + private final int value; + TxnType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + // Lock states static final protected char LOCK_ACQUIRED = 'a'; static final protected char LOCK_WAITING = 'w'; @@ -608,6 +621,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { throws SQLException, MetaException { int numTxns = rqst.getNum_txns(); ResultSet rs = null; + TxnType txnType = TxnType.DEFAULT; try { if (rqst.isSetReplPolicy()) { List targetTxnIdList = getTargetTxnIdList(rqst.getReplPolicy(), rqst.getReplSrcTxnIds(), stmt); @@ -621,6 +635,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { rqst.getReplPolicy() + " and Source transaction id : " + rqst.getReplSrcTxnIds().toString()); return targetTxnIdList; } + txnType = TxnType.REPL_CREATED; } String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID"); @@ -642,10 +657,10 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { for (long i = first; i < first + numTxns; i++) { txnIds.add(i); rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," - + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname())); + + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()) + "," + txnType.getValue()); } List queries = sqlGenerator.createInsertValuesStmt( - "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows); + "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host, txn_type)", rows); for (String q : queries) { LOG.debug("Going to execute update <" + q + ">"); stmt.execute(q); @@ -4235,7 +4250,7 @@ public void performTimeOuts() { while(true) { stmt = dbConn.createStatement(); String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); + "' and txn_last_heartbeat < " + (now - timeout) + " and txn_type != " + TxnType.REPL_CREATED.getValue(); //safety valve for extreme cases s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s); LOG.debug("Going to execute query <" + s + ">"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index e818e1b7e2..158e51e311 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -503,7 +503,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 7b7a8a24d9..ff44d448c3 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -279,6 +279,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index c88fb18973..926f305633 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1084,6 +1084,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, + TXN_TYPE int NULL, PRIMARY KEY CLUSTERED ( TXN_ID ASC diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index 14c3deb44c..12ecc7b9e0 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -346,6 +346,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE int; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index c54df55ed9..7f74d61612 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -977,7 +977,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT int + TXN_HEARTBEAT_COUNT int, + TXN_TYPE int ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 9b87563b8a..b556f5c55c 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -319,8 +319,11 @@ UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL; +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE int; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; -ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL; \ No newline at end of file +ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 63cc1f75e2..e5acd2d953 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -954,7 +954,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar2(128), TXN_META_INFO varchar2(128), - TXN_HEARTBEAT_COUNT number(10) + TXN_HEARTBEAT_COUNT number(10), + TXN_TYPE number(10) ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index ce3437f723..64cb74128b 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -335,8 +335,11 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE NUMBER(10); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; -ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL; \ No newline at end of file +ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index d210a55fcf..2c94f37de3 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1641,7 +1641,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index f2571d8f2b..e53db10713 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -354,6 +354,9 @@ INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0';