diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java index 372c709bed..9294c2b32c 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java @@ -1169,8 +1169,37 @@ public void testRecoverManyTimeouts() throws Exception { } finally { txnHandler.setTimeout(timeout); } + } - + @Test + public void testReplTimeouts() throws Exception { + long timeout = txnHandler.setTimeout(1); + try { + OpenTxnRequest request = new OpenTxnRequest(3, "me", "localhost"); + OpenTxnsResponse response = txnHandler.openTxns(request); + request.setReplPolicy("default.*"); + request.setReplSrcTxnIds(response.getTxn_ids()); + OpenTxnsResponse responseRepl = txnHandler.openTxns(request); + Thread.sleep(10); + txnHandler.performTimeOuts(); + GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo(); + int numAborted = 0; + int numOpen = 0; + for (TxnInfo txnInfo : rsp.getOpen_txns()) { + if (TxnState.ABORTED == txnInfo.getState()) { + assertTrue(response.getTxn_ids().contains(txnInfo.getId())); + numAborted++; + } + if (TxnState.OPEN == txnInfo.getState()) { + assertTrue(responseRepl.getTxn_ids().contains(txnInfo.getId())); + numOpen++; + } + } + assertEquals(3, numAborted); + assertEquals(3, numOpen); + } finally { + txnHandler.setTimeout(timeout); + } } @Test diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index cf89ab2166..97ccbfa505 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -79,6 +79,7 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_STARTED bigint NOT NULL," + " TXN_LAST_HEARTBEAT bigint NOT NULL," + " TXN_USER varchar(128) NOT NULL," + + " TXN_TYPE integer," + " TXN_HOST varchar(128) NOT NULL)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index db596a6a13..107aa39a4d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -213,6 +213,19 @@ //todo: make these like OperationType and remove above char constatns enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN} + public enum TxnType { + REPL_CREATED(0), READ_ONLY(1), UNKNOWN(2); + + private final int value; + TxnType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + // Lock states static final protected char LOCK_ACQUIRED = 'a'; static final protected char LOCK_WAITING = 'w'; @@ -553,6 +566,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { if (numTxns <= 0) { throw new MetaException("Invalid input for number of txns: " + numTxns); } + TxnType txnType = TxnType.UNKNOWN; try { Connection dbConn = null; @@ -595,6 +609,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { rqst.getReplPolicy() + " and Source transaction id : " + rqst.getReplSrcTxnIds().toString()); return new OpenTxnsResponse(targetTxnIdList); } + txnType = TxnType.REPL_CREATED; } String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID"); @@ -615,10 +630,11 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { List rows = new ArrayList<>(); for (long i = first; i < first + numTxns; i++) { txnIds.add(i); - rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname())); + rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," + + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()) + "," + txnType.getValue()); } List queries = sqlGenerator.createInsertValuesStmt( - "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows); + "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host, txn_type)", rows); for (String q : queries) { LOG.debug("Going to execute update <" + q + ">"); stmt.execute(q); @@ -3914,7 +3930,7 @@ public void performTimeOuts() { while(true) { stmt = dbConn.createStatement(); String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); + "' and txn_last_heartbeat < " + (now - timeout) + " and txn_type != " + TxnType.REPL_CREATED.getValue(); //safety valve for extreme cases s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s); LOG.debug("Going to execute query <" + s + ">"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 48d28cb7d8..62e61deec5 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -503,7 +503,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index ed6c4cd3d7..159351a040 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -248,6 +248,8 @@ CREATE TABLE "APP"."RUNTIME_STATS" ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index 6e31b16c52..9492dc429e 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1083,6 +1083,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, + TXN_TYPE int NULL, PRIMARY KEY CLUSTERED ( TXN_ID ASC diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index c2504d33c6..a131bf70fb 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -315,6 +315,8 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +ALTER TABLE TXNS ADD TXN_TYPE int; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index 4309911dbf..e20aaef017 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -976,7 +976,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT int + TXN_HEARTBEAT_COUNT int, + TXN_TYPE int ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index e01b4da7fd..2e7d403085 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -288,6 +288,8 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +ALTER TABLE TXNS ADD COLUMN TXN_TYPE int; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index a45c7bbb0f..7a11119dac 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -953,7 +953,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar2(128), TXN_META_INFO varchar2(128), - TXN_HEARTBEAT_COUNT number(10) + TXN_HEARTBEAT_COUNT number(10), + TXN_TYPE number(10) ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 327800bb0d..4746d4469c 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -306,6 +306,8 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +ALTER TABLE TXNS ADD TXN_TYPE NUMBER(10); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 2484744adf..6925888a9e 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1640,7 +1640,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index 63932a9cf3..534a9e2f82 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -323,6 +323,8 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0';