diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java index 372c709bed..9294c2b32c 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java @@ -1169,8 +1169,37 @@ public void testRecoverManyTimeouts() throws Exception { } finally { txnHandler.setTimeout(timeout); } + } - + @Test + public void testReplTimeouts() throws Exception { + long timeout = txnHandler.setTimeout(1); + try { + OpenTxnRequest request = new OpenTxnRequest(3, "me", "localhost"); + OpenTxnsResponse response = txnHandler.openTxns(request); + request.setReplPolicy("default.*"); + request.setReplSrcTxnIds(response.getTxn_ids()); + OpenTxnsResponse responseRepl = txnHandler.openTxns(request); + Thread.sleep(10); + txnHandler.performTimeOuts(); + GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo(); + int numAborted = 0; + int numOpen = 0; + for (TxnInfo txnInfo : rsp.getOpen_txns()) { + if (TxnState.ABORTED == txnInfo.getState()) { + assertTrue(response.getTxn_ids().contains(txnInfo.getId())); + numAborted++; + } + if (TxnState.OPEN == txnInfo.getState()) { + assertTrue(responseRepl.getTxn_ids().contains(txnInfo.getId())); + numOpen++; + } + } + assertEquals(3, numAborted); + assertEquals(3, numOpen); + } finally { + txnHandler.setTimeout(timeout); + } } @Test diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 4597166359..50bfca3885 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -79,7 +79,8 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_STARTED bigint NOT NULL," + " TXN_LAST_HEARTBEAT bigint NOT NULL," + " TXN_USER varchar(128) NOT NULL," + - " TXN_HOST varchar(128) NOT NULL)"); + " TXN_HOST varchar(128) NOT NULL," + + " TXN_TYPE integer)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + " TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID)," + diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index d1b0d32614..565fb89d16 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -209,6 +209,19 @@ //todo: make these like OperationType and remove above char constatns enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN} + public enum TxnType { + DEFAULT(0), REPL_CREATED(1), READ_ONLY(2); + + private final int value; + TxnType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + // Lock states static final protected char LOCK_ACQUIRED = 'a'; static final protected char LOCK_WAITING = 'w'; @@ -602,6 +615,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { throws SQLException, MetaException { int numTxns = rqst.getNum_txns(); ResultSet rs = null; + TxnType txnType = TxnType.DEFAULT; try { if (rqst.isSetReplPolicy()) { List targetTxnIdList = getTargetTxnIdList(rqst.getReplPolicy(), rqst.getReplSrcTxnIds(), stmt); @@ -615,6 +629,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { rqst.getReplPolicy() + " and Source transaction id : " + rqst.getReplSrcTxnIds().toString()); return targetTxnIdList; } + txnType = TxnType.REPL_CREATED; } String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID"); @@ -636,10 +651,10 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { for (long i = first; i < first + numTxns; i++) { txnIds.add(i); rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," - + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname())); + + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()) + "," + txnType.getValue()); } List queries = sqlGenerator.createInsertValuesStmt( - "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows); + "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host, txn_type)", rows); for (String q : queries) { LOG.debug("Going to execute update <" + q + ">"); stmt.execute(q); @@ -4224,7 +4239,7 @@ public void performTimeOuts() { while(true) { stmt = dbConn.createStatement(); String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); + "' and txn_last_heartbeat < " + (now - timeout) + " and txn_type != " + TxnType.REPL_CREATED.getValue(); //safety valve for extreme cases s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s); LOG.debug("Going to execute query <" + s + ">"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql index d679658272..76d72db31f 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -503,7 +503,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 24740f9e4a..93930078b2 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -503,7 +503,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql index 047abdbf4b..4c5cb7c43b 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql @@ -24,5 +24,8 @@ ALTER TABLE "APP"."PART_COL_PRIVS" ADD "AUTHORIZER" VARCHAR(128); DROP INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX"; CREATE INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX" ON "APP"."PART_COL_PRIVS" ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index 1bb3c1acb3..e8e2d3e95c 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -1090,6 +1090,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, + TXN_TYPE int NULL, PRIMARY KEY CLUSTERED ( TXN_ID ASC diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 7a5cec8aff..c5dc67dc3a 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -1090,6 +1090,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, + TXN_TYPE int NULL, PRIMARY KEY CLUSTERED ( TXN_ID ASC diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql index d3f2794d72..0a8481e248 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql @@ -25,6 +25,9 @@ ALTER TABLE PART_COL_PRIVS ADD AUTHORIZER nvarchar(128) NULL; DROP INDEX PART_COL_PRIVS.PARTITIONCOLUMNPRIVILEGEINDEX; CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE int NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index 1f0450330b..68554f28ca 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -983,7 +983,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT int + TXN_HEARTBEAT_COUNT int, + TXN_TYPE int ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index f0d2fa1db9..a3be35313f 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -983,7 +983,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT int + TXN_HEARTBEAT_COUNT int, + TXN_TYPE int ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql index df5485edcf..30e9b8f106 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql @@ -25,6 +25,9 @@ ALTER TABLE `PART_COL_PRIVS` ADD `AUTHORIZER` varchar(128) CHARACTER SET latin1 ALTER TABLE `PART_COL_PRIVS` DROP INDEX `PARTITIONCOLUMNPRIVILEGEINDEX`; ALTER TABLE `PART_COL_PRIVS` ADD INDEX `PARTITIONCOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE int DEFAULT NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index 2d9b2b7844..c7c549a3f8 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -960,7 +960,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar2(128), TXN_META_INFO varchar2(128), - TXN_HEARTBEAT_COUNT number(10) + TXN_HEARTBEAT_COUNT number(10), + TXN_TYPE number(10) ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index 2877c79842..5b577baab4 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -960,7 +960,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar2(128), TXN_META_INFO varchar2(128), - TXN_HEARTBEAT_COUNT number(10) + TXN_HEARTBEAT_COUNT number(10), + TXN_TYPE number(10) ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql index 6c4c5be180..0145bdbf6e 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql @@ -25,6 +25,9 @@ ALTER TABLE PART_COL_PRIVS ADD AUTHORIZER VARCHAR2(128) NULL; DROP INDEX PARTITIONCOLUMNPRIVILEGEINDEX; CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE number(10) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index f8a073a57d..54baf52b5b 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -1647,7 +1647,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index 5f93ae07ce..4551a39236 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -1647,7 +1647,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql index 81f695c8d4..5319306367 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql @@ -27,6 +27,9 @@ ALTER TABLE "PART_COL_PRIVS" ADD COLUMN "AUTHORIZER" character varying(128) DEFA DROP INDEX "PARTITIONCOLUMNPRIVILEGEINDEX"; CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON "PART_COL_PRIVS" USING btree ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer DEFAULT NULL; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0';