diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java index 372c709bed..9294c2b32c 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java @@ -1169,8 +1169,37 @@ public void testRecoverManyTimeouts() throws Exception { } finally { txnHandler.setTimeout(timeout); } + } - + @Test + public void testReplTimeouts() throws Exception { + long timeout = txnHandler.setTimeout(1); + try { + OpenTxnRequest request = new OpenTxnRequest(3, "me", "localhost"); + OpenTxnsResponse response = txnHandler.openTxns(request); + request.setReplPolicy("default.*"); + request.setReplSrcTxnIds(response.getTxn_ids()); + OpenTxnsResponse responseRepl = txnHandler.openTxns(request); + Thread.sleep(10); + txnHandler.performTimeOuts(); + GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo(); + int numAborted = 0; + int numOpen = 0; + for (TxnInfo txnInfo : rsp.getOpen_txns()) { + if (TxnState.ABORTED == txnInfo.getState()) { + assertTrue(response.getTxn_ids().contains(txnInfo.getId())); + numAborted++; + } + if (TxnState.OPEN == txnInfo.getState()) { + assertTrue(responseRepl.getTxn_ids().contains(txnInfo.getId())); + numOpen++; + } + } + assertEquals(3, numAborted); + assertEquals(3, numOpen); + } finally { + txnHandler.setTimeout(timeout); + } } @Test diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 4597166359..50bfca3885 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -79,7 +79,8 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_STARTED bigint NOT NULL," + " TXN_LAST_HEARTBEAT bigint NOT NULL," + " TXN_USER varchar(128) NOT NULL," + - " TXN_HOST varchar(128) NOT NULL)"); + " TXN_HOST varchar(128) NOT NULL," + + " TXN_TYPE integer)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + " TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID)," + diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 3c254c491e..ea2db521a5 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -168,6 +168,19 @@ //todo: make these like OperationType and remove above char constatns enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN} + public enum TxnType { + DEFAULT(0), REPL_CREATED(1), READ_ONLY(2); + + private final int value; + TxnType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + // Lock states static final protected char LOCK_ACQUIRED = 'a'; static final protected char LOCK_WAITING = 'w'; @@ -561,6 +574,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { throws SQLException, MetaException { int numTxns = rqst.getNum_txns(); ResultSet rs = null; + TxnType txnType = TxnType.DEFAULT; try { if (rqst.isSetReplPolicy()) { List targetTxnIdList = getTargetTxnIdList(rqst.getReplPolicy(), rqst.getReplSrcTxnIds(), stmt); @@ -574,6 +588,7 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { rqst.getReplPolicy() + " and Source transaction id : " + rqst.getReplSrcTxnIds().toString()); return targetTxnIdList; } + txnType = TxnType.REPL_CREATED; } String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID"); @@ -595,10 +610,10 @@ public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException { for (long i = first; i < first + numTxns; i++) { txnIds.add(i); rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," - + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname())); + + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()) + "," + txnType.getValue()); } List queries = sqlGenerator.createInsertValuesStmt( - "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows); + "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host, txn_type)", rows); for (String q : queries) { LOG.debug("Going to execute update <" + q + ">"); stmt.execute(q); @@ -4226,7 +4241,7 @@ public void performTimeOuts() { while(true) { stmt = dbConn.createStatement(); String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN + - "' and txn_last_heartbeat < " + (now - timeout); + "' and txn_last_heartbeat < " + (now - timeout) + " and txn_type != " + TxnType.REPL_CREATED.getValue(); //safety valve for extreme cases s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s); LOG.debug("Going to execute query <" + s + ">"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql index f52be79a3d..352b43e9cd 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -505,7 +505,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql index efb970afd9..e45fb642de 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql @@ -24,8 +24,10 @@ ALTER TABLE "APP"."PART_COL_PRIVS" ADD "AUTHORIZER" VARCHAR(128); DROP INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX"; CREATE INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX" ON "APP"."PART_COL_PRIVS" ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); - CREATE INDEX "APP"."TAB_COL_STATS_IDX" ON "APP"."TAB_COL_STATS" ("CAT_NAME", "DB_NAME", "TABLE_NAME", "COLUMN_NAME"); +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index 7daaf6e972..bc11b4092a 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -1091,6 +1091,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, + TXN_TYPE int NULL, PRIMARY KEY CLUSTERED ( TXN_ID ASC diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql index c0dd467755..3480edb48b 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql @@ -25,9 +25,11 @@ ALTER TABLE PART_COL_PRIVS ADD AUTHORIZER nvarchar(128) NULL; DROP INDEX PART_COL_PRIVS.PARTITIONCOLUMNPRIVILEGEINDEX; CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); - CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME); +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE int NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index 21dcd2c7d9..af955dcf7f 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -984,7 +984,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT int + TXN_HEARTBEAT_COUNT int, + TXN_TYPE int ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql index e53c9d4ecb..61d50d2d05 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql @@ -25,9 +25,11 @@ ALTER TABLE `PART_COL_PRIVS` ADD `AUTHORIZER` varchar(128) CHARACTER SET latin1 ALTER TABLE `PART_COL_PRIVS` DROP INDEX `PARTITIONCOLUMNPRIVILEGEINDEX`; ALTER TABLE `PART_COL_PRIVS` ADD INDEX `PARTITIONCOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); - CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME) USING BTREE; +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE int DEFAULT NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index c4c138abcf..bc137039c6 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -962,7 +962,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar2(128), TXN_META_INFO varchar2(128), - TXN_HEARTBEAT_COUNT number(10) + TXN_HEARTBEAT_COUNT number(10), + TXN_TYPE number(10) ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql index 47a80f49b5..a9712589c6 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql @@ -25,9 +25,11 @@ ALTER TABLE PART_COL_PRIVS ADD AUTHORIZER VARCHAR2(128) NULL; DROP INDEX PARTITIONCOLUMNPRIVILEGEINDEX; CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); - CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME); +-- HIVE-19340 +ALTER TABLE TXNS ADD TXN_TYPE number(10) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index 4570201977..36bab70eda 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -1652,7 +1652,8 @@ CREATE TABLE TXNS ( TXN_HOST varchar(128) NOT NULL, TXN_AGENT_INFO varchar(128), TXN_META_INFO varchar(128), - TXN_HEARTBEAT_COUNT integer + TXN_HEARTBEAT_COUNT integer, + TXN_TYPE integer ); CREATE TABLE TXN_COMPONENTS ( diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql index 9d7ce0990e..fe960472c0 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql @@ -27,9 +27,11 @@ ALTER TABLE "PART_COL_PRIVS" ADD COLUMN "AUTHORIZER" character varying(128) DEFA DROP INDEX "PARTITIONCOLUMNPRIVILEGEINDEX"; CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON "PART_COL_PRIVS" USING btree ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); - CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME"); +-- HIVE-19340 +ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer DEFAULT NULL; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0';