diff --git a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql index 3c20d1e..dbaaab7 100644 --- a/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql +++ b/metastore/scripts/upgrade/derby/050-HIVE-18192.derby.sql @@ -18,8 +18,8 @@ CREATE TABLE NEXT_WRITE_ID ( CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- Modify Compaction related tables to use write id instead of txn id RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; diff --git a/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql new file mode 100644 index 0000000..946cce0 --- /dev/null +++ b/metastore/scripts/upgrade/derby/057-HIVE-18193.derby.sql @@ -0,0 +1,24 @@ + +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_TXN_ID; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index cf89ab2..838be7d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -82,20 +82,20 @@ public static void prepDb(Configuration conf) throws Exception { " TXN_HOST varchar(128) NOT NULL)"); stmt.execute("CREATE TABLE TXN_COMPONENTS (" + - " TC_TXNID bigint REFERENCES TXNS (TXN_ID)," + + " TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID)," + " TC_DATABASE varchar(128) NOT NULL," + " TC_TABLE varchar(128)," + " TC_PARTITION varchar(767)," + " TC_OPERATION_TYPE char(1) NOT NULL," + - " TC_WRITEID bigint)"); + " TC_WRITEID bigint NOT NULL)"); stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" + - " CTC_TXNID bigint," + + " CTC_TXNID bigint NOT NULL," + " CTC_DATABASE varchar(128) NOT NULL," + " CTC_TABLE varchar(128)," + " CTC_PARTITION varchar(767)," + " CTC_ID bigint GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) NOT NULL," + " CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL," + - " CTC_WRITEID bigint)"); + " CTC_WRITEID bigint NOT NULL)"); stmt.execute("CREATE TABLE NEXT_TXN_ID (" + " NTXN_NEXT bigint NOT NULL)"); stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)"); diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 48d28cb..b01089f 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -507,23 +507,23 @@ CREATE TABLE TXNS ( ); CREATE TABLE TXN_COMPONENTS ( - TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID bigint, + CTC_TXNID bigint NOT NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index b4c46cd..2e476f0 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -152,8 +152,8 @@ RENAME COLUMN COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; RENAME COLUMN COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -252,6 +252,31 @@ CREATE TABLE "APP"."RUNTIME_STATS" ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index 4c5739f..ee4b7bb 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1021,12 +1021,12 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE COMPLETED_TXN_COMPONENTS( - CTC_TXNID bigint NULL, + CTC_TXNID bigint NOT NULL, CTC_DATABASE nvarchar(128) NOT NULL, CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1090,12 +1090,12 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE TXN_COMPONENTS( - TC_TXNID bigint NULL, + TC_TXNID bigint NOT NULL, TC_DATABASE nvarchar(128) NOT NULL, TC_TABLE nvarchar(128) NULL, TC_PARTITION nvarchar(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index b338f0d..2c7c801 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -204,8 +204,8 @@ EXEC SP_RENAME 'COMPACTION_QUEUE.CQ_HIGHEST_TXN_ID', 'CQ_HIGHEST_WRITE_ID', 'COL EXEC SP_RENAME 'COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID', 'CC_HIGHEST_WRITE_ID', 'COLUMN'; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -319,6 +319,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index fd71a1d..2d14f9f 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -982,10 +982,10 @@ CREATE TABLE TXNS ( CREATE TABLE TXN_COMPONENTS ( TC_TXNID bigint NOT NULL, TC_DATABASE varchar(128) NOT NULL, - TC_TABLE varchar(128) NOT NULL, + TC_TABLE varchar(128), TC_PARTITION varchar(767), TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint, + TC_WRITEID bigint NOT NULL, FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -997,7 +997,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index b0d1927..6a1fdf2 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -194,8 +194,8 @@ ALTER TABLE COMPACTION_QUEUE CHANGE `CQ_HIGHEST_TXN_ID` `CQ_HIGHEST_WRITE_ID` bi ALTER TABLE COMPLETED_COMPACTIONS CHANGE `CC_HIGHEST_TXN_ID` `CC_HIGHEST_WRITE_ID` bigint; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -292,6 +292,33 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND PARAM_VALUE='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + +ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index a45c7bb..85be655 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -957,23 +957,23 @@ CREATE TABLE TXNS ( ) ROWDEPENDENCIES; CREATE TABLE TXN_COMPONENTS ( - TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_TXNID NUMBER(19) NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE VARCHAR2(128) NOT NULL, TC_TABLE VARCHAR2(128), TC_PARTITION VARCHAR2(767) NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID NUMBER(19) + TC_WRITEID NUMBER(19) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID NUMBER(19), + CTC_TXNID NUMBER(19) NOT NULL, CTC_DATABASE VARCHAR2(128) NOT NULL, CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) + CTC_WRITEID NUMBER(19) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1065,26 +1065,26 @@ CREATE TABLE WRITE_SET ( ); CREATE TABLE TXN_TO_WRITE_ID ( - T2W_TXNID number(19) NOT NULL, - T2W_DATABASE varchar(128) NOT NULL, - T2W_TABLE varchar(256) NOT NULL, - T2W_WRITEID number(19) NOT NULL + T2W_TXNID NUMBER(19) NOT NULL, + T2W_DATABASE VARCHAR2(128) NOT NULL, + T2W_TABLE VARCHAR2(256) NOT NULL, + T2W_WRITEID NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); CREATE TABLE NEXT_WRITE_ID ( - NWI_DATABASE varchar(128) NOT NULL, - NWI_TABLE varchar(256) NOT NULL, - NWI_NEXT number(19) NOT NULL + NWI_DATABASE VARCHAR2(128) NOT NULL, + NWI_TABLE VARCHAR2(256) NOT NULL, + NWI_NEXT NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); CREATE TABLE MIN_HISTORY_LEVEL ( - MHL_TXNID number(19) NOT NULL, - MHL_MIN_OPEN_TXNID number(19) NOT NULL, + MHL_TXNID NUMBER(19) NOT NULL, + MHL_MIN_OPEN_TXNID NUMBER(19) NOT NULL, PRIMARY KEY(MHL_TXNID) ); diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 0b7c793..fce83e1 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -191,19 +191,19 @@ UPDATE DBS -- HIVE-18192 CREATE TABLE TXN_TO_WRITE_ID ( - T2W_TXNID number(19) NOT NULL, - T2W_DATABASE varchar(128) NOT NULL, - T2W_TABLE varchar(256) NOT NULL, - T2W_WRITEID number(19) NOT NULL + T2W_TXNID NUMBER(19) NOT NULL, + T2W_DATABASE VARCHAR2(128) NOT NULL, + T2W_TABLE VARCHAR2(256) NOT NULL, + T2W_WRITEID NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); CREATE TABLE NEXT_WRITE_ID ( - NWI_DATABASE varchar(128) NOT NULL, - NWI_TABLE varchar(256) NOT NULL, - NWI_NEXT number(19) NOT NULL + NWI_DATABASE VARCHAR2(128) NOT NULL, + NWI_TABLE VARCHAR2(256) NOT NULL, + NWI_NEXT NUMBER(19) NOT NULL ); CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); @@ -213,8 +213,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME COLUMN CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE ALTER TABLE COMPLETED_COMPACTIONS RENAME COLUMN CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19); -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19); +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID number(19) NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19) NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -310,6 +310,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND to_char(PARAM_VALUE)='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) NEXT_WRITE; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT DB.NAME, TBL_INFO.TBL_NAME FROM DBS DB, + (SELECT TBL.DB_ID, TBL.TBL_NAME FROM TBLS TBL, + (SELECT TBL_ID FROM TABLE_PARAMS WHERE PARAM_KEY='transactional' AND to_char(PARAM_VALUE)='true') TBL_PARAM + WHERE TBL.TBL_ID=TBL_PARAM.TBL_ID) TBL_INFO + where DB.DB_ID=TBL_INFO.DB_ID) DB_TBL_NAME, + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) TXN_INFO; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 2484744..b55d4e2 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1644,23 +1644,23 @@ CREATE TABLE TXNS ( ); CREATE TABLE TXN_COMPONENTS ( - TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID), TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128), TC_PARTITION varchar(767) DEFAULT NULL, TC_OPERATION_TYPE char(1) NOT NULL, - TC_WRITEID bigint + TC_WRITEID bigint NOT NULL ); CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS USING hash (TC_TXNID); CREATE TABLE COMPLETED_TXN_COMPONENTS ( - CTC_TXNID bigint, + CTC_TXNID bigint NOT NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index f06f0dd..7bbd3c6 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -229,8 +229,8 @@ ALTER TABLE COMPACTION_QUEUE RENAME CQ_HIGHEST_TXN_ID TO CQ_HIGHEST_WRITE_ID; ALTER TABLE COMPLETED_COMPACTIONS RENAME CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_ID; -- Modify txn_components/completed_txn_components tables to add write id. -ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; -ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; +ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint NOT NULL; +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint NOT NULL; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint @@ -327,6 +327,31 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); +-- HIVE-18193 +-- Populate NEXT_WRITE_ID for each Transactional table and set next write ID same as next txn ID +INSERT INTO NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE, NWI_NEXT) + SELECT * FROM + (SELECT "DB"."NAME", "TBL_INFO"."TBL_NAME" FROM "DBS" "DB", + (SELECT "TBL"."DB_ID", "TBL"."TBL_NAME" FROM "TBLS" "TBL", + (SELECT "TBL_ID" FROM "TABLE_PARAMS" WHERE "PARAM_KEY"='transactional' AND "PARAM_VALUE"='true') "TBL_PARAM" + WHERE "TBL"."TBL_ID"="TBL_PARAM"."TBL_ID") "TBL_INFO" + where "DB"."DB_ID"="TBL_INFO"."DB_ID") "DB_TBL_NAME", + (SELECT NTXN_NEXT FROM NEXT_TXN_ID) "NEXT_WRITE"; + +-- Populate TXN_TO_WRITE_ID for each aborted/open txns and set write ID equal to txn ID +INSERT INTO TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID, T2W_WRITEID) + SELECT * FROM + (SELECT "DB"."NAME", "TBL_INFO"."TBL_NAME" FROM "DBS" "DB", + (SELECT "TBL"."DB_ID", "TBL"."TBL_NAME" FROM "TBLS" "TBL", + (SELECT "TBL_ID" FROM "TABLE_PARAMS" WHERE "PARAM_KEY"='transactional' AND "PARAM_VALUE"='true') "TBL_PARAM" + WHERE "TBL"."TBL_ID"="TBL_PARAM"."TBL_ID") "TBL_INFO" + where "DB"."DB_ID"="TBL_INFO"."DB_ID") "DB_TBL_NAME", + (SELECT TXN_ID, TXN_ID as WRITE_ID FROM TXNS) "TXN_INFO"; + +-- Update TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS for write ID which is same as txn ID +UPDATE TXN_COMPONENTS SET TC_WRITEID = TC_TXNID; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0';