diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 6f0ba07..2d7910d 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -17,18 +17,6 @@ */ package org.apache.hive.beeline; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintStream; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -39,6 +27,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.output.NullOutputStream; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaException; @@ -46,6 +35,22 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; public class HiveSchemaTool { private String userName = null; @@ -57,6 +62,8 @@ private final String dbType; private final MetaStoreSchemaInfo metaStoreSchemaInfo; + static final private Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName()); + public HiveSchemaTool(String dbType) throws HiveMetaException { this(System.getenv("HIVE_HOME"), new HiveConf(HiveSchemaTool.class), dbType); } @@ -356,6 +363,16 @@ public void runBeeLine(String sqlScriptFile) throws IOException { argList.add("-f"); argList.add(sqlScriptFile); + if (LOG.isDebugEnabled()) { + LOG.debug("Going to invoke file that contains:"); + FileReader fr = new FileReader(sqlScriptFile); + BufferedReader reader = new BufferedReader(fr); + String line; + while ((line = reader.readLine()) != null) { + LOG.debug("script: " + line); + } + } + // run the script using Beeline BeeLine beeLine = new BeeLine(); if (!verbose) { @@ -367,6 +384,7 @@ public void runBeeLine(String sqlScriptFile) throws IOException { // We can be pretty sure that an entire line can be processed as a single command since // we always add a line separator at the end while calling dbCommandParser.buildCommand. beeLine.getOpts().setEntireLineAsCommand(true); + LOG.debug("Going to run command <" + StringUtils.join(argList, " ") + ">"); int status = beeLine.begin(argList.toArray(new String[0]), null); if (status != 0) { throw new IOException("Schema script failed, errorcode " + status); diff --git metastore/if/hive_metastore.thrift metastore/if/hive_metastore.thrift index 56ec9aa..ef49ff3 100755 --- metastore/if/hive_metastore.thrift +++ metastore/if/hive_metastore.thrift @@ -657,6 +657,7 @@ struct ShowCompactResponseElement { 6: optional string workerid, 7: optional i64 start, 8: optional string runAs, + 9: optional i64 hightestTxnId, // Highest Txn ID handled by this compaction } struct ShowCompactResponse { diff --git metastore/scripts/upgrade/derby/023-HIVE-12807.derby.sql metastore/scripts/upgrade/derby/023-HIVE-12807.derby.sql new file mode 100644 index 0000000..3dfc6fe --- /dev/null +++ metastore/scripts/upgrade/derby/023-HIVE-12807.derby.sql @@ -0,0 +1 @@ +ALTER TABLE "COMPACTION_QUEUE" ADD "CQ_HIGHEST_TXN_ID" bigint; diff --git metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql index 20eb326..3893194 100644 --- metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql @@ -327,7 +327,7 @@ ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -RUN 'hive-txn-schema-0.13.0.derby.sql'; +RUN 'hive-txn-schema-1.3.0.derby.sql'; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql index abc6bf4..2c2f1e7 100644 --- metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql @@ -327,7 +327,7 @@ ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -RUN 'hive-txn-schema-0.13.0.derby.sql'; +RUN 'hive-txn-schema-2.0.0.derby.sql'; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/derby/hive-schema-2.1.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-2.1.0.derby.sql index 8083199..660a85a 100644 --- metastore/scripts/upgrade/derby/hive-schema-2.1.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-2.1.0.derby.sql @@ -325,7 +325,7 @@ ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -RUN 'hive-txn-schema-0.13.0.derby.sql'; +RUN 'hive-txn-schema-2.0.0.derby.sql'; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql new file mode 100644 index 0000000..de16db8 --- /dev/null +++ metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql @@ -0,0 +1,89 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + + diff --git metastore/scripts/upgrade/derby/hive-txn-schema-2.0.0.derby.sql metastore/scripts/upgrade/derby/hive-txn-schema-2.0.0.derby.sql new file mode 100644 index 0000000..de16db8 --- /dev/null +++ metastore/scripts/upgrade/derby/hive-txn-schema-2.0.0.derby.sql @@ -0,0 +1,89 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + + diff --git metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql index 6359de4..2447ea3 100644 --- metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql +++ metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql @@ -1,4 +1,5 @@ -- Upgrade MetaStore schema from 1.2.0 to 1.3.0 RUN '021-HIVE-11970.derby.sql'; +RUN '023-HIVE-12807.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; diff --git metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql index adf3cb0..8030dee 100644 --- metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql +++ metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql @@ -1,4 +1,5 @@ -- Upgrade MetaStore schema from 1.2.0 to 2.0.0 RUN '021-HIVE-11970.derby.sql'; +RUN '023-HIVE-12807.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1; diff --git metastore/scripts/upgrade/mssql/008-HIVE-12807.mssql.sql metastore/scripts/upgrade/mssql/008-HIVE-12807.mssql.sql new file mode 100644 index 0000000..e2d7615 --- /dev/null +++ metastore/scripts/upgrade/mssql/008-HIVE-12807.mssql.sql @@ -0,0 +1 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_HIGHEST_TXN_ID bigint NULL; diff --git metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql index 01c1376..c773aa4 100644 --- metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql +++ metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql @@ -866,6 +866,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, + CQ_HIGHEST_TXN_ID bigint NULL, PRIMARY KEY CLUSTERED ( CQ_ID ASC diff --git metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql index 1ec8632..11c3674 100644 --- metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql +++ metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql @@ -866,6 +866,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, + CQ_HIGHEST_TXN_ID bigint NULL, PRIMARY KEY CLUSTERED ( CQ_ID ASC diff --git metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql index 731cc25..a9cef1c 100644 --- metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql +++ metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql @@ -866,6 +866,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, + CQ_HIGHEST_TXN_ID bigint NULL, PRIMARY KEY CLUSTERED ( CQ_ID ASC diff --git metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql index 9cf9d25..f019a2c 100644 --- metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql +++ metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; :r 007-HIVE-11970.mssql.sql; +:r 008-HIVE-12807.mssql.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; diff --git metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql index de3d29d..86b20f3 100644 --- metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql +++ metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE; :r 007-HIVE-11970.mssql.sql; +:r 008-HIVE-12807.mssql.sql; UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE; diff --git metastore/scripts/upgrade/mysql/023-HIVE-12807.mysql.sql metastore/scripts/upgrade/mysql/023-HIVE-12807.mysql.sql new file mode 100644 index 0000000..31a2dd1 --- /dev/null +++ metastore/scripts/upgrade/mysql/023-HIVE-12807.mysql.sql @@ -0,0 +1 @@ +ALTER TABLE `COMPACTION_QUEUE` ADD `CQ_HIGHEST_TXN_ID` bigint; diff --git metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql index 6c211a6..d9232d2 100644 --- metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql @@ -813,7 +813,7 @@ CREATE TABLE IF NOT EXISTS `NOTIFICATION_SEQUENCE` -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -SOURCE hive-txn-schema-0.13.0.mysql.sql; +SOURCE hive-txn-schema-1.3.0.mysql.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql index ff0b643..82e0128 100644 --- metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql @@ -813,7 +813,7 @@ CREATE TABLE IF NOT EXISTS `NOTIFICATION_SEQUENCE` -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -SOURCE hive-txn-schema-0.13.0.mysql.sql; +SOURCE hive-txn-schema-2.0.0.mysql.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/mysql/hive-schema-2.1.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-2.1.0.mysql.sql index d7e384c..06c3d90 100644 --- metastore/scripts/upgrade/mysql/hive-schema-2.1.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-2.1.0.mysql.sql @@ -813,7 +813,7 @@ CREATE TABLE IF NOT EXISTS `NOTIFICATION_SEQUENCE` -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- -SOURCE hive-txn-schema-0.13.0.mysql.sql; +SOURCE hive-txn-schema-2.0.0.mysql.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql new file mode 100644 index 0000000..3a530fc --- /dev/null +++ metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql @@ -0,0 +1,91 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767), + FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) not null, + HL_LOCK_TYPE char(1) not null, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), + KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint, +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git metastore/scripts/upgrade/mysql/hive-txn-schema-2.0.0.mysql.sql metastore/scripts/upgrade/mysql/hive-txn-schema-2.0.0.mysql.sql new file mode 100644 index 0000000..3a530fc --- /dev/null +++ metastore/scripts/upgrade/mysql/hive-txn-schema-2.0.0.mysql.sql @@ -0,0 +1,91 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767), + FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) not null, + HL_LOCK_TYPE char(1) not null, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), + KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint, +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql index 1b32d93..09941a1 100644 --- metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql +++ metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql @@ -1,5 +1,6 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' '; SOURCE 021-HIVE-7018.mysql.sql; SOURCE 022-HIVE-11970.mysql.sql; +SOURCE 023-HIVE-12807.mysql.sql UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' '; diff --git metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql index 1340f27..53c3ef3 100644 --- metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql +++ metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql @@ -1,6 +1,8 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' '; SOURCE 021-HIVE-7018.mysql.sql; SOURCE 022-HIVE-11970.mysql.sql; +SOURCE 023-HIVE-12807.mysql.sql + UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' '; diff --git metastore/scripts/upgrade/oracle/023-HIVE-12807.oracle.sql metastore/scripts/upgrade/oracle/023-HIVE-12807.oracle.sql new file mode 100644 index 0000000..0313e96 --- /dev/null +++ metastore/scripts/upgrade/oracle/023-HIVE-12807.oracle.sql @@ -0,0 +1 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_HIGHEST_TXN_ID NUMBER(19); diff --git metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql index d2e7945..e162988 100644 --- metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql @@ -780,7 +780,7 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); ------------------------------ -- Transaction and lock tables ------------------------------ -@hive-txn-schema-0.13.0.oracle.sql; +@hive-txn-schema-1.3.0.oracle.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql index 2dcdd77..168473a 100644 --- metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql @@ -780,7 +780,7 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); ------------------------------ -- Transaction and lock tables ------------------------------ -@hive-txn-schema-0.13.0.oracle.sql; +@hive-txn-schema-2.0.0.oracle.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/oracle/hive-schema-2.1.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-2.1.0.oracle.sql index 88f2ca6..644463d 100644 --- metastore/scripts/upgrade/oracle/hive-schema-2.1.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-2.1.0.oracle.sql @@ -780,7 +780,7 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); ------------------------------ -- Transaction and lock tables ------------------------------ -@hive-txn-schema-0.13.0.oracle.sql; +@hive-txn-schema-2.0.0.oracle.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql new file mode 100644 index 0000000..3e7a8e5 --- /dev/null +++ metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql @@ -0,0 +1,89 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID NUMBER(19) PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED NUMBER(19) NOT NULL, + TXN_LAST_HEARTBEAT NUMBER(19) NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ROWDEPENDENCIES; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_DATABASE VARCHAR2(128) NOT NULL, + TC_TABLE VARCHAR2(128), + TC_PARTITION VARCHAR2(767) NULL +) ROWDEPENDENCIES; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID NUMBER(19), + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ROWDEPENDENCIES; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID NUMBER(19) NOT NULL, + HL_LOCK_INT_ID NUMBER(19) NOT NULL, + HL_TXNID NUMBER(19), + HL_DB VARCHAR2(128) NOT NULL, + HL_TABLE VARCHAR2(128), + HL_PARTITION VARCHAR2(767), + HL_LOCK_STATE CHAR(1) NOT NULL, + HL_LOCK_TYPE CHAR(1) NOT NULL, + HL_LAST_HEARTBEAT NUMBER(19) NOT NULL, + HL_ACQUIRED_AT NUMBER(19), + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +) ROWDEPENDENCIES; + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID NUMBER(19) PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START NUMBER(19), + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID NUMBER(19), +) ROWDEPENDENCIES; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git metastore/scripts/upgrade/oracle/hive-txn-schema-2.0.0.oracle.sql metastore/scripts/upgrade/oracle/hive-txn-schema-2.0.0.oracle.sql new file mode 100644 index 0000000..3e7a8e5 --- /dev/null +++ metastore/scripts/upgrade/oracle/hive-txn-schema-2.0.0.oracle.sql @@ -0,0 +1,89 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID NUMBER(19) PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED NUMBER(19) NOT NULL, + TXN_LAST_HEARTBEAT NUMBER(19) NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ROWDEPENDENCIES; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_DATABASE VARCHAR2(128) NOT NULL, + TC_TABLE VARCHAR2(128), + TC_PARTITION VARCHAR2(767) NULL +) ROWDEPENDENCIES; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID NUMBER(19), + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ROWDEPENDENCIES; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID NUMBER(19) NOT NULL, + HL_LOCK_INT_ID NUMBER(19) NOT NULL, + HL_TXNID NUMBER(19), + HL_DB VARCHAR2(128) NOT NULL, + HL_TABLE VARCHAR2(128), + HL_PARTITION VARCHAR2(767), + HL_LOCK_STATE CHAR(1) NOT NULL, + HL_LOCK_TYPE CHAR(1) NOT NULL, + HL_LAST_HEARTBEAT NUMBER(19) NOT NULL, + HL_ACQUIRED_AT NUMBER(19), + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +) ROWDEPENDENCIES; + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID NUMBER(19) PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START NUMBER(19), + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID NUMBER(19), +) ROWDEPENDENCIES; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql index bd283d4..4554219 100644 --- metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; @022-HIVE-11970.oracle.sql; +@023-HIVE-12807.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; diff --git metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql index ceb09be..495500f 100644 --- metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual; @022-HIVE-11970.oracle.sql; +@023-HIVE-12807.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual; diff --git metastore/scripts/upgrade/postgres/022-HIVE-12807.postgres.sql metastore/scripts/upgrade/postgres/022-HIVE-12807.postgres.sql new file mode 100644 index 0000000..11cbe11 --- /dev/null +++ metastore/scripts/upgrade/postgres/022-HIVE-12807.postgres.sql @@ -0,0 +1 @@ +ALTER TABLE "COMPACTION_QUEUE" ADD COLUMN "CQ_HIGHEST_TXN_ID" bigint; diff --git metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql index 9bb5765..1ea19c3 100644 --- metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql @@ -1449,7 +1449,7 @@ GRANT ALL ON SCHEMA public TO PUBLIC; ------------------------------ -- Transaction and lock tables ------------------------------ -\i hive-txn-schema-0.13.0.postgres.sql; +\i hive-txn-schema-1.3.0.postgres.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql index c749a29..4f034d7 100644 --- metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql @@ -1449,7 +1449,7 @@ GRANT ALL ON SCHEMA public TO PUBLIC; ------------------------------ -- Transaction and lock tables ------------------------------ -\i hive-txn-schema-0.13.0.postgres.sql; +\i hive-txn-schema-2.0.0.postgres.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/postgres/hive-schema-2.1.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-2.1.0.postgres.sql index efc0f0c..1db9993 100644 --- metastore/scripts/upgrade/postgres/hive-schema-2.1.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-2.1.0.postgres.sql @@ -1449,7 +1449,7 @@ GRANT ALL ON SCHEMA public TO PUBLIC; ------------------------------ -- Transaction and lock tables ------------------------------ -\i hive-txn-schema-0.13.0.postgres.sql; +\i hive-txn-schema-2.0.0.postgres.sql; -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql new file mode 100644 index 0000000..8bf151c --- /dev/null +++ metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql @@ -0,0 +1,90 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) DEFAULT NULL +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767) DEFAULT NULL, + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS USING hash (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + + diff --git metastore/scripts/upgrade/postgres/hive-txn-schema-2.0.0.postgres.sql metastore/scripts/upgrade/postgres/hive-txn-schema-2.0.0.postgres.sql new file mode 100644 index 0000000..8bf151c --- /dev/null +++ metastore/scripts/upgrade/postgres/hive-txn-schema-2.0.0.postgres.sql @@ -0,0 +1,90 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) DEFAULT NULL +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767) DEFAULT NULL, + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS USING hash (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + + diff --git metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql index b1ec241..32c5505 100644 --- metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql +++ metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0'; \i 021-HIVE-11970.postgres.sql; +\i 022-HIVE-12807.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='1.3.0', "VERSION_COMMENT"='Hive release version 1.3.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0'; diff --git metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql index 628444c..d66549b 100644 --- metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql +++ metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0'; \i 021-HIVE-11970.postgres.sql; +\i 022-HIVE-12807.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='2.0.0', "VERSION_COMMENT"='Hive release version 2.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0'; diff --git metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index d997e33..e31516e 100644 --- metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -12564,6 +12564,11 @@ void ShowCompactResponseElement::__set_runAs(const std::string& val) { __isset.runAs = true; } +void ShowCompactResponseElement::__set_hightestTxnId(const int64_t val) { + this->hightestTxnId = val; +__isset.hightestTxnId = true; +} + uint32_t ShowCompactResponseElement::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -12655,6 +12660,14 @@ uint32_t ShowCompactResponseElement::read(::apache::thrift::protocol::TProtocol* xfer += iprot->skip(ftype); } break; + case 9: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->hightestTxnId); + this->__isset.hightestTxnId = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -12716,6 +12729,11 @@ uint32_t ShowCompactResponseElement::write(::apache::thrift::protocol::TProtocol xfer += oprot->writeString(this->runAs); xfer += oprot->writeFieldEnd(); } + if (this->__isset.hightestTxnId) { + xfer += oprot->writeFieldBegin("hightestTxnId", ::apache::thrift::protocol::T_I64, 9); + xfer += oprot->writeI64(this->hightestTxnId); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -12731,6 +12749,7 @@ void swap(ShowCompactResponseElement &a, ShowCompactResponseElement &b) { swap(a.workerid, b.workerid); swap(a.start, b.start); swap(a.runAs, b.runAs); + swap(a.hightestTxnId, b.hightestTxnId); swap(a.__isset, b.__isset); } @@ -12743,6 +12762,7 @@ ShowCompactResponseElement::ShowCompactResponseElement(const ShowCompactResponse workerid = other557.workerid; start = other557.start; runAs = other557.runAs; + hightestTxnId = other557.hightestTxnId; __isset = other557.__isset; } ShowCompactResponseElement& ShowCompactResponseElement::operator=(const ShowCompactResponseElement& other558) { @@ -12754,6 +12774,7 @@ ShowCompactResponseElement& ShowCompactResponseElement::operator=(const ShowComp workerid = other558.workerid; start = other558.start; runAs = other558.runAs; + hightestTxnId = other558.hightestTxnId; __isset = other558.__isset; return *this; } @@ -12768,6 +12789,7 @@ void ShowCompactResponseElement::printTo(std::ostream& out) const { out << ", " << "workerid="; (__isset.workerid ? (out << to_string(workerid)) : (out << "")); out << ", " << "start="; (__isset.start ? (out << to_string(start)) : (out << "")); out << ", " << "runAs="; (__isset.runAs ? (out << to_string(runAs)) : (out << "")); + out << ", " << "hightestTxnId="; (__isset.hightestTxnId ? (out << to_string(hightestTxnId)) : (out << "")); out << ")"; } diff --git metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 97d3362..c3b8d5c 100644 --- metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -5119,11 +5119,12 @@ inline std::ostream& operator<<(std::ostream& out, const ShowCompactRequest& obj } typedef struct _ShowCompactResponseElement__isset { - _ShowCompactResponseElement__isset() : partitionname(false), workerid(false), start(false), runAs(false) {} + _ShowCompactResponseElement__isset() : partitionname(false), workerid(false), start(false), runAs(false), hightestTxnId(false) {} bool partitionname :1; bool workerid :1; bool start :1; bool runAs :1; + bool hightestTxnId :1; } _ShowCompactResponseElement__isset; class ShowCompactResponseElement { @@ -5131,7 +5132,7 @@ class ShowCompactResponseElement { ShowCompactResponseElement(const ShowCompactResponseElement&); ShowCompactResponseElement& operator=(const ShowCompactResponseElement&); - ShowCompactResponseElement() : dbname(), tablename(), partitionname(), type((CompactionType::type)0), state(), workerid(), start(0), runAs() { + ShowCompactResponseElement() : dbname(), tablename(), partitionname(), type((CompactionType::type)0), state(), workerid(), start(0), runAs(), hightestTxnId(0) { } virtual ~ShowCompactResponseElement() throw(); @@ -5143,6 +5144,7 @@ class ShowCompactResponseElement { std::string workerid; int64_t start; std::string runAs; + int64_t hightestTxnId; _ShowCompactResponseElement__isset __isset; @@ -5162,6 +5164,8 @@ class ShowCompactResponseElement { void __set_runAs(const std::string& val); + void __set_hightestTxnId(const int64_t val); + bool operator == (const ShowCompactResponseElement & rhs) const { if (!(dbname == rhs.dbname)) @@ -5188,6 +5192,10 @@ class ShowCompactResponseElement { return false; else if (__isset.runAs && !(runAs == rhs.runAs)) return false; + if (__isset.hightestTxnId != rhs.__isset.hightestTxnId) + return false; + else if (__isset.hightestTxnId && !(hightestTxnId == rhs.hightestTxnId)) + return false; return true; } bool operator != (const ShowCompactResponseElement &rhs) const { diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java index dd7dd8e..50ef764 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ShowCompactResponseElement.java @@ -46,6 +46,7 @@ private static final org.apache.thrift.protocol.TField WORKERID_FIELD_DESC = new org.apache.thrift.protocol.TField("workerid", org.apache.thrift.protocol.TType.STRING, (short)6); private static final org.apache.thrift.protocol.TField START_FIELD_DESC = new org.apache.thrift.protocol.TField("start", org.apache.thrift.protocol.TType.I64, (short)7); private static final org.apache.thrift.protocol.TField RUN_AS_FIELD_DESC = new org.apache.thrift.protocol.TField("runAs", org.apache.thrift.protocol.TType.STRING, (short)8); + private static final org.apache.thrift.protocol.TField HIGHTEST_TXN_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("hightestTxnId", org.apache.thrift.protocol.TType.I64, (short)9); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -61,6 +62,7 @@ private String workerid; // optional private long start; // optional private String runAs; // optional + private long hightestTxnId; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -75,7 +77,8 @@ STATE((short)5, "state"), WORKERID((short)6, "workerid"), START((short)7, "start"), - RUN_AS((short)8, "runAs"); + RUN_AS((short)8, "runAs"), + HIGHTEST_TXN_ID((short)9, "hightestTxnId"); private static final Map byName = new HashMap(); @@ -106,6 +109,8 @@ public static _Fields findByThriftId(int fieldId) { return START; case 8: // RUN_AS return RUN_AS; + case 9: // HIGHTEST_TXN_ID + return HIGHTEST_TXN_ID; default: return null; } @@ -147,8 +152,9 @@ public String getFieldName() { // isset id assignments private static final int __START_ISSET_ID = 0; + private static final int __HIGHTESTTXNID_ISSET_ID = 1; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.PARTITIONNAME,_Fields.WORKERID,_Fields.START,_Fields.RUN_AS}; + private static final _Fields optionals[] = {_Fields.PARTITIONNAME,_Fields.WORKERID,_Fields.START,_Fields.RUN_AS,_Fields.HIGHTEST_TXN_ID}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -168,6 +174,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); tmpMap.put(_Fields.RUN_AS, new org.apache.thrift.meta_data.FieldMetaData("runAs", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.HIGHTEST_TXN_ID, new org.apache.thrift.meta_data.FieldMetaData("hightestTxnId", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ShowCompactResponseElement.class, metaDataMap); } @@ -215,6 +223,7 @@ public ShowCompactResponseElement(ShowCompactResponseElement other) { if (other.isSetRunAs()) { this.runAs = other.runAs; } + this.hightestTxnId = other.hightestTxnId; } public ShowCompactResponseElement deepCopy() { @@ -232,6 +241,8 @@ public void clear() { setStartIsSet(false); this.start = 0; this.runAs = null; + setHightestTxnIdIsSet(false); + this.hightestTxnId = 0; } public String getDbname() { @@ -425,6 +436,28 @@ public void setRunAsIsSet(boolean value) { } } + public long getHightestTxnId() { + return this.hightestTxnId; + } + + public void setHightestTxnId(long hightestTxnId) { + this.hightestTxnId = hightestTxnId; + setHightestTxnIdIsSet(true); + } + + public void unsetHightestTxnId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __HIGHTESTTXNID_ISSET_ID); + } + + /** Returns true if field hightestTxnId is set (has been assigned a value) and false otherwise */ + public boolean isSetHightestTxnId() { + return EncodingUtils.testBit(__isset_bitfield, __HIGHTESTTXNID_ISSET_ID); + } + + public void setHightestTxnIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HIGHTESTTXNID_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case DBNAME: @@ -491,6 +524,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case HIGHTEST_TXN_ID: + if (value == null) { + unsetHightestTxnId(); + } else { + setHightestTxnId((Long)value); + } + break; + } } @@ -520,6 +561,9 @@ public Object getFieldValue(_Fields field) { case RUN_AS: return getRunAs(); + case HIGHTEST_TXN_ID: + return getHightestTxnId(); + } throw new IllegalStateException(); } @@ -547,6 +591,8 @@ public boolean isSet(_Fields field) { return isSetStart(); case RUN_AS: return isSetRunAs(); + case HIGHTEST_TXN_ID: + return isSetHightestTxnId(); } throw new IllegalStateException(); } @@ -636,6 +682,15 @@ public boolean equals(ShowCompactResponseElement that) { return false; } + boolean this_present_hightestTxnId = true && this.isSetHightestTxnId(); + boolean that_present_hightestTxnId = true && that.isSetHightestTxnId(); + if (this_present_hightestTxnId || that_present_hightestTxnId) { + if (!(this_present_hightestTxnId && that_present_hightestTxnId)) + return false; + if (this.hightestTxnId != that.hightestTxnId) + return false; + } + return true; } @@ -683,6 +738,11 @@ public int hashCode() { if (present_runAs) list.add(runAs); + boolean present_hightestTxnId = true && (isSetHightestTxnId()); + list.add(present_hightestTxnId); + if (present_hightestTxnId) + list.add(hightestTxnId); + return list.hashCode(); } @@ -774,6 +834,16 @@ public int compareTo(ShowCompactResponseElement other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetHightestTxnId()).compareTo(other.isSetHightestTxnId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetHightestTxnId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.hightestTxnId, other.hightestTxnId); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -861,6 +931,12 @@ public String toString() { } first = false; } + if (isSetHightestTxnId()) { + if (!first) sb.append(", "); + sb.append("hightestTxnId:"); + sb.append(this.hightestTxnId); + first = false; + } sb.append(")"); return sb.toString(); } @@ -986,6 +1062,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, ShowCompactResponse org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 9: // HIGHTEST_TXN_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.hightestTxnId = iprot.readI64(); + struct.setHightestTxnIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1045,6 +1129,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, ShowCompactRespons oprot.writeFieldEnd(); } } + if (struct.isSetHightestTxnId()) { + oprot.writeFieldBegin(HIGHTEST_TXN_ID_FIELD_DESC); + oprot.writeI64(struct.hightestTxnId); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1079,7 +1168,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, ShowCompactResponse if (struct.isSetRunAs()) { optionals.set(3); } - oprot.writeBitSet(optionals, 4); + if (struct.isSetHightestTxnId()) { + optionals.set(4); + } + oprot.writeBitSet(optionals, 5); if (struct.isSetPartitionname()) { oprot.writeString(struct.partitionname); } @@ -1092,6 +1184,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, ShowCompactResponse if (struct.isSetRunAs()) { oprot.writeString(struct.runAs); } + if (struct.isSetHightestTxnId()) { + oprot.writeI64(struct.hightestTxnId); + } } @Override @@ -1105,7 +1200,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, ShowCompactResponseE struct.setTypeIsSet(true); struct.state = iprot.readString(); struct.setStateIsSet(true); - BitSet incoming = iprot.readBitSet(4); + BitSet incoming = iprot.readBitSet(5); if (incoming.get(0)) { struct.partitionname = iprot.readString(); struct.setPartitionnameIsSet(true); @@ -1122,6 +1217,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, ShowCompactResponseE struct.runAs = iprot.readString(); struct.setRunAsIsSet(true); } + if (incoming.get(4)) { + struct.hightestTxnId = iprot.readI64(); + struct.setHightestTxnIdIsSet(true); + } } } diff --git metastore/src/gen/thrift/gen-php/metastore/Types.php metastore/src/gen/thrift/gen-php/metastore/Types.php index fe4c966..dbe0a2b 100644 --- metastore/src/gen/thrift/gen-php/metastore/Types.php +++ metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -12499,6 +12499,10 @@ class ShowCompactResponseElement { * @var string */ public $runAs = null; + /** + * @var int + */ + public $hightestTxnId = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -12535,6 +12539,10 @@ class ShowCompactResponseElement { 'var' => 'runAs', 'type' => TType::STRING, ), + 9 => array( + 'var' => 'hightestTxnId', + 'type' => TType::I64, + ), ); } if (is_array($vals)) { @@ -12562,6 +12570,9 @@ class ShowCompactResponseElement { if (isset($vals['runAs'])) { $this->runAs = $vals['runAs']; } + if (isset($vals['hightestTxnId'])) { + $this->hightestTxnId = $vals['hightestTxnId']; + } } } @@ -12640,6 +12651,13 @@ class ShowCompactResponseElement { $xfer += $input->skip($ftype); } break; + case 9: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->hightestTxnId); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -12693,6 +12711,11 @@ class ShowCompactResponseElement { $xfer += $output->writeString($this->runAs); $xfer += $output->writeFieldEnd(); } + if ($this->hightestTxnId !== null) { + $xfer += $output->writeFieldBegin('hightestTxnId', TType::I64, 9); + $xfer += $output->writeI64($this->hightestTxnId); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index fbeab5e..6f20716 100644 --- metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -8734,6 +8734,7 @@ class ShowCompactResponseElement: - workerid - start - runAs + - hightestTxnId """ thrift_spec = ( @@ -8746,9 +8747,10 @@ class ShowCompactResponseElement: (6, TType.STRING, 'workerid', None, None, ), # 6 (7, TType.I64, 'start', None, None, ), # 7 (8, TType.STRING, 'runAs', None, None, ), # 8 + (9, TType.I64, 'hightestTxnId', None, None, ), # 9 ) - def __init__(self, dbname=None, tablename=None, partitionname=None, type=None, state=None, workerid=None, start=None, runAs=None,): + def __init__(self, dbname=None, tablename=None, partitionname=None, type=None, state=None, workerid=None, start=None, runAs=None, hightestTxnId=None,): self.dbname = dbname self.tablename = tablename self.partitionname = partitionname @@ -8757,6 +8759,7 @@ def __init__(self, dbname=None, tablename=None, partitionname=None, type=None, s self.workerid = workerid self.start = start self.runAs = runAs + self.hightestTxnId = hightestTxnId def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -8807,6 +8810,11 @@ def read(self, iprot): self.runAs = iprot.readString() else: iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.hightestTxnId = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -8849,6 +8857,10 @@ def write(self, oprot): oprot.writeFieldBegin('runAs', TType.STRING, 8) oprot.writeString(self.runAs) oprot.writeFieldEnd() + if self.hightestTxnId is not None: + oprot.writeFieldBegin('hightestTxnId', TType.I64, 9) + oprot.writeI64(self.hightestTxnId) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -8874,6 +8886,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.workerid) value = (value * 31) ^ hash(self.start) value = (value * 31) ^ hash(self.runAs) + value = (value * 31) ^ hash(self.hightestTxnId) return value def __repr__(self): diff --git metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index a7b1e86..23c31e6 100644 --- metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -2003,6 +2003,7 @@ class ShowCompactResponseElement WORKERID = 6 START = 7 RUNAS = 8 + HIGHTESTTXNID = 9 FIELDS = { DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'}, @@ -2012,7 +2013,8 @@ class ShowCompactResponseElement STATE => {:type => ::Thrift::Types::STRING, :name => 'state'}, WORKERID => {:type => ::Thrift::Types::STRING, :name => 'workerid', :optional => true}, START => {:type => ::Thrift::Types::I64, :name => 'start', :optional => true}, - RUNAS => {:type => ::Thrift::Types::STRING, :name => 'runAs', :optional => true} + RUNAS => {:type => ::Thrift::Types::STRING, :name => 'runAs', :optional => true}, + HIGHTESTTXNID => {:type => ::Thrift::Types::I64, :name => 'hightestTxnId', :optional => true} } def struct_fields; FIELDS; end