From aa8d28f70deb5d415287da24913ed1a0ac3a8126 Mon Sep 17 00:00:00 2001 From: Deepesh Khandelwal Date: Tue, 23 Sep 2014 14:58:09 -0700 Subject: [PATCH] HIVE-8239: MSSQL upgrade schema scripts does not map Java long datatype columns correctly for transaction related tables --- .../scripts/upgrade/mssql/003-HIVE-8239.mssql.sql | 103 +++++++++++++++++++++ .../upgrade/mssql/hive-schema-0.14.0.mssql.sql | 30 +++--- .../mssql/upgrade-0.13.0-to-0.14.0.mssql.sql | 1 + 3 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 metastore/scripts/upgrade/mssql/003-HIVE-8239.mssql.sql diff --git a/metastore/scripts/upgrade/mssql/003-HIVE-8239.mssql.sql b/metastore/scripts/upgrade/mssql/003-HIVE-8239.mssql.sql new file mode 100644 index 0000000..0f850e2 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/003-HIVE-8239.mssql.sql @@ -0,0 +1,103 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Drop the primary key constraint on table COMPACTION_QUEUE +DECLARE @sqlcmd NVARCHAR(MAX) + +SELECT @sqlcmd = 'ALTER TABLE COMPACTION_QUEUE DROP CONSTRAINT ' + name + ';' + FROM sys.key_constraints + WHERE [type] = 'PK' + AND [parent_object_id] = OBJECT_ID('COMPACTION_QUEUE') + +EXECUTE (@sqlcmd); + +ALTER TABLE COMPACTION_QUEUE ALTER COLUMN CQ_ID bigint NOT NULL; + +-- Restore the primary key constraint on table COMPACTION_QUEUE +ALTER TABLE COMPACTION_QUEUE ADD CONSTRAINT PK_COMPACTION_CQID PRIMARY KEY CLUSTERED (CQ_ID ASC); + +ALTER TABLE COMPACTION_QUEUE ALTER COLUMN CQ_START bigint NULL; + + +ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_TXNID bigint NULL; + + +-- Drop the primary key constraint on table HIVE_LOCKS +DECLARE @sqlcmd NVARCHAR(MAX) + +SELECT @sqlcmd = 'ALTER TABLE HIVE_LOCKS DROP CONSTRAINT ' + name + ';' + FROM sys.key_constraints + WHERE [type] = 'PK' + AND [parent_object_id] = OBJECT_ID('HIVE_LOCKS') + +EXECUTE (@sqlcmd); + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_LOCK_EXT_ID bigint NOT NULL; + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_LOCK_INT_ID bigint NOT NULL; + +-- Restore the composite primary key constraint on table HIVE_LOCKS +ALTER TABLE HIVE_LOCKS ADD CONSTRAINT PK_HL_LOCKEXTID_LOCKINTID + PRIMARY KEY CLUSTERED (HL_LOCK_EXT_ID ASC, HL_LOCK_INT_ID ASC); + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_TXNID bigint NULL; + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_LAST_HEARTBEAT bigint NOT NULL; + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_ACQUIRED_AT bigint NULL; + + +ALTER TABLE NEXT_COMPACTION_QUEUE_ID ALTER COLUMN NCQ_NEXT bigint NOT NULL; + + +ALTER TABLE NEXT_LOCK_ID ALTER COLUMN NL_NEXT bigint NOT NULL; + + +ALTER TABLE NEXT_TXN_ID ALTER COLUMN NTXN_NEXT bigint NOT NULL; + + +-- Drop the foreign key constraint on table TXN_COMPONENTS, this is required +-- before we drop the primary key constraint on table TXNS +DECLARE @sqlcmd NVARCHAR(MAX) + +SELECT @sqlcmd = 'ALTER TABLE TXN_COMPONENTS DROP CONSTRAINT ' + name + ';' + FROM sys.foreign_keys + WHERE [parent_object_id] = OBJECT_ID('TXN_COMPONENTS') + +EXECUTE (@sqlcmd); + +-- Drop the primary key constraint on table TXNS +DECLARE @sqlcmd NVARCHAR(MAX) + +SELECT @sqlcmd = 'ALTER TABLE TXNS DROP CONSTRAINT ' + name + ';' + FROM sys.key_constraints + WHERE [type] = 'PK' + AND [parent_object_id] = OBJECT_ID('TXNS') + +EXECUTE (@sqlcmd); + +ALTER TABLE TXNS ALTER COLUMN TXN_ID bigint NOT NULL; + +-- Restore the primary key constraint on table TXNS +ALTER TABLE TXNS ADD CONSTRAINT PK_TXNS_TXNID PRIMARY KEY CLUSTERED (TXN_ID ASC); + +ALTER TABLE TXNS ALTER COLUMN TXN_STARTED bigint NOT NULL; + +ALTER TABLE TXNS ALTER COLUMN TXN_LAST_HEARTBEAT bigint NOT NULL; + +ALTER TABLE TXN_COMPONENTS ALTER COLUMN TC_TXNID bigint NULL; + +-- Restore the foreign key constraint on table TXN_COMPONENTS +ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); diff --git a/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql index a9f7b83..174ed39 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql @@ -836,14 +836,14 @@ CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); -- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. -- ----------------------------------------------------------------------------------------------------------------------------------------------- CREATE TABLE COMPACTION_QUEUE( - CQ_ID int NOT NULL, + CQ_ID bigint NOT NULL, CQ_DATABASE varchar(128) NOT NULL, CQ_TABLE varchar(128) NOT NULL, CQ_PARTITION varchar(767) NULL, CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, CQ_WORKER_ID varchar(128) NULL, - CQ_START int NULL, + CQ_START bigint NULL, CQ_RUN_AS varchar(128) NULL, PRIMARY KEY CLUSTERED ( @@ -852,23 +852,23 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE COMPLETED_TXN_COMPONENTS( - CTC_TXNID int NULL, + CTC_TXNID bigint NULL, CTC_DATABASE varchar(128) NOT NULL, CTC_TABLE varchar(128) NULL, CTC_PARTITION varchar(767) NULL ); CREATE TABLE HIVE_LOCKS( - HL_LOCK_EXT_ID int NOT NULL, - HL_LOCK_INT_ID int NOT NULL, - HL_TXNID int NULL, + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128) NULL, HL_PARTITION varchar(767) NULL, HL_LOCK_STATE char(1) NOT NULL, HL_LOCK_TYPE char(1) NOT NULL, - HL_LAST_HEARTBEAT int NOT NULL, - HL_ACQUIRED_AT int NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint NULL, HL_USER varchar(128) NOT NULL, HL_HOST varchar(128) NOT NULL, PRIMARY KEY CLUSTERED @@ -879,28 +879,28 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE NEXT_COMPACTION_QUEUE_ID( - NCQ_NEXT int NOT NULL + NCQ_NEXT bigint NOT NULL ); INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); CREATE TABLE NEXT_LOCK_ID( - NL_NEXT int NOT NULL + NL_NEXT bigint NOT NULL ); INSERT INTO NEXT_LOCK_ID VALUES(1); CREATE TABLE NEXT_TXN_ID( - NTXN_NEXT int NOT NULL + NTXN_NEXT bigint NOT NULL ); INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE TXNS( - TXN_ID int NOT NULL, + TXN_ID bigint NOT NULL, TXN_STATE char(1) NOT NULL, - TXN_STARTED int NOT NULL, - TXN_LAST_HEARTBEAT int NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, TXN_USER varchar(128) NOT NULL, TXN_HOST varchar(128) NOT NULL, PRIMARY KEY CLUSTERED @@ -910,7 +910,7 @@ PRIMARY KEY CLUSTERED ); CREATE TABLE TXN_COMPONENTS( - TC_TXNID int NULL, + TC_TXNID bigint NULL, TC_DATABASE varchar(128) NOT NULL, TC_TABLE varchar(128) NULL, TC_PARTITION varchar(767) NULL diff --git a/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql index 2b5f3b8..1bda6d5 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql @@ -1,6 +1,7 @@ SELECT 'Upgrading MetaStore schema from 0.13.0 to 0.14.0' AS MESSAGE; :r 002-HIVE-7784.mssql.sql; +:r 003-HIVE-8239.mssql.sql; UPDATE VERSION SET SCHEMA_VERSION='0.14.0', VERSION_COMMENT='Hive release version 0.14.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 0.13.0 to 0.14.0' AS MESSAGE; -- 1.8.5.2 (Apple Git-48)