commit 76941312c660cd32a08cb87e54d29f787f9caea0 Author: Vihang Karajgaonkar Date: Mon May 1 14:39:22 2017 -0700 HIVE-16556 : Modify schematool scripts to initialize and create METASTORE_DB_PROPERTIES table diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 7dd4d5f0a25ce4306b85facb87b0c02f59d29a16..3be63ba7028cde7e25683086ba5b597361eef8c4 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -952,8 +952,12 @@ private void runBeeLine(String scriptDir, String scriptFile) runBeeLine(tmpFile.getPath()); } - // Generate the beeline args per hive conf and execute the given script public void runBeeLine(String sqlScriptFile) throws IOException { + runBeeLine(sqlScriptFile, (PrintStream) null); + } + + // Generate the beeline args per hive conf and execute the given script + void runBeeLine(String sqlScriptFile, PrintStream errStream) throws IOException { CommandBuilder builder = new CommandBuilder(hiveConf, userName, passWord, sqlScriptFile); // run the script using Beeline @@ -961,6 +965,8 @@ public void runBeeLine(String sqlScriptFile) throws IOException { if (!verbose) { beeLine.setOutputStream(new PrintStream(new NullOutputStream())); beeLine.getOpts().setSilent(true); + } else if(errStream != null) { + beeLine.setErrorStream(errStream); } beeLine.getOpts().setAllowMultiLineCommand(false); beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED"); diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 604c2345fcf04c877a8d257b9eef4610a58aea51..ebb002b79f0ed8ae369f929d567f77ce22293cc9 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -663,6 +663,17 @@ public void testValidateLocations() throws Exception { assertFalse(isValid); } + public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException { + schemaTool.doInit("3.0.0"); + validateMetastoreDbPropertiesTable(); + } + + public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException { + schemaTool.doInit("2.0.0"); + schemaTool.doUpgrade(); + validateMetastoreDbPropertiesTable(); + } + private File generateTestScript(String [] stmts) throws IOException { File testScriptFile = File.createTempFile("schematest", ".sql"); testScriptFile.deleteOnExit(); @@ -676,6 +687,22 @@ private File generateTestScript(String [] stmts) throws IOException { return testScriptFile; } + private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException { + boolean isValid = (boolean) schemaTool.validateSchemaTables(conn); + assertTrue(isValid); + // adding same property key twice should throw unique key constraint violation exception + String[] scripts = new String[] { + "insert into METASTORE_DB_PROPERTIES values (1, 'guid', 'test-uuid-1', 'dummy uuid 1')", + "insert into METASTORE_DB_PROPERTIES values (2, 'guid', 'test-uuid-2', 'dummy uuid 2')", }; + File scriptFile = generateTestScript(scripts); + Exception ex = null; + try { + schemaTool.runBeeLine(scriptFile.getPath(), errStream); + } catch (Exception iox) { + ex = iox; + } + assertTrue(ex != null && ex instanceof IOException); + } /** * Write out a dummy pre-upgrade script with given SQL statement. */ diff --git metastore/scripts/upgrade/derby/041-HIVE-16556.derby.sql metastore/scripts/upgrade/derby/041-HIVE-16556.derby.sql new file mode 100644 index 0000000000000000000000000000000000000000..69e0d41b9b73836aab67a856bd0a5361fa35f36b --- /dev/null +++ metastore/scripts/upgrade/derby/041-HIVE-16556.derby.sql @@ -0,0 +1,5 @@ +CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_ID" BIGINT NOT NULL, "PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESC" VARCHAR(1000)); + +ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "UNIQUE_PROPERTY_KEY" UNIQUE ("PROPERTY_KEY"); + +ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "PROPERTY_ID_PK" PRIMARY KEY ("PROPERTY_ID"); diff --git metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql index 8877681e05f9d9ecb3bec5d8579772b386dc7f08..ec9dcb64d0249660e14a560a0dcb03cb588c4954 100644 --- metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql @@ -106,6 +106,8 @@ CREATE TABLE "APP"."NOTIFICATION_LOG" ("NL_ID" BIGINT NOT NULL, "DB_NAME" VARCHA CREATE TABLE "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID" BIGINT NOT NULL, "NEXT_EVENT_ID" BIGINT NOT NULL); +CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_ID" BIGINT NOT NULL, "PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESC" VARCHAR(1000)); + CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT, "CHILD_INTEGER_IDX" INTEGER NOT NULL, "CHILD_TBL_ID" BIGINT, "PARENT_CD_ID" BIGINT NOT NULL, "PARENT_INTEGER_IDX" INTEGER, "PARENT_TBL_ID" BIGINT NOT NULL, "POSITION" BIGINT NOT NULL, "CONSTRAINT_NAME" VARCHAR(400) NOT NULL, "CONSTRAINT_TYPE" SMALLINT NOT NULL, "UPDATE_RULE" SMALLINT, "DELETE_RULE" SMALLINT, "ENABLE_VALIDATE_RELY" SMALLINT NOT NULL); ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY ("CONSTRAINT_NAME", "POSITION"); @@ -241,6 +243,10 @@ ALTER TABLE "APP"."NOTIFICATION_LOG" ADD CONSTRAINT "NOTIFICATION_LOG_PK" PRIMAR ALTER TABLE "APP"."NOTIFICATION_SEQUENCE" ADD CONSTRAINT "NOTIFICATION_SEQUENCE_PK" PRIMARY KEY ("NNI_ID"); +ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "PROPERTY_ID_PK" PRIMARY KEY ("PROPERTY_ID"); + +ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "UNIQUE_PROPERTY_KEY" UNIQUE ("PROPERTY_KEY"); + -- foreign ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; diff --git metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 3bba5235eb630e4ee5d14bf7f163dfb7166df0ed..455651f76b73e738af54cee4fc768766d17fd9a9 100644 --- metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -1,3 +1,5 @@ -- Upgrade MetaStore schema from 2.3.0 to 3.0.0 +RUN '041-HIVE-16556.derby.sql'; + UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git metastore/scripts/upgrade/mssql/026-HIVE-16556.mssql.sql metastore/scripts/upgrade/mssql/026-HIVE-16556.mssql.sql new file mode 100644 index 0000000000000000000000000000000000000000..2f669c1ac601d9f773d80bf356f62295408f2909 --- /dev/null +++ metastore/scripts/upgrade/mssql/026-HIVE-16556.mssql.sql @@ -0,0 +1,10 @@ +CREATE TABLE METASTORE_DB_PROPERTIES ( + PROPERTY_ID BIGINT NOT NULL, + PROPERTY_KEY VARCHAR(255) NOT NULL, + PROPERTY_VALUE VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT METASTORE_DB_PROPERTIES_PK PRIMARY KEY (PROPERTY_ID); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT UNIQUE_PROPERTY_KEY UNIQUE (PROPERTY_KEY); diff --git metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql index 98682a83760f89aec441af3f6cb5ff03f5251496..38c94ee5bad76476cb45924540d0d5d8a3774664 100644 --- metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql +++ metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql @@ -1016,6 +1016,16 @@ CREATE TABLE WRITE_SET ( WS_OPERATION_TYPE char(1) NOT NULL ); +CREATE TABLE METASTORE_DB_PROPERTIES ( + PROPERTY_ID BIGINT NOT NULL, + PROPERTY_KEY VARCHAR(255) NOT NULL, + PROPERTY_VALUE VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT METASTORE_DB_PROPERTIES_PK PRIMARY KEY (PROPERTY_ID); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT UNIQUE_PROPERTY_KEY UNIQUE (PROPERTY_KEY); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index 94d18a34d37dea44a684242cbe1bdc5c3784d2e6..e579dac43a9fb02400171e00ce363639379abe56 100644 --- metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -1,4 +1,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; +:r 026-HIVE-16556.mssql.sql + UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git metastore/scripts/upgrade/mysql/041-HIVE-16556.mysql.sql metastore/scripts/upgrade/mysql/041-HIVE-16556.mysql.sql new file mode 100644 index 0000000000000000000000000000000000000000..fe29aa9b97ecac17327b260123462c55158341c5 --- /dev/null +++ metastore/scripts/upgrade/mysql/041-HIVE-16556.mysql.sql @@ -0,0 +1,11 @@ +-- +-- Table structure for table METASTORE_DB_PROPERTIES +-- +CREATE TABLE IF NOT EXISTS `METASTORE_DB_PROPERTIES` ( + `PROPERTY_ID` BIGINT(20) NOT NULL, + `PROPERTY_KEY` varchar(255) NOT NULL, + `PROPERTY_VALUE` varchar(1000) NOT NULL, + `DESC` varchar(1000), + PRIMARY KEY(`PROPERTY_ID`), + UNIQUE KEY `UNIQUE_PROPERTY_KEY` (`PROPERTY_KEY`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql index 4db2f34159d6650fab4179d1478c6c91c72b5351..9458605276ce551589363b5bade452e968e1acd6 100644 --- metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql @@ -829,6 +829,18 @@ CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` CREATE INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX` ON KEY_CONSTRAINTS (`PARENT_TBL_ID`) USING BTREE; +-- ----------------------------- +-- Metastore DB Properties table +-- ----------------------------- +CREATE TABLE IF NOT EXISTS `METASTORE_DB_PROPERTIES` ( + `PROPERTY_ID` BIGINT(20) NOT NULL, + `PROPERTY_KEY` varchar(255) NOT NULL, + `PROPERTY_VALUE` varchar(1000) NOT NULL, + `DESC` varchar(1000), + PRIMARY KEY(`PROPERTY_ID`), + UNIQUE KEY `UNIQUE_PROPERTY_KEY` (`PROPERTY_KEY`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- diff --git metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index e5d82e10160c2eab8571b3cd8f3078fbebcf3c5c..288438790b574d48f5754744c28126a9b450ce89 100644 --- metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -1,5 +1,7 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; +SOURCE 041-HIVE-16556.mysql.sql; + UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git metastore/scripts/upgrade/oracle/041-HIVE-16556.oracle.sql metastore/scripts/upgrade/oracle/041-HIVE-16556.oracle.sql new file mode 100644 index 0000000000000000000000000000000000000000..eb02572e9609bfd99000af666404d18d1f3e7812 --- /dev/null +++ metastore/scripts/upgrade/oracle/041-HIVE-16556.oracle.sql @@ -0,0 +1,11 @@ +CREATE TABLE METASTORE_DB_PROPERTIES +( + PROPERTY_ID NUMBER NOT NULL, + PROPERTY_KEY VARCHAR(255) NOT NULL, + PROPERTY_VALUE VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT UNIQUE_PROPERTY_KEY UNIQUE (PROPERTY_KEY); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT PROPERTY_ID_PK PRIMARY KEY (PROPERTY_ID); diff --git metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql index 7907c59f35a4575a614fdfd0419486156130ab23..5ad9544ddebcc7b891703c0fbca8f13c31abb60f 100644 --- metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql @@ -799,7 +799,18 @@ ALTER TABLE KEY_CONSTRAINTS ADD CONSTRAINT CONSTRAINTS_PK PRIMARY KEY (CONSTRAIN CREATE INDEX CONSTRAINTS_PT_INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID); +-- Table for METASTORE_DB_PROPERTIES and its constraints +CREATE TABLE METASTORE_DB_PROPERTIES +( + PROPERTY_ID NUMBER NOT NULL, + PROPERTY_KEY VARCHAR(255) NOT NULL, + PROPERTY_VALUE VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT UNIQUE_PROPERTY_KEY UNIQUE (PROPERTY_KEY); +ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT PROPERTY_ID_PK PRIMARY KEY (PROPERTY_ID); ------------------------------ -- Transaction and lock tables ------------------------------ diff --git metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 31c4f5dfbd94fd52317ad0cd65517e09925767f7..a15f2884d18e332e6a2714555f782ddd7c769e3c 100644 --- metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -1,4 +1,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; +@041-HIVE-16556.oracle.sql; + UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; diff --git metastore/scripts/upgrade/postgres/040-HIVE-16556.postgres.sql metastore/scripts/upgrade/postgres/040-HIVE-16556.postgres.sql new file mode 100644 index 0000000000000000000000000000000000000000..4eec3d41f8ddd39c5caa5a3407bee5f76a756575 --- /dev/null +++ metastore/scripts/upgrade/postgres/040-HIVE-16556.postgres.sql @@ -0,0 +1,13 @@ +CREATE TABLE "METASTORE_DB_PROPERTIES" +( + "PROPERTY_ID" BIGINT NOT NULL, + "PROPERTY_KEY" VARCHAR(255) NOT NULL, + "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + +ALTER TABLE ONLY "METASTORE_DB_PROPERTIES" + ADD CONSTRAINT "UNIQUE_PROPERTY_KEY" UNIQUE ("PROPERTY_KEY"); + +ALTER TABLE ONLY "METASTORE_DB_PROPERTIES" + ADD CONSTRAINT "METASTORE_DB_PROPERTIES_pkey" PRIMARY KEY ("PROPERTY_ID"); diff --git metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql index 49976d070d8a7ac825bb3920fa01f7d328aada9d..62b3d815f111ef589fb8e7dd93c334a6994b759a 100644 --- metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql @@ -606,6 +606,14 @@ CREATE TABLE "KEY_CONSTRAINTS" CREATE INDEX "CONSTRAINTS_PARENT_TBLID_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("PARENT_TBL_ID"); +CREATE TABLE "METASTORE_DB_PROPERTIES" +( + "PROPERTY_ID" BIGINT NOT NULL, + "PROPERTY_KEY" VARCHAR(255) NOT NULL, + "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, + "DESC" VARCHAR(1000) +); + -- -- Name: BUCKETING_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: -- @@ -950,7 +958,11 @@ ALTER TABLE ONLY "TYPES" ALTER TABLE ONLY "ROLE_MAP" ADD CONSTRAINT "USERROLEMAPINDEX" UNIQUE ("PRINCIPAL_NAME", "ROLE_ID", "GRANTOR", "GRANTOR_TYPE"); +ALTER TABLE ONLY "METASTORE_DB_PROPERTIES" + ADD CONSTRAINT "UNIQUE_PROPERTY_KEY" UNIQUE ("PROPERTY_KEY"); +ALTER TABLE ONLY "METASTORE_DB_PROPERTIES" + ADD CONSTRAINT "METASTORE_DB_PROPERTIES_pkey" PRIMARY KEY ("PROPERTY_ID"); -- -- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: -- diff --git metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index 2dd9bb9bfb607529f3ab9be4dccab34f26375e4f..4d8f0cde1a54769c320a3491f6fb44456abf5763 100644 --- metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -1,5 +1,7 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0'; +\i 040-HIVE-16556.postgres.sql; + UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0';