diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 22149e4..4ddd484 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -322,6 +322,8 @@ METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false), METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms"), METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", true), + METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", false), + METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false), METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked"), METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed"), METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false), @@ -1140,6 +1142,11 @@ private void initialize(Class cls) { auxJars = this.get(ConfVars.HIVEAUXJARS.varname); } + if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) { + setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false); + setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true); + } + // setup list of conf vars that are not allowed to change runtime String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString()); if (restrictListStr != null) { diff --git conf/hive-default.xml.template conf/hive-default.xml.template index 9a3fc1d..4046b0e 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1976,4 +1976,17 @@ + + hive.metastore.schema.verification + true + + Enforce metastore schema version consistency. + True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic + schema migration attempt. Users are required to manully migrate schema after Hive upgrade which ensures + proper metastore schema migration. (Default) + False: Warn if the version information stored in metastore doesn't match with one from in Hive jars. + + + + diff --git metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql new file mode 100644 index 0000000..4e08fc1 --- /dev/null +++ metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql @@ -0,0 +1,8 @@ +-- +-- Hive HIVE-3764 +-- Support metastore version consistency check + +CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "COMMENT" VARCHAR(255)); +ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID"); + +INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value'); diff --git metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql index cce544f..8f61aeb 100644 --- metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql @@ -98,6 +98,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL); +CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "COMMENT" VARCHAR(255)); + -- ---------------------------------------------- -- DDL Statements for indexes -- ---------------------------------------------- @@ -284,6 +286,8 @@ ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_FK" FOREIGN KEY ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID"); + -- ---------------------------------------------- -- DDL Statements for checks -- ---------------------------------------------- @@ -292,3 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "SQL110318025504980" CHECK (DEFERRED_REB ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N')); +INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0'); diff --git metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql index cae7936..aaef099 100644 --- metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql +++ metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql @@ -1,2 +1 @@ -- Upgrade MetaStore schema from 0.10.0 to 0.11.0 - diff --git metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql index 492cc93..f256c60 100644 --- metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql +++ metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql @@ -1,2 +1,4 @@ -- Upgrade MetaStore schema from 0.11.0 to 0.12.0 RUN '013-HIVE-3255.derby.sql'; +RUN '014-HIVE-3764.derby.sql'; +UPDATE "APP".VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1; diff --git metastore/scripts/upgrade/derby/upgrade.order.derby metastore/scripts/upgrade/derby/upgrade.order.derby new file mode 100644 index 0000000..59f02b9 --- /dev/null +++ metastore/scripts/upgrade/derby/upgrade.order.derby @@ -0,0 +1,7 @@ +0.5.0-to-0.6.0 +0.6.0-to-0.7.0 +0.7.0-to-0.8.0 +0.8.0-to-0.9.0 +0.9.0-to-0.10.0 +0.10.0-to-0.11.0 +0.11.0-to-0.12.0 diff --git metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql new file mode 100644 index 0000000..08c73f6 --- /dev/null +++ metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql @@ -0,0 +1,9 @@ +-- Table structure for VERSION +CREATE TABLE IF NOT EXISTS `VERSION` ( + `VER_ID` BIGINT NOT NULL, + `SCHEMA_VERSION` VARCHAR(127) NOT NULL, + `COMMENT` VARCHAR(255), + PRIMARY KEY (`VER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value'); diff --git metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql index 22a77fe..1daafb9 100644 --- metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql @@ -751,6 +751,17 @@ CREATE TABLE IF NOT EXISTS `DELEGATION_TOKENS` PRIMARY KEY (`TOKEN_IDENT`) ) ENGINE=INNODB DEFAULT CHARSET=latin1; +-- +-- Table structure for VERSION +-- +CREATE TABLE IF NOT EXISTS `VERSION` ( + `VER_ID` BIGINT NOT NULL, + `SCHEMA_VERSION` VARCHAR(127) NOT NULL, + `COMMENT` VARCHAR(255), + PRIMARY KEY (`VER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0'); /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; diff --git metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql index 375a05f..afa4912 100644 --- metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql +++ metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql @@ -1,3 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' '; SOURCE 013-HIVE-3255.mysql.sql; +SOURCE 014-HIVE-3764.mysql.sql; +UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' '; diff --git metastore/scripts/upgrade/mysql/upgrade.order.mysql metastore/scripts/upgrade/mysql/upgrade.order.mysql new file mode 100644 index 0000000..59f02b9 --- /dev/null +++ metastore/scripts/upgrade/mysql/upgrade.order.mysql @@ -0,0 +1,7 @@ +0.5.0-to-0.6.0 +0.6.0-to-0.7.0 +0.7.0-to-0.8.0 +0.8.0-to-0.9.0 +0.9.0-to-0.10.0 +0.10.0-to-0.11.0 +0.11.0-to-0.12.0 diff --git metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql new file mode 100644 index 0000000..7e8530d --- /dev/null +++ metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql @@ -0,0 +1,10 @@ +-- HIVE-3764 Support metastore version consistency check + +CREATE TABLE IF NOT EXISTS VERSION ( + VER_ID NUMBER NOT NULL, + SCHEMA_VERSION VARCHAR(127) NOT NULL, + COMMENT VARCHAR(255) +) +ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); + +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value'); diff --git metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql index 85a0178..6e191a1 100644 --- metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql @@ -483,6 +483,13 @@ CREATE TABLE TAB_COL_STATS ( LAST_ANALYZED NUMBER NOT NULL ); +CREATE TABLE IF NOT EXISTS VERSION ( + VER_ID NUMBER NOT NULL, + SCHEMA_VERSION VARCHAR(127) NOT NULL, + COMMENT VARCHAR(255) +) +ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); + ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; @@ -707,4 +714,5 @@ CREATE INDEX TBLS_N50 ON TBLS (SD_ID); -- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent] CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME); +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0'); diff --git metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql new file mode 100644 index 0000000..6cd6522 --- /dev/null +++ metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql @@ -0,0 +1,2 @@ +SELECT 'Upgrading MetaStore schema from 0.10.0 to 0.11.0' AS ' '; +SELECT 'Finished upgrading MetaStore schema from 0.10.0 to 0.11.0' AS Status from dual; diff --git metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql index a2d0901..e2df6ae 100644 --- metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql @@ -1,3 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' '; @013-HIVE-3255.oracle.sql; +@014-HIVE-3764.oracle.sql; +UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual; diff --git metastore/scripts/upgrade/oracle/upgrade.order.oracle metastore/scripts/upgrade/oracle/upgrade.order.oracle new file mode 100644 index 0000000..bcba11f --- /dev/null +++ metastore/scripts/upgrade/oracle/upgrade.order.oracle @@ -0,0 +1,3 @@ +0.9.0-to-0.10.0 +0.10.0-to-0.11.0 +0.11.0-to-0.12.0 diff --git metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql new file mode 100644 index 0000000..a6f1537 --- /dev/null +++ metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql @@ -0,0 +1,12 @@ +-- +-- Table structure for VERSION +-- +CREATE TABLE "VERSION" ( + "VER_ID" bigint, + "SCHEMA_VERSION" character varying(127) NOT NULL, + "COMMENT" character varying(255) NOT NULL, + PRIMARY KEY ("VER_ID") +); +ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID"); + +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value'); diff --git metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql index 7b319ba..b31a8d1 100644 --- metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql @@ -516,6 +516,16 @@ CREATE TABLE "TAB_COL_STATS" ( ); -- +-- Table structure for VERSION +-- +CREATE TABLE "VERSION" ( + "VER_ID" bigint, + "SCHEMA_VERSION" character varying(127) NOT NULL, + "COMMENT" character varying(255) NOT NULL, + PRIMARY KEY ("VER_ID") +); + +-- -- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: -- @@ -1379,6 +1389,9 @@ ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_fkey" FOREIGN KEY -- ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + +ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID"); + -- -- Name: public; Type: ACL; Schema: -; Owner: hiveuser -- @@ -1387,6 +1400,7 @@ REVOKE ALL ON SCHEMA public FROM PUBLIC; GRANT ALL ON SCHEMA public TO PUBLIC; +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0'); -- -- PostgreSQL database dump complete -- diff --git metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql index 9da0a1b..ad46939 100644 --- metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql +++ metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql @@ -1,3 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0'; \i 013-HIVE-3255.postgres.sql; +\i 014-HIVE-3764.postgres.sql; +UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0'; diff --git metastore/scripts/upgrade/postgres/upgrade.order.postgres metastore/scripts/upgrade/postgres/upgrade.order.postgres new file mode 100644 index 0000000..59f02b9 --- /dev/null +++ metastore/scripts/upgrade/postgres/upgrade.order.postgres @@ -0,0 +1,7 @@ +0.5.0-to-0.6.0 +0.6.0-to-0.7.0 +0.7.0-to-0.8.0 +0.8.0-to-0.9.0 +0.9.0-to-0.10.0 +0.10.0-to-0.11.0 +0.11.0-to-0.12.0 diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 39dda92..e7104db 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -400,6 +400,7 @@ public RawStore getMS() throws MetaException { RawStore ms = threadLocalMS.get(); if (ms == null) { ms = newRawStore(); + ms.verifySchema(); threadLocalMS.set(ms); ms = threadLocalMS.get(); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java new file mode 100644 index 0000000..3a40f04 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.common.util.HiveVersionInfo; + + +public class MetaStoreSchemaInfo { + private static String SQL_FILE_EXTENSION=".sql"; + private static String UPGRADE_FILE_PREFIX="upgrade-"; + private static String INIT_FILE_PREFIX="hive-schema-"; + private static String VERSION_UPGRADE_LIST = "upgrade.order"; + + private final String dbType; + private final String hiveVersions[]; + private final HiveConf hiveConf; + private final String hiveHome; + + public MetaStoreSchemaInfo(String hiveHome, HiveConf hiveConf, String dbType) throws HiveMetaException { + this.hiveHome = hiveHome; + this.dbType = dbType; + this.hiveConf = hiveConf; + hiveConf.getJar(); + // load upgrade overder for the given dbType + List upgradeOrderList = new ArrayList(); + String upgradeListFile = getMetaStoreScriptDir() + File.separator + + VERSION_UPGRADE_LIST + "." + dbType; + try { + BufferedReader bfReader = + new BufferedReader(new FileReader(upgradeListFile)); + String currVersion; + while ((currVersion = bfReader.readLine()) != null) { + upgradeOrderList.add(currVersion); + } + } catch (FileNotFoundException e) { + throw new HiveMetaException("File " + upgradeListFile + "not found ", e); + } catch (IOException e) { + throw new HiveMetaException("Error reading " + upgradeListFile, e); + } + hiveVersions = upgradeOrderList.toArray(new String[0]); + } + + /*** + * Get the list of sql scripts required to upgrade from the give version to current + * @param fromVersion + * @return + * @throws HiveMetaException + */ + public List getUpgradeScripts(String fromVersion) + throws HiveMetaException { + List upgradeScriptList = new ArrayList(); + + // check if we are already at current schema level + if (getHiveSchemaVersion().equals(fromVersion)) { + return upgradeScriptList; + } + // Find the list of scripts to execute for this upgrade + int firstScript = hiveVersions.length; + for (int i=0; i < hiveVersions.length; i++) { + if (hiveVersions[i].startsWith(fromVersion)) { + firstScript = i; + } + } + if (firstScript == hiveVersions.length) { + throw new HiveMetaException("Unknown version specified for upgrade " + + fromVersion + " Metastore schema may be too old or newer"); + } + + for (int i=firstScript; i < hiveVersions.length; i++) { + String scriptFile = generateUpgradeFileName(hiveVersions[i]); + upgradeScriptList.add(scriptFile); + } + return upgradeScriptList; + } + + /*** + * Get the name of the script to initialize the schema for given version + * @param toVersion Target version. If it's null, then the current server version is used + * @return + * @throws HiveMetaException + */ + public String generateInitFileName(String toVersion) throws HiveMetaException { + if (toVersion == null) { + toVersion = getHiveSchemaVersion(); + } + String initScriptName = INIT_FILE_PREFIX + toVersion + "." + + dbType + SQL_FILE_EXTENSION; + // check if the file exists + if (!(new File(getMetaStoreScriptDir() + File.separatorChar + + initScriptName).exists())) { + throw new HiveMetaException("Unknown version specified for initialization: " + toVersion); + } + return initScriptName; + } + + /** + * Find the directory of metastore scripts + * @return + */ + public String getMetaStoreScriptDir() { + return hiveHome + File.separatorChar + + "scripts" + File.separatorChar + "metastore" + + File.separatorChar + "upgrade" + File.separatorChar + dbType; + } + + // format the upgrade script name eg upgrade-x-y-dbType.sql + private String generateUpgradeFileName(String fileVersion) { + return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION; + } + + // Current hive version, remove the 'SNAPSHOT' part if needed + public static String getHiveSchemaVersion() { + return HiveVersionInfo.getVersion().replace("-SNAPSHOT", ""); + } + +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index a27243d..6ce016b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; -import static org.apache.commons.lang.StringUtils.repeat; import java.net.URI; import java.net.URISyntaxException; @@ -34,10 +33,11 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.jdo.JDODataStoreException; import javax.jdo.JDOHelper; import javax.jdo.JDOObjectNotFoundException; import javax.jdo.PersistenceManager; @@ -119,10 +119,12 @@ import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; +import org.apache.hadoop.hive.metastore.model.MVersionTable; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; import org.apache.hadoop.util.StringUtils; +import org.datanucleus.store.rdbms.exceptions.MissingTableException; /** * This class is the interface between the application logic and the database @@ -163,6 +165,7 @@ int openTrasactionCalls = 0; private Transaction currentTransaction = null; private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; + private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false); public ObjectStore() { } @@ -5609,4 +5612,120 @@ public boolean removeMasterKey(Integer id) { return masterKeys; } + // compare hive version and metastore version + @Override + public void verifySchema() throws MetaException { + // If the schema version is already checked, then go ahead and use this metastore + if (isSchemaVerified.get()) { + return; + } + checkSchema(); + } + + private synchronized void checkSchema() throws MetaException { + if (isSchemaVerified.get()) { + // recheck if it got verified + return; + } + + boolean strictValidation = + HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION); + // read the schema version stored in metastore db + String schemaVer = getMetaStoreSchemaVersion(); + if (schemaVer == null) { + if (strictValidation) { + throw new MetaException("Version information not found in metastore. "); + } else { + LOG.warn("Version information not found in metastore. " + + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() + + "is not enabled so recording the schema version " + + MetaStoreSchemaInfo.getHiveSchemaVersion()); + setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), + "Set by MetaStore"); + } + } else { + if (!schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) { + throw new MetaException("Hive Schema version " + + MetaStoreSchemaInfo.getHiveSchemaVersion() + + " does not match metastore's schema version " + schemaVer + + " Metastore is not upgraded or corrupt"); + } + } + isSchemaVerified.set(true); + return; + } + + // load the schema version stored in metastore db + @Override + public String getMetaStoreSchemaVersion() throws MetaException { + + MVersionTable mSchemaVer; + try { + mSchemaVer = getMSchemaVersion(); + } catch (NoSuchObjectException e) { + return null; + } + return mSchemaVer.getSchemaVersion(); + } + + @SuppressWarnings("unchecked") + private MVersionTable getMSchemaVersion() + throws NoSuchObjectException, MetaException { + boolean committed = false; + List mVerTables = new ArrayList(); + + try { + openTransaction(); + Query query = pm.newQuery(MVersionTable.class); + + try { + mVerTables = (List)query.execute(); + pm.retrieveAll(mVerTables); + } catch (JDODataStoreException e) { + if (e.getCause() instanceof MissingTableException) { + throw new MetaException("Version table not found. " + + "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion()); + } else { + throw e; + } + } + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + } + } + if (mVerTables.isEmpty()) { + throw new NoSuchObjectException("No matching version found"); + } + if (mVerTables.size() > 1) { + throw new MetaException("Metastore contains multiple versions"); + } + return mVerTables.get(0); + } + + @Override + public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException { + MVersionTable mSchemaVer; + boolean commited = false; + + try { + mSchemaVer = getMSchemaVersion(); + } catch (NoSuchObjectException e) { + // if the version doesn't exist, then create it + mSchemaVer = new MVersionTable(); + } + + mSchemaVer.setSchemaVersion(schemaVersion); + mSchemaVer.setVersionComment(comment); + try { + openTransaction(); + pm.makePersistent(mSchemaVer); + commited = commitTransaction(); + } finally { + if (!commited) { + rollbackTransaction(); + } + } + } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java index e410c3a..bf2b5ed 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -436,4 +436,10 @@ public abstract void updateMasterKey(Integer seqNo, String key) public abstract String[] getMasterKeys(); + public abstract void verifySchema() throws MetaException; + + public abstract String getMetaStoreSchemaVersion() throws MetaException; + + public abstract void setMetaStoreSchemaVersion(String version, String comment) throws MetaException; + } diff --git metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java new file mode 100644 index 0000000..c6c4289 --- /dev/null +++ metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.model; + +public class MVersionTable { + private String schemaVersion; + private String versionComment; + + public MVersionTable() {} + + public MVersionTable(String schemaVersion, String versionComment) { + this.schemaVersion = schemaVersion; + this.versionComment = versionComment; + } + /** + * @return the versionComment + */ + public String getVersionComment() { + return versionComment; + } + /** + * @param versionComment the versionComment to set + */ + public void setVersionComment(String versionComment) { + this.versionComment = versionComment; + } + + /** + * @return the schemaVersion + */ + public String getSchemaVersion() { + return schemaVersion; + } + /** + * @param schemaVersion the schemaVersion to set + */ + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + +} diff --git metastore/src/model/package.jdo metastore/src/model/package.jdo index c42b5b0..91f8020 100644 --- metastore/src/model/package.jdo +++ metastore/src/model/package.jdo @@ -899,6 +899,17 @@ + + + + + + + + + + + diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 8066784..3a3891c 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hive.metastore; -import java.util.List; import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configurable; @@ -532,11 +532,11 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj, public boolean addToken(String tokenIdentifier, String delegationToken) { return false; } - + public boolean removeToken(String tokenIdentifier) { return false; } - + public String getToken(String tokenIdentifier) { return ""; } @@ -560,4 +560,19 @@ public boolean removeMasterKey(Integer keySeq) { return new String[0]; } + @Override + public void verifySchema() throws MetaException { + } + + @Override + public String getMetaStoreSchemaVersion() throws MetaException { + return objectStore.getMetaStoreSchemaVersion(); + } + + @Override + public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException { + objectStore.setMetaStoreSchemaVersion(schemaVersion, comment); + + } + } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 0f9b16c..f696155 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -593,6 +593,19 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj,List