diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 22149e4..4ddd484 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -322,6 +322,8 @@
METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false),
METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms"),
METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", true),
+ METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", false),
+ METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false),
METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked"),
METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed"),
METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false),
@@ -1140,6 +1142,11 @@ private void initialize(Class> cls) {
auxJars = this.get(ConfVars.HIVEAUXJARS.varname);
}
+ if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) {
+ setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false);
+ setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true);
+ }
+
// setup list of conf vars that are not allowed to change runtime
String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString());
if (restrictListStr != null) {
diff --git conf/hive-default.xml.template conf/hive-default.xml.template
index 9a3fc1d..4046b0e 100644
--- conf/hive-default.xml.template
+++ conf/hive-default.xml.template
@@ -1976,4 +1976,17 @@
+
+ hive.metastore.schema.verification
+ true
+
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manully migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+
+
+
+
diff --git metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
new file mode 100644
index 0000000..4e08fc1
--- /dev/null
+++ metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
@@ -0,0 +1,8 @@
+--
+-- Hive HIVE-3764
+-- Support metastore version consistency check
+
+CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "COMMENT" VARCHAR(255));
+ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID");
+
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
diff --git metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
index cce544f..8f61aeb 100644
--- metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
+++ metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
@@ -98,6 +98,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME"
CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "COMMENT" VARCHAR(255));
+
-- ----------------------------------------------
-- DDL Statements for indexes
-- ----------------------------------------------
@@ -284,6 +286,8 @@ ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_FK" FOREIGN KEY
ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
+ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID");
+
-- ----------------------------------------------
-- DDL Statements for checks
-- ----------------------------------------------
@@ -292,3 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "SQL110318025504980" CHECK (DEFERRED_REB
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N'));
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
diff --git metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
index cae7936..aaef099 100644
--- metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
+++ metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
@@ -1,2 +1 @@
-- Upgrade MetaStore schema from 0.10.0 to 0.11.0
-
diff --git metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
index 492cc93..f256c60 100644
--- metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
+++ metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
@@ -1,2 +1,4 @@
-- Upgrade MetaStore schema from 0.11.0 to 0.12.0
RUN '013-HIVE-3255.derby.sql';
+RUN '014-HIVE-3764.derby.sql';
+UPDATE "APP".VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1;
diff --git metastore/scripts/upgrade/derby/upgrade.order.derby metastore/scripts/upgrade/derby/upgrade.order.derby
new file mode 100644
index 0000000..59f02b9
--- /dev/null
+++ metastore/scripts/upgrade/derby/upgrade.order.derby
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
diff --git metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
new file mode 100644
index 0000000..08c73f6
--- /dev/null
+++ metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
@@ -0,0 +1,9 @@
+-- Table structure for VERSION
+CREATE TABLE IF NOT EXISTS `VERSION` (
+ `VER_ID` BIGINT NOT NULL,
+ `SCHEMA_VERSION` VARCHAR(127) NOT NULL,
+ `COMMENT` VARCHAR(255),
+ PRIMARY KEY (`VER_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
diff --git metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
index 22a77fe..1daafb9 100644
--- metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
+++ metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
@@ -751,6 +751,17 @@ CREATE TABLE IF NOT EXISTS `DELEGATION_TOKENS`
PRIMARY KEY (`TOKEN_IDENT`)
) ENGINE=INNODB DEFAULT CHARSET=latin1;
+--
+-- Table structure for VERSION
+--
+CREATE TABLE IF NOT EXISTS `VERSION` (
+ `VER_ID` BIGINT NOT NULL,
+ `SCHEMA_VERSION` VARCHAR(127) NOT NULL,
+ `COMMENT` VARCHAR(255),
+ PRIMARY KEY (`VER_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
diff --git metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
index 375a05f..afa4912 100644
--- metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
+++ metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
SOURCE 013-HIVE-3255.mysql.sql;
+SOURCE 014-HIVE-3764.mysql.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
diff --git metastore/scripts/upgrade/mysql/upgrade.order.mysql metastore/scripts/upgrade/mysql/upgrade.order.mysql
new file mode 100644
index 0000000..59f02b9
--- /dev/null
+++ metastore/scripts/upgrade/mysql/upgrade.order.mysql
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
diff --git metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
new file mode 100644
index 0000000..7e8530d
--- /dev/null
+++ metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
@@ -0,0 +1,10 @@
+-- HIVE-3764 Support metastore version consistency check
+
+CREATE TABLE IF NOT EXISTS VERSION (
+ VER_ID NUMBER NOT NULL,
+ SCHEMA_VERSION VARCHAR(127) NOT NULL,
+ COMMENT VARCHAR(255)
+)
+ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
diff --git metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
index 85a0178..6e191a1 100644
--- metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
+++ metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
@@ -483,6 +483,13 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
+CREATE TABLE IF NOT EXISTS VERSION (
+ VER_ID NUMBER NOT NULL,
+ SCHEMA_VERSION VARCHAR(127) NOT NULL,
+ COMMENT VARCHAR(255)
+)
+ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
+
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
@@ -707,4 +714,5 @@ CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
-- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent]
CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME);
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
diff --git metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
new file mode 100644
index 0000000..6cd6522
--- /dev/null
+++ metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
@@ -0,0 +1,2 @@
+SELECT 'Upgrading MetaStore schema from 0.10.0 to 0.11.0' AS ' ';
+SELECT 'Finished upgrading MetaStore schema from 0.10.0 to 0.11.0' AS Status from dual;
diff --git metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
index a2d0901..e2df6ae 100644
--- metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
+++ metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
@013-HIVE-3255.oracle.sql;
+@014-HIVE-3764.oracle.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
diff --git metastore/scripts/upgrade/oracle/upgrade.order.oracle metastore/scripts/upgrade/oracle/upgrade.order.oracle
new file mode 100644
index 0000000..bcba11f
--- /dev/null
+++ metastore/scripts/upgrade/oracle/upgrade.order.oracle
@@ -0,0 +1,3 @@
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
diff --git metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
new file mode 100644
index 0000000..a6f1537
--- /dev/null
+++ metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
@@ -0,0 +1,12 @@
+--
+-- Table structure for VERSION
+--
+CREATE TABLE "VERSION" (
+ "VER_ID" bigint,
+ "SCHEMA_VERSION" character varying(127) NOT NULL,
+ "COMMENT" character varying(255) NOT NULL,
+ PRIMARY KEY ("VER_ID")
+);
+ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID");
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
diff --git metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
index 7b319ba..b31a8d1 100644
--- metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
+++ metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
@@ -516,6 +516,16 @@ CREATE TABLE "TAB_COL_STATS" (
);
--
+-- Table structure for VERSION
+--
+CREATE TABLE "VERSION" (
+ "VER_ID" bigint,
+ "SCHEMA_VERSION" character varying(127) NOT NULL,
+ "COMMENT" character varying(255) NOT NULL,
+ PRIMARY KEY ("VER_ID")
+);
+
+--
-- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1379,6 +1389,9 @@ ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_fkey" FOREIGN KEY
--
ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE;
+
+ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID");
+
--
-- Name: public; Type: ACL; Schema: -; Owner: hiveuser
--
@@ -1387,6 +1400,7 @@ REVOKE ALL ON SCHEMA public FROM PUBLIC;
GRANT ALL ON SCHEMA public TO PUBLIC;
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
--
-- PostgreSQL database dump complete
--
diff --git metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
index 9da0a1b..ad46939 100644
--- metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
+++ metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0';
\i 013-HIVE-3255.postgres.sql;
+\i 014-HIVE-3764.postgres.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0';
diff --git metastore/scripts/upgrade/postgres/upgrade.order.postgres metastore/scripts/upgrade/postgres/upgrade.order.postgres
new file mode 100644
index 0000000..59f02b9
--- /dev/null
+++ metastore/scripts/upgrade/postgres/upgrade.order.postgres
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 39dda92..e7104db 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -400,6 +400,7 @@ public RawStore getMS() throws MetaException {
RawStore ms = threadLocalMS.get();
if (ms == null) {
ms = newRawStore();
+ ms.verifySchema();
threadLocalMS.set(ms);
ms = threadLocalMS.get();
}
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
new file mode 100644
index 0000000..3a40f04
--- /dev/null
+++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.common.util.HiveVersionInfo;
+
+
+public class MetaStoreSchemaInfo {
+ private static String SQL_FILE_EXTENSION=".sql";
+ private static String UPGRADE_FILE_PREFIX="upgrade-";
+ private static String INIT_FILE_PREFIX="hive-schema-";
+ private static String VERSION_UPGRADE_LIST = "upgrade.order";
+
+ private final String dbType;
+ private final String hiveVersions[];
+ private final HiveConf hiveConf;
+ private final String hiveHome;
+
+ public MetaStoreSchemaInfo(String hiveHome, HiveConf hiveConf, String dbType) throws HiveMetaException {
+ this.hiveHome = hiveHome;
+ this.dbType = dbType;
+ this.hiveConf = hiveConf;
+ hiveConf.getJar();
+ // load upgrade overder for the given dbType
+ List upgradeOrderList = new ArrayList();
+ String upgradeListFile = getMetaStoreScriptDir() + File.separator +
+ VERSION_UPGRADE_LIST + "." + dbType;
+ try {
+ BufferedReader bfReader =
+ new BufferedReader(new FileReader(upgradeListFile));
+ String currVersion;
+ while ((currVersion = bfReader.readLine()) != null) {
+ upgradeOrderList.add(currVersion);
+ }
+ } catch (FileNotFoundException e) {
+ throw new HiveMetaException("File " + upgradeListFile + "not found ", e);
+ } catch (IOException e) {
+ throw new HiveMetaException("Error reading " + upgradeListFile, e);
+ }
+ hiveVersions = upgradeOrderList.toArray(new String[0]);
+ }
+
+ /***
+ * Get the list of sql scripts required to upgrade from the give version to current
+ * @param fromVersion
+ * @return
+ * @throws HiveMetaException
+ */
+ public List getUpgradeScripts(String fromVersion)
+ throws HiveMetaException {
+ List upgradeScriptList = new ArrayList();
+
+ // check if we are already at current schema level
+ if (getHiveSchemaVersion().equals(fromVersion)) {
+ return upgradeScriptList;
+ }
+ // Find the list of scripts to execute for this upgrade
+ int firstScript = hiveVersions.length;
+ for (int i=0; i < hiveVersions.length; i++) {
+ if (hiveVersions[i].startsWith(fromVersion)) {
+ firstScript = i;
+ }
+ }
+ if (firstScript == hiveVersions.length) {
+ throw new HiveMetaException("Unknown version specified for upgrade " +
+ fromVersion + " Metastore schema may be too old or newer");
+ }
+
+ for (int i=firstScript; i < hiveVersions.length; i++) {
+ String scriptFile = generateUpgradeFileName(hiveVersions[i]);
+ upgradeScriptList.add(scriptFile);
+ }
+ return upgradeScriptList;
+ }
+
+ /***
+ * Get the name of the script to initialize the schema for given version
+ * @param toVersion Target version. If it's null, then the current server version is used
+ * @return
+ * @throws HiveMetaException
+ */
+ public String generateInitFileName(String toVersion) throws HiveMetaException {
+ if (toVersion == null) {
+ toVersion = getHiveSchemaVersion();
+ }
+ String initScriptName = INIT_FILE_PREFIX + toVersion + "." +
+ dbType + SQL_FILE_EXTENSION;
+ // check if the file exists
+ if (!(new File(getMetaStoreScriptDir() + File.separatorChar +
+ initScriptName).exists())) {
+ throw new HiveMetaException("Unknown version specified for initialization: " + toVersion);
+ }
+ return initScriptName;
+ }
+
+ /**
+ * Find the directory of metastore scripts
+ * @return
+ */
+ public String getMetaStoreScriptDir() {
+ return hiveHome + File.separatorChar +
+ "scripts" + File.separatorChar + "metastore" +
+ File.separatorChar + "upgrade" + File.separatorChar + dbType;
+ }
+
+ // format the upgrade script name eg upgrade-x-y-dbType.sql
+ private String generateUpgradeFileName(String fileVersion) {
+ return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION;
+ }
+
+ // Current hive version, remove the 'SNAPSHOT' part if needed
+ public static String getHiveSchemaVersion() {
+ return HiveVersionInfo.getVersion().replace("-SNAPSHOT", "");
+ }
+
+}
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index a27243d..6ce016b 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.commons.lang.StringUtils.repeat;
import java.net.URI;
import java.net.URISyntaxException;
@@ -34,10 +33,11 @@
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import javax.jdo.JDODataStoreException;
import javax.jdo.JDOHelper;
import javax.jdo.JDOObjectNotFoundException;
import javax.jdo.PersistenceManager;
@@ -119,10 +119,12 @@
import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
+import org.apache.hadoop.hive.metastore.model.MVersionTable;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
import org.apache.hadoop.util.StringUtils;
+import org.datanucleus.store.rdbms.exceptions.MissingTableException;
/**
* This class is the interface between the application logic and the database
@@ -163,6 +165,7 @@
int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
+ private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false);
public ObjectStore() {
}
@@ -5609,4 +5612,120 @@ public boolean removeMasterKey(Integer id) {
return masterKeys;
}
+ // compare hive version and metastore version
+ @Override
+ public void verifySchema() throws MetaException {
+ // If the schema version is already checked, then go ahead and use this metastore
+ if (isSchemaVerified.get()) {
+ return;
+ }
+ checkSchema();
+ }
+
+ private synchronized void checkSchema() throws MetaException {
+ if (isSchemaVerified.get()) {
+ // recheck if it got verified
+ return;
+ }
+
+ boolean strictValidation =
+ HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
+ // read the schema version stored in metastore db
+ String schemaVer = getMetaStoreSchemaVersion();
+ if (schemaVer == null) {
+ if (strictValidation) {
+ throw new MetaException("Version information not found in metastore. ");
+ } else {
+ LOG.warn("Version information not found in metastore. "
+ + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
+ "is not enabled so recording the schema version " +
+ MetaStoreSchemaInfo.getHiveSchemaVersion());
+ setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+ "Set by MetaStore");
+ }
+ } else {
+ if (!schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) {
+ throw new MetaException("Hive Schema version "
+ + MetaStoreSchemaInfo.getHiveSchemaVersion() +
+ " does not match metastore's schema version " + schemaVer +
+ " Metastore is not upgraded or corrupt");
+ }
+ }
+ isSchemaVerified.set(true);
+ return;
+ }
+
+ // load the schema version stored in metastore db
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+
+ MVersionTable mSchemaVer;
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ return null;
+ }
+ return mSchemaVer.getSchemaVersion();
+ }
+
+ @SuppressWarnings("unchecked")
+ private MVersionTable getMSchemaVersion()
+ throws NoSuchObjectException, MetaException {
+ boolean committed = false;
+ List mVerTables = new ArrayList();
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MVersionTable.class);
+
+ try {
+ mVerTables = (List)query.execute();
+ pm.retrieveAll(mVerTables);
+ } catch (JDODataStoreException e) {
+ if (e.getCause() instanceof MissingTableException) {
+ throw new MetaException("Version table not found. " +
+ "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion());
+ } else {
+ throw e;
+ }
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ if (mVerTables.isEmpty()) {
+ throw new NoSuchObjectException("No matching version found");
+ }
+ if (mVerTables.size() > 1) {
+ throw new MetaException("Metastore contains multiple versions");
+ }
+ return mVerTables.get(0);
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ MVersionTable mSchemaVer;
+ boolean commited = false;
+
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ // if the version doesn't exist, then create it
+ mSchemaVer = new MVersionTable();
+ }
+
+ mSchemaVer.setSchemaVersion(schemaVersion);
+ mSchemaVer.setVersionComment(comment);
+ try {
+ openTransaction();
+ pm.makePersistent(mSchemaVer);
+ commited = commitTransaction();
+ } finally {
+ if (!commited) {
+ rollbackTransaction();
+ }
+ }
+ }
}
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index e410c3a..bf2b5ed 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -436,4 +436,10 @@ public abstract void updateMasterKey(Integer seqNo, String key)
public abstract String[] getMasterKeys();
+ public abstract void verifySchema() throws MetaException;
+
+ public abstract String getMetaStoreSchemaVersion() throws MetaException;
+
+ public abstract void setMetaStoreSchemaVersion(String version, String comment) throws MetaException;
+
}
diff --git metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
new file mode 100644
index 0000000..c6c4289
--- /dev/null
+++ metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.model;
+
+public class MVersionTable {
+ private String schemaVersion;
+ private String versionComment;
+
+ public MVersionTable() {}
+
+ public MVersionTable(String schemaVersion, String versionComment) {
+ this.schemaVersion = schemaVersion;
+ this.versionComment = versionComment;
+ }
+ /**
+ * @return the versionComment
+ */
+ public String getVersionComment() {
+ return versionComment;
+ }
+ /**
+ * @param versionComment the versionComment to set
+ */
+ public void setVersionComment(String versionComment) {
+ this.versionComment = versionComment;
+ }
+
+ /**
+ * @return the schemaVersion
+ */
+ public String getSchemaVersion() {
+ return schemaVersion;
+ }
+ /**
+ * @param schemaVersion the schemaVersion to set
+ */
+ public void setSchemaVersion(String schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+}
diff --git metastore/src/model/package.jdo metastore/src/model/package.jdo
index c42b5b0..91f8020 100644
--- metastore/src/model/package.jdo
+++ metastore/src/model/package.jdo
@@ -899,6 +899,17 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 8066784..3a3891c 100644
--- metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.metastore;
-import java.util.List;
import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configurable;
@@ -532,11 +532,11 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj,
public boolean addToken(String tokenIdentifier, String delegationToken) {
return false;
}
-
+
public boolean removeToken(String tokenIdentifier) {
return false;
}
-
+
public String getToken(String tokenIdentifier) {
return "";
}
@@ -560,4 +560,19 @@ public boolean removeMasterKey(Integer keySeq) {
return new String[0];
}
+ @Override
+ public void verifySchema() throws MetaException {
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+ return objectStore.getMetaStoreSchemaVersion();
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ objectStore.setMetaStoreSchemaVersion(schemaVersion, comment);
+
+ }
+
}
diff --git metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 0f9b16c..f696155 100644
--- metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -593,6 +593,19 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj,List