diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 453b0ea..5f310d6 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -24,10 +24,15 @@ Internal marker for test. Used for masking env-dependent values - - datanucleus.schema.autoCreateAll - true - + + datanucleus.schema.autoCreateAll + true + + + + hive.metastore.schema.verification + false + javax.jdo.option.ConnectionURL diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 283ef2e..bae1825 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -232,7 +232,6 @@ private static URL checkConfigFile(File f) { HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS, HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE, HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, - HiveConf.ConfVars.METASTORE_AUTO_START_MECHANISM_MODE, HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION, HiveConf.ConfVars.METASTORE_CACHE_LEVEL2, HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE, @@ -712,18 +711,18 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "validates existing schema against code. turn this on if you want to verify existing schema"), METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms", "metadata store type"), METASTORE_AUTO_CREATE_ALL("datanucleus.schema.autoCreateAll", false, - "creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once"), - METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false, + "Auto creates necessary schema on a startup if one doesn't exist. Set this to false, after creating it once." + + "To enable auto create also set hive.metastore.schema.verification=false. Auto creation is not " + + "recommended for production use cases, run schematool command instead." ), + METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", true, "Enforce metastore schema version consistency.\n" + - "True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic\n" + + "True: Verify that version information stored in is compatible with one from Hive jars. Also disable automatic\n" + " schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures\n" + " proper metastore schema migration. (Default)\n" + "False: Warn if the version information stored in metastore doesn't match with one from in Hive jars."), - METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", true, + METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", false, "When true the current MS version is recorded in the VERSION table. If this is disabled and verification is\n" + " enabled the MS will be unusable."), - METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked", - "throw exception if metadata tables are incorrect"), METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed", "Default transaction isolation level for identity generation."), METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false, diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 041b67d..775f5c2 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -64,6 +64,11 @@ + hive.metastore.schema.verification + false + + + javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 72bdcfb..8b6dc95 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -74,6 +74,11 @@ + hive.metastore.schema.verification + false + + + javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml index bc8c0e5..989e65e 100644 --- a/data/conf/spark/standalone/hive-site.xml +++ b/data/conf/spark/standalone/hive-site.xml @@ -50,6 +50,11 @@ + hive.metastore.schema.verification + false + + + javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index f1d9ddc..8c66f87 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -50,6 +50,11 @@ + hive.metastore.schema.verification + false + + + javax.jdo.option.ConnectionURL jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 1352198..6d1673d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -107,12 +107,14 @@ public void testVersionRestriction () throws Exception { } /*** - * Test that with no verification, hive populates the schema and version correctly + * Test that with no verification, and record verification enabled, hive populates the schema + * and version correctly * @throws Exception */ public void testMetastoreVersion () throws Exception { // let the schema and version be auto created System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); + System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION.toString(), "true"); hiveConf = new HiveConf(this.getClass()); SessionState.start(new CliSessionState(hiveConf)); driver = new Driver(hiveConf); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 82de857..5adfa02 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -385,6 +385,7 @@ private static void configureSSL(Configuration conf) { @SuppressWarnings("nls") private static Properties getDataSourceProps(Configuration conf) { Properties prop = new Properties(); + correctAutoStartMechanism(conf); Iterator> iter = conf.iterator(); while (iter.hasNext()) { @@ -420,6 +421,24 @@ private static Properties getDataSourceProps(Configuration conf) { return prop; } + /** + * Update conf to set datanucleus.autoStartMechanismMode=ignored. + * This is necessary to able to use older version of hive against + * an upgraded but compatible metastore schema in db from new version + * of hive + * @param conf + */ + private static void correctAutoStartMechanism(Configuration conf) { + final String autoStartKey = "datanucleus.autoStartMechanismMode"; + final String autoStartIgnore = "ignored"; + String currentAutoStartVal = conf.get(autoStartKey); + if(currentAutoStartVal != null && !currentAutoStartVal.equalsIgnoreCase(autoStartIgnore)) { + LOG.warn(autoStartKey + " is set to unsupported value " + conf.get(autoStartKey) + + " . Setting it to value " + autoStartIgnore); + } + conf.set(autoStartKey, autoStartIgnore); + } + private static synchronized PersistenceManagerFactory getPMF() { if (pmf == null) { pmf = JDOHelper.getPersistenceManagerFactory(prop); @@ -7769,10 +7788,10 @@ private synchronized void checkSchema() throws MetaException { "Set by MetaStore " + USER + "@" + HOSTNAME); } } else { - // metastore schema version is different than Hive distribution needs if (MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) { LOG.debug("Found expected HMS version of " + dbSchemaVer); } else { + // metastore schema version is different than Hive distribution needs if (strictValidation) { throw new MetaException("Hive Schema version " + hiveSchemaVer + " does not match metastore's schema version " + dbSchemaVer + @@ -7781,7 +7800,7 @@ private synchronized void checkSchema() throws MetaException { LOG.error("Version information found in metastore differs " + dbSchemaVer + " from expected schema version " + hiveSchemaVer + ". Schema verififcation is disabled " + - HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version."); + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION); setMetaStoreSchemaVersion(hiveSchemaVer, "Set by MetaStore " + USER + "@" + HOSTNAME); } @@ -7858,6 +7877,7 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro "version = " + schemaVersion + ", comment = " + comment); return; } + LOG.warn("Setting metastore schema version in db to " + schemaVersion); try { mSchemaVer = getMSchemaVersion(); diff --git a/ql/src/test/templates/TestPerfCliDriver.vm b/ql/src/test/templates/TestPerfCliDriver.vm index 97dde2c..d2946cb 100644 --- a/ql/src/test/templates/TestPerfCliDriver.vm +++ b/ql/src/test/templates/TestPerfCliDriver.vm @@ -57,6 +57,7 @@ public class $className extends TestCase { static { System.setProperty("datanucleus.schema.autoCreateAll", "true"); + System.setProperty("hive.metastore.schema.verification", "false"); MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript";