diff --git a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java index d306e29..a737cd2 100644 --- a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java +++ b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java @@ -277,7 +277,9 @@ private File generateTmpFile(String context) { @Before public void setup() { - System.setProperty("datanucleus.schema.autoCreateAll", "true"); + System.setProperty("datanucleus.fixedDatastore", "false"); + System.setProperty("datanucleus.autoCreateSchema", "true"); + System.setProperty("hive.metastore.schema.verification", "false"); cli = new HiveCli(); redirectOutputStream(); initFromFile(); diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 5f310d6..7992a04 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -25,12 +25,17 @@ - datanucleus.schema.autoCreateAll + hive.metastore.schema.verification + false + + + + datanucleus.autoCreateSchema true - hive.metastore.schema.verification + datanucleus.fixedDatastore false diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e92466f..1979f1d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -232,7 +232,7 @@ private static URL checkConfigFile(File f) { HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS, HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS, HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE, - HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, + HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA, HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION, HiveConf.ConfVars.METASTORE_CACHE_LEVEL2, HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE, @@ -705,17 +705,18 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "List of comma separated metastore object types that should be pinned in the cache"), METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", "Specify connection pool library for datanucleus"), - METASTORE_VALIDATE_TABLES("datanucleus.schema.validateTables", false, + METASTORE_VALIDATE_TABLES("datanucleus.validateTables", false, "validates existing schema against code. turn this on if you want to verify existing schema"), - METASTORE_VALIDATE_COLUMNS("datanucleus.schema.validateColumns", false, + METASTORE_VALIDATE_COLUMNS("datanucleus.validateColumns", false, "validates existing schema against code. turn this on if you want to verify existing schema"), - METASTORE_VALIDATE_CONSTRAINTS("datanucleus.schema.validateConstraints", false, + METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false, "validates existing schema against code. turn this on if you want to verify existing schema"), METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms", "metadata store type"), - METASTORE_AUTO_CREATE_ALL("datanucleus.schema.autoCreateAll", false, - "Auto creates necessary schema on a startup if one doesn't exist. Set this to false, after creating it once." + METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", false, + "creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once" + "To enable auto create also set hive.metastore.schema.verification=false. Auto creation is not " + "recommended for production use cases, run schematool command instead." ), + METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", true, "Dictates whether to allow updates to schema or not."), METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", true, "Enforce metastore schema version consistency.\n" + "True: Verify that version information stored in is compatible with one from Hive jars. Also disable automatic\n" + @@ -3775,7 +3776,8 @@ private void initialize(Class cls) { } if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) { - setBoolVar(ConfVars.METASTORE_AUTO_CREATE_ALL, false); + setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false); + setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true); } if (getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) { diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index b09c159..be45bac 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -59,11 +59,16 @@ - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true + datanucleus.fixedDatastore + false + + + hive.metastore.schema.verification false diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 8b6dc95..305b9d8 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -69,11 +69,16 @@ - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true + datanucleus.fixedDatastore + false + + + hive.metastore.schema.verification false diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml index 989e65e..ce72e6a 100644 --- a/data/conf/spark/standalone/hive-site.xml +++ b/data/conf/spark/standalone/hive-site.xml @@ -45,11 +45,16 @@ - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true + datanucleus.fixedDatastore + false + + + hive.metastore.schema.verification false diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index 8c66f87..ea6fdf4 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -45,11 +45,16 @@ - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true + datanucleus.fixedDatastore + false + + + hive.metastore.schema.verification false diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index 85f82b5..96466b0 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -63,11 +63,21 @@ - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true + datanucleus.fixedDatastore + false + + + + hive.metastore.schema.verification + false + + + hive.exec.local.scratchdir ${test.tmp.dir}/localscratchdir/ Local scratch space for Hive jobs diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml index 5bef8c8..8473d99 100644 --- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml +++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml @@ -45,7 +45,7 @@ com.microsoft.sqlserver.jdbc.SQLServerDriver - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema false hive.support.concurrency diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 6d1673d..4ab680c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -51,7 +51,8 @@ protected void setUp() throws Exception { // reset defaults ObjectStore.setSchemaVerified(false); System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); - System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true"); + System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA.toString(), "true"); + System.setProperty(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE.toString(), "false"); hiveConf = new HiveConf(this.getClass()); System.setProperty("hive.support.concurrency", "false"); System.setProperty("hive.metastore.event.listeners", @@ -80,7 +81,8 @@ public void testDefaults() { System.clearProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString()); hiveConf = new HiveConf(this.getClass()); assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); - assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); + assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA)); + assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE)); } /*** @@ -91,7 +93,8 @@ public void testVersionRestriction () throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true"); hiveConf = new HiveConf(this.getClass()); assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION)); - assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL)); + assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA)); + assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE)); // session creation should fail since the schema didn't get created try { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 0c313a2..c378c7f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -64,7 +64,6 @@ import org.datanucleus.ClassLoaderResolver; import org.datanucleus.NucleusContext; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; -import org.datanucleus.AbstractNucleusContext; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -935,8 +934,8 @@ public void testHttpHeaderSize() throws Exception { } /** - * Tests that DataNucleus' NucleusContext.classLoaderResolverMap clears cached class objects - * (& hence doesn't leak classloaders) on closing any session + * Tests that DataNucleus' NucleusContext.classLoaderResolverMap clears cached class objects (& hence + * doesn't leak classloaders) on closing any session * * @throws Exception */ @@ -999,7 +998,7 @@ private int getNucleusClassLoaderResolverMapSize() { } if (nc != null) { try { - classLoaderResolverMap = AbstractNucleusContext.class.getDeclaredField("classLoaderResolverMap"); + classLoaderResolverMap = NucleusContext.class.getDeclaredField("classLoaderResolverMap"); if (classLoaderResolverMap != null) { classLoaderResolverMap.setAccessible(true); cMap = (Map) classLoaderResolverMap.get(nc); diff --git a/metastore/pom.xml b/metastore/pom.xml index 7f75d4db..ec12cf9 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -127,11 +127,6 @@ ${jdo-api.version} - org.datanucleus - javax.jdo - ${datanucleus-jdo.version} - - org.antlr antlr-runtime ${antlr.version} diff --git a/metastore/scripts/upgrade/mssql/README b/metastore/scripts/upgrade/mssql/README index 0b62a01..8e5a33e 100644 --- a/metastore/scripts/upgrade/mssql/README +++ b/metastore/scripts/upgrade/mssql/README @@ -77,7 +77,7 @@ Upgrading from 0.11.0 to 0.13.0 2) Make sure you have - datanucleus.schema.autoCreateAll + datanucleus.autoCreateSchema true in your hive-site.xml. This will cause DataNucleus to create diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 5adfa02..318fc2d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -153,7 +153,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.HiveStringUtils; import org.apache.thrift.TException; -import org.datanucleus.AbstractNucleusContext; import org.datanucleus.ClassLoaderResolver; import org.datanucleus.NucleusContext; import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; @@ -8359,8 +8358,8 @@ public static void unCacheDataNucleusClassLoaders() { JDOPersistenceManagerFactory jdoPmf = (JDOPersistenceManagerFactory) pmf; NucleusContext nc = jdoPmf.getNucleusContext(); try { - Field classLoaderResolverMap = AbstractNucleusContext.class.getDeclaredField( - "classLoaderResolverMap"); + Field classLoaderResolverMap = + NucleusContext.class.getDeclaredField("classLoaderResolverMap"); classLoaderResolverMap.setAccessible(true); classLoaderResolverMap.set(nc, new HashMap()); LOG.debug("Removed cached classloaders from DataNucleus NucleusContext"); diff --git a/pom.xml b/pom.xml index 0217ede..5f173fb 100644 --- a/pom.xml +++ b/pom.xml @@ -112,10 +112,9 @@ 1.7.7 0.8.0.RELEASE 1.6.0 - 4.2.1 - 4.1.6 - 4.1.7 - 3.2.0-m3 + 3.2.6 + 3.2.10 + 3.2.9 1.2 1.4 3.2.2 @@ -595,11 +594,6 @@ ${datanucleus-rdbms.version} - org.datanucleus - javax.jdo - ${datanucleus-jdo.version} - - org.iq80.snappy snappy ${snappy.version} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index cadda8f..84d6473 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3686,4 +3686,5 @@ public static boolean checkLlapIOSupportedTypes(final List readColumnNam } return result; } + } diff --git a/ql/src/test/templates/TestPerfCliDriver.vm b/ql/src/test/templates/TestPerfCliDriver.vm index d2946cb..bde7b9e 100644 --- a/ql/src/test/templates/TestPerfCliDriver.vm +++ b/ql/src/test/templates/TestPerfCliDriver.vm @@ -56,8 +56,9 @@ public class $className extends TestCase { } static { - System.setProperty("datanucleus.schema.autoCreateAll", "true"); + System.setProperty("datanucleus.autoCreateSchema", "true"); System.setProperty("hive.metastore.schema.verification", "false"); + MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript";