diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e64e8fc..8e072f7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -472,6 +472,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { " schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures\n" + " proper metastore schema migration. (Default)\n" + "False: Warn if the version information stored in metastore doesn't match with one from in Hive jars."), + METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", true, + "When true the current MS version is recorded in the VERSION table. If this is disabled and verification is\n" + + " enabled the MS will be unusable."), METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked", "throw exception if metadata tables are incorrect"), METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed", diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index 95c1413..5514228 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -23,21 +23,24 @@ import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.common.util.HiveStringUtils; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; public class TestMetastoreVersion extends TestCase { - + private static final Log LOG = LogFactory.getLog(TestMetastoreVersion.class); protected HiveConf hiveConf; private Driver driver; private String metaStoreRoot; private String testMetastoreDB; - Random randomNum = new Random(); @Override protected void setUp() throws Exception { @@ -45,13 +48,19 @@ protected void setUp() throws Exception { Field defDb = HiveMetaStore.HMSHandler.class.getDeclaredField("currentUrl"); defDb.setAccessible(true); defDb.set(null, null); + // reset defaults + ObjectStore.setSchemaVerified(false); + System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false"); + System.setProperty(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA.toString(), "true"); + System.setProperty(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE.toString(), "false"); hiveConf = new HiveConf(this.getClass()); + System.setProperty("hive.support.concurrency", "false"); System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); testMetastoreDB = System.getProperty("java.io.tmpdir") + - File.separator + "test_metastore-" + randomNum.nextInt(); + File.separator + "test_metastore-" + System.currentTimeMillis(); System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + testMetastoreDB + ";create=true"); metaStoreRoot = System.getProperty("test.tmp.dir"); @@ -92,7 +101,10 @@ public void testVersionRestriction () throws Exception { SessionState.start(new CliSessionState(hiveConf)); fail("Expected exception"); } catch (RuntimeException re) { - assertTrue(re.getCause().getCause().getCause() instanceof MetaException); + LOG.info("Exception in testVersionRestriction: " + re, re); + String msg = HiveStringUtils.stringifyException(re); + assertTrue("Expected 'Version information not found in metastore' in: " + msg, msg + .contains("Version information not found in metastore")); } } @@ -149,7 +161,7 @@ public void testVersionMisMatch () throws Exception { SessionState.start(new CliSessionState(hiveConf)); driver = new Driver(hiveConf); CommandProcessorResponse proc = driver.run("show tables"); - assertFalse(proc.getResponseCode() == 0); + assertEquals(0, proc.getResponseCode()); } // write the given version to metastore diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index fcaffc7..f700353 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -21,6 +21,7 @@ import static org.apache.commons.lang.StringUtils.join; import java.io.IOException; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -161,6 +162,10 @@ private static PersistenceManagerFactory pmf = null; private static Lock pmfPropLock = new ReentrantLock(); + /** + * Verify the schema only once per JVM since the db connection info is static + */ + private final static AtomicBoolean isSchemaVerified = new AtomicBoolean(false); private static final Log LOG = LogFactory.getLog(ObjectStore.class.getName()); private static enum TXN_STATUS { @@ -168,6 +173,8 @@ } private static final Map PINCLASSMAP; + private static final String HOSTNAME; + private static final String USER; static { Map map = new HashMap(); map.put("table", MTable.class); @@ -179,8 +186,22 @@ map.put("fieldschema", MFieldSchema.class); map.put("order", MOrder.class); PINCLASSMAP = Collections.unmodifiableMap(map); + String hostname = "UNKNOWN"; + try { + InetAddress clientAddr = InetAddress.getLocalHost(); + hostname = clientAddr.getHostAddress(); + } catch (IOException e) { + } + HOSTNAME = hostname; + String user = System.getenv("USER"); + if (user == null) { + USER = "UNKNOWN"; + } else { + USER = user; + } } + private boolean isInitialized = false; private PersistenceManager pm = null; private MetaStoreDirectSql directSql = null; @@ -189,7 +210,6 @@ int openTrasactionCalls = 0; private Transaction currentTransaction = null; private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; - private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false); private Pattern partitionValidationPattern; @@ -6581,6 +6601,10 @@ public void verifySchema() throws MetaException { checkSchema(); } + public static void setSchemaVerified(boolean val) { + isSchemaVerified.set(val); + } + private synchronized void checkSchema() throws MetaException { // recheck if it got verified by another thread while we were waiting if (isSchemaVerified.get()) { @@ -6592,32 +6616,33 @@ private synchronized void checkSchema() throws MetaException { // read the schema version stored in metastore db String schemaVer = getMetaStoreSchemaVersion(); if (schemaVer == null) { - // metastore has no schema version information if (strictValidation) { - throw new MetaException("Version information not found in metastore. "); - } else { - LOG.warn("Version information not found in metastore. " - + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() + - " is not enabled so recording the schema version " + - MetaStoreSchemaInfo.getHiveSchemaVersion()); - setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), - "Set by MetaStore"); - } + throw new MetaException("Version information not found in metastore. "); + } else { + LOG.warn("Version information not found in metastore. " + + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() + + " is not enabled so recording the schema version " + + MetaStoreSchemaInfo.getHiveSchemaVersion()); + setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), + "Set by MetaStore " + USER + "@" + HOSTNAME); + } } else { // metastore schema version is different than Hive distribution needs - if (strictValidation) { - if (!schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) { + if (schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) { + LOG.debug("Found expected HMS version of " + schemaVer); + } else { + if (strictValidation) { throw new MetaException("Hive Schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion() + " does not match metastore's schema version " + schemaVer + " Metastore is not upgraded or corrupt"); } else { - LOG.warn("Metastore version was " + schemaVer + " " + - HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() + - " is not enabled so recording the new schema version " + - MetaStoreSchemaInfo.getHiveSchemaVersion()); + LOG.error("Version information found in metastore differs " + schemaVer + + " from expected schema version " + MetaStoreSchemaInfo.getHiveSchemaVersion() + + ". Schema verififcation is disabled " + + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version."); setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), - "Set by MetaStore"); + "Set by MetaStore " + USER + "@" + HOSTNAME); } } } @@ -6669,7 +6694,11 @@ private MVersionTable getMSchemaVersion() throw new NoSuchObjectException("No matching version found"); } if (mVerTables.size() > 1) { - throw new MetaException("Metastore contains multiple versions"); + String msg = "Metastore contains multiple versions (" + mVerTables.size() + ") "; + for (MVersionTable version : mVerTables) { + msg += "[ version = " + version.getSchemaVersion() + ", comment = " + version.getVersionComment() + " ] "; + } + throw new MetaException(msg.trim()); } return mVerTables.get(0); } @@ -6678,6 +6707,13 @@ private MVersionTable getMSchemaVersion() public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException { MVersionTable mSchemaVer; boolean commited = false; + boolean recordVersion = + HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION); + if (!recordVersion) { + LOG.warn("setMetaStoreSchemaVersion called but recording version is disabled: " + + "version = " + schemaVersion + ", comment = " + comment); + return; + } try { mSchemaVer = getMSchemaVersion();