diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java index b86d736a89..3b52ac2e26 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMetaStoreHandler.java @@ -93,9 +93,7 @@ private String getDbTypeConfString() {// "ORACLE", "MYSQL", "MSSQL", "POSTGRES" public void beforeTest() throws Exception { getRule().before(); - if (!isDerby()) {// derby is handled with old QTestUtil logic (TxnDbUtil stuff) - getRule().install(); - } + getRule().install(); } public void afterTest(QTestUtil qt) throws Exception { diff --git pom.xml pom.xml index 13f4510240..f55b0e223a 100644 --- pom.xml +++ pom.xml @@ -84,6 +84,8 @@ + set-this-to-colon-separated-full-path-list-of-jars-to-run-integration-tests + ${maven.test.classpath} file:// @@ -1367,6 +1369,7 @@ ${test.conf.dir} ${basedir}/${hive.path.to.root}/conf + ${itest.jdbc.jars} US/Pacific diff --git ql/pom.xml ql/pom.xml index 3cb96e9dc8..518093ce0e 100644 --- ql/pom.xml +++ ql/pom.xml @@ -31,6 +31,7 @@ .. 1.6.6 0.9.10 + @@ -904,6 +905,24 @@ run + + setup-metastore-scripts + process-test-resources + + run + + + + + + + + + + + + + diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java new file mode 100644 index 0000000000..d54688d72f --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.lockmgr; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres; +import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test class to run DbTxnManager tests against different dbms types. + * Example: mvn test -Dtest=ITestDbTxnManager -Dtest.metastore.db=postgres -Ditest.jdbc.jars=yourPathtoJdbcDriver + * @throws Exception + */ +public class ITestDbTxnManager extends TestDbTxnManager2 { + + private static final String SYS_PROP_METASTORE_DB = "test.metastore.db"; + private static final Logger LOG = LoggerFactory.getLogger(TestDbTxnManager2.class); + private static DatabaseRule rule; + + + @BeforeClass + public static void setupDb() throws Exception { + String metastoreType = + System.getProperty(SYS_PROP_METASTORE_DB) == null ? "derby" : System.getProperty(SYS_PROP_METASTORE_DB) + .toLowerCase(); + rule = getDatabaseRule(metastoreType).setVerbose(false); + + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); + conf.setVar(HiveConf.ConfVars.METASTOREDBTYPE, metastoreType.toUpperCase()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY, rule.getJdbcUrl()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECTION_DRIVER, rule.getJdbcDriver()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME, rule.getHiveUser()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.PWD, rule.getHivePassword()); + + LOG.info("Set metastore connection to url: {}", + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + TxnDbUtil.setConfValues(conf); + // Start the docker container and create the hive user + rule.before(); + rule.createUser(); + } + + @AfterClass + public static void tearDownDb() { + rule.after(); + } + + private static DatabaseRule getDatabaseRule(String metastoreType) { + switch (metastoreType) { + case "postgres": + return new Postgres(); + case "oracle": + return new Oracle(); + case "mysql": + return new Mysql(); + case "mssql": + return new Mssql(); + default: + return new Derby(); + } + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 80fb1aff78..aaa481e8ed 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -31,11 +31,18 @@ import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle; +import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres; import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.TestTxnCommands2; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -47,10 +54,13 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ComparisonFailure; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -83,19 +93,14 @@ * each thread. */ public class TestDbTxnManager2 { - private static HiveConf conf = new HiveConf(Driver.class); + protected static HiveConf conf = new HiveConf(Driver.class); + private HiveTxnManager txnMgr; private Context ctx; private Driver driver, driver2; private TxnStore txnHandler; - public TestDbTxnManager2() throws Exception { - conf - .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, - "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - conf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false); - TxnDbUtil.setConfValues(conf); - } + @Before public void setUp() throws Exception { SessionState.start(conf); @@ -470,9 +475,9 @@ public void testMetastoreTablesCleanup() throws Exception { driver.run("insert into temp.T12p partition (ds='tomorrow', hour='2') values (13, 13)"); driver.run("insert into temp.T13p partition (ds='today', hour='1') values (7, 7)"); driver.run("insert into temp.T13p partition (ds='tomorrow', hour='2') values (8, 8)"); - int count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')"); + int count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\" in ('t10', 't11')"); Assert.assertEquals(4, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\" in ('t12p', 't13p')"); Assert.assertEquals(5, count); // Fail some inserts, so that we have records in TXN_COMPONENTS @@ -481,54 +486,54 @@ public void testMetastoreTablesCleanup() throws Exception { driver.run("insert into temp.T11 values (10, 10)"); driver.run("insert into temp.T12p partition (ds='today', hour='1') values (11, 11)"); driver.run("insert into temp.T13p partition (ds='today', hour='1') values (12, 12)"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(4, count); conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); // Drop a table/partition; corresponding records in TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS should disappear - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\"='t10'"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\"='t10'"); Assert.assertEquals(2, count); driver.run("drop table temp.T10"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\"='t10'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\"='t10'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\"='t12p' and \"TC_PARTITION\"='ds=today/hour=1'"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\"='t12p' and \"CTC_PARTITION\"='ds=today/hour=1'"); Assert.assertEquals(1, count); driver.run("alter table temp.T12p drop partition (ds='today', hour='1')"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\"='t12p' and \"TC_PARTITION\"='ds=today/hour=1'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\"='t12p' and \"CTC_PARTITION\"='ds=today/hour=1'"); Assert.assertEquals(0, count); // Successfully perform compaction on a table/partition, so that we have successful records in COMPLETED_COMPACTIONS driver.run("alter table temp.T11 compact 'minor'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='i'"); Assert.assertEquals(1, count); TestTxnCommands2.runWorker(conf); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11' and \"CQ_STATE\"='r' and \"CQ_TYPE\"='i'"); Assert.assertEquals(1, count); TestTxnCommands2.runCleaner(conf); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t11' and \"CC_STATE\"='s' and \"CC_TYPE\"='i'"); Assert.assertEquals(1, count); driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'minor'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p' and \"CQ_PARTITION\"='ds=tomorrow/hour=2' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='i'"); Assert.assertEquals(1, count); TestTxnCommands2.runWorker(conf); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p' and \"CQ_PARTITION\"='ds=tomorrow/hour=2' and \"CQ_STATE\"='r' and \"CQ_TYPE\"='i'"); Assert.assertEquals(1, count); TestTxnCommands2.runCleaner(conf); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t12p' and \"CC_STATE\"='s' and \"CC_TYPE\"='i'"); Assert.assertEquals(1, count); // Fail compaction, so that we have failed records in COMPLETED_COMPACTIONS. @@ -537,67 +542,67 @@ public void testMetastoreTablesCleanup() throws Exception { driver.run("insert into temp.T12p partition (ds='tomorrow', hour='2') values (15, 15)"); conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, true); driver.run("alter table temp.T11 compact 'major'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(1, count); TestTxnCommands2.runWorker(conf); // will fail - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t11' and \"CC_STATE\"='f' and \"CC_TYPE\"='a'"); Assert.assertEquals(1, count); driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p' and \"CQ_PARTITION\"='ds=tomorrow/hour=2' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(1, count); TestTxnCommands2.runWorker(conf); // will fail - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p' and \"CQ_PARTITION\"='ds=tomorrow/hour=2' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t12p' and \"CC_STATE\"='f' and \"CC_TYPE\"='a'"); Assert.assertEquals(1, count); conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false); // Put 2 records into COMPACTION_QUEUE and do nothing driver.run("alter table temp.T11 compact 'major'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(1, count); driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p' and \"CQ_PARTITION\"='ds=tomorrow/hour=2' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(1, count); // Drop a table/partition, corresponding records in COMPACTION_QUEUE and COMPLETED_COMPACTIONS should disappear driver.run("drop table temp.T11"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t11'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t11'"); Assert.assertEquals(0, count); driver.run("alter table temp.T12p drop partition (ds='tomorrow', hour='2')"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t12p'"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\"='t12p'"); Assert.assertEquals(0, count); // Put 1 record into COMPACTION_QUEUE and do nothing driver.run("alter table temp.T13p partition (ds='today', hour='1') compact 'major'"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\"='t13p' and \"CQ_STATE\"='i' and \"CQ_TYPE\"='a'"); Assert.assertEquals(1, count); // Drop database, everything in all 4 meta tables should disappear - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(2, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(1, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); driver.run("drop database if exists temp cascade"); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_DATABASE\"='temp' and \"TC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_DATABASE\"='temp' and \"CTC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPACTION_QUEUE\" where \"CQ_DATABASE\"='temp' and \"CQ_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); - count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')"); + count = TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_COMPACTIONS\" where \"CC_DATABASE\"='temp' and \"CC_TABLE\" in ('t10', 't11', 't12p', 't13p')"); Assert.assertEquals(0, count); } @@ -1018,7 +1023,7 @@ public void testWriteSetTracking3() throws Exception { @Test public void testWriteSetTracking4() throws Exception { dropTable(new String[] {"TAB_PART", "TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + @@ -1043,7 +1048,7 @@ public void testWriteSetTracking4() throws Exception { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks); //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); AllocateTableWriteIdsRequest rqst = new AllocateTableWriteIdsRequest("default", "tab2"); rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId())); @@ -1056,7 +1061,7 @@ public void testWriteSetTracking4() throws Exception { txnHandler.addDynamicPartitions(adp); txnMgr2.commitTxn(); //Short Running updated nothing, so we expect 0 rows in WRITE_SET - Assert.assertEquals( 0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals( 0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); txnMgr2.openTxn(ctx, "T3"); driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'", true); //pretend this partition exists @@ -1066,7 +1071,7 @@ public void testWriteSetTracking4() throws Exception { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks); //since TAB2 is empty //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); rqst = new AllocateTableWriteIdsRequest("default", "tab2"); rqst.setTxnIds(Collections.singletonList(txnMgr2.getCurrentTxnId())); @@ -1078,14 +1083,14 @@ public void testWriteSetTracking4() throws Exception { adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); //simulate partition update txnMgr2.commitTxn(); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); AcidWriteSetService houseKeeper = new AcidWriteSetService(); houseKeeper.setConf(conf); houseKeeper.run(); //since T3 overlaps with Long Running (still open) GC does nothing - Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); driver.compileAndRespond("update TAB2 set b = 17 where a = 1", true); //no rows match txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running"); @@ -1104,7 +1109,7 @@ public void testWriteSetTracking4() throws Exception { locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 0, locks.size()); houseKeeper.run(); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); } /** * overlapping txns updating the same resource but 1st one rolls back; 2nd commits @@ -1113,7 +1118,7 @@ public void testWriteSetTracking4() throws Exception { @Test public void testWriteSetTracking5() throws Exception { dropTable(new String[] {"TAB_PART"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); driver.run("insert into TAB_PART partition(p='blah') values(1,2)"); @@ -1144,9 +1149,9 @@ public void testWriteSetTracking5() throws Exception { Arrays.asList("p=blah")); adp.setOperationType(DataOperationType.UPDATE); txnHandler.addDynamicPartitions(adp); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); txnMgr2.commitTxn(); //since conflicting txn rolled back, commit succeeds - Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); } /** * check that read query concurrent with txn works ok @@ -1154,7 +1159,7 @@ public void testWriteSetTracking5() throws Exception { @Test public void testWriteSetTracking6() throws Exception { dropTable(new String[] {"TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); driver.run("create table if not exists TAB2(a int, b int) clustered " + "by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); driver.compileAndRespond("select * from TAB2 where a = 113", true); @@ -1166,13 +1171,13 @@ public void testWriteSetTracking6() throws Exception { swapTxnManager(txnMgr2); driver.compileAndRespond("update TAB2 set b = 17 where a = 101", true); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Horton"); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr2.commitTxn(); //no conflict - Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); @@ -1180,7 +1185,7 @@ public void testWriteSetTracking6() throws Exception { MetastoreTaskThread writeSetService = new AcidWriteSetService(); writeSetService.setConf(conf); writeSetService.run(); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); } /** @@ -1190,7 +1195,7 @@ public void testWriteSetTracking6() throws Exception { @Test public void testWriteSetTracking7() throws Exception { dropTable(new String[] {"tab2", "TAB2"}); - Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); + Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); driver.run("create table if not exists tab2 (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); driver.run("insert into tab2 partition(p)(a,b,p) values(1,1,'one'),(2,2,'two')"); //txnid:1 @@ -1233,13 +1238,13 @@ public void testWriteSetTracking7() throws Exception { txnMgr.commitTxn(); //txnid:idTxnUpdate2 //now both txns concurrently updated TAB2 but different partitions. - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=one' and \"WS_OPERATION_TYPE\"='u'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='u'")); //2 from txnid:1, 1 from txnid:2, 1 from txnid:3 - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab2' and \"CTC_PARTITION\" is not null")); //================ //test with predicates such that partition pruning doesn't kick in @@ -1289,13 +1294,13 @@ public void testWriteSetTracking7() throws Exception { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn(); //txnid:idTxnUpdate4 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=one' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); //2 from insert + 1 for each update stmt - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\" is not null")); } /** * Concurrent updates with partition pruning predicate and w/o one @@ -1347,12 +1352,12 @@ public void testWriteSetTracking8() throws Exception { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn(); //txnid:idTxnUpdate2 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=one' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\" is not null")); } /** * Concurrent update/delete of different partitions - should pass @@ -1404,18 +1409,18 @@ public void testWriteSetTracking9() throws Exception { txnHandler.addDynamicPartitions(adp); txnMgr.commitTxn(); //txnid:idTxnUpdate2 - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + " and ctc_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + " and ctc_table='tab1' and ctc_partition='p=one'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + (idTxnUpdate1 - 1) + " and \"CTC_TABLE\"='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + idTxnUpdate1 + " and \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\"='p=one'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + idTxnDelete1 + " and \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\"='p=two'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=one' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='d' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\" is not null")); } /** * Concurrent update/delete of same partition - should fail to commit @@ -1475,10 +1480,10 @@ public void testWriteSetTracking10() throws Exception { "Aborting [txnid:5,5] due to a write conflict on default/tab1/p=two committed by [txnid:4,5] d/u", exception.getCause().getMessage()); - Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'")); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='u' and \"WS_TABLE\"='tab1'")); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\" is not null")); } /** * Concurrent delete/delete of same partition - should NOT pass @@ -1547,15 +1552,15 @@ public void testWriteSetTracking11() throws Exception { "Reason: Aborting [txnid:5,5] due to a write conflict on default/tab1/p=two " + "committed by [txnid:4,5] d/d", expectedException.getMessage()); Assert.assertEquals("WRITE_SET mismatch: " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, TxnDbUtil.countQueryAgent(conf, - "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete)); + "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='d' and \"WS_TABLE\"='tab1' and \"WS_TXNID\"=" + txnIdDelete)); Assert.assertEquals("WRITE_SET mismatch: " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 0, TxnDbUtil.countQueryAgent(conf, - "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect)); - Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null")); + "select count(*) from \"WRITE_SET\" where \"WS_PARTITION\"='p=two' and \"WS_OPERATION_TYPE\"='d' and \"WS_TABLE\"='tab1' and \"WS_TXNID\"=" + txnIdSelect)); + Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TABLE\"='tab1' and \"CTC_PARTITION\" is not null")); } @Test public void testCompletedTxnComponents() throws Exception { @@ -1566,11 +1571,11 @@ public void testCompletedTxnComponents() throws Exception { driver.run("insert into tab_not_acid2 values(1,1),(2,2)"); //writing both acid and non-acid resources in the same txn driver.run("from tab_not_acid2 insert into tab1 partition(p='two')(a,b) select a,b insert into tab_not_acid2(a,b) select a,b "); //txnid:1 - Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\"")); //only expect transactional components to be in COMPLETED_TXN_COMPONENTS - Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=6 and \"CTC_TABLE\"='tab1'")); } /** @@ -1589,13 +1594,13 @@ public void testMultiInsert() throws Exception { //writing both acid and non-acid resources in the same txn //tab1 write is a dynamic partition insert driver.run("from tab_not_acid insert into tab1 partition(p)(a,b,p) select a,b,p insert into tab_not_acid(a,b) select a,b where p='two'"); //txnid:9 - Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\"")); //only expect transactional components to be in COMPLETED_TXN_COMPONENTS - Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9")); - Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=9")); + Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=9 and \"CTC_TABLE\"='tab1'")); } //todo: Concurrent insert/update of same partition - should pass @@ -1613,18 +1618,18 @@ public void testMultiInsert() throws Exception { + "insert into tabMmDp select a,b,p"); //txnid: 6 (2 drops, 2 creates, 2 inserts) final String completedTxnComponentsContents = - TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"); + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""); Assert.assertEquals(completedTxnComponentsContents, - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\"")); Assert.assertEquals(completedTxnComponentsContents, - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6")); + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=6")); Assert.assertEquals(completedTxnComponentsContents, - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 " - + "and ctc_table='tabmmdp'")); + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=6 " + + "and \"CTC_TABLE\"='tabmmdp'")); // ctc_update_delete value should be "N" for both partitions since these are inserts Assert.assertEquals(completedTxnComponentsContents, - 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 " - + "and ctc_table='tabmmdp' and ctc_update_delete='N'")); + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=6 " + + "and \"CTC_TABLE\"='tabmmdp' and \"CTC_UPDATE_DELETE\"='N'")); } private List getLocksWithFilterOptions(HiveTxnManager txnMgr, @@ -1732,9 +1737,9 @@ private void testMerge3Way(boolean cc) throws Exception { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1)); //complete 1st txn long writeId = txnMgr.getTableWriteId("default", "target"); AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, writeId, "default", "target", @@ -1751,40 +1756,40 @@ private void testMerge3Way(boolean cc) throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + - " and tc_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1 + + " and \"TC_OPERATION_TYPE\"='u'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 2, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + - " and tc_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1 + + " and \"TC_OPERATION_TYPE\"='d'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 3, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + - " and tc_operation_type='i'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1 + + " and \"TC_OPERATION_TYPE\"='i'")); txnMgr.commitTxn(); //commit T1 Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), 6, - TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + txnId1)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + - " and ws_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId1 + + " and \"WS_OPERATION_TYPE\"='u'")); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 2, - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + - " and ws_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId1 + + " and \"WS_OPERATION_TYPE\"='d'")); //re-check locks which were in Waiting state - should now be Acquired ((DbLockManager)txnMgr2.getLockManager()).checkLock(extLockId); @@ -1798,9 +1803,9 @@ private void testMerge3Way(boolean cc) throws Exception { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId2)); //complete 2nd txn writeId = txnMgr2.getTableWriteId("default", "target"); adp = new AddDynamicPartitions(txnId2, writeId, "default", "target", @@ -1819,22 +1824,22 @@ private void testMerge3Way(boolean cc) throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + - " and tc_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId2 + + " and \"TC_OPERATION_TYPE\"='u'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), (cc ? 2 : 0), - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + - " and tc_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId2 + + " and \"TC_OPERATION_TYPE\"='d'")); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 3, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 + - " and tc_operation_type='i'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId2 + + " and \"TC_OPERATION_TYPE\"='i'")); LockException expectedException = null; try { @@ -1846,47 +1851,54 @@ private void testMerge3Way(boolean cc) throws Exception { if(cc) { Assert.assertNotNull("didn't get exception", expectedException); try { - Assert.assertEquals("Transaction manager has aborted the transaction txnid:11. Reason: " + - "Aborting [txnid:11,11] due to a write conflict on default/target/p=1/q=3 " + - "committed by [txnid:10,11] u/u", expectedException.getMessage()); - } - catch(ComparisonFailure ex) { - //the 2 txns have 2 conflicts between them so check for either failure since which one is - //reported (among the 2) is not deterministic Assert.assertEquals("Transaction manager has aborted the transaction txnid:11. Reason: " + "Aborting [txnid:11,11] due to a write conflict on default/target/p=1/q=2 " + "committed by [txnid:10,11] d/d", expectedException.getMessage()); + } catch (ComparisonFailure ex) { + //the 2 txns have 3 conflicts between them so check for either failure since which one is + //reported (among the 3) is not deterministic + try { + Assert.assertEquals("Transaction manager has aborted the transaction txnid:11. Reason: " + + "Aborting [txnid:11,11] due to a write conflict on default/target/p=2/q=2 " + + "committed by [txnid:10,11] d/d", expectedException.getMessage()); + } catch (ComparisonFailure ex2) { + Assert.assertEquals("Transaction manager has aborted the transaction txnid:11. Reason: " + + "Aborting [txnid:11,11] due to a write conflict on default/target/p=1/q=3 " + + "committed by [txnid:10,11] u/u", expectedException.getMessage()); + } } + + Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + txnId2)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId2)); } else { Assert.assertNull("Unexpected exception " + expectedException, expectedException); Assert.assertEquals( "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), 4, - TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + txnId2)); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 + - " and ws_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId2 + + " and \"WS_OPERATION_TYPE\"='u'")); Assert.assertEquals( "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 + - " and ws_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId2 + + " and \"WS_OPERATION_TYPE\"='d'")); } @@ -1924,9 +1936,9 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, //no DP, so it's populated from lock info - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid1)); List locks = getLocks(txnMgr); if (causeConflict) { @@ -1957,10 +1969,10 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { txnMgr.commitTxn(); //commit T1 Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), causeConflict ? 1 : 0, //Inserts are not tracked by WRITE_SET - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid1 + - " and ws_operation_type=" + (causeConflict ? "'u'" : "'i'"))); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnid1 + + " and \"WS_OPERATION_TYPE\"=" + (causeConflict ? "'u'" : "'i'"))); //re-check locks which were in Waiting state - should now be Acquired @@ -1972,15 +1984,15 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, // - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2)); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, // - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + - "and tc_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2 + + "and \"TC_OPERATION_TYPE\"='d'")); //complete T2 txn LockException expectedException = null; @@ -1999,10 +2011,10 @@ private void testMergeUnpartitioned(boolean causeConflict) throws Exception { } else { Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, //Unpartitioned table: 1 row for Delete; Inserts are not tracked in WRITE_SET - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 + - " and ws_operation_type='d'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnid2 + + " and \"WS_OPERATION_TYPE\"='d'")); } } /** @@ -2024,15 +2036,15 @@ public void testDynamicPartitionInsert() throws Exception { checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks); Assert.assertEquals( "HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString(conf, "select * from HIVE_LOCKS"), + TxnDbUtil.queryToString(conf, "select * from \"HIVE_LOCKS\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"HIVE_LOCKS\" where \"HL_TXNID\"=" + txnid1)); txnMgr.rollbackTxn(); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 0, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid1)); //now actually write to table to generate some partitions driver.run("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)"); driver.run("select count(*) from target"); @@ -2041,10 +2053,10 @@ public void testDynamicPartitionInsert() throws Exception { Assert.assertEquals("", "4", r.get(0)); Assert.assertEquals(//look in COMPLETED_TXN_COMPONENTS because driver.run() committed!!!! "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1 + 1) + "): " + - TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"COMPLETED_TXN_COMPONENTS\""), 2, //2 distinct partitions created //txnid+1 because we want txn used by previous driver.run("insert....) - TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1))); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"COMPLETED_TXN_COMPONENTS\" where \"CTC_TXNID\"=" + (txnid1 + 1))); long txnid2 = txnMgr.openTxn(ctx, "T1"); @@ -2060,9 +2072,9 @@ public void testDynamicPartitionInsert() throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 2, //2 distinct partitions modified - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2)); txnMgr.commitTxn(); } @Test @@ -2124,9 +2136,9 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 0, //because it's using a DP write - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1)); //complete T1 transaction (simulate writing to 2 partitions) long writeId = txnMgr.getTableWriteId("default", "target"); AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, writeId, "default", "target", @@ -2135,16 +2147,16 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 2, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 + - " and tc_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnId1 + + " and \"TC_OPERATION_TYPE\"='u'")); txnMgr.commitTxn(); //commit T1 Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 2, //2 partitions updated - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 + - " and ws_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnId1 + + " and \"WS_OPERATION_TYPE\"='u'")); //re-check locks which were in Waiting state - should now be Acquired @@ -2160,9 +2172,9 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 0, //because it's using a DP write - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2)); //complete T2 txn //simulate Insert into 2 partitions writeId = txnMgr2.getTableWriteId("default", "target"); @@ -2172,9 +2184,9 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 2, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2 + " and \"TC_OPERATION_TYPE\"='i'")); //simulate Update of 1 partitions; depending on causeConflict, choose one of the partitions //which was modified by the T1 update stmt or choose a non-conflicting one adp = new AddDynamicPartitions(txnid2, writeId, "default", "target", @@ -2183,9 +2195,9 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { txnHandler.addDynamicPartitions(adp); Assert.assertEquals( "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"), + TxnDbUtil.queryToString(conf, "select * from \"TXN_COMPONENTS\""), 1, - TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"TXN_COMPONENTS\" where \"TC_TXNID\"=" + txnid2 + " and \"TC_OPERATION_TYPE\"='u'")); LockException expectedException = null; @@ -2204,14 +2216,14 @@ private void testMergePartitioned(boolean causeConflict) throws Exception { } else { Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, //1 partitions updated - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 + - " and ws_operation_type='u'")); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnid2 + + " and \"WS_OPERATION_TYPE\"='u'")); Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " + - TxnDbUtil.queryToString(conf, "select * from WRITE_SET"), + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, //1 partitions updated (and no other entries) - TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2)); + TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\" where \"WS_TXNID\"=" + txnid2)); } } diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java index 9a9ab53fcc..311c39fb44 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java @@ -103,6 +103,7 @@ public void setup() throws Exception { conf = new HiveConf(); TxnDbUtil.setConfValues(conf); TxnDbUtil.cleanDb(conf); + TxnDbUtil.prepDb(conf); ms = new HiveMetaStoreClient(conf); txnHandler = TxnUtils.getTxnStore(conf); tmpdir = new File(Files.createTempDirectory("compactor_test_table_").toString()); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java index 3e56ad513c..b798cdd041 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java @@ -72,4 +72,19 @@ public static boolean needsInBatching(DatabaseProduct dbType) { public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) { return dbType == DERBY || dbType == ORACLE || dbType == POSTGRES; } + + public static String getHiveSchemaPostfix(DatabaseProduct dbType) { + switch (dbType) { + case SQLSERVER: + return "mssql"; + case DERBY: + case MYSQL: + case POSTGRES: + case ORACLE: + return dbType.name().toLowerCase(); + case OTHER: + default: + return null; + } + } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 620c77e589..f5a5adcdd4 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hive.metastore.txn; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; import java.sql.Connection; import java.sql.Driver; import java.sql.PreparedStatement; @@ -29,12 +32,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; +import java.util.HashSet; import java.util.List; import java.util.Properties; +import java.util.Scanner; +import java.util.Set; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.DatabaseProduct; +import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; @@ -83,318 +92,38 @@ public static void setConfValues(Configuration conf) { MetastoreConf.setBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY, true); } + /** + * Prepares the metastore database for unit tests. + * Runs the latest init schema against the database configured in the CONNECT_URL_KEY param. + * Ignores any duplication (table, index etc.) So it can be called multiple times for the same database. + * @param conf Metastore configuration + * @throws Exception + */ public static synchronized void prepDb(Configuration conf) throws Exception { - // This is a bogus hack because it copies the contents of the SQL file - // intended for creating derby databases, and thus will inexorably get - // out of date with it. I'm open to any suggestions on how to make this - // read the file in a build friendly way. - Connection conn = null; Statement stmt = null; try { conn = getConnection(conf); + String s = conn.getMetaData().getDatabaseProductName(); + DatabaseProduct dbProduct = DatabaseProduct.determineDatabaseProduct(s); stmt = conn.createStatement(); - stmt.execute("CREATE TABLE TXNS (" + - " TXN_ID bigint PRIMARY KEY," + - " TXN_STATE char(1) NOT NULL," + - " TXN_STARTED bigint NOT NULL," + - " TXN_LAST_HEARTBEAT bigint NOT NULL," + - " TXN_USER varchar(128) NOT NULL," + - " TXN_HOST varchar(128) NOT NULL," + - " TXN_TYPE integer)"); - - stmt.execute("CREATE TABLE TXN_COMPONENTS (" + - " TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID)," + - " TC_DATABASE varchar(128) NOT NULL," + - " TC_TABLE varchar(128)," + - " TC_PARTITION varchar(767)," + - " TC_OPERATION_TYPE char(1) NOT NULL," + - " TC_WRITEID bigint)"); - stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" + - " CTC_TXNID bigint NOT NULL," + - " CTC_DATABASE varchar(128) NOT NULL," + - " CTC_TABLE varchar(128)," + - " CTC_PARTITION varchar(767)," + - " CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL," + - " CTC_WRITEID bigint," + - " CTC_UPDATE_DELETE char(1) NOT NULL)"); - stmt.execute("CREATE TABLE NEXT_TXN_ID (" + " NTXN_NEXT bigint NOT NULL)"); - stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)"); - - stmt.execute("CREATE TABLE TXN_TO_WRITE_ID (" + - " T2W_TXNID bigint NOT NULL," + - " T2W_DATABASE varchar(128) NOT NULL," + - " T2W_TABLE varchar(256) NOT NULL," + - " T2W_WRITEID bigint NOT NULL)"); - stmt.execute("CREATE TABLE NEXT_WRITE_ID (" + - " NWI_DATABASE varchar(128) NOT NULL," + - " NWI_TABLE varchar(256) NOT NULL," + - " NWI_NEXT bigint NOT NULL)"); - - stmt.execute("CREATE TABLE MIN_HISTORY_LEVEL (" + - " MHL_TXNID bigint NOT NULL," + - " MHL_MIN_OPEN_TXNID bigint NOT NULL," + - " PRIMARY KEY(MHL_TXNID))"); - - stmt.execute("CREATE TABLE HIVE_LOCKS (" + - " HL_LOCK_EXT_ID bigint NOT NULL," + - " HL_LOCK_INT_ID bigint NOT NULL," + - " HL_TXNID bigint NOT NULL," + - " HL_DB varchar(128) NOT NULL," + - " HL_TABLE varchar(128)," + - " HL_PARTITION varchar(767)," + - " HL_LOCK_STATE char(1) NOT NULL," + - " HL_LOCK_TYPE char(1) NOT NULL," + - " HL_LAST_HEARTBEAT bigint NOT NULL," + - " HL_ACQUIRED_AT bigint," + - " HL_USER varchar(128) NOT NULL," + - " HL_HOST varchar(128) NOT NULL," + - " HL_HEARTBEAT_COUNT integer," + - " HL_AGENT_INFO varchar(128)," + - " HL_BLOCKEDBY_EXT_ID bigint," + - " HL_BLOCKEDBY_INT_ID bigint," + - " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))"); - stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)"); - - stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)"); - stmt.execute("INSERT INTO NEXT_LOCK_ID VALUES(1)"); - - stmt.execute("CREATE TABLE COMPACTION_QUEUE (" + - " CQ_ID bigint PRIMARY KEY," + - " CQ_DATABASE varchar(128) NOT NULL," + - " CQ_TABLE varchar(128) NOT NULL," + - " CQ_PARTITION varchar(767)," + - " CQ_STATE char(1) NOT NULL," + - " CQ_TYPE char(1) NOT NULL," + - " CQ_TBLPROPERTIES varchar(2048)," + - " CQ_WORKER_ID varchar(128)," + - " CQ_START bigint," + - " CQ_RUN_AS varchar(128)," + - " CQ_HIGHEST_WRITE_ID bigint," + - " CQ_META_INFO varchar(2048) for bit data," + - " CQ_HADOOP_JOB_ID varchar(32)," + - " CQ_ERROR_MESSAGE clob)"); - - stmt.execute("CREATE TABLE NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT bigint NOT NULL)"); - stmt.execute("INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1)"); - - stmt.execute("CREATE TABLE COMPLETED_COMPACTIONS (" + - " CC_ID bigint PRIMARY KEY," + - " CC_DATABASE varchar(128) NOT NULL," + - " CC_TABLE varchar(128) NOT NULL," + - " CC_PARTITION varchar(767)," + - " CC_STATE char(1) NOT NULL," + - " CC_TYPE char(1) NOT NULL," + - " CC_TBLPROPERTIES varchar(2048)," + - " CC_WORKER_ID varchar(128)," + - " CC_START bigint," + - " CC_END bigint," + - " CC_RUN_AS varchar(128)," + - " CC_HIGHEST_WRITE_ID bigint," + - " CC_META_INFO varchar(2048) for bit data," + - " CC_HADOOP_JOB_ID varchar(32)," + - " CC_ERROR_MESSAGE clob)"); - - stmt.execute("CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (" - + "CC_DATABASE,CC_TABLE,CC_PARTITION)"); - - stmt.execute("CREATE TABLE AUX_TABLE (" + - " MT_KEY1 varchar(128) NOT NULL," + - " MT_KEY2 bigint NOT NULL," + - " MT_COMMENT varchar(255)," + - " PRIMARY KEY(MT_KEY1, MT_KEY2))"); - - stmt.execute("CREATE TABLE WRITE_SET (" + - " WS_DATABASE varchar(128) NOT NULL," + - " WS_TABLE varchar(128) NOT NULL," + - " WS_PARTITION varchar(767)," + - " WS_TXNID bigint NOT NULL," + - " WS_COMMIT_ID bigint NOT NULL," + - " WS_OPERATION_TYPE char(1) NOT NULL)" - ); - - stmt.execute("CREATE TABLE REPL_TXN_MAP (" + - " RTM_REPL_POLICY varchar(256) NOT NULL, " + - " RTM_SRC_TXN_ID bigint NOT NULL, " + - " RTM_TARGET_TXN_ID bigint NOT NULL, " + - " PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID))" - ); - - stmt.execute("CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (" + - " MRL_TXN_ID BIGINT NOT NULL, " + - " MRL_DB_NAME VARCHAR(128) NOT NULL, " + - " MRL_TBL_NAME VARCHAR(256) NOT NULL, " + - " MRL_LAST_HEARTBEAT BIGINT NOT NULL, " + - " PRIMARY KEY(MRL_TXN_ID))" - ); - - try { - stmt.execute("CREATE TABLE \"APP\".\"TBLS\" (\"TBL_ID\" BIGINT NOT NULL, " + - " \"CREATE_TIME\" INTEGER NOT NULL, \"DB_ID\" BIGINT, \"LAST_ACCESS_TIME\" INTEGER NOT NULL, " + - " \"OWNER\" VARCHAR(767), \"OWNER_TYPE\" VARCHAR(10), \"RETENTION\" INTEGER NOT NULL, " + - " \"SD_ID\" BIGINT, \"TBL_NAME\" VARCHAR(256), \"TBL_TYPE\" VARCHAR(128), " + - " \"VIEW_EXPANDED_TEXT\" LONG VARCHAR, \"VIEW_ORIGINAL_TEXT\" LONG VARCHAR, " + - " \"IS_REWRITE_ENABLED\" CHAR(1) NOT NULL DEFAULT \'N\', " + - " \"WRITE_ID\" BIGINT DEFAULT 0, " + - " PRIMARY KEY (TBL_ID))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("TBLS table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"DBS\" (\"DB_ID\" BIGINT NOT NULL, \"DESC\" " + - "VARCHAR(4000), \"DB_LOCATION_URI\" VARCHAR(4000) NOT NULL, \"NAME\" VARCHAR(128), " + - "\"OWNER_NAME\" VARCHAR(128), \"OWNER_TYPE\" VARCHAR(10), " + - "\"CTLG_NAME\" VARCHAR(256) NOT NULL, PRIMARY KEY (DB_ID))"); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("TBLS table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"PARTITIONS\" (" + - " \"PART_ID\" BIGINT NOT NULL, \"CREATE_TIME\" INTEGER NOT NULL, " + - " \"LAST_ACCESS_TIME\" INTEGER NOT NULL, \"PART_NAME\" VARCHAR(767), " + - " \"SD_ID\" BIGINT, \"TBL_ID\" BIGINT, " + - " \"WRITE_ID\" BIGINT DEFAULT 0, " + - " PRIMARY KEY (PART_ID))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("PARTITIONS table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"TABLE_PARAMS\" (" + - " \"TBL_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " + - " \"PARAM_VALUE\" CLOB, " + - " PRIMARY KEY (TBL_ID, PARAM_KEY))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("TABLE_PARAMS table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"PARTITION_PARAMS\" (" + - " \"PART_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " + - " \"PARAM_VALUE\" CLOB, " + - " PRIMARY KEY (PART_ID, PARAM_KEY))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("PARTITION_PARAMS table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"SEQUENCE_TABLE\" (\"SEQUENCE_NAME\" VARCHAR(256) NOT " + - - "NULL, \"NEXT_VAL\" BIGINT NOT NULL)" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("SEQUENCE_TABLE table already exist, ignoring"); - } else { - throw e; - } - } - - try { - stmt.execute("CREATE TABLE \"APP\".\"NOTIFICATION_SEQUENCE\" (\"NNI_ID\" BIGINT NOT NULL, " + - - "\"NEXT_EVENT_ID\" BIGINT NOT NULL)" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("NOTIFICATION_SEQUENCE table already exist, ignoring"); - } else { - throw e; - } + if (checkDbPrepared(stmt)) { + return; } - - try { - stmt.execute("CREATE TABLE \"APP\".\"NOTIFICATION_LOG\" (\"NL_ID\" BIGINT NOT NULL, " + - "\"DB_NAME\" VARCHAR(128), \"EVENT_ID\" BIGINT NOT NULL, \"EVENT_TIME\" INTEGER NOT" + - - " NULL, \"EVENT_TYPE\" VARCHAR(32) NOT NULL, \"MESSAGE\" CLOB, \"TBL_NAME\" " + - "VARCHAR" + - "(256), \"MESSAGE_FORMAT\" VARCHAR(16))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("NOTIFICATION_LOG table already exist, ignoring"); - } else { - throw e; - } + String schemaRootPath = getSchemaRootPath(); + IMetaStoreSchemaInfo metaStoreSchemaInfo = + MetaStoreSchemaInfoFactory.get(conf, schemaRootPath, DatabaseProduct.getHiveSchemaPostfix(dbProduct)); + String initFile = metaStoreSchemaInfo.generateInitFileName(null); + try (InputStream is = new FileInputStream( + metaStoreSchemaInfo.getMetaStoreScriptDir() + File.separator + initFile)) { + importSQL(stmt, is); } - - stmt.execute("INSERT INTO \"APP\".\"SEQUENCE_TABLE\" (\"SEQUENCE_NAME\", \"NEXT_VAL\") " + - "SELECT * FROM (VALUES ('org.apache.hadoop.hive.metastore.model.MNotificationLog', " + - "1)) tmp_table WHERE NOT EXISTS ( SELECT \"NEXT_VAL\" FROM \"APP\"" + - ".\"SEQUENCE_TABLE\" WHERE \"SEQUENCE_NAME\" = 'org.apache.hadoop.hive.metastore" + - ".model.MNotificationLog')"); - - stmt.execute("INSERT INTO \"APP\".\"NOTIFICATION_SEQUENCE\" (\"NNI_ID\", \"NEXT_EVENT_ID\")" + - " SELECT * FROM (VALUES (1,1)) tmp_table WHERE NOT EXISTS ( SELECT " + - "\"NEXT_EVENT_ID\" FROM \"APP\".\"NOTIFICATION_SEQUENCE\")"); - - try { - stmt.execute("CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (" + - "WNL_ID bigint NOT NULL," + - "WNL_TXNID bigint NOT NULL," + - "WNL_WRITEID bigint NOT NULL," + - "WNL_DATABASE varchar(128) NOT NULL," + - "WNL_TABLE varchar(128) NOT NULL," + - "WNL_PARTITION varchar(1024) NOT NULL," + - "WNL_TABLE_OBJ clob NOT NULL," + - "WNL_PARTITION_OBJ clob," + - "WNL_FILES clob," + - "WNL_EVENT_TIME integer NOT NULL," + - "PRIMARY KEY (WNL_TXNID, WNL_DATABASE, WNL_TABLE, WNL_PARTITION))" - ); - } catch (SQLException e) { - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("TXN_WRITE_NOTIFICATION_LOG table already exist, ignoring"); - } else { - throw e; - } - } - - stmt.execute("INSERT INTO \"APP\".\"SEQUENCE_TABLE\" (\"SEQUENCE_NAME\", \"NEXT_VAL\") " + - "SELECT * FROM (VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', " + - "1)) tmp_table WHERE NOT EXISTS ( SELECT \"NEXT_VAL\" FROM \"APP\"" + - ".\"SEQUENCE_TABLE\" WHERE \"SEQUENCE_NAME\" = 'org.apache.hadoop.hive.metastore" + - ".model.MTxnWriteNotificationLog')"); } catch (SQLException e) { try { conn.rollback(); } catch (SQLException re) { LOG.error("Error rolling back: " + re.getMessage()); } - - // Another thread might have already created these tables. - if (e.getMessage() != null && e.getMessage().contains("already exists")) { - LOG.info("Txn tables already exist, returning"); - return; - } - // This might be a deadlock, if so, let's retry if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) { LOG.warn("Caught deadlock, retrying db creation"); @@ -408,6 +137,76 @@ public static synchronized void prepDb(Configuration conf) throws Exception { } } + private static boolean checkDbPrepared(Statement stmt) { + /* + * The prepDb is called many times in some test cases involuntary, since it is always called in TxnHandler.setConf(). + * This could consume a lots of time, so if the TXNS table is already created we assume the database is ready. + */ + try { + stmt.execute("SELECT * FROM \"TXNS\""); + } catch (SQLException e) { + return false; + } + return true; + } + + private static void importSQL(Statement stmt, InputStream in) throws SQLException { + Set knownErrors = getAlreadyExistsErrorCodes(); + Scanner s = new Scanner(in); + s.useDelimiter("(;(\r)?\n)|(--.*\n)"); + while (s.hasNext()) { + String line = s.next(); + + if (line.trim().length() > 0) { + try { + stmt.execute(line); + } catch (SQLException e) { + if (knownErrors.contains(e.getSQLState())) { + LOG.info("Ignoring sql error {}", e.getMessage()); + } else { + throw e; + } + } + } + } + } + + private static Set getAlreadyExistsErrorCodes() { + // function already exists, table already exists, index already exists, duplicate key + Set knownErrors = new HashSet<>(); + // derby + knownErrors.addAll(Arrays.asList("X0Y68", "X0Y32", "X0Y44", "42Z93", "23505")); + // postgres + knownErrors.addAll(Arrays.asList("42P07", "42P16", "42710")); + // mssql + knownErrors.addAll(Arrays.asList("S0000", "S0001", "23000")); + // mysql + knownErrors.addAll(Arrays.asList("42S01", "HY000")); + // oracle + knownErrors.addAll(Arrays.asList("42000")); + return knownErrors; + } + + private static String getSchemaRootPath() { + String hiveRoot = System.getProperty("hive.root"); + if (StringUtils.isNotEmpty(hiveRoot)) { + return ensurePathEndsInSlash(hiveRoot) + "standalone-metastore/metastore-server/target/tmp/"; + } else { + return ensurePathEndsInSlash(System.getProperty("test.tmp.dir", "target/tmp")); + } + } + + private static String ensurePathEndsInSlash(String path) { + if (path == null) { + throw new NullPointerException("Path cannot be null"); + } + if (path.endsWith(File.separator)) { + return path; + } else { + return path + File.separator; + } + } + public static void cleanDb(Configuration conf) throws Exception { int retryCount = 0; while(++retryCount <= 3) { @@ -419,24 +218,24 @@ public static void cleanDb(Configuration conf) throws Exception { stmt = conn.createStatement(); // We want to try these, whether they succeed or fail. - success &= dropIndex(stmt, "HL_TXNID_INDEX", retryCount); - - success &= dropTable(stmt, "TXN_COMPONENTS", retryCount); - success &= dropTable(stmt, "COMPLETED_TXN_COMPONENTS", retryCount); - success &= dropTable(stmt, "TXNS", retryCount); - success &= dropTable(stmt, "NEXT_TXN_ID", retryCount); - success &= dropTable(stmt, "TXN_TO_WRITE_ID", retryCount); - success &= dropTable(stmt, "NEXT_WRITE_ID", retryCount); - success &= dropTable(stmt, "MIN_HISTORY_LEVEL", retryCount); - success &= dropTable(stmt, "HIVE_LOCKS", retryCount); - success &= dropTable(stmt, "NEXT_LOCK_ID", retryCount); - success &= dropTable(stmt, "COMPACTION_QUEUE", retryCount); - success &= dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID", retryCount); - success &= dropTable(stmt, "COMPLETED_COMPACTIONS", retryCount); - success &= dropTable(stmt, "AUX_TABLE", retryCount); - success &= dropTable(stmt, "WRITE_SET", retryCount); - success &= dropTable(stmt, "REPL_TXN_MAP", retryCount); - success &= dropTable(stmt, "MATERIALIZATION_REBUILD_LOCKS", retryCount); + success &= dropIndex(conn, stmt, "HL_TXNID_INDEX", "HIVE_LOCKS", retryCount); + + success &= dropTable(conn, stmt, "TXN_COMPONENTS", retryCount); + success &= dropTable(conn, stmt, "COMPLETED_TXN_COMPONENTS", retryCount); + success &= dropTable(conn, stmt, "TXNS", retryCount); + success &= dropTable(conn, stmt, "NEXT_TXN_ID", retryCount); + success &= dropTable(conn, stmt, "TXN_TO_WRITE_ID", retryCount); + success &= dropTable(conn, stmt, "NEXT_WRITE_ID", retryCount); + success &= dropTable(conn, stmt, "MIN_HISTORY_LEVEL", retryCount); + success &= dropTable(conn, stmt, "HIVE_LOCKS", retryCount); + success &= dropTable(conn, stmt, "NEXT_LOCK_ID", retryCount); + success &= dropTable(conn, stmt, "COMPACTION_QUEUE", retryCount); + success &= dropTable(conn, stmt, "NEXT_COMPACTION_QUEUE_ID", retryCount); + success &= dropTable(conn, stmt, "COMPLETED_COMPACTIONS", retryCount); + success &= dropTable(conn, stmt, "AUX_TABLE", retryCount); + success &= dropTable(conn, stmt, "WRITE_SET", retryCount); + success &= dropTable(conn, stmt, "REPL_TXN_MAP", retryCount); + success &= dropTable(conn, stmt, "MATERIALIZATION_REBUILD_LOCKS", retryCount); /* * Don't drop NOTIFICATION_LOG, SEQUENCE_TABLE and NOTIFICATION_SEQUENCE as its used by other * table which are not txn related to generate primary key. So if these tables are dropped @@ -453,12 +252,21 @@ public static void cleanDb(Configuration conf) throws Exception { throw new RuntimeException("Failed to clean up txn tables"); } - private static boolean dropIndex(Statement stmt, String index, int retryCount) { + private static boolean dropIndex(Connection conn, Statement stmt, String index, String table, int retryCount) { + Set knownErrors = new HashSet<>(); + knownErrors.addAll(Arrays.asList("42X65", "42704", "42000", "S0006", "S0007", "72000")); try { - stmt.execute("DROP INDEX " + index); + String s = conn.getMetaData().getDatabaseProductName(); + DatabaseProduct dbProduct = DatabaseProduct.determineDatabaseProduct(s); + if (dbProduct == MYSQL) { + stmt.execute("DROP INDEX " + index + " ON " + table); + } else if (dbProduct == SQLSERVER) { + stmt.execute("DROP INDEX "+ table + "." + index); + } else { + stmt.execute("DROP INDEX " + index); + } } catch (SQLException e) { - if (!("42X65".equals(e.getSQLState()) && 30000 == e.getErrorCode())) { - //42X65/3000 means index doesn't exist + if (!knownErrors.contains(e.getSQLState())) { LOG.error("Unable to drop index {} {} State={} code={} retryCount={}", index, e.getMessage(), e.getSQLState(), e.getErrorCode(), retryCount); return false; @@ -467,14 +275,23 @@ private static boolean dropIndex(Statement stmt, String index, int retryCount) { return true; } - private static boolean dropTable(Statement stmt, String name, int retryCount) throws SQLException { + private static boolean dropTable(Connection conn, Statement stmt, String name, int retryCount) throws SQLException { + Set knownErrors = new HashSet<>(); + knownErrors.addAll(Arrays.asList("42Y55", "42P01", "42S02", "S0005", "42000")); for (int i = 0; i < 3; i++) { try { - stmt.execute("DROP TABLE " + name); + String s = conn.getMetaData().getDatabaseProductName(); + DatabaseProduct dbProduct = DatabaseProduct.determineDatabaseProduct(s); + if (dbProduct == POSTGRES) { + stmt.execute("DROP TABLE \"" + name +"\""); + } else { + stmt.execute("DROP TABLE " + name); + } + LOG.debug("Successfully dropped table " + name); return true; } catch (SQLException e) { - if ("42Y55".equals(e.getSQLState()) && 30000 == e.getErrorCode()) { + if (knownErrors.contains(e.getSQLState())) { LOG.debug("Not dropping " + name + " because it doesn't exist"); //failed because object doesn't exist return true; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 7d0db0c3a0..e9a03ccb8f 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -4224,15 +4224,6 @@ private void checkQFileTestHack() { LOG.info("Hacking in canned values for transaction manager"); // Set up the transaction/locking db in the derby metastore TxnDbUtil.setConfValues(conf); - try { - TxnDbUtil.prepDb(conf); - } catch (Exception e) { - // We may have already created the tables and thus don't need to redo it. - if (e.getMessage() != null && !e.getMessage().contains("already exists")) { - throw new RuntimeException("Unable to set up transaction database for" + - " testing: " + e.getMessage(), e); - } - } } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java index c537d95470..afa8f2a781 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/Mysql.java @@ -54,7 +54,7 @@ public String getJdbcDriver() { @Override public String getJdbcUrl() { - return "jdbc:mysql://localhost:3306/" + HIVE_DB; + return "jdbc:mysql://localhost:3306/" + HIVE_DB + "?sessionVariables=sql_mode=ANSI_QUOTES"; } @Override