diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java index b0f8b15..23e2fc7 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java @@ -22,7 +22,7 @@ public class HiveSchemaHelper { public static final String DB_DERBY = "derby"; public static final String DB_MYSQL = "mysql"; - public static final String DB_POSTGRACE = "postgrace"; + public static final String DB_POSTGRACE = "postgres"; public static final String DB_ORACLE = "oracle"; public interface NestedScriptParser { @@ -225,8 +225,8 @@ public String getScriptName(String dbCommand) throws IllegalArgumentException { if (!isNestedScript(dbCommand)) { throw new IllegalArgumentException("Not a nested script format " + dbCommand); } - // remove ending ';' - return dbCommand.replace(";", ""); + // remove ending ';' and starting '@' + return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, ""); } @Override diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 318e7ed..a1f9a6a 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -366,6 +366,7 @@ public void runBeeLine(String sqlScriptFile) throws IOException { beeLine.getOpts().setSilent(true); } beeLine.getOpts().setAllowMultiLineCommand(false); + beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED"); int status = beeLine.begin(argList.toArray(new String[0]), null); if (status != 0) { throw new IOException("Schema script failed, errorcode " + status); diff --git beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java index 09d96d7..338dc42 100644 --- beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java +++ beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java @@ -30,6 +30,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; import org.apache.hive.beeline.HiveSchemaHelper; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; import org.apache.hive.beeline.HiveSchemaTool; @@ -100,7 +101,8 @@ public void testSchemaUpgradeDryRun() throws Exception { * @throws Exception */ public void testSchemaInit() throws Exception { - schemaTool.doInit("0.12.0"); + schemaTool.doInit(MetaStoreSchemaInfo.getHiveSchemaVersion()); + schemaTool.verifySchemaVersion(); } /** @@ -321,6 +323,51 @@ public void testScriptMultiRowComment() throws Exception { assertEquals(expectedSQL, flattenedSql); } + /** + * Test nested script formatting + * @throws Exception + */ + public void testNestedScriptsForOracle() throws Exception { + String childTab1 = "childTab1"; + String childTab2 = "childTab2"; + String parentTab = "fooTab"; + + String childTestScript1[] = { + "-- this is a comment ", + "DROP TABLE IF EXISTS " + childTab1 + ";", + "CREATE TABLE " + childTab1 + "(id INTEGER);", + "DROP TABLE " + childTab1 + ";" + }; + String childTestScript2[] = { + "-- this is a comment", + "DROP TABLE IF EXISTS " + childTab2 + ";", + "CREATE TABLE " + childTab2 + "(id INTEGER);", + "-- this is also a comment", + "DROP TABLE " + childTab2 + ";" + }; + + String parentTestScript[] = { + " -- this is a comment", + "DROP TABLE IF EXISTS " + parentTab + ";", + " -- this is another comment ", + "CREATE TABLE " + parentTab + "(id INTEGER);", + "@" + generateTestScript(childTestScript1).getName() + ";", + "DROP TABLE " + parentTab + ";", + "@" + generateTestScript(childTestScript2).getName() + ";", + "--ending comment ", + }; + + File testScriptFile = generateTestScript(parentTestScript); + String flattenedSql = HiveSchemaTool.buildCommand( + HiveSchemaHelper.getDbCommandParser("oracle"), + testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + assertFalse(flattenedSql.contains("@")); + assertFalse(flattenedSql.contains("comment")); + assertTrue(flattenedSql.contains(childTab1)); + assertTrue(flattenedSql.contains(childTab2)); + assertTrue(flattenedSql.contains(parentTab)); + } + private File generateTestScript(String [] stmts) throws IOException { File testScriptFile = File.createTempFile("schematest", ".sql"); testScriptFile.deleteOnExit(); diff --git metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql index ca890e6..f70a7c0 100644 --- metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql +++ metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql @@ -296,4 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "SQL110318025504980" CHECK (DEFERRED_REB ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N')); -INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.13.0', 'Hive release version 0.13.0'); +INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.13.0', 'Hive release version 0.13.0'); diff --git metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql index 4cad6a0..113f147 100644 --- metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql +++ metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql @@ -1,10 +1,10 @@ -- HIVE-3764 Support metastore version consistency check -CREATE TABLE IF NOT EXISTS VERSION ( +CREATE TABLE VERSION ( VER_ID NUMBER NOT NULL, SCHEMA_VERSION VARCHAR(127) NOT NULL, VERSION_COMMENT VARCHAR(255) -) +); ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); -INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '', 'Initial value'); +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, ' ', 'Initial value'); diff --git metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql index c9c7218..812b897 100644 --- metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql @@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS ( LAST_ANALYZED NUMBER NOT NULL ); -CREATE TABLE IF NOT EXISTS VERSION ( +CREATE TABLE VERSION ( VER_ID NUMBER NOT NULL, SCHEMA_VERSION VARCHAR(127) NOT NULL, VERSION_COMMENT VARCHAR(255) -) +); ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID); diff --git metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql index 8c24b0b..fa483bf 100644 --- metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql +++ metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql @@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS ( LAST_ANALYZED NUMBER NOT NULL ); -CREATE TABLE IF NOT EXISTS VERSION ( +CREATE TABLE VERSION ( VER_ID NUMBER NOT NULL, SCHEMA_VERSION VARCHAR(127) NOT NULL, - COMMENT VARCHAR(255) -) + VERSION_COMMENT VARCHAR(255) +); ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID); diff --git metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql index f090f35..c77eaa4 100644 --- metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql @@ -1,4 +1,4 @@ -SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' '; +SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual; @013-HIVE-3255.oracle.sql; @014-HIVE-3764.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1; diff --git metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql index 0520f19..8847d3e 100644 --- metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql +++ metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql @@ -1,3 +1,3 @@ -SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' '; +SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual; UPDATE VERSION SET SCHEMA_VERSION='0.13.0', VERSION_COMMENT='Hive release version 0.13.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual; diff --git metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql index 15cb4ee..bc6486b 100644 --- metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql +++ metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql @@ -521,7 +521,7 @@ CREATE TABLE "TAB_COL_STATS" ( CREATE TABLE "VERSION" ( "VER_ID" bigint, "SCHEMA_VERSION" character varying(127) NOT NULL, - "VERSION_COMMENT" character varying(255) NOT NULL, + "VERSION_COMMENT" character varying(255) NOT NULL ); -- diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java index dba5364..653424f 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java @@ -17,8 +17,13 @@ */ package org.apache.hadoop.hive.metastore; +import java.io.File; +import java.lang.reflect.Field; +import java.util.Random; + import junit.framework.TestCase; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -31,18 +36,35 @@ protected HiveConf hiveConf; private Driver driver; private String hiveHome; + private String testMetastoreDB; + Random randomNum = new Random(); @Override protected void setUp() throws Exception { super.setUp(); + Field defDb = HiveMetaStore.HMSHandler.class.getDeclaredField("createDefaultDB"); + defDb.setAccessible(true); + defDb.setBoolean(null, false); hiveConf = new HiveConf(this.getClass()); System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); + testMetastoreDB = System.getProperty("java.io.tmpdir") + + File.separator + "test_metastore-" + randomNum.nextInt(); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + "jdbc:derby:" + testMetastoreDB + ";create=true"); hiveHome = System.getProperty("hive.home"); } + @Override + protected void tearDown() throws Exception { + File metaStoreDir = new File(testMetastoreDB); + if (metaStoreDir.exists()) { + FileUtils.deleteDirectory(metaStoreDir); + } + } + /*** * Test config defaults */