diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java index a21fa652e9c1aa3fd763afa4136f6fcb7c7b7459..46254c13981e17d9e36d50d85c1b730ee091f46d 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java @@ -17,7 +17,11 @@ */ package org.apache.hive.beeline; +import com.google.common.collect.Lists; + +import java.util.ArrayList; import java.util.IllegalFormatException; +import java.util.List; public class HiveSchemaHelper { public static final String DB_DERBY = "derby"; @@ -80,6 +84,12 @@ * @return */ public boolean needsQuotedIdentifier(); + + /*** + * Set DB specific options if any + * @param dbOpts db options + */ + public void setDbOpts(String dbOpts); } @@ -89,6 +99,7 @@ * */ private static abstract class AbstractCommandParser implements NestedScriptParser { + private List dbOpts = new ArrayList(); @Override public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ @@ -127,6 +138,15 @@ public String cleanseCommand(String dbCommand) { public boolean needsQuotedIdentifier() { return false; } + + @Override + public void setDbOpts(String dbOpts) { + this.dbOpts = Lists.newArrayList(dbOpts.split(",")); + } + + protected List getDbOpts() { + return dbOpts; + } } diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index c376687cb47332323912e4c6dbe713b7b37ae834..5f9ac9e6b3b631f28de38abdbc3f12270500fe1b 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -56,6 +56,7 @@ private String passWord = null; private boolean dryRun = false; private boolean verbose = false; + private String dbOpts = null; private final HiveConf hiveConf; private final String dbType; private final MetaStoreSchemaInfo metaStoreSchemaInfo; @@ -101,6 +102,10 @@ public void setVerbose(boolean verbose) { this.verbose = verbose; } + public void setDbOpts(String dbOpts) { + this.dbOpts = dbOpts; + } + private static void printAndExit(Options cmdLineOptions) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("schemaTool", cmdLineOptions); @@ -240,6 +245,7 @@ public void doUpgrade(String fromSchemaVer) throws HiveMetaException { for (String scriptFile : upgradeScripts) { System.out.println("Upgrade script " + scriptFile); if (!dryRun) { + runPreUpgrade(scriptDir, scriptFile); runBeeLine(scriptDir, scriptFile); System.out.println("Completed " + scriptFile); } @@ -337,6 +343,39 @@ public static String buildCommand(NestedScriptParser dbCommandParser, return sb.toString(); } + + /** + * Run pre-upgrade scripts corresponding to a given upgrade script, + * if any exist. The errors from pre-upgrade are ignored. + * Pre-upgrade scripts typically contain setup statements which + * may fail on some database versions and failure is ignorable. + * + * @param scriptDir upgrade script directory name + * @param scriptFile upgrade script file name + */ + private void runPreUpgrade(String scriptDir, String scriptFile) { + for (int i = 0;; i++) { + String preUpgradeScript = + MetaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile); + File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); + if (!preUpgradeScriptFile.isFile()) { + break; + } + + try { + runBeeLine(scriptDir, preUpgradeScript); + System.out.println("Completed " + preUpgradeScript); + } catch (Exception e) { + // Ignore the pre-upgrade script errors + System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + + e.getMessage()); + if (verbose) { + e.printStackTrace(); + } + } + } + } + // run beeline on the given metastore scrip, flatten the nested scripts into single file private void runBeeLine(String scriptDir, String scriptFile) throws IOException { NestedScriptParser dbCommandParser = @@ -423,6 +462,9 @@ private static void initOptions(Options cmdLineOptions) { Option dbTypeOpt = OptionBuilder.withArgName("databaseType") .hasArgs().withDescription("Metastore database type") .create("dbType"); + Option dbOpts = OptionBuilder.withArgName("databaseOpts") + .hasArgs().withDescription("Backend DB specific options") + .create("dbOpts"); Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)"); Option verboseOpt = new Option("verbose", "only print SQL statements"); @@ -432,6 +474,7 @@ private static void initOptions(Options cmdLineOptions) { cmdLineOptions.addOption(passwdOpt); cmdLineOptions.addOption(dbTypeOpt); cmdLineOptions.addOption(verboseOpt); + cmdLineOptions.addOption(dbOpts); cmdLineOptions.addOptionGroup(optGroup); } @@ -488,7 +531,9 @@ public static void main(String[] args) { if (line.hasOption("verbose")) { schemaTool.setVerbose(true); } - + if (line.hasOption("dbOpts")) { + schemaTool.setDbOpts(line.getOptionValue("dbOpts")); + } if (line.hasOption("info")) { schemaTool.showInfo(); } else if (line.hasOption("upgradeSchema")) { diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 749fb9e86b4f74f768da356cf82f621fdef399cd..12ea226dc1e43ab8e7abea21e5ab0eed990321ce 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -19,9 +19,12 @@ package org.apache.hive.beeline; import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; import java.util.Random; import junit.framework.TestCase; @@ -31,14 +34,15 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; -import org.apache.hive.beeline.HiveSchemaHelper; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; -import org.apache.hive.beeline.HiveSchemaTool; +import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser; public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; private HiveConf hiveConf; private String testMetastoreDB; + private PrintStream errStream; + private PrintStream outStream; @Override protected void setUp() throws Exception { @@ -48,8 +52,11 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + testMetastoreDB + ";create=true"); hiveConf = new HiveConf(this.getClass()); - schemaTool = new HiveSchemaTool(System.getProperty("test.tmp.dir"), hiveConf, "derby"); + schemaTool = new HiveSchemaTool( + System.getProperty("test.tmp.dir", "target/tmp"), hiveConf, "derby"); System.setProperty("beeLine.system.exit", "true"); + errStream = System.err; + outStream = System.out; } @Override @@ -58,6 +65,8 @@ protected void tearDown() throws Exception { if (metaStoreDir.exists()) { FileUtils.forceDeleteOnExit(metaStoreDir); } + System.setOut(outStream); + System.setErr(errStream); } /** @@ -121,12 +130,42 @@ public void testSchemaUpgrade() throws Exception { foundException = true; } if (!foundException) { - throw new Exception("Hive operations shouldn't pass with older version schema"); + throw new Exception( + "Hive operations shouldn't pass with older version schema"); } - // upgrade schema from 0.7.0 to latest + // Generate dummy pre-upgrade script with errors + String invalidPreUpgradeScript = writeDummyPreUpgradeScript( + 0, "upgrade-0.11.0-to-0.12.0.derby.sql", "foo bar;"); + // Generate dummy pre-upgrade scripts with valid SQL + String validPreUpgradeScript0 = writeDummyPreUpgradeScript( + 0, "upgrade-0.12.0-to-0.13.0.derby.sql", + "CREATE TABLE schema_test0 (id integer);"); + String validPreUpgradeScript1 = writeDummyPreUpgradeScript( + 1, "upgrade-0.12.0-to-0.13.0.derby.sql", + "CREATE TABLE schema_test1 (id integer);"); + + // Capture system out and err + schemaTool.setVerbose(true); + OutputStream stderr = new ByteArrayOutputStream(); + PrintStream errPrintStream = new PrintStream(stderr); + System.setErr(errPrintStream); + OutputStream stdout = new ByteArrayOutputStream(); + PrintStream outPrintStream = new PrintStream(stdout); + System.setOut(outPrintStream); + + // Upgrade schema from 0.7.0 to latest schemaTool.doUpgrade("0.7.0"); - // verify that driver works fine with latest schema + + // Verify that the schemaTool ran pre-upgrade scripts and ignored errors + assertTrue(stderr.toString().contains(invalidPreUpgradeScript)); + assertTrue(stderr.toString().contains("foo")); + assertFalse(stderr.toString().contains(validPreUpgradeScript0)); + assertFalse(stderr.toString().contains(validPreUpgradeScript1)); + assertTrue(stdout.toString().contains(validPreUpgradeScript0)); + assertTrue(stdout.toString().contains(validPreUpgradeScript1)); + + // Verify that driver works fine with latest schema schemaTool.verifySchemaVersion(); } @@ -380,4 +419,21 @@ private File generateTestScript(String [] stmts) throws IOException { out.close(); return testScriptFile; } -} + + /** + * Write out a dummy pre-upgrade script with given SQL statement. + */ + private String writeDummyPreUpgradeScript(int index, String upgradeScriptName, + String sql) throws Exception { + String preUpgradeScript = "pre-" + index + "-" + upgradeScriptName; + String dummyPreScriptPath = System.getProperty("test.tmp.dir", "target/tmp") + + File.separatorChar + "scripts" + File.separatorChar + "metastore" + + File.separatorChar + "upgrade" + File.separatorChar + "derby" + + File.separatorChar + preUpgradeScript; + FileWriter fstream = new FileWriter(dummyPreScriptPath); + BufferedWriter out = new BufferedWriter(fstream); + out.write(sql + System.getProperty("line.separator") + ";"); + out.close(); + return preUpgradeScript; + } +} \ No newline at end of file diff --git metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql index aedb2b5dbf9b513cb7c124f59e05ea9831455a72..9e761e55843f3eef195babf3f0e7db1f0e835a2d 100644 --- metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql +++ metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql @@ -11,5 +11,4 @@ FROM "PARTITION_KEY_VALS" src AND "PARTITION_KEYS"."INTEGER_IDX" = src."INTEGER_IDX" AND "PARTITION_KEYS"."PKEY_TYPE" = 'date'; -DROP FUNCTION hive13_to_date(date_str text); - +DROP FUNCTION hive13_to_date(date_str text); \ No newline at end of file diff --git metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql new file mode 100644 index 0000000000000000000000000000000000000000..bc028fbf2c7c5c14aa2ace43033da1919fb2fbb4 --- /dev/null +++ metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql @@ -0,0 +1 @@ +CREATE LANGUAGE plpgsql; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index af56ef6a7c6e27ead312a68d30d94802095f2c60..50d03a83d3f88219fa7e3471396a2737d79158d5 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -37,6 +37,7 @@ private static String UPGRADE_FILE_PREFIX="upgrade-"; private static String INIT_FILE_PREFIX="hive-schema-"; private static String VERSION_UPGRADE_LIST = "upgrade.order"; + private static String PRE_UPGRADE_PREFIX = "pre-"; private final String dbType; private final String hiveSchemaVersions[]; private final HiveConf hiveConf; @@ -138,6 +139,10 @@ private String generateUpgradeFileName(String fileVersion) { return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION; } + public static String getPreUpgradeScriptName(int index, String upgradeScriptName) { + return PRE_UPGRADE_PREFIX + index + "-" + upgradeScriptName; + } + public static String getHiveSchemaVersion() { String hiveVersion = HiveVersionInfo.getShortVersion(); // if there is an equivalent version, return that, else return this version @@ -149,4 +154,4 @@ public static String getHiveSchemaVersion() { } } -} +} \ No newline at end of file