diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java index a21fa652e9c1aa3fd763afa4136f6fcb7c7b7459..8cd51b926385d85c982996ca5d7f8ac7efebc4be 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java @@ -17,7 +17,11 @@ */ package org.apache.hive.beeline; +import com.google.common.collect.Lists; + +import java.util.ArrayList; import java.util.IllegalFormatException; +import java.util.List; public class HiveSchemaHelper { public static final String DB_DERBY = "derby"; @@ -80,6 +84,12 @@ * @return */ public boolean needsQuotedIdentifier(); + + /*** + * Set DB specific options if any + * @param dbOpts db options + */ + public void setDbOpts(String dbOpts); } @@ -89,6 +99,7 @@ * */ private static abstract class AbstractCommandParser implements NestedScriptParser { + private List dbOpts = new ArrayList(); @Override public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ @@ -127,6 +138,15 @@ public String cleanseCommand(String dbCommand) { public boolean needsQuotedIdentifier() { return false; } + + @Override + public void setDbOpts(String dbOpts) { + this.dbOpts = Lists.newArrayList(dbOpts.split(",")); + } + + protected List getDbOpts() { + return dbOpts; + } } @@ -212,6 +232,8 @@ public String cleanseCommand(String dbCommand) { // Postgres specific parser public static class PostgresCommandParser extends AbstractCommandParser { private static String POSTGRES_NESTING_TOKEN = "\\i"; + public static String POSTGRES_CREATE_LANGUAGE = "CREATE LANGUAGE plpgsql"; + public static String POSTGRES_INCLUDE_CREATE_LANG = "postgres.filter.pre.9"; @Override public String getScriptName(String dbCommand) throws IllegalArgumentException { @@ -232,6 +254,19 @@ public boolean isNestedScript(String dbCommand) { public boolean needsQuotedIdentifier() { return true; } + + @Override + public boolean isNonExecCommand(String dbCommand) { + // The upgrade script for HIVE-5700 creates a UDF with language 'plpgsql', + // which is available by default for Postgres 9.2 onwards. + // Execute "CREATE LANGUAGE" for pre-9.x postgres + if (!getDbOpts().contains(POSTGRES_INCLUDE_CREATE_LANG)) { + if (dbCommand.startsWith(POSTGRES_CREATE_LANGUAGE)) { + return true; + } + } + return super.isNonExecCommand(dbCommand); + } } //Oracle specific parser diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index c376687cb47332323912e4c6dbe713b7b37ae834..5260150bd05bf2f7dfc3166e60e67f3bcf150900 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -56,6 +56,7 @@ private String passWord = null; private boolean dryRun = false; private boolean verbose = false; + private String dbOpts = null; private final HiveConf hiveConf; private final String dbType; private final MetaStoreSchemaInfo metaStoreSchemaInfo; @@ -101,6 +102,10 @@ public void setVerbose(boolean verbose) { this.verbose = verbose; } + public void setDbOpts(String dbOpts) { + this.dbOpts = dbOpts; + } + private static void printAndExit(Options cmdLineOptions) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("schemaTool", cmdLineOptions); @@ -423,6 +428,9 @@ private static void initOptions(Options cmdLineOptions) { Option dbTypeOpt = OptionBuilder.withArgName("databaseType") .hasArgs().withDescription("Metastore database type") .create("dbType"); + Option dbOpts = OptionBuilder.withArgName("databaseOpts") + .hasArgs().withDescription("Backend DB specific options") + .create("dbOpts"); Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)"); Option verboseOpt = new Option("verbose", "only print SQL statements"); @@ -432,6 +440,7 @@ private static void initOptions(Options cmdLineOptions) { cmdLineOptions.addOption(passwdOpt); cmdLineOptions.addOption(dbTypeOpt); cmdLineOptions.addOption(verboseOpt); + cmdLineOptions.addOption(dbOpts); cmdLineOptions.addOptionGroup(optGroup); } @@ -488,7 +497,9 @@ public static void main(String[] args) { if (line.hasOption("verbose")) { schemaTool.setVerbose(true); } - + if (line.hasOption("dbOpts")) { + schemaTool.setDbOpts(line.getOptionValue("dbOpts")); + } if (line.hasOption("info")) { schemaTool.showInfo(); } else if (line.hasOption("upgradeSchema")) { diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 749fb9e86b4f74f768da356cf82f621fdef399cd..12585c7a3697551c6691647dedb01c58f52f6be6 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -31,9 +31,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; -import org.apache.hive.beeline.HiveSchemaHelper; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; -import org.apache.hive.beeline.HiveSchemaTool; +import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser; public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; @@ -368,6 +367,64 @@ public void testNestedScriptsForOracle() throws Exception { assertTrue(flattenedSql.contains(parentTab)); } + /** + * Test postgres filter: + * The upgrade script for HIVE-5700 creates a UDF with language 'plpgsql'. + * Language plpgsql is available by default in Postgres 9.2 and above. + * For versions prior to 9.2, passing a dboption will create the language. + * + * @throws Exception + */ + public void testPostgresFilter() throws Exception { + String testScript[] = { + "-- this is a comment", + "DROP TABLE IF EXISTS fooTab;", + "CREATE TABLE fooTab(id INTEGER);", + PostgresCommandParser.POSTGRES_CREATE_LANGUAGE + ";", + "DROP TABLE footab;", + "-- ending comment" + }; + + String resultScriptWithoutFilter[] = { + "DROP TABLE IF EXISTS fooTab", + "CREATE TABLE fooTab(id INTEGER)", + "DROP TABLE footab", + }; + + String resultScriptWithFilter[] = { + "DROP TABLE IF EXISTS fooTab", + "CREATE TABLE fooTab(id INTEGER)", + PostgresCommandParser.POSTGRES_CREATE_LANGUAGE, + "DROP TABLE footab", + }; + + String expectedSQL = StringUtils.join( + resultScriptWithoutFilter, + System.getProperty("line.separator")) + + System.getProperty("line.separator"); + File testScriptFile = generateTestScript(testScript); + String flattenedSql = HiveSchemaTool.buildCommand( + HiveSchemaHelper.getDbCommandParser("postgres"), + testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); + assertEquals(expectedSQL, flattenedSql); + + // Set the dboption to create the language. + NestedScriptParser postgresParser = + HiveSchemaHelper.getDbCommandParser("postgres"); + postgresParser.setDbOpts(PostgresCommandParser.POSTGRES_INCLUDE_CREATE_LANG); + expectedSQL = StringUtils.join( + resultScriptWithFilter, + System.getProperty("line.separator")) + + System.getProperty("line.separator"); + testScriptFile = generateTestScript(testScript); + flattenedSql = HiveSchemaTool.buildCommand( + postgresParser, + testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); + assertEquals(expectedSQL, flattenedSql); + } + private File generateTestScript(String [] stmts) throws IOException { File testScriptFile = File.createTempFile("schematest", ".sql"); testScriptFile.deleteOnExit(); @@ -380,4 +437,4 @@ private File generateTestScript(String [] stmts) throws IOException { out.close(); return testScriptFile; } -} +} \ No newline at end of file diff --git metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql index aedb2b5dbf9b513cb7c124f59e05ea9831455a72..04145c516b8997183555808e93749d5d1d75ba84 100644 --- metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql +++ metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql @@ -1,5 +1,6 @@ SELECT '< HIVE-5700 enforce single date format for partition column storage >'; +CREATE LANGUAGE plpgsql; -- Normalize the date partition column values as best we can. No schema changes. CREATE FUNCTION hive13_to_date(date_str text) RETURNS DATE AS $$ DECLARE dt DATE; BEGIN dt := date_str::DATE; RETURN dt; EXCEPTION WHEN others THEN RETURN null; END; $$ LANGUAGE plpgsql;