commit 509f0afd6727ad15d505d5af234d8c98a14ccd29 Author: Vihang Karajgaonkar Date: Thu May 25 17:55:02 2017 -0700 HIVE-16771 : Schematool should use MetastoreSchemaInfo to get the metastore schema version from database diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java deleted file mode 100644 index a4ecc089f8a19bd94af2eae17e534cc6e8d2a9ca..0000000000000000000000000000000000000000 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java +++ /dev/null @@ -1,581 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.beeline; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaException; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.IllegalFormatException; -import java.util.List; - -public class HiveSchemaHelper { - public static final String DB_DERBY = "derby"; - public static final String DB_HIVE = "hive"; - public static final String DB_MSSQL = "mssql"; - public static final String DB_MYSQL = "mysql"; - public static final String DB_POSTGRACE = "postgres"; - public static final String DB_ORACLE = "oracle"; - - /*** - * Get JDBC connection to metastore db - * - * @param userName metastore connection username - * @param password metastore connection password - * @param printInfo print connection parameters - * @param hiveConf hive config object - * @return metastore connection object - * @throws org.apache.hadoop.hive.metastore.api.MetaException - */ - public static Connection getConnectionToMetastore(String userName, - String password, String url, String driver, boolean printInfo, - HiveConf hiveConf) - throws HiveMetaException { - try { - url = url == null ? getValidConfVar( - HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url; - driver = driver == null ? getValidConfVar( - HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver; - if (printInfo) { - System.out.println("Metastore connection URL:\t " + url); - System.out.println("Metastore Connection Driver :\t " + driver); - System.out.println("Metastore connection User:\t " + userName); - } - if ((userName == null) || userName.isEmpty()) { - throw new HiveMetaException("UserName empty "); - } - - // load required JDBC driver - Class.forName(driver); - - // Connect using the JDBC URL and user/pass from conf - return DriverManager.getConnection(url, userName, password); - } catch (IOException e) { - throw new HiveMetaException("Failed to get schema version.", e); - } catch (SQLException e) { - throw new HiveMetaException("Failed to get schema version.", e); - } catch (ClassNotFoundException e) { - throw new HiveMetaException("Failed to load driver", e); - } - } - - public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf) - throws IOException { - String confVarStr = hiveConf.get(confVar.varname); - if (confVarStr == null || confVarStr.isEmpty()) { - throw new IOException("Empty " + confVar.varname); - } - return confVarStr.trim(); - } - - public interface NestedScriptParser { - - public enum CommandType { - PARTIAL_STATEMENT, - TERMINATED_STATEMENT, - COMMENT - } - - static final String DEFAULT_DELIMITER = ";"; - static final String DEFAULT_QUOTE = "\""; - - /** - * Find the type of given command - * - * @param dbCommand - * @return - */ - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; - - /** - * Parse the DB specific nesting format and extract the inner script name if any - * - * @param dbCommand command from parent script - * @return - * @throws IllegalFormatException - */ - public String getScriptName(String dbCommand) throws IllegalArgumentException; - - /** - * Find if the given command is a nested script execution - * - * @param dbCommand - * @return - */ - public boolean isNestedScript(String dbCommand); - - /** - * Find if the given command should not be passed to DB - * - * @param dbCommand - * @return - */ - public boolean isNonExecCommand(String dbCommand); - - /** - * Get the SQL statement delimiter - * - * @return - */ - public String getDelimiter(); - - /** - * Get the SQL indentifier quotation character - * - * @return - */ - public String getQuoteCharacter(); - - /** - * Clear any client specific tags - * - * @return - */ - public String cleanseCommand(String dbCommand); - - /** - * Does the DB required table/column names quoted - * - * @return - */ - public boolean needsQuotedIdentifier(); - - /** - * Flatten the nested upgrade script into a buffer - * - * @param scriptDir upgrade script directory - * @param scriptFile upgrade script file - * @return string of sql commands - */ - public String buildCommand(String scriptDir, String scriptFile) - throws IllegalFormatException, IOException; - - /** - * Flatten the nested upgrade script into a buffer - * - * @param scriptDir upgrade script directory - * @param scriptFile upgrade script file - * @param fixQuotes whether to replace quote characters - * @return string of sql commands - */ - public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes) - throws IllegalFormatException, IOException; - } - - /*** - * Base implementation of NestedScriptParser - * abstractCommandParser. - * - */ - private static abstract class AbstractCommandParser implements NestedScriptParser { - private List dbOpts; - private String msUsername; - private String msPassword; - private HiveConf hiveConf; - - public AbstractCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - setDbOpts(dbOpts); - this.msUsername = msUsername; - this.msPassword = msPassword; - this.hiveConf = hiveConf; - } - - @Override - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ - if (dbCommand == null || dbCommand.isEmpty()) { - throw new IllegalArgumentException("invalid command line " + dbCommand); - } - dbCommand = dbCommand.trim(); - if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) { - return false; - } else { - return true; - } - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - return (dbCommand.startsWith("--") || dbCommand.startsWith("#")); - } - - @Override - public String getDelimiter() { - return DEFAULT_DELIMITER; - } - - @Override - public String getQuoteCharacter() { - return DEFAULT_QUOTE; - } - - - @Override - public String cleanseCommand(String dbCommand) { - // strip off the delimiter - if (dbCommand.endsWith(getDelimiter())) { - dbCommand = dbCommand.substring(0, - dbCommand.length() - getDelimiter().length()); - } - return dbCommand; - } - - @Override - public boolean needsQuotedIdentifier() { - return false; - } - - @Override - public String buildCommand( - String scriptDir, String scriptFile) throws IllegalFormatException, IOException { - return buildCommand(scriptDir, scriptFile, false); - } - - @Override - public String buildCommand( - String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException { - BufferedReader bfReader = - new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile)); - String currLine; - StringBuilder sb = new StringBuilder(); - String currentCommand = null; - while ((currLine = bfReader.readLine()) != null) { - currLine = currLine.trim(); - - if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) { - currLine = currLine.replace("\\\"", getQuoteCharacter()); - } - - if (currLine.isEmpty()) { - continue; // skip empty lines - } - - if (currentCommand == null) { - currentCommand = currLine; - } else { - currentCommand = currentCommand + " " + currLine; - } - if (isPartialCommand(currLine)) { - // if its a partial line, continue collecting the pieces - continue; - } - - // if this is a valid executable command then add it to the buffer - if (!isNonExecCommand(currentCommand)) { - currentCommand = cleanseCommand(currentCommand); - if (isNestedScript(currentCommand)) { - // if this is a nested sql script then flatten it - String currScript = getScriptName(currentCommand); - sb.append(buildCommand(scriptDir, currScript)); - } else { - // Now we have a complete statement, process it - // write the line to buffer - sb.append(currentCommand); - sb.append(System.getProperty("line.separator")); - } - } - currentCommand = null; - } - bfReader.close(); - return sb.toString(); - } - - private void setDbOpts(String dbOpts) { - if (dbOpts != null) { - this.dbOpts = Lists.newArrayList(dbOpts.split(",")); - } else { - this.dbOpts = Lists.newArrayList(); - } - } - - protected List getDbOpts() { - return dbOpts; - } - - protected String getMsUsername() { - return msUsername; - } - - protected String getMsPassword() { - return msPassword; - } - - protected HiveConf getHiveConf() { - return hiveConf; - } - } - - // Derby commandline parser - public static class DerbyCommandParser extends AbstractCommandParser { - private static final String DERBY_NESTING_TOKEN = "RUN"; - - public DerbyCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - - if (!isNestedScript(dbCommand)) { - throw new IllegalArgumentException("Not a script format " + dbCommand); - } - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - return tokens[1].replace(";", "").replaceAll("'", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - // Derby script format is RUN '' - return dbCommand.startsWith(DERBY_NESTING_TOKEN); - } - } - - // Derby commandline parser - public static class HiveCommandParser extends AbstractCommandParser { - private static String HIVE_NESTING_TOKEN = "SOURCE"; - private final NestedScriptParser nestedDbCommandParser; - - public HiveCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf, String metaDbType) { - super(dbOpts, msUsername, msPassword, hiveConf); - nestedDbCommandParser = getDbCommandParser(metaDbType); - } - - @Override - public String getQuoteCharacter() { - return nestedDbCommandParser.getQuoteCharacter(); - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - - if (!isNestedScript(dbCommand)) { - throw new IllegalArgumentException("Not a script format " + dbCommand); - } - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - return tokens[1].replace(";", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(HIVE_NESTING_TOKEN); - } - } - - // MySQL parser - public static class MySqlCommandParser extends AbstractCommandParser { - private static final String MYSQL_NESTING_TOKEN = "SOURCE"; - private static final String DELIMITER_TOKEN = "DELIMITER"; - private String delimiter = DEFAULT_DELIMITER; - - public MySqlCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); - } - - @Override - public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ - boolean isPartial = super.isPartialCommand(dbCommand); - // if this is a delimiter directive, reset our delimiter - if (dbCommand.startsWith(DELIMITER_TOKEN)) { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - delimiter = tokens[1]; - } - return isPartial; - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - // remove ending ';' - return tokens[1].replace(";", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(MYSQL_NESTING_TOKEN); - } - - @Override - public String getDelimiter() { - return delimiter; - } - - @Override - public String getQuoteCharacter() { - return "`"; - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - return super.isNonExecCommand(dbCommand) || - (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) || - dbCommand.startsWith(DELIMITER_TOKEN); - } - - @Override - public String cleanseCommand(String dbCommand) { - return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", ""); - } - - } - - // Postgres specific parser - public static class PostgresCommandParser extends AbstractCommandParser { - private static final String POSTGRES_NESTING_TOKEN = "\\i"; - @VisibleForTesting - public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings"; - @VisibleForTesting - public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81"; - - public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - // remove ending ';' - return tokens[1].replace(";", ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(POSTGRES_NESTING_TOKEN); - } - - @Override - public boolean needsQuotedIdentifier() { - return true; - } - - @Override - public boolean isNonExecCommand(String dbCommand) { - // Skip "standard_conforming_strings" command which is read-only in older - // Postgres versions like 8.1 - // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html - if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) { - if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) { - return true; - } - } - return super.isNonExecCommand(dbCommand); - } - } - - //Oracle specific parser - public static class OracleCommandParser extends AbstractCommandParser { - private static final String ORACLE_NESTING_TOKEN = "@"; - - public OracleCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - if (!isNestedScript(dbCommand)) { - throw new IllegalArgumentException("Not a nested script format " + dbCommand); - } - // remove ending ';' and starting '@' - return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, ""); - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(ORACLE_NESTING_TOKEN); - } - } - - //MSSQL specific parser - public static class MSSQLCommandParser extends AbstractCommandParser { - private static final String MSSQL_NESTING_TOKEN = ":r"; - - public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf) { - super(dbOpts, msUsername, msPassword, hiveConf); - } - - @Override - public String getScriptName(String dbCommand) throws IllegalArgumentException { - String[] tokens = dbCommand.split(" "); - if (tokens.length != 2) { - throw new IllegalArgumentException("Couldn't parse line " + dbCommand); - } - return tokens[1]; - } - - @Override - public boolean isNestedScript(String dbCommand) { - return dbCommand.startsWith(MSSQL_NESTING_TOKEN); - } - } - - public static NestedScriptParser getDbCommandParser(String dbName) { - return getDbCommandParser(dbName, null); - } - - public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) { - return getDbCommandParser(dbName, null, null, null, null, metaDbName); - } - - public static NestedScriptParser getDbCommandParser(String dbName, - String dbOpts, String msUsername, String msPassword, - HiveConf hiveConf, String metaDbType) { - if (dbName.equalsIgnoreCase(DB_DERBY)) { - return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf); - } else if (dbName.equalsIgnoreCase(DB_HIVE)) { - return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType); - } else if (dbName.equalsIgnoreCase(DB_MSSQL)) { - return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf); - } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { - return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf); - } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { - return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf); - } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { - return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf); - } else { - throw new IllegalArgumentException("Unknown dbType " + dbName); - } - } -} diff --git beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index f312e46d755fc843369167af6d4da3de7dbe4a61..6ff179a4d1497a7d852093a46d3f62128e43fb7e 100644 --- beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -38,8 +38,10 @@ import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +82,7 @@ private final String dbType; private final String metaDbType; private final IMetaStoreSchemaInfo metaStoreSchemaInfo; + private boolean needsQuotedIdentifier; static final private Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName()); @@ -95,6 +98,7 @@ public HiveSchemaTool(String hiveHome, HiveConf hiveConf, String dbType, String this.hiveConf = hiveConf; this.dbType = dbType; this.metaDbType = metaDbType; + this.needsQuotedIdentifier = getDbCommandParser(dbType).needsQuotedIdentifier(); this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, hiveHome, dbType); } @@ -169,42 +173,13 @@ private NestedScriptParser getDbCommandParser(String dbType) { public void showInfo() throws HiveMetaException { Connection metastoreConn = getConnectionToMetastore(true); String hiveVersion = metaStoreSchemaInfo.getHiveSchemaVersion(); - String dbVersion = getMetaStoreSchemaVersion(metastoreConn); + String dbVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); System.out.println("Hive distribution version:\t " + hiveVersion); System.out.println("Metastore schema version:\t " + dbVersion); assertCompatibleVersion(hiveVersion, dbVersion); } - private String getMetaStoreSchemaVersion(Connection metastoreConn) - throws HiveMetaException { - return getMetaStoreSchemaVersion(metastoreConn, false); - } - - // read schema version from metastore - private String getMetaStoreSchemaVersion(Connection metastoreConn, - boolean checkDuplicatedVersion) throws HiveMetaException { - String versionQuery; - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { - versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t"; - } else { - versionQuery = "select t.SCHEMA_VERSION from VERSION t"; - } - try(Statement stmt = metastoreConn.createStatement(); - ResultSet res = stmt.executeQuery(versionQuery)) { - if (!res.next()) { - throw new HiveMetaException("Could not find version info in metastore VERSION table."); - } - String currentSchemaVersion = res.getString(1); - if (checkDuplicatedVersion && res.next()) { - throw new HiveMetaException("Multiple versions were found in metastore."); - } - return currentSchemaVersion; - } catch (SQLException e) { - throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage()); - } - } - boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException { System.out.println("Validating database/table/partition locations"); boolean rtn; @@ -226,7 +201,7 @@ private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) String dbLoc; boolean isValid = true; int numOfInvalid = 0; - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt"; } else { dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt"; @@ -255,13 +230,13 @@ private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServer String tabLoc, tabIDRange; boolean isValid = true; int numOfInvalid = 0; - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" "; } else { tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS"; } - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " + "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW + "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" "; @@ -315,13 +290,13 @@ private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultSe String partLoc, partIDRange; boolean isValid = true; int numOfInvalid = 0; - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" "; } else { partIDRange = "select max(PART_ID), min(PART_ID) from PARTITIONS"; } - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { partLoc = "select pt.\"PART_ID\", pt.\"PART_NAME\", sd.\"LOCATION\", tbl.\"TBL_ID\", tbl.\"TBL_NAME\",dbt.\"DB_ID\", dbt.\"NAME\" from \"PARTITIONS\" pt " + "inner join \"SDS\" sd on pt.\"SD_ID\" = sd.\"SD_ID\" and pt.\"PART_ID\" >= ? and pt.\"PART_ID\"<= ? " + " inner join \"TBLS\" tbl on pt.\"TBL_ID\" = tbl.\"TBL_ID\" inner join " @@ -376,13 +351,13 @@ private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defau String skewedColLoc, skewedColIDRange; boolean isValid = true; int numOfInvalid = 0; - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" "; } else { skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP"; } - if (getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (needsQuotedIdentifier) { skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk " + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? "; } else { @@ -497,11 +472,9 @@ public void verifySchemaVersion() throws HiveMetaException { if (dryRun) { return; } - String newSchemaVersion = getMetaStoreSchemaVersion( - getConnectionToMetastore(false)); + String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); // verify that the new version is added to schema assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); - } private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion) @@ -517,8 +490,8 @@ private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVe * @throws MetaException */ public void doUpgrade() throws HiveMetaException { - String fromVersion = getMetaStoreSchemaVersion( - getConnectionToMetastore(false)); + String fromVersion = + metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); if (fromVersion == null || fromVersion.isEmpty()) { throw new HiveMetaException("Schema version not stored in the metastore. " + "Metastore schema is too old or corrupt. Try specifying the version manually"); @@ -526,6 +499,10 @@ public void doUpgrade() throws HiveMetaException { doUpgrade(fromVersion); } + private MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) { + return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, hiveConf, + dbType); + } /** * Perform metastore schema upgrade * @@ -682,10 +659,10 @@ boolean validateSequences(Connection conn) throws HiveMetaException { for (String seqName : seqNameToTable.keySet()) { String tableName = seqNameToTable.get(seqName).getLeft(); String tableKey = seqNameToTable.get(seqName).getRight(); - String seqQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ? + String seqQuery = needsQuotedIdentifier ? ("select t.\"NEXT_VAL\" from \"SEQUENCE_TABLE\" t WHERE t.\"SEQUENCE_NAME\"='org.apache.hadoop.hive.metastore.model." + seqName + "'") : ("select t.NEXT_VAL from SEQUENCE_TABLE t WHERE t.SEQUENCE_NAME='org.apache.hadoop.hive.metastore.model." + seqName + "'"); - String maxIdQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ? + String maxIdQuery = needsQuotedIdentifier ? ("select max(\"" + tableKey + "\") from \"" + tableName + "\"") : ("select max(" + tableKey + ") from " + tableName); @@ -715,7 +692,7 @@ boolean validateSequences(Connection conn) throws HiveMetaException { boolean validateSchemaVersions(Connection conn) throws HiveMetaException { System.out.println("Validating schema version"); try { - String newSchemaVersion = getMetaStoreSchemaVersion(conn, true); + String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); } catch (HiveMetaException hme) { if (hme.getMessage().contains("Metastore schema version is not compatible") @@ -743,7 +720,7 @@ boolean validateSchemaTables(Connection conn) throws HiveMetaException { System.out.println("Validating metastore schema tables"); try { - version = getMetaStoreSchemaVersion(hmsConn); + version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); } catch (HiveMetaException he) { System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage()); System.out.println("Failed in schema version validation."); @@ -781,7 +758,8 @@ boolean validateSchemaTables(Connection conn) throws HiveMetaException { // parse the schema file to determine the tables that are expected to exist // we are using oracle schema because it is simpler to parse, no quotes or backticks etc String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent(); - String schemaFile = baseDir + "/" + dbType + "/hive-schema-" + version + "." + dbType + ".sql"; + String schemaFile = metaStoreSchemaInfo.getMetaStoreScriptDir() + File.separatorChar + + metaStoreSchemaInfo.generateInitFileName(version); try { LOG.debug("Parsing schema script " + schemaFile); @@ -896,7 +874,7 @@ boolean validateColumnNullValues(Connection conn) throws HiveMetaException { boolean isValid = true; try { Statement stmt = conn.createStatement(); - String tblQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ? + String tblQuery = needsQuotedIdentifier ? ("select t.* from \"TBLS\" t WHERE t.\"SD_ID\" IS NULL and (t.\"TBL_TYPE\"='" + TableType.EXTERNAL_TABLE + "' or t.\"TBL_TYPE\"='" + TableType.MANAGED_TABLE + "')") : ("select t.* from TBLS t WHERE t.SD_ID IS NULL and (t.TBL_TYPE='" + TableType.EXTERNAL_TABLE + "' or t.TBL_TYPE='" + TableType.MANAGED_TABLE + "')"); diff --git beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java index 716bce7289b6e7c4d4bcea38b06f591a9ad9c098..d9b8f423ae9bb55cdc7e8f64af978ff868397f97 100644 --- beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java +++ beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java @@ -18,6 +18,7 @@ package org.apache.hive.beeline; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index d1658f7bb2e5306b6b5954165759a217d1ba235c..2d3ee3d8b90a730b2f8861373df8be47aa0e2053 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -38,9 +38,10 @@ import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.PostgresCommandParser; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; -import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser; public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java index d6627433cb0f3f1c8fe868b3748fd754a7bff821..296d04b9d42e8c00be01b7e0a35f3998a6ad274c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.common.classification.InterfaceAudience; + +import java.sql.Connection; import java.util.List; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; /** * Defines the method which must be implemented to be used using schema tool to support metastore @@ -78,6 +81,18 @@ String getHiveSchemaVersion(); /** + * Get the schema version from the backend database. This version is used by SchemaTool to to + * compare the version returned by getHiveSchemaVersion and determine the upgrade order and + * scripts needed to upgrade the metastore schema + * + * @param metastoreDbConnectionInfo Connection information needed to connect to the backend + * database + * @return + * @throws HiveMetaException when unable to fetch the schema version + */ + String getMetaStoreSchemaVersion( + HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException; + /** * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This * is result of the db schema upgrade design principles followed in hive project. The state where * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index 6bddb8e864d26a6cb7e2fe9b8b8becf427779944..9f09bc35507a3e0d25a7ef921fde08b9db2ed363 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -22,10 +22,16 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper; +import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo; import org.apache.hive.common.util.HiveVersionInfo; import com.google.common.collect.ImmutableMap; @@ -198,4 +204,30 @@ public boolean isVersionCompatible(String hiveVersion, String dbVersion) { return true; } + @Override + public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo) throws HiveMetaException { + String versionQuery; + Connection metastoreDbConnection = HiveSchemaHelper.getConnectionToMetastore(connectionInfo); + boolean needsQuotedIdentifier = + HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType()).needsQuotedIdentifier(); + if (needsQuotedIdentifier) { + versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t"; + } else { + versionQuery = "select t.SCHEMA_VERSION from VERSION t"; + } + try { + Statement stmt = metastoreDbConnection.createStatement(); + ResultSet res = stmt.executeQuery(versionQuery); + if (!res.next()) { + throw new HiveMetaException("Could not find version info in metastore VERSION table."); + } + String currentSchemaVersion = res.getString(1); + if (res.next()) { + throw new HiveMetaException("Multiple versions were found in metastore."); + } + return currentSchemaVersion; + } catch (SQLException e) { + throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage()); + } + } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java new file mode 100644 index 0000000000000000000000000000000000000000..0127bdd56cfe2aaea9a361cee7ea8c398c902016 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java @@ -0,0 +1,640 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.IllegalFormatException; +import java.util.List; + +public class HiveSchemaHelper { + public static final String DB_DERBY = "derby"; + public static final String DB_HIVE = "hive"; + public static final String DB_MSSQL = "mssql"; + public static final String DB_MYSQL = "mysql"; + public static final String DB_POSTGRACE = "postgres"; + public static final String DB_ORACLE = "oracle"; + + /*** + * Get JDBC connection to metastore db + * + * @param userName metastore connection username + * @param password metastore connection password + * @param printInfo print connection parameters + * @param hiveConf hive config object + * @return metastore connection object + * @throws org.apache.hadoop.hive.metastore.api.MetaException + */ + public static Connection getConnectionToMetastore(String userName, + String password, String url, String driver, boolean printInfo, + HiveConf hiveConf) + throws HiveMetaException { + try { + url = url == null ? getValidConfVar( + HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url; + driver = driver == null ? getValidConfVar( + HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver; + if (printInfo) { + System.out.println("Metastore connection URL:\t " + url); + System.out.println("Metastore Connection Driver :\t " + driver); + System.out.println("Metastore connection User:\t " + userName); + } + if ((userName == null) || userName.isEmpty()) { + throw new HiveMetaException("UserName empty "); + } + + // load required JDBC driver + Class.forName(driver); + + // Connect using the JDBC URL and user/pass from conf + return DriverManager.getConnection(url, userName, password); + } catch (IOException e) { + throw new HiveMetaException("Failed to get schema version.", e); + } catch (SQLException e) { + throw new HiveMetaException("Failed to get schema version.", e); + } catch (ClassNotFoundException e) { + throw new HiveMetaException("Failed to load driver", e); + } + } + + public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException { + return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(), + info.getDriver(), info.getPrintInfo(), info.getHiveConf()); + } + + public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf) + throws IOException { + String confVarStr = hiveConf.get(confVar.varname); + if (confVarStr == null || confVarStr.isEmpty()) { + throw new IOException("Empty " + confVar.varname); + } + return confVarStr.trim(); + } + + public interface NestedScriptParser { + + public enum CommandType { + PARTIAL_STATEMENT, + TERMINATED_STATEMENT, + COMMENT + } + + static final String DEFAULT_DELIMITER = ";"; + static final String DEFAULT_QUOTE = "\""; + + /** + * Find the type of given command + * + * @param dbCommand + * @return + */ + public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; + + /** + * Parse the DB specific nesting format and extract the inner script name if any + * + * @param dbCommand command from parent script + * @return + * @throws IllegalFormatException + */ + public String getScriptName(String dbCommand) throws IllegalArgumentException; + + /** + * Find if the given command is a nested script execution + * + * @param dbCommand + * @return + */ + public boolean isNestedScript(String dbCommand); + + /** + * Find if the given command should not be passed to DB + * + * @param dbCommand + * @return + */ + public boolean isNonExecCommand(String dbCommand); + + /** + * Get the SQL statement delimiter + * + * @return + */ + public String getDelimiter(); + + /** + * Get the SQL indentifier quotation character + * + * @return + */ + public String getQuoteCharacter(); + + /** + * Clear any client specific tags + * + * @return + */ + public String cleanseCommand(String dbCommand); + + /** + * Does the DB required table/column names quoted + * + * @return + */ + public boolean needsQuotedIdentifier(); + + /** + * Flatten the nested upgrade script into a buffer + * + * @param scriptDir upgrade script directory + * @param scriptFile upgrade script file + * @return string of sql commands + */ + public String buildCommand(String scriptDir, String scriptFile) + throws IllegalFormatException, IOException; + + /** + * Flatten the nested upgrade script into a buffer + * + * @param scriptDir upgrade script directory + * @param scriptFile upgrade script file + * @param fixQuotes whether to replace quote characters + * @return string of sql commands + */ + public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes) + throws IllegalFormatException, IOException; + } + + /*** + * Base implementation of NestedScriptParser + * abstractCommandParser. + * + */ + private static abstract class AbstractCommandParser implements NestedScriptParser { + private List dbOpts; + private String msUsername; + private String msPassword; + private HiveConf hiveConf; + + public AbstractCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + setDbOpts(dbOpts); + this.msUsername = msUsername; + this.msPassword = msPassword; + this.hiveConf = hiveConf; + } + + @Override + public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ + if (dbCommand == null || dbCommand.isEmpty()) { + throw new IllegalArgumentException("invalid command line " + dbCommand); + } + dbCommand = dbCommand.trim(); + if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) { + return false; + } else { + return true; + } + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + return (dbCommand.startsWith("--") || dbCommand.startsWith("#")); + } + + @Override + public String getDelimiter() { + return DEFAULT_DELIMITER; + } + + @Override + public String getQuoteCharacter() { + return DEFAULT_QUOTE; + } + + + @Override + public String cleanseCommand(String dbCommand) { + // strip off the delimiter + if (dbCommand.endsWith(getDelimiter())) { + dbCommand = dbCommand.substring(0, + dbCommand.length() - getDelimiter().length()); + } + return dbCommand; + } + + @Override + public boolean needsQuotedIdentifier() { + return false; + } + + @Override + public String buildCommand( + String scriptDir, String scriptFile) throws IllegalFormatException, IOException { + return buildCommand(scriptDir, scriptFile, false); + } + + @Override + public String buildCommand( + String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException { + BufferedReader bfReader = + new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile)); + String currLine; + StringBuilder sb = new StringBuilder(); + String currentCommand = null; + while ((currLine = bfReader.readLine()) != null) { + currLine = currLine.trim(); + + if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) { + currLine = currLine.replace("\\\"", getQuoteCharacter()); + } + + if (currLine.isEmpty()) { + continue; // skip empty lines + } + + if (currentCommand == null) { + currentCommand = currLine; + } else { + currentCommand = currentCommand + " " + currLine; + } + if (isPartialCommand(currLine)) { + // if its a partial line, continue collecting the pieces + continue; + } + + // if this is a valid executable command then add it to the buffer + if (!isNonExecCommand(currentCommand)) { + currentCommand = cleanseCommand(currentCommand); + if (isNestedScript(currentCommand)) { + // if this is a nested sql script then flatten it + String currScript = getScriptName(currentCommand); + sb.append(buildCommand(scriptDir, currScript)); + } else { + // Now we have a complete statement, process it + // write the line to buffer + sb.append(currentCommand); + sb.append(System.getProperty("line.separator")); + } + } + currentCommand = null; + } + bfReader.close(); + return sb.toString(); + } + + private void setDbOpts(String dbOpts) { + if (dbOpts != null) { + this.dbOpts = Lists.newArrayList(dbOpts.split(",")); + } else { + this.dbOpts = Lists.newArrayList(); + } + } + + protected List getDbOpts() { + return dbOpts; + } + + protected String getMsUsername() { + return msUsername; + } + + protected String getMsPassword() { + return msPassword; + } + + protected HiveConf getHiveConf() { + return hiveConf; + } + } + + // Derby commandline parser + public static class DerbyCommandParser extends AbstractCommandParser { + private static final String DERBY_NESTING_TOKEN = "RUN"; + + public DerbyCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + + if (!isNestedScript(dbCommand)) { + throw new IllegalArgumentException("Not a script format " + dbCommand); + } + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + return tokens[1].replace(";", "").replaceAll("'", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + // Derby script format is RUN '' + return dbCommand.startsWith(DERBY_NESTING_TOKEN); + } + } + + // Derby commandline parser + public static class HiveCommandParser extends AbstractCommandParser { + private static String HIVE_NESTING_TOKEN = "SOURCE"; + private final NestedScriptParser nestedDbCommandParser; + + public HiveCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf, String metaDbType) { + super(dbOpts, msUsername, msPassword, hiveConf); + nestedDbCommandParser = getDbCommandParser(metaDbType); + } + + @Override + public String getQuoteCharacter() { + return nestedDbCommandParser.getQuoteCharacter(); + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + + if (!isNestedScript(dbCommand)) { + throw new IllegalArgumentException("Not a script format " + dbCommand); + } + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + return tokens[1].replace(";", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(HIVE_NESTING_TOKEN); + } + } + + // MySQL parser + public static class MySqlCommandParser extends AbstractCommandParser { + private static final String MYSQL_NESTING_TOKEN = "SOURCE"; + private static final String DELIMITER_TOKEN = "DELIMITER"; + private String delimiter = DEFAULT_DELIMITER; + + public MySqlCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + + @Override + public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ + boolean isPartial = super.isPartialCommand(dbCommand); + // if this is a delimiter directive, reset our delimiter + if (dbCommand.startsWith(DELIMITER_TOKEN)) { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + delimiter = tokens[1]; + } + return isPartial; + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + // remove ending ';' + return tokens[1].replace(";", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(MYSQL_NESTING_TOKEN); + } + + @Override + public String getDelimiter() { + return delimiter; + } + + @Override + public String getQuoteCharacter() { + return "`"; + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + return super.isNonExecCommand(dbCommand) || + (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) || + dbCommand.startsWith(DELIMITER_TOKEN); + } + + @Override + public String cleanseCommand(String dbCommand) { + return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", ""); + } + + } + + // Postgres specific parser + public static class PostgresCommandParser extends AbstractCommandParser { + private static final String POSTGRES_NESTING_TOKEN = "\\i"; + @VisibleForTesting + public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings"; + @VisibleForTesting + public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81"; + + public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + // remove ending ';' + return tokens[1].replace(";", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(POSTGRES_NESTING_TOKEN); + } + + @Override + public boolean needsQuotedIdentifier() { + return true; + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + // Skip "standard_conforming_strings" command which is read-only in older + // Postgres versions like 8.1 + // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html + if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) { + if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) { + return true; + } + } + return super.isNonExecCommand(dbCommand); + } + } + + //Oracle specific parser + public static class OracleCommandParser extends AbstractCommandParser { + private static final String ORACLE_NESTING_TOKEN = "@"; + + public OracleCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + if (!isNestedScript(dbCommand)) { + throw new IllegalArgumentException("Not a nested script format " + dbCommand); + } + // remove ending ';' and starting '@' + return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(ORACLE_NESTING_TOKEN); + } + } + + //MSSQL specific parser + public static class MSSQLCommandParser extends AbstractCommandParser { + private static final String MSSQL_NESTING_TOKEN = ":r"; + + public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + return tokens[1]; + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(MSSQL_NESTING_TOKEN); + } + } + + public static NestedScriptParser getDbCommandParser(String dbName) { + return getDbCommandParser(dbName, null); + } + + public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) { + return getDbCommandParser(dbName, null, null, null, null, metaDbName); + } + + public static NestedScriptParser getDbCommandParser(String dbName, + String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf, String metaDbType) { + if (dbName.equalsIgnoreCase(DB_DERBY)) { + return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf); + } else if (dbName.equalsIgnoreCase(DB_HIVE)) { + return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType); + } else if (dbName.equalsIgnoreCase(DB_MSSQL)) { + return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf); + } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { + return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf); + } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { + return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf); + } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { + return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf); + } else { + throw new IllegalArgumentException("Unknown dbType " + dbName); + } + } + + public static class MetaStoreConnectionInfo { + private final String userName; + private final String password; + private final String url; + private final String driver; + private final boolean printInfo; + private final HiveConf hiveConf; + private final String dbType; + + public MetaStoreConnectionInfo(String userName, String password, String url, String driver, + boolean printInfo, HiveConf hiveConf, String dbType) { + super(); + this.userName = userName; + this.password = password; + this.url = url; + this.driver = driver; + this.printInfo = printInfo; + this.hiveConf = hiveConf; + this.dbType = dbType; + } + + public String getPassword() { + return password; + } + + public String getUrl() { + return url; + } + + public String getDriver() { + return driver; + } + + public boolean isPrintInfo() { + return printInfo; + } + + public HiveConf getHiveConf() { + return hiveConf; + } + + public String getUsername() { + return userName; + } + + public boolean getPrintInfo() { + return printInfo; + } + + public String getDbType() { + return dbType; + } + } +}