diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index 88b6b2b..32a7b32 100644 --- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -127,7 +127,7 @@ private static void printAndExit(Options cmdLineOptions) { System.exit(1); } - private Connection getConnectionToMetastore(boolean printInfo) + Connection getConnectionToMetastore(boolean printInfo) throws HiveMetaException { return HiveSchemaHelper.getConnectionToMetastore(userName, passWord, printInfo, hiveConf); @@ -179,21 +179,15 @@ private String getMetaStoreSchemaVersion(Connection metastoreConn, } catch (SQLException e) { throw new HiveMetaException("Failed to get schema version.", e); } - finally { - try { - metastoreConn.close(); - } catch (SQLException e) { - System.err.println("Failed to close the metastore connection"); - e.printStackTrace(System.err); - } - } } - boolean validateLocations(String defaultLocPrefix) throws HiveMetaException { + boolean validateLocations(Connection conn, String defaultLocPrefix) throws HiveMetaException { + System.out.println("Validating database/table/partition locations"); boolean rtn; - rtn = checkMetaStoreDBLocation(defaultLocPrefix); - rtn = checkMetaStoreTableLocation(defaultLocPrefix) && rtn; - rtn = checkMetaStorePartitionLocation(defaultLocPrefix) && rtn; + rtn = checkMetaStoreDBLocation(conn, defaultLocPrefix); + rtn = checkMetaStoreTableLocation(conn, defaultLocPrefix) && rtn; + rtn = checkMetaStorePartitionLocation(conn, defaultLocPrefix) && rtn; + System.out.println((rtn ? "Succeeded" : "Failed") + " in database/table/partition location validation"); return rtn; } @@ -203,24 +197,22 @@ private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLExce } // read schema version from metastore - private boolean checkMetaStoreDBLocation(String locHeader) + private boolean checkMetaStoreDBLocation(Connection conn, String locHeader) throws HiveMetaException { String defaultPrefix = locHeader; String dbLoc; boolean isValid = true; int numOfInvalid = 0; - Connection metastoreConn = getConnectionToMetastore(true); if (getDbCommandParser(dbType).needsQuotedIdentifier()) { dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt"; } else { dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt"; } - String locValue; - String dbName; - try(Statement stmt = metastoreConn.createStatement(); + + try(Statement stmt = conn.createStatement(); ResultSet res = stmt.executeQuery(dbLoc)) { while (res.next()) { - locValue = res.getString(3); + String locValue = res.getString(3); if (locValue == null) { System.err.println("NULL Location for DB with " + getNameOrID(res,2,1)); numOfInvalid++; @@ -251,14 +243,6 @@ private boolean checkMetaStoreDBLocation(String locHeader) } catch (SQLException e) { throw new HiveMetaException("Failed to get DB Location Info.", e); } - finally { - try { - metastoreConn.close(); - } catch (SQLException e) { - System.err.println("Failed to close the metastore connection"); - e.printStackTrace(System.err); - } - } if (numOfInvalid > 0) { isValid = false; System.err.println("Total number of invalid DB locations is: "+ numOfInvalid); @@ -266,12 +250,11 @@ private boolean checkMetaStoreDBLocation(String locHeader) return isValid; } - private boolean checkMetaStoreTableLocation(String locHeader) + private boolean checkMetaStoreTableLocation(Connection conn, String locHeader) throws HiveMetaException { String defaultPrefix = locHeader; String tabLoc, tabIDRange; boolean isValid = true; - Connection metastoreConn = getConnectionToMetastore(true); int numOfInvalid = 0; if (getDbCommandParser(dbType).needsQuotedIdentifier()) { tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" "; @@ -286,13 +269,12 @@ private boolean checkMetaStoreTableLocation(String locHeader) } else { tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_ID >= ? and tbl.TBL_ID <= ? inner join DBS dbt on tbl.DB_ID = dbt.DB_ID"; } - String locValue; - String tabName; - long maxID = 0, minID = 0, curID; + + long maxID = 0, minID = 0; long rtnSize = 2000; try { - Statement stmt = metastoreConn.createStatement(); + Statement stmt = conn.createStatement(); ResultSet res = stmt.executeQuery(tabIDRange); if (res.next()) { maxID = res.getLong(1); @@ -300,14 +282,13 @@ private boolean checkMetaStoreTableLocation(String locHeader) } res.close(); stmt.close(); - curID = minID; - PreparedStatement pStmt = metastoreConn.prepareStatement(tabLoc); + PreparedStatement pStmt = conn.prepareStatement(tabLoc); while (minID <= maxID) { pStmt.setLong(1, minID); pStmt.setLong(2, minID + rtnSize); res = pStmt.executeQuery(); while (res.next()) { - locValue = res.getString(3); + String locValue = res.getString(3); if (locValue == null) { System.err.println("In DB with " + getNameOrID(res,5,4)); System.err.println("NULL Location for TABLE with " + getNameOrID(res,2,1)); @@ -346,14 +327,6 @@ private boolean checkMetaStoreTableLocation(String locHeader) } catch (SQLException e) { throw new HiveMetaException("Failed to get Table Location Info.", e); } - finally { - try { - metastoreConn.close(); - } catch (SQLException e) { - System.err.println("Failed to close the metastore connection"); - e.printStackTrace(System.err); - } - } if (numOfInvalid > 0) { isValid = false; System.err.println("Total number of invalid TABLE locations is: "+ numOfInvalid); @@ -361,13 +334,12 @@ private boolean checkMetaStoreTableLocation(String locHeader) return isValid; } - private boolean checkMetaStorePartitionLocation(String locHeader) + private boolean checkMetaStorePartitionLocation(Connection conn, String locHeader) throws HiveMetaException { String defaultPrefix = locHeader; String partLoc, partIDRange; boolean isValid = true; int numOfInvalid = 0; - Connection metastoreConn = getConnectionToMetastore(true); if (getDbCommandParser(dbType).needsQuotedIdentifier()) { partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" "; } else { @@ -384,13 +356,12 @@ private boolean checkMetaStorePartitionLocation(String locHeader) + "inner join SDS sd on pt.SD_ID = sd.SD_ID and pt.PART_ID >= ? and pt.PART_ID <= ? " + "inner join TBLS tbl on tbl.TBL_ID = pt.TBL_ID inner join DBS dbt on tbl.DB_ID = dbt.DB_ID "; } - String locValue; - String tabName; - long maxID = 0, minID = 0, curID; + + long maxID = 0, minID = 0; long rtnSize = 2000; try { - Statement stmt = metastoreConn.createStatement(); + Statement stmt = conn.createStatement(); ResultSet res = stmt.executeQuery(partIDRange); if (res.next()) { maxID = res.getLong(1); @@ -398,14 +369,13 @@ private boolean checkMetaStorePartitionLocation(String locHeader) } res.close(); stmt.close(); - curID = minID; - PreparedStatement pStmt = metastoreConn.prepareStatement(partLoc); + PreparedStatement pStmt = conn.prepareStatement(partLoc); while (minID <= maxID) { pStmt.setLong(1, minID); pStmt.setLong(2, minID + rtnSize); res = pStmt.executeQuery(); while (res.next()) { - locValue = res.getString(3); + String locValue = res.getString(3); if (locValue == null) { System.err.println("In DB with " + getNameOrID(res,7,6) + ", TABLE with " + getNameOrID(res,5,4)); System.err.println("NULL Location for PARTITION with " + getNameOrID(res,2,1)); @@ -442,14 +412,6 @@ private boolean checkMetaStorePartitionLocation(String locHeader) } catch (SQLException e) { throw new HiveMetaException("Failed to get Partiton Location Info.", e); } - finally { - try { - metastoreConn.close(); - } catch (SQLException e) { - System.err.println("Failed to close the metastore connection"); - e.printStackTrace(System.err); - } - } if (numOfInvalid > 0) { isValid = false; System.err.println("Total number of invalid PARTITION locations is: "+ numOfInvalid); @@ -582,16 +544,28 @@ public void doInit(String toVersion) throws HiveMetaException { } public void doValidate() throws HiveMetaException { - System.out.print("Starting metastore validation"); - validateSchemaVersions(); - validateSequences(); - validateSchemaTables(); - validateLocations(null); - validateColumnNullValues(); - System.out.print("Done with metastore validation"); + System.out.println("Starting metastore validation"); + Connection conn = getConnectionToMetastore(false); + try { + validateSchemaVersions(conn); + validateSequences(conn); + validateSchemaTables(conn); + validateLocations(conn, null); + validateColumnNullValues(conn); + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + throw new HiveMetaException("Failed to close metastore connection", e); + } + } + } + + System.out.println("Done with metastore validation"); } - boolean validateSequences() throws HiveMetaException { + boolean validateSequences(Connection conn) throws HiveMetaException { Map> seqNameToTable = new ImmutableMap.Builder>() .put("MDatabase", Pair.of("DBS", "DB_ID")) @@ -611,7 +585,7 @@ boolean validateSequences() throws HiveMetaException { .build(); System.out.println("Validating sequence number for SEQUENCE_TABLE"); - Connection conn = getConnectionToMetastore(true); + boolean isValid = true; try { Statement stmt = conn.createStatement(); @@ -638,25 +612,17 @@ boolean validateSequences() throws HiveMetaException { } } + System.out.println((isValid ? "Succeeded" :"Failed") + " in sequence number validation for SEQUENCE_TABLE"); return isValid; } catch(SQLException e) { throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e); - } finally { - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - throw new HiveMetaException("Failed to close metastore connection", e); - } - } } } - boolean validateSchemaVersions() throws HiveMetaException { + boolean validateSchemaVersions(Connection conn) throws HiveMetaException { System.out.println("Validating schema version"); try { - String newSchemaVersion = getMetaStoreSchemaVersion( - getConnectionToMetastore(false), true); + String newSchemaVersion = getMetaStoreSchemaVersion(conn, true); assertCompatibleVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); } catch (HiveMetaException hme) { if (hme.getMessage().contains("Metastore schema version is not compatible") @@ -672,19 +638,17 @@ boolean validateSchemaVersions() throws HiveMetaException { return true; } - boolean validateSchemaTables() throws HiveMetaException { + boolean validateSchemaTables(Connection conn) throws HiveMetaException { ResultSet rs = null; DatabaseMetaData metadata = null; List dbTables = new ArrayList(); List schemaTables = new ArrayList(); List subScripts = new ArrayList(); - Connection hmsConn = getConnectionToMetastore(false); - String version = getMetaStoreSchemaVersion(hmsConn); - hmsConn = getConnectionToMetastore(false); + String version = getMetaStoreSchemaVersion(conn); System.out.println("Validating tables in the schema for version " + version); try { - metadata = hmsConn.getMetaData(); + metadata = conn.getMetaData(); String[] types = {"TABLE"}; rs = metadata.getTables(null, null, "%", types); String table = null; @@ -704,15 +668,6 @@ boolean validateSchemaTables() throws HiveMetaException { throw new HiveMetaException("Failed to close resultset", e); } } - - if (hmsConn != null) { - try { - hmsConn.close(); - - } catch (SQLException e) { - throw new HiveMetaException("Failed to close metastore connection", e); - } - } } // parse the schema file to determine the tables that are expected to exist @@ -721,11 +676,11 @@ boolean validateSchemaTables() throws HiveMetaException { String schemaFile = baseDir + "/oracle/hive-schema-" + version + ".oracle.sql"; try { - LOG.info("Parsing schema script " + schemaFile); + LOG.debug("Parsing schema script " + schemaFile); subScripts.addAll(findCreateTable(schemaFile, schemaTables)); while (subScripts.size() > 0) { schemaFile = baseDir + "/oracle/" + subScripts.remove(0); - LOG.info("Parsing subscript " + schemaFile); + LOG.debug("Parsing subscript " + schemaFile); subScripts.addAll(findCreateTable(schemaFile, schemaTables)); } } catch (Exception e) { @@ -742,7 +697,7 @@ boolean validateSchemaTables() throws HiveMetaException { + " ] are missing from the database schema."); return false; } else { - System.out.println("Schema table validation successful"); + System.out.println("Succeeded in schema table validation"); return true; } } @@ -778,9 +733,8 @@ boolean validateSchemaTables() throws HiveMetaException { return subs; } - boolean validateColumnNullValues() throws HiveMetaException { + boolean validateColumnNullValues(Connection conn) throws HiveMetaException { System.out.println("Validating columns for incorrect NULL values"); - Connection conn = getConnectionToMetastore(true); boolean isValid = true; try { Statement stmt = conn.createStatement(); @@ -797,15 +751,10 @@ boolean validateColumnNullValues() throws HiveMetaException { System.err.println("Value of SD_ID in TBLS should not be NULL: hive table - " + tableName + " tableId - " + tableId + " tableType - " + tableType); } + System.out.println((isValid ? "Succeeded" : "Failed") + " in column validation for incorrect NULL values"); return isValid; } catch(SQLException e) { throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e); - } finally { - try { - conn.close(); - } catch (SQLException e) { - throw new HiveMetaException("Failed to close metastore connection", e); - } } } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index ac2c927..3d585ac 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.sql.Connection; import java.util.Random; import junit.framework.TestCase; @@ -39,6 +40,7 @@ public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; + private Connection conn; private HiveConf hiveConf; private String testMetastoreDB; private PrintStream errStream; @@ -57,6 +59,7 @@ protected void setUp() throws Exception { System.setProperty("beeLine.system.exit", "true"); errStream = System.err; outStream = System.out; + conn = schemaTool.getConnectionToMetastore(false); } @Override @@ -67,6 +70,9 @@ protected void tearDown() throws Exception { } System.setOut(outStream); System.setErr(errStream); + if (conn != null) { + conn.close(); + } } /** @@ -77,7 +83,7 @@ public void testValidateSequences() throws Exception { schemaTool.doInit(); // Test empty database - boolean isValid = schemaTool.validateSequences(); + boolean isValid = schemaTool.validateSequences(conn); assertTrue(isValid); // Test valid case @@ -87,7 +93,7 @@ public void testValidateSequences() throws Exception { }; File scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSequences(); + isValid = schemaTool.validateSequences(conn); assertTrue(isValid); // Test invalid case @@ -99,7 +105,7 @@ public void testValidateSequences() throws Exception { }; scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSequences(); + isValid = schemaTool.validateSequences(conn); assertFalse(isValid); } @@ -110,12 +116,12 @@ public void testValidateSequences() throws Exception { public void testValidateSchemaTables() throws Exception { schemaTool.doInit("2.0.0"); - boolean isValid = (boolean)schemaTool.validateSchemaTables(); + boolean isValid = (boolean)schemaTool.validateSchemaTables(conn); assertTrue(isValid); // upgrade to 2.2.0 schema and re-validate schemaTool.doUpgrade("2.2.0"); - isValid = (boolean)schemaTool.validateSchemaTables(); + isValid = (boolean)schemaTool.validateSchemaTables(conn); assertTrue(isValid); // Simulate a missing table scenario by renaming a couple of tables @@ -126,7 +132,7 @@ public void testValidateSchemaTables() throws Exception { File scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaTables(); + isValid = schemaTool.validateSchemaTables(conn); assertFalse(isValid); // Restored the renamed tables @@ -137,7 +143,7 @@ public void testValidateSchemaTables() throws Exception { scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaTables(); + isValid = schemaTool.validateSchemaTables(conn); assertTrue(isValid); } @@ -149,12 +155,12 @@ public void testValidateNullValues() throws Exception { schemaTool.doInit(); // Test empty database - boolean isValid = schemaTool.validateColumnNullValues(); + boolean isValid = schemaTool.validateColumnNullValues(conn); assertTrue(isValid); // Test valid case createTestHiveTableSchemas(); - isValid = schemaTool.validateColumnNullValues(); + isValid = schemaTool.validateColumnNullValues(conn); // Test invalid case String[] scripts = new String[] { @@ -162,7 +168,7 @@ public void testValidateNullValues() throws Exception { }; File scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateColumnNullValues(); + isValid = schemaTool.validateColumnNullValues(conn); assertFalse(isValid); } @@ -217,14 +223,14 @@ public void testSchemaInit() throws Exception { */ public void testValidateSchemaVersions() throws Exception { schemaTool.doInit(); - boolean isValid = schemaTool.validateSchemaVersions(); + boolean isValid = schemaTool.validateSchemaVersions(conn); // Test an invalid case with multiple versions String[] scripts = new String[] { "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')" }; File scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); + isValid = schemaTool.validateSchemaVersions(conn); assertFalse(isValid); scripts = new String[] { @@ -232,7 +238,7 @@ public void testValidateSchemaVersions() throws Exception { }; scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); + isValid = schemaTool.validateSchemaVersions(conn); assertTrue(isValid); // Test an invalid case without version @@ -241,7 +247,7 @@ public void testValidateSchemaVersions() throws Exception { }; scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateSchemaVersions(); + isValid = schemaTool.validateSchemaVersions(conn); assertFalse(isValid); } @@ -596,12 +602,11 @@ public void testValidateLocations() throws Exception { schemaTool.doInit(); String defaultRoot = "hdfs://myhost.com:8020"; //check empty DB - boolean isValid = schemaTool.validateLocations(null); + boolean isValid = schemaTool.validateLocations(conn, null); assertTrue(isValid); - isValid = schemaTool.validateLocations(defaultRoot); + isValid = schemaTool.validateLocations(conn, defaultRoot); assertTrue(isValid); - String dbmydbLocation = defaultRoot + "/user/hive/warehouse/mydb"; // Test valid case String[] scripts = new String[] { "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role')", @@ -613,9 +618,9 @@ public void testValidateLocations() throws Exception { }; File scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateLocations(null); + isValid = schemaTool.validateLocations(conn, null); assertTrue(isValid); - isValid = schemaTool.validateLocations(defaultRoot); + isValid = schemaTool.validateLocations(conn, defaultRoot); assertTrue(isValid); scripts = new String[] { "delete from PARTITIONS", @@ -634,9 +639,9 @@ public void testValidateLocations() throws Exception { }; scriptFile = generateTestScript(scripts); schemaTool.runBeeLine(scriptFile.getPath()); - isValid = schemaTool.validateLocations(null); + isValid = schemaTool.validateLocations(conn, null); assertFalse(isValid); - isValid = schemaTool.validateLocations(defaultRoot); + isValid = schemaTool.validateLocations(conn, defaultRoot); assertFalse(isValid); }