diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java index 1fdce17..16c501f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java @@ -49,7 +49,7 @@ private final File testDataDirectory; private final File testScriptDirectory; private boolean overwrite = false; - private boolean rewriteSourceTables = true; + private boolean useSharedDatabase = false; private MiniHS2 miniHS2; private QFileClientBuilder clientBuilder; private QFileBuilder fileBuilder; @@ -101,7 +101,7 @@ boolean getBooleanPropertyValue(String name, boolean defaultValue) { public void beforeClass() throws Exception { overwrite = getBooleanPropertyValue("test.output.overwrite", Boolean.FALSE); - rewriteSourceTables = getBooleanPropertyValue("test.rewrite.source.tables", Boolean.TRUE); + useSharedDatabase = getBooleanPropertyValue("test.beeline.shared.database", Boolean.FALSE); String beeLineUrl = System.getProperty("test.beeline.url"); if (StringUtils.isEmpty(beeLineUrl)) { @@ -122,7 +122,7 @@ public void beforeClass() throws Exception { .setLogDirectory(logDirectory) .setQueryDirectory(queryDirectory) .setResultsDirectory(resultsDirectory) - .setRewriteSourceTables(rewriteSourceTables) + .setUseSharedDatabase(useSharedDatabase) .setComparePortable(comparePortable); runInfraScript(initScript, new File(logDirectory, "init.beeline"), diff --git itests/util/src/main/java/org/apache/hive/beeline/QFile.java itests/util/src/main/java/org/apache/hive/beeline/QFile.java index 38b0d91..21be8b0 100644 --- itests/util/src/main/java/org/apache/hive/beeline/QFile.java +++ itests/util/src/main/java/org/apache/hive/beeline/QFile.java @@ -81,7 +81,7 @@ private static RegexFilterSet staticFilterSet = getStaticFilterSet(); private static RegexFilterSet portableFilterSet = getPortableFilterSet(); private RegexFilterSet specificFilterSet; - private boolean rewriteSourceTables; + private boolean useSharedDatabase; private Converter converter; private boolean comparePortable; @@ -91,6 +91,7 @@ public String getName() { return name; } + public String getDatabaseName() { return databaseName; } @@ -127,6 +128,10 @@ public Converter getConverter() { return converter; } + public boolean isUseSharedDatabase() { + return useSharedDatabase; + } + public String getDebugHint() { return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile, logFile, beforeExecuteLogFile, afterExecuteLogFile, @@ -134,13 +139,13 @@ public String getDebugHint() { } /** - * Filters the sql commands if necessary. + * Filters the sql commands if necessary - eg. not using the shared database. * @param commands The array of the sql commands before filtering * @return The filtered array of the sql command strings * @throws IOException File read error */ public String[] filterCommands(String[] commands) throws IOException { - if (rewriteSourceTables) { + if (!useSharedDatabase) { for (int i=0; i getDatabases() throws SQLException { + Set databases = new HashSet(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the databases + try (ResultSet schemasResultSet = metaData.getSchemas()) { + while (schemasResultSet.next()) { + databases.add(schemasResultSet.getString("TABLE_SCHEM")); + } + } + return databases; + } + + private Set getTables() throws SQLException { + Set tables = new HashSet(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the tables in the default database + String[] types = new String[] {"TABLE"}; + try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) { + while (tablesResultSet.next()) { + tables.add(tablesResultSet.getString("TABLE_NAME")); + } + } + return tables; + } + + private Set getViews() throws SQLException { + Set views = new HashSet(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the tables in the default database + String[] types = new String[] {"VIEW"}; + try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) { + while (tablesResultSet.next()) { + views.add(tablesResultSet.getString("TABLE_NAME")); + } + } + return views; + } + public void execute(String[] commands, File resultFile, Converter converter) throws Exception { beeLine.runCommands( @@ -69,39 +137,53 @@ public void execute(String[] commands, File resultFile, Converter converter) } private void beforeExecute(QFile qFile) throws Exception { - execute( - new String[] { - "!set outputformat tsv2", - "!set verbose false", - "!set silent true", - "!set showheader false", - "USE default;", - "SHOW TABLES;", - "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", - "CREATE DATABASE `" + qFile.getDatabaseName() + "`;", - "USE `" + qFile.getDatabaseName() + "`;", - "set hive.testing.short.logs=true;", - "set hive.testing.remove.logs=false;", - }, - qFile.getBeforeExecuteLogFile(), - Converter.NONE); + String[] commands = TEST_FIRST_COMMANDS; + + String[] extraCommands; + if (qFile.isUseSharedDatabase()) { + // If we are using a shared database, then remove not known databases, tables, views. + Set dropCommands = getDatabases().stream() + .filter(database -> !database.equals("default")) + .map(database -> "DROP DATABASE `" + database + "` CASCADE;") + .collect(Collectors.toSet()); + + Set srcTables = QTestUtil.getSrcTables(); + dropCommands.addAll(getTables().stream() + .filter(table -> !srcTables.contains(table)) + .map(table -> "DROP TABLE `" + table + "` PURGE;") + .collect(Collectors.toSet())); + + dropCommands.addAll(getViews().stream() + .map(view -> "DROP VIEW `" + view + "`;") + .collect(Collectors.toSet())); + extraCommands = dropCommands.toArray(new String[]{}); + } else { + // If we are using a test specific database, then we just drop the database, and recreate + extraCommands = new String[] { + "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", + "CREATE DATABASE `" + qFile.getDatabaseName() + "`;", + "USE `" + qFile.getDatabaseName() + "`;" + }; + } + commands = ArrayUtils.addAll(commands, extraCommands); + commands = ArrayUtils.addAll(commands, TEST_SET_LOG_COMMANDS); + execute(commands, qFile.getBeforeExecuteLogFile(), Converter.NONE); beeLine.setIsTestMode(true); } private void afterExecute(QFile qFile) throws Exception { beeLine.setIsTestMode(false); - execute( - new String[] { - "set hive.testing.short.logs=false;", - "!set verbose true", - "!set silent false", - "!set showheader true", - "!set outputformat table", - "USE default;", - "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", - }, - qFile.getAfterExecuteLogFile(), - Converter.NONE); + String[] commands = TEST_RESET_COMMANDS; + + if (!qFile.isUseSharedDatabase()) { + // If we are using a test specific database, then we just drop the database + String[] extraCommands = new String[] { + "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;" + }; + commands = ArrayUtils.addAll(commands, extraCommands); + } + + execute(commands, qFile.getAfterExecuteLogFile(), Converter.NONE); } public void execute(QFile qFile) throws Exception {