diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java index 3d8eb83b04..62c037edc8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java @@ -46,7 +46,7 @@ public void testMTQueries1() throws Exception { util.getConf().set("hive.stats.dbclass", "fs"); util.getConf().set("hive.mapred.mode", "nonstrict"); util.getConf().set("hive.stats.column.autogather", "false"); - util.newSession(true); + util.newSession(); } boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts); if (!success) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java index 18269ebc92..9915580871 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java @@ -21,7 +21,7 @@ /** * QTestArguments composite used as arguments holder for QTestUtil initialization. */ -public class QTestArguments { +public final class QTestArguments { private String outDir; private String logDir; @@ -34,19 +34,22 @@ private QTestUtil.FsType fsType; private QTestUtil.QTestSetup qtestSetup; - public void setClusterType(QTestUtil.MiniClusterType clusterType) { - this.clusterType = clusterType; + private QTestArguments() { } public QTestUtil.MiniClusterType getClusterType() { return clusterType; } + private void setClusterType(QTestUtil.MiniClusterType clusterType) { + this.clusterType = clusterType; + } + public String getOutDir() { return outDir; } - public void setOutDir(String outDir) { + private void setOutDir(String outDir) { this.outDir = outDir; } @@ -54,27 +57,27 @@ public String getLogDir() { return logDir; } - public void setLogDir(String logDir) { + private void setLogDir(String logDir) { this.logDir = logDir; } - public void setConfDir(String confDir) { - this.confDir = confDir; - } - public String getConfDir() { return confDir; } - public void setHadoopVer(String hadoopVer) { - this.hadoopVer = hadoopVer; + private void setConfDir(String confDir) { + this.confDir = confDir; } public String getHadoopVer() { return hadoopVer; } - public void setInitScript(String initScript) { + private void setHadoopVer(String hadoopVer) { + this.hadoopVer = hadoopVer; + } + + private void setInitScript(String initScript) { this.initScript = initScript; } @@ -82,7 +85,7 @@ public String getInitScript() { return initScript; } - public void setCleanupScript(String cleanupScript) { + private void setCleanupScript(String cleanupScript) { this.cleanupScript = cleanupScript; } @@ -90,34 +93,35 @@ public String getCleanupScript() { return cleanupScript; } - public void setWithLlapIo(boolean withLlapIo) { - this.withLlapIo = withLlapIo; - } - public boolean isWithLlapIo() { return withLlapIo; } - public void setFsType(QTestUtil.FsType fsType) { - this.fsType = fsType; + private void setWithLlapIo(boolean withLlapIo) { + this.withLlapIo = withLlapIo; } public QTestUtil.FsType getFsType() { return fsType; } - public void setQTestSetup(QTestUtil.QTestSetup qtestSetup) { - this.qtestSetup = qtestSetup; + private void setFsType(QTestUtil.FsType fsType) { + this.fsType = fsType; } public QTestUtil.QTestSetup getQTestSetup() { return qtestSetup; } + private void setQTestSetup(QTestUtil.QTestSetup qtestSetup) { + this.qtestSetup = qtestSetup; + } + /** * QTestArgumentsBuilder used for QTestArguments construction. */ public static final class QTestArgumentsBuilder { + private String outDir; private String logDir; private String confDir; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0e8b82930e..f85984617c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -33,7 +33,6 @@ import java.io.OutputStream; import java.io.PrintStream; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -84,7 +83,6 @@ import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.llap.io.api.LlapProxy; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.dataset.Dataset; @@ -135,22 +133,20 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import junit.framework.TestSuite; - /** * QTestUtil. * */ public class QTestUtil { - public static final String UTF_8 = "UTF-8"; + private static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); + private static final String UTF_8 = "UTF-8"; // security property names private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private static final String CRLF = System.getProperty("line.separator"); public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; - static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); private final static String defaultInitScript = "q_test_init.sql"; private final static String defaultCleanupScript = "q_test_cleanup.sql"; private static SimpleDateFormat formatter = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); @@ -158,9 +154,7 @@ public static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target - - public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; - public static final String TEST_HIVE_USER_PROPERTY = "test.hive.user"; + private static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; /** * The default Erasure Coding Policy to use in Erasure Coding tests. @@ -171,8 +165,7 @@ @Deprecated private final String testFiles; private final File datasetDir; - protected final String outDir; - protected String overrideResultsDir; + private final String outDir; protected final String logDir; private final TreeMap qMap; private final Set qSortSet; @@ -190,23 +183,22 @@ private final FsType fsType; private ParseDriver pd; protected Hive db; - protected QueryState queryState; + private QueryState queryState; protected HiveConf conf; protected HiveConf savedConf; private IDriver drv; private BaseSemanticAnalyzer sem; - protected final boolean overWrite; + private final boolean overWrite; private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private FileSystem fs; private HadoopShims.HdfsEncryptionShim hes = null; private MiniLlapCluster llapCluster = null; - private String hadoopVer = null; - private QTestSetup setup = null; + private final String hadoopVer; + private final QTestSetup setup; private SparkSession sparkSession = null; private boolean isSessionStateStarted = false; - private static final String javaVersion = getJavaVersion(); private QOutProcessor qOutProcessor; private final String initScript; private final String cleanupScript; @@ -214,10 +206,6 @@ private MiniDruidCluster druidCluster; private SingleNodeKafkaCluster kafkaCluster; - public interface SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName); - } - public static Set getSrcTables() { if (srcTables == null){ initSrcTables(); @@ -262,7 +250,6 @@ private CliDriver getCliDriver() { return cliDriver; } - /** * Returns the default UDF names which should not be removed when resetting the test database * @return The list of the UDF names not to remove @@ -482,7 +469,6 @@ public static MiniClusterType valueForString(String type) { } } - private String getKeyProviderURI() { // Use the target directory if it is not specified String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT; @@ -496,10 +482,11 @@ public QTestUtil(QTestArguments testArgs) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," + " fsType={}", - testArgs.getOutDir(), testArgs.getLogDir(), testArgs.getClusterType(), testArgs.getConfDir(), hadoopVer, - testArgs.getInitScript(), testArgs.getCleanupScript(), testArgs.isWithLlapIo(), testArgs.getFsType()); - + testArgs.getOutDir(), testArgs.getLogDir(), testArgs.getClusterType(), testArgs.getConfDir(), + testArgs.getHadoopVer(), testArgs.getInitScript(), testArgs.getCleanupScript(), + testArgs.isWithLlapIo(), testArgs.getFsType()); Preconditions.checkNotNull(testArgs.getClusterType(), "ClusterType cannot be null"); + this.fsType = testArgs.getFsType(); this.outDir = testArgs.getOutDir(); this.logDir = testArgs.getLogDir(); @@ -516,7 +503,7 @@ public QTestUtil(QTestArguments testArgs) throws Exception { queryState = new QueryState.Builder().withHiveConf(new HiveConf(IDriver.class)).build(); conf = queryState.getConf(); - this.hadoopVer = getHadoopMainVersion(hadoopVer); + this.hadoopVer = getHadoopMainVersion(testArgs.getHadoopVer()); qMap = new TreeMap(); qSortSet = new HashSet(); qSortQuerySet = new HashSet(); @@ -529,7 +516,6 @@ public QTestUtil(QTestArguments testArgs) throws Exception { this.clusterType = testArgs.getClusterType(); HadoopShims shims = ShimLoader.getHadoopShims(); - setupFileSystem(shims); setup = testArgs.getQTestSetup(); @@ -568,6 +554,7 @@ public QTestUtil(QTestArguments testArgs) throws Exception { init(); savedConf = new HiveConf(conf); } + private String getScriptsDir() { // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); @@ -682,7 +669,6 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws MiniClusterType.druidKafka ).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - } else { } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local, MiniClusterType.druidLocal) .contains(clusterType)) { @@ -853,27 +839,6 @@ private boolean matches(Pattern pattern, String query) { return false; } - /** - * Get formatted Java version to include minor version, but - * exclude patch level. - * - * @return Java version formatted as major_version.minor_version - */ - private static String getJavaVersion() { - String version = System.getProperty("java.version"); - if (version == null) { - throw new NullPointerException("No java version could be determined " + - "from system properties"); - } - - // "java version" system property is formatted - // major_version.minor_version.patch_level. - // Find second dot, instead of last dot, to be safe - int pos = version.indexOf('.'); - pos = version.indexOf('.', pos + 1); - return version.substring(0, pos); - } - /** * Clear out any side effects of running tests */ @@ -1001,13 +966,10 @@ public void newSession(boolean canReuseSession) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); CliSessionState ss = new CliSessionState(conf); - assert ss != null; ss.in = System.in; SessionState oldSs = SessionState.get(); - restartSessions(canReuseSession, ss, oldSs); - closeSession(oldSs); SessionState.start(ss); @@ -1097,28 +1059,6 @@ private void cleanupFromFile() throws IOException { } } - protected void runCreateTableCmd(String createTableCmd) throws Exception { - int ecode = 0; - ecode = drv.run(createTableCmd).getResponseCode(); - if (ecode != 0) { - throw new Exception("create table command: " + createTableCmd - + " failed with exit code= " + ecode); - } - - return; - } - - protected void runCmd(String cmd) throws Exception { - int ecode = 0; - ecode = drv.run(cmd).getResponseCode(); - drv.close(); - if (ecode != 0) { - throw new Exception("command: " + cmd - + " failed with exit code= " + ecode); - } - return; - } - public void createSources() throws Exception { createSources(null); } @@ -1234,7 +1174,6 @@ public String cliInit(File file) throws Exception { CliSessionState ss = (CliSessionState) SessionState.get(); - String outFileExtension = getOutFileExtension(fileName); String stdoutName = null; @@ -1257,7 +1196,7 @@ public String cliInit(File file) throws Exception { } private void setSessionOutputs(String fileName, CliSessionState ss, File outf) - throws FileNotFoundException, Exception, UnsupportedEncodingException { + throws Exception { OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf)); if (ss.out != null) { ss.out.flush(); @@ -1304,8 +1243,8 @@ private CliSessionState startSessionState(boolean canReuseSession) throws IOExce String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); + CliSessionState ss = new CliSessionState(conf); - assert ss != null; ss.in = System.in; ss.out = System.out; ss.err = System.out; @@ -1343,21 +1282,6 @@ public int executeAdhocCommand(String q) { return cliDriver.processLine(q1); } - public int executeOne(String tname) { - String q = qMap.get(tname); - - if (q.indexOf(";") == -1) { - return -1; - } - - String q1 = q.substring(0, q.indexOf(";") + 1); - String qrest = q.substring(q.indexOf(";") + 1); - qMap.put(tname, qrest); - - System.out.println("Executing " + q1); - return cliDriver.processLine(q1); - } - public int execute(String tname) { return drv.run(qMap.get(tname)).getResponseCode(); } @@ -1514,20 +1438,6 @@ private String getOutFileExtension(String fname) { return ".out"; } - public void convertSequenceFileToTextFile() throws Exception { - // Create an instance of hive in order to create the tables - testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); - db = Hive.get(conf); - - // Move all data from dest4_sequencefile to dest4 - drv - .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); - - // Drop dest4_sequencefile - db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, "dest4_sequencefile", - true, true); - } - public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1588,35 +1498,6 @@ public QTestProcessExecResult checkNegativeResults(String tname, Error e) throws return result; } - public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { - - if (tree != null) { - String outFileExtension = getOutFileExtension(tname); - - File parseDir = new File(outDir, "parse"); - String expf = outPath(parseDir.toString(), tname.concat(outFileExtension)); - - File outf = null; - outf = new File(logDir); - outf = new File(outf, tname.concat(outFileExtension)); - - FileWriter outfd = new FileWriter(outf); - outfd.write(tree.toStringTree()); - outfd.close(); - - QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - - if (overWrite) { - overwriteResults(outf.getPath(), expf); - return QTestProcessExecResult.createWithoutOutput(0); - } - - return exitVal; - } else { - throw new Exception("Parse tree is null"); - } - } - /** * Given the current configurations (e.g., hadoop version and execution mode), return * the correct file name to compare with the current test run output. @@ -1675,7 +1556,6 @@ public QTestProcessExecResult checkCliDriverResults(String tname) throws Excepti return exitVal; } - public QTestProcessExecResult checkCompareCliDriverResults(String tname, List outputs) throws Exception { assert outputs.size() > 1; @@ -1831,13 +1711,6 @@ public ASTNode parseQuery(String tname) throws Exception { return pd.parse(qMap.get(tname)); } - public void resetParser() throws SemanticException { - pd = new ParseDriver(); - queryState = new QueryState.Builder().withHiveConf(conf).build(); - sem = new SemanticAnalyzer(queryState); - } - - public List> analyzeAST(ASTNode ast) throws Exception { // Do semantic analysis and plan generation @@ -2186,10 +2059,6 @@ public QOutProcessor getQOutProcessor() { return qOutProcessor; } - public static void initEventNotificationPoll() throws Exception { - NotificationEventPoll.initialize(SessionState.get().getConf()); - } - /** * Should deleted test tables have their data purged. * @return true if data should be purged