diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java index b7e563aebf..405d21cdb2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java @@ -86,8 +86,7 @@ public QTestProcessExecResult checkCliDriverResults(String tname) throws Excepti return QTestProcessExecResult.create(failedCount, fileNames.toString()); } - public CheckResults(String outDir, String logDir, MiniClusterType miniMr, - String hadoopVer, String locationSubdir) + public CheckResults(String outDir, String logDir, MiniClusterType miniMr, String locationSubdir) throws Exception { super( @@ -96,7 +95,6 @@ public CheckResults(String outDir, String logDir, MiniClusterType miniMr, .withLogDir(logDir) .withClusterType(miniMr) .withConfDir(null) - .withHadoopVer(hadoopVer) .withInitScript("") .withCleanupScript("") .withLlapIo(false) @@ -119,7 +117,7 @@ public void testAlterTablePartitionLocation_alter5() throws Exception { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { - qt[i] = new CheckResults(resDir, logDir, MiniClusterType.none, "0.20", "parta"); + qt[i] = new CheckResults(resDir, logDir, MiniClusterType.none, "parta"); qt[i].newSession(); qt[i].addFile(qfiles[i], false); qt[i].clearTestSideEffects(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java index 3d8eb83b04..62c037edc8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java @@ -46,7 +46,7 @@ public void testMTQueries1() throws Exception { util.getConf().set("hive.stats.dbclass", "fs"); util.getConf().set("hive.mapred.mode", "nonstrict"); util.getConf().set("hive.stats.column.autogather", "false"); - util.newSession(true); + util.newSession(); } boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts); if (!success) { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java index 060e0cd3e2..01c3c0bebb 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.accumulo; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.QTestArguments; import org.apache.hadoop.hive.ql.QTestUtil; @@ -25,7 +24,6 @@ * AccumuloQTestUtil initializes Accumulo-specific test fixtures. */ public class AccumuloQTestUtil extends QTestUtil { - AccumuloTestSetup setup = null; public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception { @@ -36,22 +34,10 @@ public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr, .withLogDir(logDir) .withClusterType(miniMr) .withConfDir(null) - .withHadoopVer("0.20") .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) + .withQTestSetup(setup) .build()); - - setup.setupWithHiveConf(conf); - this.setup = setup; - this.savedConf = new HiveConf(conf); - } - - @Override - public void initConf() throws Exception { - if (setup != null) { - setup.updateConf(conf); - } - super.initConf(); } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java index 47cf7ac79a..c5998145e2 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java @@ -37,20 +37,28 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QTestUtil; /** * Start and stop an AccumuloMiniCluster for testing purposes */ -public class AccumuloTestSetup { +public class AccumuloTestSetup extends QTestUtil.QTestSetup { + public static final String PASSWORD = "password"; public static final String TABLE_NAME = "accumuloHiveTable"; - protected MiniAccumuloCluster miniCluster; + private MiniAccumuloCluster miniCluster; public AccumuloTestSetup() { } - protected void setupWithHiveConf(HiveConf conf) throws Exception { + @Override + public void preTest(HiveConf conf) throws Exception { + super.preTest(conf); + setupWithHiveConf(conf); + } + + private void setupWithHiveConf(HiveConf conf) throws Exception { if (null == miniCluster) { String testTmpDir = System.getProperty("test.tmp.dir"); File tmpDir = new File(testTmpDir, "accumulo"); @@ -63,7 +71,6 @@ protected void setupWithHiveConf(HiveConf conf) throws Exception { cfg.setNumTservers(1); miniCluster = new MiniAccumuloCluster(cfg); - miniCluster.start(); createAccumuloTable(miniCluster.getConnector("root", PASSWORD)); @@ -76,7 +83,7 @@ protected void setupWithHiveConf(HiveConf conf) throws Exception { * Update hiveConf with the Accumulo specific parameters * @param conf The hiveconf to update */ - public void updateConf(HiveConf conf) { + private void updateConf(HiveConf conf) { // Setup connection information conf.set(AccumuloConnectionParameters.USER_NAME, "root"); conf.set(AccumuloConnectionParameters.USER_PASS, PASSWORD); @@ -86,7 +93,7 @@ public void updateConf(HiveConf conf) { } } - protected void createAccumuloTable(Connector conn) throws TableExistsException, + private void createAccumuloTable(Connector conn) throws TableExistsException, TableNotFoundException, AccumuloException, AccumuloSecurityException { TableOperations tops = conn.tableOperations(); if (tops.exists(TABLE_NAME)) { @@ -131,10 +138,12 @@ protected void createAccumuloTable(Connector conn) throws TableExistsException, } } + @Override public void tearDown() throws Exception { if (null != miniCluster) { miniCluster.stop(); miniCluster = null; } + super.tearDown(); } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 64f29194d5..aa2c7a7ea3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -58,16 +58,14 @@ public void beforeClass() { String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); - try { - String hadoopVer = cliConfig.getHadoopVersion(); + try { qt = new QTestUtil( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(hiveConfDir) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(true) @@ -82,11 +80,12 @@ public void beforeClass() { qt.newSession(); qt.cleanUp(); qt.createSources(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in static initialization",e); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -95,6 +94,7 @@ public void beforeClass() { public void setUp() { try { qt.newSession(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -109,6 +109,7 @@ public void tearDown() { try { qt.clearTestSideEffects(); qt.clearPostTestEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -119,7 +120,7 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { qt.shutdown(); if (System.getenv(QTestUtil.QTEST_LEAVE_FILES) == null) { @@ -134,10 +135,10 @@ public void shutdown() throws Exception { } } - static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log, " + private static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log, " + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs."; - protected void runTestHelper(String tname, String fname, String fpath, boolean expectSuccess) throws Exception { + protected void runTestHelper(String tname, String fname, String fpath, boolean expectSuccess) { long startTime = System.currentTimeMillis(); qt.getConf().set(HCONF_TEST_BLOBSTORE_PATH_UNIQUE, testBlobstorePathUnique); try { @@ -145,10 +146,12 @@ protected void runTestHelper(String tname, String fname, String fpath, boolean e qt.addFile(fpath); qt.cliInit(new File(fpath)); + int ecode = qt.executeClient(fname); if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); } + QTestProcessExecResult result = qt.checkCliDriverResults(fname); if (result.getReturnCode() != 0) { String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 0d64cfa1ac..6bbcf3d474 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -34,7 +34,6 @@ public class CoreAccumuloCliDriver extends CliAdapter { private AccumuloQTestUtil qt; - private static AccumuloTestSetup setup; public CoreAccumuloCliDriver(AbstractCliConfig cliConfig) { super(cliConfig); @@ -43,35 +42,32 @@ public CoreAccumuloCliDriver(AbstractCliConfig cliConfig) { @Override @BeforeClass public void beforeClass() { - setup = new AccumuloTestSetup(); - MiniClusterType miniMR = cliConfig.getClusterType(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); try { qt = new AccumuloQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, - setup, initScript, cleanupScript); + new AccumuloTestSetup(), initScript, cleanupScript); // do a one time initialization qt.newSession(); qt.cleanUp(); qt.createSources(); + } catch (Exception e) { - throw new RuntimeException("Unexpected exception in setUp",e); + throw new RuntimeException("Unexpected exception in setUp", e); } } @Override @AfterClass - public void shutdown() throws Exception { - setup.tearDown(); - + public void shutdown() { try { qt.shutdown(); - } - catch (Exception e) { - throw new RuntimeException("Unexpected exception in tearDown",e); + + } catch (Exception e) { + throw new RuntimeException("Unexpected exception in tearDown", e); } } @@ -80,6 +76,7 @@ public void shutdown() throws Exception { public void setUp() { try { qt.newSession(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -94,6 +91,7 @@ public void tearDown() { try { qt.clearPostTestEffects(); qt.clearTestSideEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -101,15 +99,16 @@ public void tearDown() { fail("Unexpected exception in tearDown"); } } + @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { long startTime = System.currentTimeMillis(); try { System.err.println("Begin query: " + fname); qt.addFile(fpath); - qt.cliInit(new File(fpath)); + int ecode = qt.executeClient(fname); if (ecode != 0) { qt.failed(ecode, fname, null); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java index bdb15b3869..cd7e46fb61 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java @@ -23,7 +23,7 @@ public CoreBlobstoreCliDriver(AbstractCliConfig testCliConfig) { } @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { super.runTestHelper(tname, fname, fpath, true); } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java index 801c44b40c..43029076f4 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java @@ -23,7 +23,7 @@ public CoreBlobstoreNegativeCliDriver(AbstractCliConfig testCliConfig) { } @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { super.runTestHelper(tname, fname, fpath, false); } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index 841344438e..7ed8388a52 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -53,14 +53,13 @@ public void beforeClass() { String message = "Starting " + CoreCliDriver.class.getName() + " run at " + System.currentTimeMillis(); LOG.info(message); System.err.println(message); - final MiniClusterType miniMR =cliConfig.getClusterType(); - final String hiveConfDir = cliConfig.getHiveConfDir(); - final String initScript = cliConfig.getInitScript(); - final String cleanupScript = cliConfig.getCleanupScript(); - try { - final String hadoopVer = cliConfig.getHadoopVersion(); + MiniClusterType miniMR =cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); + String initScript = cliConfig.getInitScript(); + String cleanupScript = cliConfig.getCleanupScript(); + try { qt = new ElapsedTimeLoggingWrapper() { @Override public QTestUtil invokeInternal() throws Exception { @@ -70,7 +69,6 @@ public QTestUtil invokeInternal() throws Exception { .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(hiveConfDir) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(true) @@ -101,7 +99,7 @@ public Void invokeInternal() throws Exception { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in static initialization",e); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -116,6 +114,7 @@ public Void invokeInternal() throws Exception { return null; } }.invoke("PerTestSetup done.", LOG, false); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -136,6 +135,7 @@ public Void invokeInternal() throws Exception { return null; } }.invoke("PerTestTearDown done.", LOG, false); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -146,7 +146,7 @@ public Void invokeInternal() throws Exception { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { new ElapsedTimeLoggingWrapper() { @Override @@ -155,6 +155,7 @@ public Void invokeInternal() throws Exception { return null; } }.invoke("Teardown done.", LOG, false); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -169,7 +170,7 @@ public Void invokeInternal() throws Exception { + "or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; @Override - public void runTest(String testName, String fname, String fpath) throws Exception { + public void runTest(String testName, String fname, String fpath) { Stopwatch sw = Stopwatch.createStarted(); boolean skipped = false; boolean failed = false; @@ -179,11 +180,13 @@ public void runTest(String testName, String fname, String fpath) throws Exceptio qt.addFile(fpath); qt.cliInit(new File(fpath)); + int ecode = qt.executeClient(fname); if (ecode != 0) { failed = true; qt.failed(ecode, fname, debugHint); } + QTestProcessExecResult result = qt.checkCliDriverResults(fname); if (result.getReturnCode() != 0) { failed = true; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 8ce43495e6..7a06768fda 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -21,9 +21,7 @@ import java.io.File; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.hadoop.hive.ql.QTestArguments; import org.apache.hadoop.hive.ql.QTestProcessExecResult; @@ -38,28 +36,26 @@ public class CoreCompareCliDriver extends CliAdapter{ private static QTestUtil qt; + public CoreCompareCliDriver(AbstractCliConfig testCliConfig) { super(testCliConfig); } - @Override @BeforeClass public void beforeClass() { - MiniClusterType miniMR = cliConfig.getClusterType(); String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); + try { - String hadoopVer = cliConfig.getHadoopVersion(); qt = new QTestUtil( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(hiveConfDir) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) @@ -74,7 +70,7 @@ public void beforeClass() { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception in static initialization"); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -83,6 +79,7 @@ public void beforeClass() { public void setUp() { try { qt.clearTestSideEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -96,6 +93,7 @@ public void setUp() { public void tearDown() { try { qt.clearPostTestEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -106,7 +104,7 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { qt.shutdown(); } catch (Exception e) { @@ -117,13 +115,11 @@ public void shutdown() throws Exception { } } - private Map> versionFiles = new HashMap<>(); - - static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + private static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { final String queryDirectory = cliConfig.getQueryDirectory(); long startTime = System.currentTimeMillis(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreDummy.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreDummy.java index 4d40ef9047..a0d1433536 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreDummy.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreDummy.java @@ -42,11 +42,11 @@ public void tearDown() { } @Override - public void shutdown() throws Exception { + public void shutdown() { } @Override - public void runTest(String name, String name2, String absolutePath) throws Exception { + public void runTest(String name, String name2, String absolutePath) { List versionFiles = QTestUtil.getVersionFiles(cliConfig.getQueryDirectory(), name); if (versionFiles.size() < 2) { fail("Cannot run " + name2 + " with only " + versionFiles.size() + " versions"); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 252e9f6b90..0d67768967 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -58,9 +58,8 @@ public void beforeClass() { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception in static initialization: "+e.getMessage()); + throw new RuntimeException("Unexpected exception in static initialization: ", e); } - } @Override @@ -91,7 +90,7 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { qt.shutdown(); } catch (Exception e) { @@ -103,7 +102,7 @@ public void shutdown() throws Exception { } @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { long startTime = System.currentTimeMillis(); try { System.err.println("Begin query: " + fname); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index c009cec57b..af170a9514 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -53,7 +53,7 @@ public void beforeClass() throws Exception { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception in static initialization: "+e.getMessage()); + throw new RuntimeException("Unexpected exception in static initialization: ", e); } } @@ -86,7 +86,7 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { qt.shutdown(); } catch (Exception e) { @@ -98,7 +98,7 @@ public void shutdown() throws Exception { } @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String tname, String fname, String fpath) { long startTime = System.currentTimeMillis(); try { System.err.println("Begin query: " + fname); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 0807da1c7c..91c3bf87b9 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -35,6 +35,7 @@ public class CoreNegativeCliDriver extends CliAdapter{ private QTestUtil qt; + public CoreNegativeCliDriver(AbstractCliConfig testCliConfig) { super(testCliConfig); } @@ -47,14 +48,12 @@ public void beforeClass(){ String cleanupScript = cliConfig.getCleanupScript(); try { - String hadoopVer = cliConfig.getHadoopVersion(); qt = new QTestUtil( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(hiveConfDir) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) @@ -63,11 +62,12 @@ public void beforeClass(){ qt.newSession(); qt.cleanUp(); qt.createSources(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception in static initialization"); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -76,6 +76,7 @@ public void beforeClass(){ public void setUp() { try { qt.newSession(); + } catch (Throwable e) { e.printStackTrace(); System.err.flush(); @@ -89,6 +90,7 @@ public void tearDown() { try { qt.clearTestSideEffects(); qt.clearPostTestEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -99,9 +101,10 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { try { qt.shutdown(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -110,17 +113,9 @@ public void shutdown() throws Exception { } } - /** - * Dummy last test. This is only meant to shutdown qt - */ - public void testNegativeCliDriver_shutdown() { - System.err.println ("Cleaning up " + "$className"); - } - - static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + private static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; - @Override public void runTest(String tname, String fname, String fpath) throws Exception { long startTime = System.currentTimeMillis(); @@ -128,9 +123,8 @@ public void runTest(String tname, String fname, String fpath) throws Exception { System.err.println("Begin query: " + fname); qt.addFile(fpath); - - qt.cliInit(new File(fpath)); + int ecode = qt.executeClient(fname); if (ecode == 0) { qt.failed(fname, debugHint); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index badb4a5888..cf3d815441 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -58,20 +58,19 @@ public CorePerfCliDriver(AbstractCliConfig testCliConfig) { public void beforeClass() { System.setProperty("datanucleus.schema.autoCreateAll", "true"); System.setProperty("hive.metastore.schema.verification", "false"); + MiniClusterType miniMR = cliConfig.getClusterType(); String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); - try { - String hadoopVer = cliConfig.getHadoopVersion(); + try { qt = new QTestUtil( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(hiveConfDir) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) @@ -85,12 +84,12 @@ public void beforeClass() { // the 30TB TPCDS scale set. This way the optimizer will generate plans for a 30 TB set. MetaStoreDumpUtility.setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(qt.getConf(), System.getProperty(QTestUtil.TEST_TMP_DIR_PROPERTY)); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in static initialization: " + e.getMessage(), - e); + throw new RuntimeException("Unexpected exception in static initialization: " + e.getMessage(), e); } } @@ -104,11 +103,12 @@ public void shutdown() throws Exception { public void setUp() { try { qt.newSession(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - fail("Unexpected exception"); + fail("Unexpected exception in setUp"); } } @@ -117,6 +117,7 @@ public void setUp() { public void tearDown() { try { qt.clearPostTestEffects(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -125,13 +126,12 @@ public void tearDown() { } } - static String debugHint = + private static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; - @Override - public void runTest(String name, String fname, String fpath) throws Exception { + public void runTest(String name, String fname, String fpath) { long startTime = System.currentTimeMillis(); try { System.err.println("Begin query: " + fname); @@ -143,6 +143,7 @@ public void runTest(String name, String fname, String fpath) throws Exception { if (ecode != 0) { qt.failed(ecode, fname, debugHint); } + QTestProcessExecResult result = qt.checkCliDriverResults(fname); if (result.getReturnCode() != 0) { String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? @@ -158,5 +159,4 @@ public void runTest(String name, String fname, String fpath) throws Exception { assertTrue("Test passed", true); } - } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 6a6b1003b6..e8e51768e3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -48,7 +48,6 @@ public HBaseQTestUtil( .withLogDir(logDir) .withClusterType(miniMr) .withConfDir(null) - .withHadoopVer("0.20") .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) @@ -63,7 +62,6 @@ public void createSources(String tname) throws Exception { super.createSources(tname); conf.setBoolean("hive.test.init.phase", true); - initDataset(HBASE_SRC_NAME); // create a snapshot @@ -72,9 +70,10 @@ public void createSources(String tname) throws Exception { admin = conn.getAdmin(); admin.snapshot(HBASE_SRC_SNAPSHOT_NAME, TableName.valueOf(HBASE_SRC_NAME)); } finally { - if (admin != null) admin.close(); + if (admin != null) { + admin.close(); + } } - conf.setBoolean("hive.test.init.phase", false); } @@ -87,7 +86,9 @@ public void cleanUp(String tname) throws Exception { admin = conn.getAdmin(); admin.deleteSnapshots(HBASE_SRC_SNAPSHOT_NAME); } finally { - if (admin != null) admin.close(); + if (admin != null) { + admin.close(); + } } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index cc63a6cc83..ec38aea250 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -94,6 +94,7 @@ private void setUpFixtures(HiveConf conf) throws Exception { Configuration hbaseConf = HBaseConfiguration.create(conf); util = new HBaseTestingUtility(hbaseConf); + util.startMiniDFSCluster(1); hbaseCluster = util.startMiniHBaseCluster(1, NUM_REGIONSERVERS); hbaseConn = util.getConnection(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java index 18269ebc92..33412b2766 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java @@ -21,32 +21,34 @@ /** * QTestArguments composite used as arguments holder for QTestUtil initialization. */ -public class QTestArguments { +public final class QTestArguments { private String outDir; private String logDir; private String confDir; private QTestUtil.MiniClusterType clusterType; - private String hadoopVer; private String initScript; private String cleanupScript; private boolean withLlapIo; private QTestUtil.FsType fsType; private QTestUtil.QTestSetup qtestSetup; - public void setClusterType(QTestUtil.MiniClusterType clusterType) { - this.clusterType = clusterType; + private QTestArguments() { } public QTestUtil.MiniClusterType getClusterType() { return clusterType; } + private void setClusterType(QTestUtil.MiniClusterType clusterType) { + this.clusterType = clusterType; + } + public String getOutDir() { return outDir; } - public void setOutDir(String outDir) { + private void setOutDir(String outDir) { this.outDir = outDir; } @@ -54,27 +56,19 @@ public String getLogDir() { return logDir; } - public void setLogDir(String logDir) { + private void setLogDir(String logDir) { this.logDir = logDir; } - public void setConfDir(String confDir) { - this.confDir = confDir; - } - public String getConfDir() { return confDir; } - public void setHadoopVer(String hadoopVer) { - this.hadoopVer = hadoopVer; - } - - public String getHadoopVer() { - return hadoopVer; + private void setConfDir(String confDir) { + this.confDir = confDir; } - public void setInitScript(String initScript) { + private void setInitScript(String initScript) { this.initScript = initScript; } @@ -82,7 +76,7 @@ public String getInitScript() { return initScript; } - public void setCleanupScript(String cleanupScript) { + private void setCleanupScript(String cleanupScript) { this.cleanupScript = cleanupScript; } @@ -90,39 +84,39 @@ public String getCleanupScript() { return cleanupScript; } - public void setWithLlapIo(boolean withLlapIo) { - this.withLlapIo = withLlapIo; - } - public boolean isWithLlapIo() { return withLlapIo; } - public void setFsType(QTestUtil.FsType fsType) { - this.fsType = fsType; + private void setWithLlapIo(boolean withLlapIo) { + this.withLlapIo = withLlapIo; } public QTestUtil.FsType getFsType() { return fsType; } - public void setQTestSetup(QTestUtil.QTestSetup qtestSetup) { - this.qtestSetup = qtestSetup; + private void setFsType(QTestUtil.FsType fsType) { + this.fsType = fsType; } public QTestUtil.QTestSetup getQTestSetup() { return qtestSetup; } + private void setQTestSetup(QTestUtil.QTestSetup qtestSetup) { + this.qtestSetup = qtestSetup; + } + /** * QTestArgumentsBuilder used for QTestArguments construction. */ public static final class QTestArgumentsBuilder { + private String outDir; private String logDir; private String confDir; private QTestUtil.MiniClusterType clusterType; - private String hadoopVer; private String initScript; private String cleanupScript; private boolean withLlapIo; @@ -156,12 +150,6 @@ public QTestArgumentsBuilder withClusterType(QTestUtil.MiniClusterType clusterTy return this; } - - public QTestArgumentsBuilder withHadoopVer(String hadoopVer) { - this.hadoopVer = hadoopVer; - return this; - } - public QTestArgumentsBuilder withInitScript(String initScript) { this.initScript = initScript; return this; @@ -193,7 +181,6 @@ public QTestArguments build() { testArguments.setLogDir(logDir); testArguments.setConfDir(confDir); testArguments.setClusterType(clusterType); - testArguments.setHadoopVer(hadoopVer); testArguments.setInitScript(initScript); testArguments.setCleanupScript(cleanupScript); testArguments.setWithLlapIo(withLlapIo); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0e8b82930e..5924d06371 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -33,7 +33,6 @@ import java.io.OutputStream; import java.io.PrintStream; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -84,7 +83,6 @@ import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.llap.io.api.LlapProxy; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.dataset.Dataset; @@ -135,22 +133,20 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import junit.framework.TestSuite; - /** * QTestUtil. * */ public class QTestUtil { - public static final String UTF_8 = "UTF-8"; + private static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); + private static final String UTF_8 = "UTF-8"; // security property names private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private static final String CRLF = System.getProperty("line.separator"); public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; - static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); private final static String defaultInitScript = "q_test_init.sql"; private final static String defaultCleanupScript = "q_test_cleanup.sql"; private static SimpleDateFormat formatter = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); @@ -158,9 +154,7 @@ public static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target - - public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; - public static final String TEST_HIVE_USER_PROPERTY = "test.hive.user"; + private static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; /** * The default Erasure Coding Policy to use in Erasure Coding tests. @@ -171,8 +165,7 @@ @Deprecated private final String testFiles; private final File datasetDir; - protected final String outDir; - protected String overrideResultsDir; + private final String outDir; protected final String logDir; private final TreeMap qMap; private final Set qSortSet; @@ -190,23 +183,21 @@ private final FsType fsType; private ParseDriver pd; protected Hive db; - protected QueryState queryState; + private QueryState queryState; protected HiveConf conf; protected HiveConf savedConf; private IDriver drv; private BaseSemanticAnalyzer sem; - protected final boolean overWrite; + private final boolean overWrite; private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private FileSystem fs; private HadoopShims.HdfsEncryptionShim hes = null; private MiniLlapCluster llapCluster = null; - private String hadoopVer = null; - private QTestSetup setup = null; + private final QTestSetup setup; private SparkSession sparkSession = null; private boolean isSessionStateStarted = false; - private static final String javaVersion = getJavaVersion(); private QOutProcessor qOutProcessor; private final String initScript; private final String cleanupScript; @@ -214,10 +205,6 @@ private MiniDruidCluster druidCluster; private SingleNodeKafkaCluster kafkaCluster; - public interface SuiteAddTestFunctor { - public void addTestToSuite(TestSuite suite, Object setup, String tName); - } - public static Set getSrcTables() { if (srcTables == null){ initSrcTables(); @@ -262,7 +249,6 @@ private CliDriver getCliDriver() { return cliDriver; } - /** * Returns the default UDF names which should not be removed when resetting the test database * @return The list of the UDF names not to remove @@ -289,18 +275,6 @@ public HiveConf getConf() { return conf; } - private String getHadoopMainVersion(String input) { - if (input == null) { - return null; - } - Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); - Matcher m = p.matcher(input); - if (m.matches()) { - return m.group(1); - } - return null; - } - public void initConf() throws Exception { String vectorizationEnabled = System.getProperty("test.vectorization.enabled"); @@ -482,7 +456,6 @@ public static MiniClusterType valueForString(String type) { } } - private String getKeyProviderURI() { // Use the target directory if it is not specified String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT; @@ -494,12 +467,12 @@ private String getKeyProviderURI() { public QTestUtil(QTestArguments testArgs) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + - " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," + - " fsType={}", - testArgs.getOutDir(), testArgs.getLogDir(), testArgs.getClusterType(), testArgs.getConfDir(), hadoopVer, - testArgs.getInitScript(), testArgs.getCleanupScript(), testArgs.isWithLlapIo(), testArgs.getFsType()); + " initScript={}, cleanupScript={}, withLlapIo={}, fsType={}", + testArgs.getOutDir(), testArgs.getLogDir(), testArgs.getClusterType(), testArgs.getConfDir(), + testArgs.getInitScript(), testArgs.getCleanupScript(), testArgs.isWithLlapIo(), testArgs.getFsType()); Preconditions.checkNotNull(testArgs.getClusterType(), "ClusterType cannot be null"); + this.fsType = testArgs.getFsType(); this.outDir = testArgs.getOutDir(); this.logDir = testArgs.getLogDir(); @@ -516,7 +489,6 @@ public QTestUtil(QTestArguments testArgs) throws Exception { queryState = new QueryState.Builder().withHiveConf(new HiveConf(IDriver.class)).build(); conf = queryState.getConf(); - this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap(); qSortSet = new HashSet(); qSortQuerySet = new HashSet(); @@ -529,7 +501,6 @@ public QTestUtil(QTestArguments testArgs) throws Exception { this.clusterType = testArgs.getClusterType(); HadoopShims shims = ShimLoader.getHadoopShims(); - setupFileSystem(shims); setup = testArgs.getQTestSetup(); @@ -568,6 +539,7 @@ public QTestUtil(QTestArguments testArgs) throws Exception { init(); savedConf = new HiveConf(conf); } + private String getScriptsDir() { // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); @@ -682,7 +654,6 @@ private void setupMiniCluster(HadoopShims shims, String confDir) throws MiniClusterType.druidKafka ).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - } else { } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local, MiniClusterType.druidLocal) .contains(clusterType)) { @@ -853,27 +824,6 @@ private boolean matches(Pattern pattern, String query) { return false; } - /** - * Get formatted Java version to include minor version, but - * exclude patch level. - * - * @return Java version formatted as major_version.minor_version - */ - private static String getJavaVersion() { - String version = System.getProperty("java.version"); - if (version == null) { - throw new NullPointerException("No java version could be determined " + - "from system properties"); - } - - // "java version" system property is formatted - // major_version.minor_version.patch_level. - // Find second dot, instead of last dot, to be safe - int pos = version.indexOf('.'); - pos = version.indexOf('.', pos + 1); - return version.substring(0, pos); - } - /** * Clear out any side effects of running tests */ @@ -1001,13 +951,10 @@ public void newSession(boolean canReuseSession) throws Exception { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); CliSessionState ss = new CliSessionState(conf); - assert ss != null; ss.in = System.in; SessionState oldSs = SessionState.get(); - restartSessions(canReuseSession, ss, oldSs); - closeSession(oldSs); SessionState.start(ss); @@ -1097,28 +1044,6 @@ private void cleanupFromFile() throws IOException { } } - protected void runCreateTableCmd(String createTableCmd) throws Exception { - int ecode = 0; - ecode = drv.run(createTableCmd).getResponseCode(); - if (ecode != 0) { - throw new Exception("create table command: " + createTableCmd - + " failed with exit code= " + ecode); - } - - return; - } - - protected void runCmd(String cmd) throws Exception { - int ecode = 0; - ecode = drv.run(cmd).getResponseCode(); - drv.close(); - if (ecode != 0) { - throw new Exception("command: " + cmd - + " failed with exit code= " + ecode); - } - return; - } - public void createSources() throws Exception { createSources(null); } @@ -1234,7 +1159,6 @@ public String cliInit(File file) throws Exception { CliSessionState ss = (CliSessionState) SessionState.get(); - String outFileExtension = getOutFileExtension(fileName); String stdoutName = null; @@ -1257,7 +1181,7 @@ public String cliInit(File file) throws Exception { } private void setSessionOutputs(String fileName, CliSessionState ss, File outf) - throws FileNotFoundException, Exception, UnsupportedEncodingException { + throws Exception { OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf)); if (ss.out != null) { ss.out.flush(); @@ -1304,8 +1228,8 @@ private CliSessionState startSessionState(boolean canReuseSession) throws IOExce String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); + CliSessionState ss = new CliSessionState(conf); - assert ss != null; ss.in = System.in; ss.out = System.out; ss.err = System.out; @@ -1343,21 +1267,6 @@ public int executeAdhocCommand(String q) { return cliDriver.processLine(q1); } - public int executeOne(String tname) { - String q = qMap.get(tname); - - if (q.indexOf(";") == -1) { - return -1; - } - - String q1 = q.substring(0, q.indexOf(";") + 1); - String qrest = q.substring(q.indexOf(";") + 1); - qMap.put(tname, qrest); - - System.out.println("Executing " + q1); - return cliDriver.processLine(q1); - } - public int execute(String tname) { return drv.run(qMap.get(tname)).getResponseCode(); } @@ -1514,20 +1423,6 @@ private String getOutFileExtension(String fname) { return ".out"; } - public void convertSequenceFileToTextFile() throws Exception { - // Create an instance of hive in order to create the tables - testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); - db = Hive.get(conf); - - // Move all data from dest4_sequencefile to dest4 - drv - .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); - - // Drop dest4_sequencefile - db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, "dest4_sequencefile", - true, true); - } - public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); @@ -1588,35 +1483,6 @@ public QTestProcessExecResult checkNegativeResults(String tname, Error e) throws return result; } - public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { - - if (tree != null) { - String outFileExtension = getOutFileExtension(tname); - - File parseDir = new File(outDir, "parse"); - String expf = outPath(parseDir.toString(), tname.concat(outFileExtension)); - - File outf = null; - outf = new File(logDir); - outf = new File(outf, tname.concat(outFileExtension)); - - FileWriter outfd = new FileWriter(outf); - outfd.write(tree.toStringTree()); - outfd.close(); - - QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); - - if (overWrite) { - overwriteResults(outf.getPath(), expf); - return QTestProcessExecResult.createWithoutOutput(0); - } - - return exitVal; - } else { - throw new Exception("Parse tree is null"); - } - } - /** * Given the current configurations (e.g., hadoop version and execution mode), return * the correct file name to compare with the current test run output. @@ -1629,7 +1495,6 @@ public String outPath(String outDir, String testName) { // List of configurations. Currently the list consists of hadoop version and execution mode only List configs = new ArrayList(); configs.add(this.clusterType.toString()); - configs.add(this.hadoopVer); Deque stack = new LinkedList(); StringBuilder sb = new StringBuilder(); @@ -1675,7 +1540,6 @@ public QTestProcessExecResult checkCliDriverResults(String tname) throws Excepti return exitVal; } - public QTestProcessExecResult checkCompareCliDriverResults(String tname, List outputs) throws Exception { assert outputs.size() > 1; @@ -1831,13 +1695,6 @@ public ASTNode parseQuery(String tname) throws Exception { return pd.parse(qMap.get(tname)); } - public void resetParser() throws SemanticException { - pd = new ParseDriver(); - queryState = new QueryState.Builder().withHiveConf(conf).build(); - sem = new SemanticAnalyzer(queryState); - } - - public List> analyzeAST(ASTNode ast) throws Exception { // Do semantic analysis and plan generation @@ -1852,10 +1709,6 @@ public void resetParser() throws SemanticException { return sem.getRootTasks(); } - public TreeMap getQMap() { - return qMap; - } - /** * QTestSetup defines test fixtures which are reused across testcases, * and are needed before any test can be run @@ -1972,7 +1825,6 @@ public void run() { .withLogDir(logDir) .withClusterType(MiniClusterType.none) .withConfDir(null) - .withHadoopVer("0.20") .withInitScript(initScript == null ? defaultInitScript : initScript) .withCleanupScript(cleanupScript == null ? defaultCleanupScript : cleanupScript) .withLlapIo(false) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 77de3faa1f..a7972ecea3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -16,12 +16,11 @@ * limitations under the License. */ package org.apache.hadoop.hive.ql.parse; + import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; -import java.io.Serializable; -import java.util.List; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; @@ -30,7 +29,6 @@ import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; -import org.apache.hadoop.hive.ql.exec.Task; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -41,8 +39,9 @@ private static QTestUtil qt; - static CliConfigs.ParseNegativeConfig cliConfig = new CliConfigs.ParseNegativeConfig(); - static boolean firstRun; + private static CliConfigs.ParseNegativeConfig cliConfig = new CliConfigs.ParseNegativeConfig(); + private static boolean firstRun; + public CoreParseNegative(AbstractCliConfig testCliConfig) { super(testCliConfig); } @@ -54,26 +53,26 @@ public void beforeClass() { String initScript = cliConfig.getInitScript(); String cleanupScript = cliConfig.getCleanupScript(); firstRun = true; + try { - String hadoopVer = cliConfig.getHadoopVersion(); qt = new QTestUtil( QTestArguments.QTestArgumentsBuilder.instance() .withOutDir(cliConfig.getResultsDir()) .withLogDir(cliConfig.getLogDir()) .withClusterType(miniMR) .withConfDir(null) - .withHadoopVer(hadoopVer) .withInitScript(initScript) .withCleanupScript(cleanupScript) .withLlapIo(false) .build()); qt.newSession(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in static initialization",e); + throw new RuntimeException("Unexpected exception in static initialization", e); } } @@ -88,24 +87,24 @@ public void tearDown() { @Override @AfterClass - public void shutdown() throws Exception { + public void shutdown() { String reason = "clear post test effects"; try { qt.clearPostTestEffects(); reason = "shutdown"; qt.shutdown(); + } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); - throw new RuntimeException("Unexpected exception in " + reason,e); + throw new RuntimeException("Unexpected exception in " + reason, e); } } - static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + private static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; - @Override public void runTest(String tname, String fname, String fpath) throws Exception { long startTime = System.currentTimeMillis(); @@ -121,7 +120,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { qt.cliInit(new File(fpath)); ASTNode tree = qt.parseQuery(fname); - List> tasks = qt.analyzeAST(tree); + qt.analyzeAST(tree); fail("Unexpected success for query: " + fname + debugHint); } catch (ParseException pe) {