diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java index 6498682..3026ea0 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java @@ -107,6 +107,10 @@ public void execute() throws Throwable { JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir); executedTests.addAll(parser.getExecutedTests()); failedTests.addAll(parser.getFailedTests()); + // if the TEST*.xml was not generated or was corrupt, let someone know + if (parser.getNumAttemptedTests() == 0) { + failedTests.add(batch.getName() + " - did not produce a TEST-*.xml file"); + } } } finally { long elapsed = System.currentTimeMillis() - start; diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java index 09147f4..b05d2c2 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java @@ -29,7 +29,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; -import org.apache.commons.io.FileUtils; import org.apache.hive.ptest.execution.conf.Host; import org.apache.hive.ptest.execution.conf.TestBatch; import org.apache.hive.ptest.execution.ssh.RSyncCommand; @@ -218,7 +217,7 @@ private boolean executeTestBatch(Drone drone, TestBatch batch, Set fa script.delete(); mLogger.info(drone + " executing " + batch + " with " + command); RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(), - drone.getHost(), drone.getInstance(), command). + drone.getHost(), drone.getInstance(), command, true). call(); File batchLogDir = null; if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { @@ -346,8 +345,22 @@ RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFil * exits with a status code of 255 until all drones have been utilized, possibly * excluding the host from future use. */ + ListenableFuture execIgnoreAllErrors(final String cmd) + throws Exception { + return exec(cmd, false); + } + /** + * Execute command on at least one drone. The method will retry when the command + * exits with a status code of 255 until all drones have been utilized, possibly + * excluding the host from future use. + */ ListenableFuture exec(final String cmd) throws Exception { + return exec(cmd, true); + } + + private ListenableFuture exec(final String cmd, final boolean reportErrors) + throws Exception { return mExecutor.submit(new Callable() { @Override public SSHResult call() throws Exception { @@ -357,8 +370,8 @@ public SSHResult call() throws Exception { templateVariables.put("localDir", drone.getLocalDirectory()); String command = Templates.getTemplateResult(cmd, templateVariables); SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(), - drone.getHost(), drone.getInstance(), command).call(); - if(result.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { + drone.getHost(), drone.getInstance(), command, reportErrors).call(); + if(reportErrors && result.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { mDrones.remove(drone); // return value not checked due to concurrent access mLogger.error("Aborting drone during exec " + command, new AbortDroneException("Drone " + drone + " exited with " @@ -388,7 +401,7 @@ public RemoteCommandResult call() throws Exception { templateVariables.put("localDir", drone.getLocalDirectory()); String command = Templates.getTemplateResult(cmd, templateVariables); SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(), - drone.getHost(), drone.getInstance(), command).call(); + drone.getHost(), drone.getInstance(), command, true).call(); if(result.getExitCode() != Constants.EXIT_CODE_SUCCESS) { mDrones.remove(drone); // return value not checked due to concurrent access mLogger.error("Aborting drone during exec " + command, diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java index 0cc816c..cc9f617 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java @@ -76,6 +76,9 @@ public JUnitReportParser(Logger logger, File directory) throws Exception { } return failedTests; } + public int getNumAttemptedTests() { + return getExecutedTests().size() + getFailedTests().size(); + } private void parse() { for(File file : getFiles(directory)) { FileInputStream stream = null; diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java index fa2d8e7..aee09b3 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java @@ -88,6 +88,14 @@ protected void execLocally(String command) } return toListOfResults(futures); } + protected List execHostsIgnoreErrors(String command) + throws Exception { + List> futures = Lists.newArrayList(); + for(HostExecutor hostExecutor : hostExecutors) { + futures.add(hostExecutor.execIgnoreAllErrors(command)); + } + return toListOfResults(futures); + } // clean prep protected List execInstances(String command) throws Exception { diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java index 02d7b6c..825f0c0 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java @@ -65,6 +65,8 @@ public void execute() throws Exception { Templates.writeTemplateResult("source-prep.vm", sourcePrepScript, getTemplateDefaults()); execLocally("bash " + sourcePrepScript.getPath()); logger.debug("Deleting " + sourcePrepScript + ": " + sourcePrepScript.delete()); + execHostsIgnoreErrors("pkill -f java"); + execHostsIgnoreErrors("pkill -9 -f java"); elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start), TimeUnit.MILLISECONDS); logger.info("PERF: source prep took " + elapsedTime + " minutes"); diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java index 9ef9cd6..e7a5752 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java @@ -286,7 +286,7 @@ private static String publicIp(NodeMetadata node) { @Override public void run() { String ip = publicIpOrHostname(node); - SSHCommand command = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mUser, ip, 0, "pkill -f java"); + SSHCommand command = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mUser, ip, 0, "pkill -f java", true); mSSHCommandExecutor.execute(command); if(command.getExitCode() == Constants.EXIT_CODE_UNKNOWN || command.getException() != null) { diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommand.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommand.java index d027e44..47c2d91 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommand.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommand.java @@ -22,12 +22,14 @@ private final SSHCommandExecutor executor; private final String command; + private final boolean reportErrors; public SSHCommand(SSHCommandExecutor executor, String privateKey, - String user, String host, int instance, String command) { + String user, String host, int instance, String command, boolean reportErrors) { super(privateKey, user, host, instance); this.executor = executor; this.command = command; + this.reportErrors = reportErrors; } @Override @@ -37,6 +39,10 @@ public SSHResult call() { getExitCode(), getException(), getOutput()); } + public boolean isReportErrors() { + return reportErrors; + } + public String getCommand() { return command; } diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java index 8c9f707..e335b55 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java @@ -63,7 +63,7 @@ public void execute(SSHCommand command) { command.setExitCode(Constants.EXIT_CODE_UNKNOWN); return; } - if(attempts++ <= 3 && cmd.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { + if(command.isReportErrors() && attempts++ <= 3 && cmd.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { mLogger.warn("Command exited with " + cmd.getExitCode() + ", will retry: " + command); retry = true; TimeUnit.SECONDS.sleep(5); diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPrepPhase.testExecute.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPrepPhase.testExecute.approved.txt index 7d35d84..4dc7fca 100644 --- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPrepPhase.testExecute.approved.txt +++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPrepPhase.testExecute.approved.txt @@ -1,3 +1,5 @@ bash /tmp/hive-ptest-units/TestPrepPhase/source-prep.sh mkdir -p /some/working/dir/scratch +pkill -9 -f java +pkill -f java rm -rf /some/working/dir/scratch \ No newline at end of file diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/ssh/TestSSHCommandExecutor.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/ssh/TestSSHCommandExecutor.java index f81535f..9499797 100644 --- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/ssh/TestSSHCommandExecutor.java +++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/ssh/TestSSHCommandExecutor.java @@ -54,7 +54,7 @@ public void testShutdownBeforeWaitFor() throws Exception { "-o StrictHostKeyChecking=no"); Assert.assertFalse(executor.isShutdown()); executor.shutdownNow(); - SSHCommand command = new SSHCommand(executor, "privateKey", "user", "host", 1, "whoami"); + SSHCommand command = new SSHCommand(executor, "privateKey", "user", "host", 1, "whoami", true); executor.execute(command); Assert.assertTrue(executor.isShutdown()); Assert.assertEquals(Constants.EXIT_CODE_UNKNOWN, command.getExitCode()); @@ -77,7 +77,7 @@ public Integer answer(InvocationOnMock invocation) throws Throwable { return Constants.EXIT_CODE_UNKNOWN; } }); - SSHCommand command = new SSHCommand(executor, "privateKey", "user", "host", 1, "whoami"); + SSHCommand command = new SSHCommand(executor, "privateKey", "user", "host", 1, "whoami", true); executor.execute(command); Assert.assertTrue(executor.isShutdown()); Assert.assertEquals(Constants.EXIT_CODE_UNKNOWN, command.getExitCode());