diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java index a33d222..b2c61f0 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java @@ -97,6 +97,7 @@ public void run() { String profile = startRequest.getProfile(); File profileConfFile = new File(mExecutionContextConfiguration.getProfileDirectory(), String.format("%s.properties", profile)); + LOG.info("Attempting to run using profile file: {}", profileConfFile); if(!profileConfFile.isFile()) { test.setStatus(Status.illegalArgument( "Profile " + profile + " not found in directory " + diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java index f41253b..3a4fa7f 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java @@ -66,6 +66,7 @@ private final File mSuccessfulTestLogDir; private final File mFailedTestLogDir; private final long mNumPollSeconds; + private final boolean fetchLogsForSuccessfulTests; private volatile boolean mShutdown; private int numParallelBatchesProcessed = 0; private int numIsolatedBatchesProcessed = 0; @@ -75,7 +76,7 @@ RSyncCommandExecutor rsyncCommandExecutor, ImmutableMap templateDefaults, File scratchDir, File succeededLogDir, File failedLogDir, long numPollSeconds, - Logger logger) { + boolean fetchLogsForSuccessfulTests, Logger logger) { List drones = Lists.newArrayList(); String[] localDirs = host.getLocalDirectories(); for (int index = 0; index < host.getThreads(); index++) { @@ -93,6 +94,7 @@ mSuccessfulTestLogDir = succeededLogDir; mFailedTestLogDir = failedLogDir; mNumPollSeconds = numPollSeconds; + this.fetchLogsForSuccessfulTests = fetchLogsForSuccessfulTests; mLogger = logger; } @@ -277,7 +279,7 @@ private boolean executeTestBatch(Drone drone, TestBatch batch, Set fa batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName())); } copyFromDroneToLocal(drone, batchLogDir.getAbsolutePath(), - drone.getLocalLogDirectory() + "/"); + drone.getLocalLogDirectory() + "/", fetchLogsForSuccessfulTests || !result); File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName())); PrintWriter writer = new PrintWriter(logFile); writer.write(String.format("result = '%s'\n", sshResult.toString())); @@ -365,7 +367,7 @@ RSyncResult copyToDroneFromLocal(Drone drone, String localFile, String remoteFil } }); } - RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFile) + RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFile, boolean fetchAllLogs) throws SSHExecutionException, IOException { Map templateVariables = Maps.newHashMap(mTemplateDefaults); templateVariables.put("instanceName", drone.getInstanceName()); @@ -374,7 +376,7 @@ RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFil drone.getHost(), drone.getInstance(), Templates.getTemplateResult(localFile, templateVariables), Templates.getTemplateResult(remoteFile, templateVariables), - RSyncCommand.Type.TO_LOCAL).call(); + fetchAllLogs ? RSyncCommand.Type.TO_LOCAL : RSyncCommand.Type.TO_LOCAL_NON_RECURSIVE).call(); if(result.getException() != null || result.getExitCode() != Constants.EXIT_CODE_SUCCESS) { throw new SSHExecutionException(result); } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index d390134..1cdfdb3 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -144,7 +144,8 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu @Override public HostExecutor build(Host host) { return new HostExecutor(host, executionContext.getPrivateKey(), mExecutor, sshCommandExecutor, - rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10, logger); + rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10, + configuration.shouldFetchLogsForSuccessfulTests(), logger); } }; diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java index f14026c..e584f9c 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java @@ -61,6 +61,7 @@ // This ends up being set to "test" | mvn ${testCasePropertyName} for instance private static final String TEST_CASE_PROPERTY_NAME = "testCasePropertyName"; private static final String BUILD_TOOL = "buildTool"; + private static final String FETCH_LOGS_FOR_SUCCESSFUL_TESTS = "fetchLogsForSuccessfulTests"; // The following parameters are not supported yet. TODO Add support private static final String APPLY_PATCH_SCRIPT_PATH = "applyPatchScriptPath"; private static final String PREP_TEMPLATE_PATH = "prepTemplatePath"; @@ -92,6 +93,7 @@ private final String jiraPassword; private final String testCasePropertyName; private final String buildTool; + private final boolean fetchLogsForSuccessfulTests; private final String applyPathScriptPath; private final String prepTemplatePath; private final String batchExecTemplatePath; @@ -137,6 +139,7 @@ public TestConfiguration(Context context, Logger logger) logsURL = context.getString(LOGS_URL, "").trim(); testCasePropertyName = context.getString(TEST_CASE_PROPERTY_NAME, "testcase").trim(); sshOpts = context.getString(SSH_OPTS, "").trim(); + fetchLogsForSuccessfulTests = context.getBoolean(FETCH_LOGS_FOR_SUCCESSFUL_TESTS, true); applyPathScriptPath = context.getString(APPLY_PATCH_SCRIPT_PATH, null); prepTemplatePath = context.getString(PREP_TEMPLATE_PATH, null); @@ -226,6 +229,7 @@ public String getPatch() { public String getTestCasePropertyName() { return testCasePropertyName; } + public boolean shouldFetchLogsForSuccessfulTests() {return fetchLogsForSuccessfulTests;} // TODO Make sure this method is eventually used to find the prep / batch scripts. public String getApplyPathScriptPath() { diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommand.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommand.java index 24e9d57..fbb1e79 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommand.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommand.java @@ -57,6 +57,7 @@ public String toString() { public static enum Type { FROM_LOCAL(), - TO_LOCAL(); + TO_LOCAL(), + TO_LOCAL_NON_RECURSIVE(); } } \ No newline at end of file diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java index 86c8796..cd7bcf9 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java @@ -66,6 +66,11 @@ public void execute(RSyncCommand command) { String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 %s@%s:%s %s", command.getPrivateKey(), command.getUser(), command.getHost(), command.getRemoteFile(), command.getLocalFile())); + } else if (command.getType() == RSyncCommand.Type.TO_LOCAL_NON_RECURSIVE) { + cmd = mLocalCommandFactory.create(collector, + String.format("timeout 1h rsync --exclude \"*/\" -vaPe \"ssh -i %s\" --timeout 600 %s@%s:%s %s", + command.getPrivateKey(), command.getUser(), command.getHost(), + command.getRemoteFile(), command.getLocalFile())); } else if(command.getType() == RSyncCommand.Type.FROM_LOCAL) { cmd = mLocalCommandFactory.create(collector, String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 --delete --delete-during --force %s %s@%s:%s", diff --git testutils/ptest2/src/main/resources/batch-exec.vm testutils/ptest2/src/main/resources/batch-exec.vm index 2cc56ea..e71a8e2 100644 --- testutils/ptest2/src/main/resources/batch-exec.vm +++ testutils/ptest2/src/main/resources/batch-exec.vm @@ -92,18 +92,19 @@ echo $pid >> batch.pid wait $pid ret=$? date +"%Y-%m-%d %T.%3N" -find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ - xargs -I {} sh -c 'f=$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f' +mkdir ${logDir}/logs find ./ -type f -name 'TEST-*.xml' | \ - xargs -I {} sh -c 'f=TEST-${batchName}-$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f' +xargs -I {} sh -c 'f=TEST-${batchName}-$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f' +find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ + xargs -I {} sh -c 'f=$(basename {}); test -f ${logDir}/logs/$f && f=$f-$(uuidgen); mv {} ${logDir}/logs/$f' find ./ -path "*/spark/work" | \ - xargs -I {} sh -c 'mv {} ${logDir}/spark-log' + xargs -I {} sh -c 'mv {} ${logDir}/logs/spark-log' find ./ -type f -name 'syslog*' | \ - xargs -I {} sh -c 'mkdir -p ${logDir}/syslogs; mv {} ${logDir}/syslogs' + xargs -I {} sh -c 'mkdir -p ${logDir}/logs/syslogs; mv {} ${logDir}/logs/syslogs' date +"%Y-%m-%d %T.%3N" if [[ -f $logDir/.log ]] then - mv $logDir/.log $logDir/dot.log + mv $logDir/.log $logDir/logs/dot.log fi exit $ret diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java index 37bc24b..71e1a64 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java @@ -106,7 +106,8 @@ public void initialize(String name) throws Exception { protected void createHostExecutor() { hostExecutor = new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor, - rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, logger); + rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, true, + logger); hostExecutors = ImmutableList.of(hostExecutor); } diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java index 65cf6a0..8964453 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java @@ -138,7 +138,8 @@ public void teardown() { } private HostExecutor createHostExecutor() { return new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor, - rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, logger); + rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, true, + logger); } private String getExecutedCommands() { diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt index 5318a83..4882732 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt @@ -91,18 +91,19 @@ echo $pid >> batch.pid wait $pid ret=$? date +"%Y-%m-%d %T.%3N" -find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ - xargs -I {} sh -c 'f=$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +mkdir /some/log/dir/logs find ./ -type f -name 'TEST-*.xml' | \ - xargs -I {} sh -c 'f=TEST-batch-1-$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +xargs -I {} sh -c 'f=TEST-batch-1-$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ + xargs -I {} sh -c 'f=$(basename {}); test -f /some/log/dir/logs/$f && f=$f-$(uuidgen); mv {} /some/log/dir/logs/$f' find ./ -path "*/spark/work" | \ - xargs -I {} sh -c 'mv {} /some/log/dir/spark-log' + xargs -I {} sh -c 'mv {} /some/log/dir/logs/spark-log' find ./ -type f -name 'syslog*' | \ - xargs -I {} sh -c 'mkdir -p /some/log/dir/syslogs; mv {} /some/log/dir/syslogs' + xargs -I {} sh -c 'mkdir -p /some/log/dir/logs/syslogs; mv {} /some/log/dir/logs/syslogs' date +"%Y-%m-%d %T.%3N" if [[ -f /some/log/dir/.log ]] then - mv /some/log/dir/.log /some/log/dir/dot.log + mv /some/log/dir/.log /some/log/dir/logs/dot.log fi exit $ret diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt index e165240..8c891f7 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt @@ -91,18 +91,19 @@ echo $pid >> batch.pid wait $pid ret=$? date +"%Y-%m-%d %T.%3N" -find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ - xargs -I {} sh -c 'f=$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +mkdir /some/log/dir/logs find ./ -type f -name 'TEST-*.xml' | \ - xargs -I {} sh -c 'f=TEST-batch-1-$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +xargs -I {} sh -c 'f=TEST-batch-1-$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f' +find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \ + xargs -I {} sh -c 'f=$(basename {}); test -f /some/log/dir/logs/$f && f=$f-$(uuidgen); mv {} /some/log/dir/logs/$f' find ./ -path "*/spark/work" | \ - xargs -I {} sh -c 'mv {} /some/log/dir/spark-log' + xargs -I {} sh -c 'mv {} /some/log/dir/logs/spark-log' find ./ -type f -name 'syslog*' | \ - xargs -I {} sh -c 'mkdir -p /some/log/dir/syslogs; mv {} /some/log/dir/syslogs' + xargs -I {} sh -c 'mkdir -p /some/log/dir/logs/syslogs; mv {} /some/log/dir/logs/syslogs' date +"%Y-%m-%d %T.%3N" if [[ -f /some/log/dir/.log ]] then - mv /some/log/dir/.log /some/log/dir/dot.log + mv /some/log/dir/.log /some/log/dir/logs/dot.log fi exit $ret