diff --git testutils/ptest2/pom.xml testutils/ptest2/pom.xml index 4dcd4a2..375c2a1 100644 --- testutils/ptest2/pom.xml +++ testutils/ptest2/pom.xml @@ -27,6 +27,7 @@ limitations under the License. UTF-8 2.4.1 + 3.2.16.RELEASE @@ -137,12 +138,12 @@ limitations under the License. org.springframework spring-web - 3.2.1.RELEASE + ${spring.framework.version} org.springframework spring-webmvc - 3.2.1.RELEASE + ${spring.framework.version} xml-apis diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java index 14a639a..817e00a 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java @@ -90,13 +90,21 @@ private final String mLogsEndpoint; private final ObjectMapper mMapper; private final DefaultHttpClient mHttpClient; + private final String testOutputDir; - public PTestClient(String logsEndpoint, String apiEndPoint, String password) + public PTestClient(String logsEndpoint, String apiEndPoint, String password, String testOutputDir) throws MalformedURLException { - if (logsEndpoint.endsWith("/")) { - this.mLogsEndpoint = logsEndpoint; + this.testOutputDir = testOutputDir; + if (!Strings.isNullOrEmpty(testOutputDir)) { + Preconditions.checkArgument(Strings.isNullOrEmpty(logsEndpoint), + "logsEndPoint must be specified if " + OUTPUT_DIR + " is specified"); + if (logsEndpoint.endsWith("/")) { + this.mLogsEndpoint = logsEndpoint; + } else { + this.mLogsEndpoint = logsEndpoint + "/"; + } } else { - this.mLogsEndpoint = logsEndpoint + "/"; + this.mLogsEndpoint = null; } if(apiEndPoint.endsWith("/")) { this.mApiEndPoint = apiEndPoint + "api/v1"; @@ -111,7 +119,7 @@ public PTestClient(String logsEndpoint, String apiEndPoint, String password) new UsernamePasswordCredentials("hive", password)); } public boolean testStart(String profile, String testHandle, - String jira, String patch, String testOutputDir, boolean clearLibraryCache) + String jira, String patch, boolean clearLibraryCache) throws Exception { patch = Strings.nullToEmpty(patch).trim(); if(!patch.isEmpty()) { @@ -301,7 +309,7 @@ public static void main(String[] args) throws Exception { ENDPOINT }); PTestClient client = new PTestClient(commandLine.getOptionValue(LOGS_ENDPOINT), commandLine.getOptionValue(ENDPOINT), - commandLine.getOptionValue(PASSWORD)); + commandLine.getOptionValue(PASSWORD), commandLine.getOptionValue(OUTPUT_DIR)); String command = commandLine.getOptionValue(COMMAND); boolean result; if("testStart".equalsIgnoreCase(command)) { @@ -310,7 +318,7 @@ public static void main(String[] args) throws Exception { TEST_HANDLE }); result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE), - commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), commandLine.getOptionValue(OUTPUT_DIR), + commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), commandLine.hasOption(CLEAR_LIBRARY_CACHE)); } else if("testTailLog".equalsIgnoreCase(command)) { result = client.testTailLog(commandLine.getOptionValue(TEST_HANDLE)); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java index 279cd72..2f96ad0 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java @@ -57,6 +57,8 @@ import com.google.common.collect.Lists; + + /** * Server interface of the ptest environment. Each request * is converted from JSON and each response is returned in JSON. @@ -79,7 +81,9 @@ public ExecutionController() throws IOException { String executionContextConfigurationFile = System.getProperty(CONF_PROPERTY, "").trim(); Preconditions.checkArgument(!executionContextConfigurationFile.isEmpty(), CONF_PROPERTY + " is required"); + LOG.info("Reading configuration from file: " + executionContextConfigurationFile); mExecutionContextConfiguration = ExecutionContextConfiguration.fromFile(executionContextConfigurationFile); + LOG.info("ExecutionContext is [{}]", mExecutionContextConfiguration); mExecutionContextProvider = mExecutionContextConfiguration.getExecutionContextProvider(); mTests = Collections.synchronizedMap(new LinkedHashMap() { private static final long serialVersionUID = 1L; diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java index 2d0939d..a33d222 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java @@ -98,7 +98,9 @@ public void run() { File profileConfFile = new File(mExecutionContextConfiguration.getProfileDirectory(), String.format("%s.properties", profile)); if(!profileConfFile.isFile()) { - test.setStatus(Status.illegalArgument("Profile " + profile + " not found")); + test.setStatus(Status.illegalArgument( + "Profile " + profile + " not found in directory " + + mExecutionContextConfiguration.getProfileDirectory())); test.setExecutionFinishTime(System.currentTimeMillis()); } else { File logDir = Dirs.create(new File(mExecutionContextConfiguration. @@ -124,7 +126,7 @@ public void run() { test.setStatus(Status.failed("Tests failed with exit code " + result)); } logStream.flush(); - // if all drones where abandoned on a host, replace it + // if all drones where abandoned on a host, try replacing them. mExecutionContext.replaceBadHosts(); } } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java index d8c6d2e..ca2c8a3 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java @@ -225,6 +225,10 @@ private String getCaller() { return ""; } + private String getThreadName() { + return Thread.currentThread().getName(); + } + private String getCallerShortName(StackTraceElement frame) { String className = frame.getClassName(); String methodName = frame.getMethodName(); @@ -242,6 +246,10 @@ private synchronized void log(LEVEL level, String msg, Throwable t) { mLog.print(" "); mLog.print(String.format("%5s", level.name())); mLog.print(" "); + mLog.print("["); + mLog.print(getThreadName()); + mLog.print("]"); + mLog.print(" "); mLog.print(getCaller()); mLog.print(" "); mLog.print(msg); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java index 2c9100e..5f84f00 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java @@ -110,10 +110,10 @@ public Void call() throws Exception { executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults); } finally { stopwatch.stop(); - mLogger.info("Finishing submitTests on host: {}. ElapsedTime(seconds)={}," + + mLogger.info("Finishing submitTests on host: {}. ElapsedTime(ms)={}," + " NumParallelBatchesProcessed={}, NumIsolatedBatchesProcessed={}", new Object[]{getHost().toString(), - stopwatch.elapsed(TimeUnit.SECONDS), numParallelBatchesProcessed, + stopwatch.elapsed(TimeUnit.MILLISECONDS), numParallelBatchesProcessed, numIsolatedBatchesProcessed}); } return null; @@ -175,8 +175,8 @@ public Void call() throws Exception { } } finally { sw.stop(); - mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(seconds)={}", - new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); + mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}", + new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)}); } } } while(!mShutdown && !parallelWorkQueue.isEmpty()); @@ -214,8 +214,8 @@ public Void call() throws Exception { } } finally { sw.stop(); - mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(seconds)={}", - new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); + mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(ms)={}", + new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)}); } } } while(!mShutdown && !isolatedWorkQueue.isEmpty()); @@ -258,9 +258,9 @@ private boolean executeTestBatch(Drone drone, TestBatch batch, Set fa drone.getHost(), drone.getInstance(), command, true). call(); sw.stop(); - mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(seconds)={}", + mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(ms)={}", new Object[]{batch.getName(), - getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); + getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)}); File batchLogDir = null; if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { throw new AbortDroneException("Drone " + drone.toString() + " exited with " + diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java index 4d397ea..b57320d 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java @@ -64,9 +64,9 @@ private void awaitProcessCompletion() throws InterruptedException { exitCode = process.waitFor(); if (stopwatch.isRunning()) { stopwatch.stop(); - logger.info("Finished LocalCommandId={}. ElapsedTime(seconds)={}", commandId, + logger.info("Finished LocalCommandId={}. ElapsedTime(ms)={}", commandId, stopwatch.elapsed( - TimeUnit.SECONDS)); + TimeUnit.MILLISECONDS)); } } } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index de5c322..0ff090d 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -98,7 +99,8 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu mExecutionContext = executionContext; mSshCommandExecutor = sshCommandExecutor; mRsyncCommandExecutor = rsyncCommandExecutor; - mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); + mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool( + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HostExecutor %d").build())); final File failedLogDir = Dirs.create(new File(logDir, "failed")); final File succeededLogDir = Dirs.create(new File(logDir, "succeeded")); final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch")); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java index 945ad77..35ddd44 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java @@ -103,4 +103,16 @@ public static ExecutionContextConfiguration fromFile(String file) throws IOExcep in.close(); } } + + @Override + public String toString() { + return "ExecutionContextConfiguration{" + + "mExecutionContextProvider=" + mExecutionContextProvider + + ", mWorkingDirectory='" + mWorkingDirectory + '\'' + + ", mGlobalLogDirectory='" + mGlobalLogDirectory + '\'' + + ", mProfileDirectory='" + mProfileDirectory + '\'' + + ", mMaxLogDirectoriesPerProfile=" + mMaxLogDirectoriesPerProfile + + ", mMaxRsyncThreads=" + mMaxRsyncThreads + + '}'; + } } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java index ac4bafb..2c5bd3a 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java @@ -60,6 +60,11 @@ private static final String LOGS_URL = "logsURL"; private static final String TEST_CASE_PROPERTY_NAME = "testCasePropertyName"; private static final String BUILD_TOOL = "buildTool"; + // The following parameters are not supported yet. TODO Add support + private static final String APPLY_PATCH_SCRIPT_PATH = "applyPatchScriptPath"; + private static final String PREP_TEMPLATE_PATH = "prepTemplatePath"; + private static final String BATCH_EXEC_TEMPLATE_PATH = "batchExecTemplatePath"; + private final Context context; private String antArgs; @@ -86,6 +91,9 @@ private final String jiraPassword; private final String testCasePropertyName; private final String buildTool; + private final String applyPathScriptPath; + private final String prepTemplatePath; + private final String batchExecTemplatePath; private String jiraName; private boolean clearLibraryCache; @@ -129,6 +137,10 @@ public TestConfiguration(Context context, Logger logger) testCasePropertyName = context.getString(TEST_CASE_PROPERTY_NAME, "testcase").trim(); sshOpts = context.getString(SSH_OPTS, "").trim(); + applyPathScriptPath = context.getString(APPLY_PATCH_SCRIPT_PATH, null); + prepTemplatePath = context.getString(PREP_TEMPLATE_PATH, null); + batchExecTemplatePath = context.getString(BATCH_EXEC_TEMPLATE_PATH, null); + } public Context getContext() { return context; @@ -213,6 +225,21 @@ public String getPatch() { public String getTestCasePropertyName() { return testCasePropertyName; } + + public String getApplyPathScriptPath() { + return applyPathScriptPath; + } + + public String getPrepTemplatePath() { + return prepTemplatePath; + } + + public String getBatchExecTemplatePath() { + return batchExecTemplatePath; + } + + // TODO - Allow the branch to be specified as a parameter to ptest, rather than requiring a separate property file. + // (will need to handle an alternate work-dir as well in this case - derive from branch?) public void setPatch(String patch) { this.patch = Strings.nullToEmpty(patch); } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java index c51ef3c..e38cbb4 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java @@ -116,7 +116,15 @@ public TestParser(Context context, String testCasePropertyName, Map properties = parseQTestProperties(); List result = Lists.newArrayList(); - for(String alias : context.getString("qFileTests", "").split(" ")) { + String qFileTestsString = context.getString("qFileTests",null); + String []aliases; + if (qFileTestsString != null) { + aliases = qFileTestsString.split(" "); + } else { + aliases = new String[0]; + } + + for(String alias : aliases) { Context testContext = new Context(context.getSubProperties( Joiner.on(".").join("qFileTest", alias, ""))); String driver = checkNotNull(testContext.getString("driver"), "driver").trim(); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java index 075ae3e..f7b50d6 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java @@ -57,7 +57,11 @@ public void terminate(ExecutionContext executionContext) { } @Override public void replaceBadHosts(ExecutionContext executionContext) throws CreateHostsFailedException { - throw new UnsupportedOperationException(); + if (!executionContext.getBadHosts().isEmpty()) { + LOG.warn( + "Found bad nodes on FixedExecutionContext. Cannot replace them. Degraded performance. badNodes={}", + executionContext.getBadHosts()); + } } @Override diff --git testutils/ptest2/src/main/resources/log4j2.properties testutils/ptest2/src/main/resources/log4j2.properties index 85f1b64..1c16919 100644 --- testutils/ptest2/src/main/resources/log4j2.properties +++ testutils/ptest2/src/main/resources/log4j2.properties @@ -40,7 +40,7 @@ appender.FILE.name = FILE appender.FILE.fileName = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file} appender.FILE.filePattern = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}.%i appender.FILE.layout.type = PatternLayout -appender.FILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +appender.FILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n appender.FILE.policies.type = Policies appender.FILE.policies.size.type = SizeBasedTriggeringPolicy appender.FILE.policies.size.size = 50MB