diff --git testutils/ptest2/pom.xml testutils/ptest2/pom.xml
index 4dcd4a2..375c2a1 100644
--- testutils/ptest2/pom.xml
+++ testutils/ptest2/pom.xml
@@ -27,6 +27,7 @@ limitations under the License.
UTF-8
2.4.1
+ 3.2.16.RELEASE
@@ -137,12 +138,12 @@ limitations under the License.
org.springframework
spring-web
- 3.2.1.RELEASE
+ ${spring.framework.version}
org.springframework
spring-webmvc
- 3.2.1.RELEASE
+ ${spring.framework.version}
xml-apis
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
index 14a639a..8e2604d 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java
@@ -90,13 +90,21 @@
private final String mLogsEndpoint;
private final ObjectMapper mMapper;
private final DefaultHttpClient mHttpClient;
+ private final String testOutputDir;
- public PTestClient(String logsEndpoint, String apiEndPoint, String password)
+ public PTestClient(String logsEndpoint, String apiEndPoint, String password, String testOutputDir)
throws MalformedURLException {
- if (logsEndpoint.endsWith("/")) {
- this.mLogsEndpoint = logsEndpoint;
+ this.testOutputDir = testOutputDir;
+ if (!Strings.isNullOrEmpty(testOutputDir)) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(logsEndpoint),
+ "logsEndPoint must be specified if " + OUTPUT_DIR + " is specified");
+ if (logsEndpoint.endsWith("/")) {
+ this.mLogsEndpoint = logsEndpoint;
+ } else {
+ this.mLogsEndpoint = logsEndpoint + "/";
+ }
} else {
- this.mLogsEndpoint = logsEndpoint + "/";
+ this.mLogsEndpoint = null;
}
if(apiEndPoint.endsWith("/")) {
this.mApiEndPoint = apiEndPoint + "api/v1";
@@ -111,7 +119,7 @@ public PTestClient(String logsEndpoint, String apiEndPoint, String password)
new UsernamePasswordCredentials("hive", password));
}
public boolean testStart(String profile, String testHandle,
- String jira, String patch, String testOutputDir, boolean clearLibraryCache)
+ String jira, String patch, boolean clearLibraryCache)
throws Exception {
patch = Strings.nullToEmpty(patch).trim();
if(!patch.isEmpty()) {
@@ -301,7 +309,7 @@ public static void main(String[] args) throws Exception {
ENDPOINT
});
PTestClient client = new PTestClient(commandLine.getOptionValue(LOGS_ENDPOINT), commandLine.getOptionValue(ENDPOINT),
- commandLine.getOptionValue(PASSWORD));
+ commandLine.getOptionValue(PASSWORD), commandLine.getOptionValue(OUTPUT_DIR));
String command = commandLine.getOptionValue(COMMAND);
boolean result;
if("testStart".equalsIgnoreCase(command)) {
@@ -310,7 +318,7 @@ public static void main(String[] args) throws Exception {
TEST_HANDLE
});
result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE),
- commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), commandLine.getOptionValue(OUTPUT_DIR),
+ commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH),
commandLine.hasOption(CLEAR_LIBRARY_CACHE));
} else if("testTailLog".equalsIgnoreCase(command)) {
result = client.testTailLog(commandLine.getOptionValue(TEST_HANDLE));
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
index 279cd72..2f96ad0 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java
@@ -57,6 +57,8 @@
import com.google.common.collect.Lists;
+
+
/**
* Server interface of the ptest environment. Each request
* is converted from JSON and each response is returned in JSON.
@@ -79,7 +81,9 @@ public ExecutionController()
throws IOException {
String executionContextConfigurationFile = System.getProperty(CONF_PROPERTY, "").trim();
Preconditions.checkArgument(!executionContextConfigurationFile.isEmpty(), CONF_PROPERTY + " is required");
+ LOG.info("Reading configuration from file: " + executionContextConfigurationFile);
mExecutionContextConfiguration = ExecutionContextConfiguration.fromFile(executionContextConfigurationFile);
+ LOG.info("ExecutionContext is [{}]", mExecutionContextConfiguration);
mExecutionContextProvider = mExecutionContextConfiguration.getExecutionContextProvider();
mTests = Collections.synchronizedMap(new LinkedHashMap() {
private static final long serialVersionUID = 1L;
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
index 2d0939d..a33d222 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
@@ -98,7 +98,9 @@ public void run() {
File profileConfFile = new File(mExecutionContextConfiguration.getProfileDirectory(),
String.format("%s.properties", profile));
if(!profileConfFile.isFile()) {
- test.setStatus(Status.illegalArgument("Profile " + profile + " not found"));
+ test.setStatus(Status.illegalArgument(
+ "Profile " + profile + " not found in directory " +
+ mExecutionContextConfiguration.getProfileDirectory()));
test.setExecutionFinishTime(System.currentTimeMillis());
} else {
File logDir = Dirs.create(new File(mExecutionContextConfiguration.
@@ -124,7 +126,7 @@ public void run() {
test.setStatus(Status.failed("Tests failed with exit code " + result));
}
logStream.flush();
- // if all drones where abandoned on a host, replace it
+ // if all drones where abandoned on a host, try replacing them.
mExecutionContext.replaceBadHosts();
}
}
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
index d8c6d2e..ca2c8a3 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
@@ -225,6 +225,10 @@ private String getCaller() {
return "";
}
+ private String getThreadName() {
+ return Thread.currentThread().getName();
+ }
+
private String getCallerShortName(StackTraceElement frame) {
String className = frame.getClassName();
String methodName = frame.getMethodName();
@@ -242,6 +246,10 @@ private synchronized void log(LEVEL level, String msg, Throwable t) {
mLog.print(" ");
mLog.print(String.format("%5s", level.name()));
mLog.print(" ");
+ mLog.print("[");
+ mLog.print(getThreadName());
+ mLog.print("]");
+ mLog.print(" ");
mLog.print(getCaller());
mLog.print(" ");
mLog.print(msg);
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 2c9100e..5f84f00 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -110,10 +110,10 @@ public Void call() throws Exception {
executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults);
} finally {
stopwatch.stop();
- mLogger.info("Finishing submitTests on host: {}. ElapsedTime(seconds)={}," +
+ mLogger.info("Finishing submitTests on host: {}. ElapsedTime(ms)={}," +
" NumParallelBatchesProcessed={}, NumIsolatedBatchesProcessed={}",
new Object[]{getHost().toString(),
- stopwatch.elapsed(TimeUnit.SECONDS), numParallelBatchesProcessed,
+ stopwatch.elapsed(TimeUnit.MILLISECONDS), numParallelBatchesProcessed,
numIsolatedBatchesProcessed});
}
return null;
@@ -175,8 +175,8 @@ public Void call() throws Exception {
}
} finally {
sw.stop();
- mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(seconds)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
+ mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}",
+ new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
}
}
} while(!mShutdown && !parallelWorkQueue.isEmpty());
@@ -214,8 +214,8 @@ public Void call() throws Exception {
}
} finally {
sw.stop();
- mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(seconds)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
+ mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(ms)={}",
+ new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
}
}
} while(!mShutdown && !isolatedWorkQueue.isEmpty());
@@ -258,9 +258,9 @@ private boolean executeTestBatch(Drone drone, TestBatch batch, Set fa
drone.getHost(), drone.getInstance(), command, true).
call();
sw.stop();
- mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(seconds)={}",
+ mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(ms)={}",
new Object[]{batch.getName(),
- getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
+ getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
File batchLogDir = null;
if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
throw new AbortDroneException("Drone " + drone.toString() + " exited with " +
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
index 4d397ea..b57320d 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
@@ -64,9 +64,9 @@ private void awaitProcessCompletion() throws InterruptedException {
exitCode = process.waitFor();
if (stopwatch.isRunning()) {
stopwatch.stop();
- logger.info("Finished LocalCommandId={}. ElapsedTime(seconds)={}", commandId,
+ logger.info("Finished LocalCommandId={}. ElapsedTime(ms)={}", commandId,
stopwatch.elapsed(
- TimeUnit.SECONDS));
+ TimeUnit.MILLISECONDS));
}
}
}
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
index de5c322..0ff090d 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
@@ -35,6 +35,7 @@
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -98,7 +99,8 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu
mExecutionContext = executionContext;
mSshCommandExecutor = sshCommandExecutor;
mRsyncCommandExecutor = rsyncCommandExecutor;
- mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
+ mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HostExecutor %d").build()));
final File failedLogDir = Dirs.create(new File(logDir, "failed"));
final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
index 945ad77..35ddd44 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java
@@ -103,4 +103,16 @@ public static ExecutionContextConfiguration fromFile(String file) throws IOExcep
in.close();
}
}
+
+ @Override
+ public String toString() {
+ return "ExecutionContextConfiguration{" +
+ "mExecutionContextProvider=" + mExecutionContextProvider +
+ ", mWorkingDirectory='" + mWorkingDirectory + '\'' +
+ ", mGlobalLogDirectory='" + mGlobalLogDirectory + '\'' +
+ ", mProfileDirectory='" + mProfileDirectory + '\'' +
+ ", mMaxLogDirectoriesPerProfile=" + mMaxLogDirectoriesPerProfile +
+ ", mMaxRsyncThreads=" + mMaxRsyncThreads +
+ '}';
+ }
}
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
index ac4bafb..2c5bd3a 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
@@ -60,6 +60,11 @@
private static final String LOGS_URL = "logsURL";
private static final String TEST_CASE_PROPERTY_NAME = "testCasePropertyName";
private static final String BUILD_TOOL = "buildTool";
+ // The following parameters are not supported yet. TODO Add support
+ private static final String APPLY_PATCH_SCRIPT_PATH = "applyPatchScriptPath";
+ private static final String PREP_TEMPLATE_PATH = "prepTemplatePath";
+ private static final String BATCH_EXEC_TEMPLATE_PATH = "batchExecTemplatePath";
+
private final Context context;
private String antArgs;
@@ -86,6 +91,9 @@
private final String jiraPassword;
private final String testCasePropertyName;
private final String buildTool;
+ private final String applyPathScriptPath;
+ private final String prepTemplatePath;
+ private final String batchExecTemplatePath;
private String jiraName;
private boolean clearLibraryCache;
@@ -129,6 +137,10 @@ public TestConfiguration(Context context, Logger logger)
testCasePropertyName = context.getString(TEST_CASE_PROPERTY_NAME, "testcase").trim();
sshOpts = context.getString(SSH_OPTS, "").trim();
+ applyPathScriptPath = context.getString(APPLY_PATCH_SCRIPT_PATH, null);
+ prepTemplatePath = context.getString(PREP_TEMPLATE_PATH, null);
+ batchExecTemplatePath = context.getString(BATCH_EXEC_TEMPLATE_PATH, null);
+
}
public Context getContext() {
return context;
@@ -213,6 +225,21 @@ public String getPatch() {
public String getTestCasePropertyName() {
return testCasePropertyName;
}
+
+ public String getApplyPathScriptPath() {
+ return applyPathScriptPath;
+ }
+
+ public String getPrepTemplatePath() {
+ return prepTemplatePath;
+ }
+
+ public String getBatchExecTemplatePath() {
+ return batchExecTemplatePath;
+ }
+
+ // TODO - Allow the branch to be specified as a parameter to ptest, rather than requiring a separate property file.
+ // (will need to handle an alternate work-dir as well in this case - derive from branch?)
public void setPatch(String patch) {
this.patch = Strings.nullToEmpty(patch);
}
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
index c51ef3c..e38cbb4 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
@@ -116,7 +116,15 @@ public TestParser(Context context, String testCasePropertyName,
Map properties = parseQTestProperties();
List result = Lists.newArrayList();
- for(String alias : context.getString("qFileTests", "").split(" ")) {
+ String qFileTestsString = context.getString("qFileTests",null);
+ String []aliases;
+ if (qFileTestsString != null) {
+ aliases = qFileTestsString.split(" ");
+ } else {
+ aliases = new String[0];
+ }
+
+ for(String alias : aliases) {
Context testContext = new Context(context.getSubProperties(
Joiner.on(".").join("qFileTest", alias, "")));
String driver = checkNotNull(testContext.getString("driver"), "driver").trim();
diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java
index 075ae3e..f7b50d6 100644
--- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java
+++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java
@@ -57,7 +57,11 @@ public void terminate(ExecutionContext executionContext) {
}
@Override
public void replaceBadHosts(ExecutionContext executionContext) throws CreateHostsFailedException {
- throw new UnsupportedOperationException();
+ if (!executionContext.getBadHosts().isEmpty()) {
+ LOG.warn(
+ "Found bad nodes on FixedExecutionContext. Cannot replace them. Degraded performance. badNodes={}",
+ executionContext.getBadHosts());
+ }
}
@Override
diff --git testutils/ptest2/src/main/resources/log4j2.properties testutils/ptest2/src/main/resources/log4j2.properties
index 85f1b64..1c16919 100644
--- testutils/ptest2/src/main/resources/log4j2.properties
+++ testutils/ptest2/src/main/resources/log4j2.properties
@@ -40,7 +40,7 @@ appender.FILE.name = FILE
appender.FILE.fileName = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}
appender.FILE.filePattern = ${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}.%i
appender.FILE.layout.type = PatternLayout
-appender.FILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+appender.FILE.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n
appender.FILE.policies.type = Policies
appender.FILE.policies.size.type = SizeBasedTriggeringPolicy
appender.FILE.policies.size.size = 50MB