diff --git testutils/ptest2/conf/deployed/master-mr2.properties testutils/ptest2/conf/deployed/master-mr2.properties index 05c040599172caf9d67c4e27affce12a335dab4b..9e8f24382ea4d7cfac602ca9e68e8e81cd125e23 100644 --- testutils/ptest2/conf/deployed/master-mr2.properties +++ testutils/ptest2/conf/deployed/master-mr2.properties @@ -15,8 +15,6 @@ mavenEnvOpts = -Dhttp.proxyHost=localhost -Dhttp.proxyPort=3128 mavenTestArgs = testCasePropertyName = test buildTool = maven -javaHome = /usr/java/jdk1.8.0_25 -javaHomeForTests = /usr/java/jdk1.8.0_25 unitTests.directories = ./ additionalProfiles = diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java index 8e2604d372ac29b94445b269f08423b058308efe..81b6b2aa77af4ac096f0db162f19f2baa10770c8 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java @@ -42,6 +42,7 @@ import org.apache.hive.ptest.api.response.TestStartResponse; import org.apache.hive.ptest.api.response.TestStatus; import org.apache.hive.ptest.api.response.TestStatusResponse; +import org.apache.hive.ptest.execution.conf.TestConfiguration; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; import org.apache.http.auth.AuthScope; @@ -85,6 +86,8 @@ private static final String OUTPUT_DIR = "outputDir"; private static final String TEST_HANDLE = "testHandle"; private static final String CLEAR_LIBRARY_CACHE = "clearLibraryCache"; + private static final String JAVA_VERSION = TestConfiguration.JAVA_VERSION; + private static final String JAVA_VERSION_FOR_TEST = TestConfiguration.JAVA_VERSION_FOR_TEST; private static final int MAX_RETRIES = 10; private final String mApiEndPoint; private final String mLogsEndpoint; @@ -119,7 +122,7 @@ public PTestClient(String logsEndpoint, String apiEndPoint, String password, Str new UsernamePasswordCredentials("hive", password)); } public boolean testStart(String profile, String testHandle, - String jira, String patch, boolean clearLibraryCache) + String jira, String patch, boolean clearLibraryCache, String javaVersion, String javaTestVersion) throws Exception { patch = Strings.nullToEmpty(patch).trim(); if(!patch.isEmpty()) { @@ -128,7 +131,8 @@ public boolean testStart(String profile, String testHandle, throw new IllegalArgumentException("Patch " + patch + " was zero bytes"); } } - TestStartRequest startRequest = new TestStartRequest(profile, testHandle, jira, patch, clearLibraryCache); + TestStartRequest startRequest = new TestStartRequest(profile, testHandle, jira, patch, clearLibraryCache, + javaVersion, javaTestVersion); post(startRequest, false); boolean result = false; try { @@ -295,6 +299,10 @@ public static void main(String[] args) throws Exception { options.addOption(null, TEST_HANDLE, true, "Server supplied test handle. (Required for testStop and testTailLog)"); options.addOption(null, OUTPUT_DIR, true, "Directory to download and save test-results.tar.gz to. (Optional for testStart)"); options.addOption(null, CLEAR_LIBRARY_CACHE, false, "Before starting the test, delete the ivy and maven directories (Optional for testStart)"); + options.addOption(null, JAVA_VERSION, true, "Java version number (eg. 7,8) to use when building Hive. " + + "We expect a matching environment property (JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set. (Optional for testStart)"); + options.addOption(null, JAVA_VERSION_FOR_TEST, true, "Java version number (eg. 7,8) to use when running tests. " + + "We expect a matching environment property (JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set. (Optional for testStart)"); options.addOption(null, LOGS_ENDPOINT, true, "URL to get the logs"); CommandLine commandLine = parser.parse(options, args); @@ -319,7 +327,8 @@ public static void main(String[] args) throws Exception { }); result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE), commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), - commandLine.hasOption(CLEAR_LIBRARY_CACHE)); + commandLine.hasOption(CLEAR_LIBRARY_CACHE), commandLine.getOptionValue(JAVA_VERSION), + commandLine.getOptionValue(JAVA_VERSION_FOR_TEST)); } else if("testTailLog".equalsIgnoreCase(command)) { result = client.testTailLog(commandLine.getOptionValue(TEST_HANDLE)); } else if("testList".equalsIgnoreCase(command)) { diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java index 8deed52ae0307d4fc075654a4d75e6cb09a5d9db..0730f5e69633fa710feb8f7ca0a53b3e83bdf5e1 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/request/TestStartRequest.java @@ -24,17 +24,21 @@ private String patchURL; private String jiraName; private boolean clearLibraryCache; + private String javaVersion; + private String javaTestVersion; public TestStartRequest() { } - public TestStartRequest(String profile, String testHandle, - String jiraName, String patchURL, boolean clearLibraryCache) { + public TestStartRequest(String profile, String testHandle, String jiraName, String patchURL, + boolean clearLibraryCache, String javaVersion, String javaTestVersion) { this.profile = profile; this.testHandle = testHandle; this.jiraName = jiraName; this.patchURL = patchURL; this.clearLibraryCache = clearLibraryCache; + this.javaVersion = javaVersion; + this.javaTestVersion = javaTestVersion; } public String getProfile() { return profile; @@ -67,6 +71,23 @@ public String getTestHandle() { public void setTestHandle(String testHandle) { this.testHandle = testHandle; } + + public String getJavaVersion() { + return javaVersion; + } + + public void setJavaVersion(String javaVersion) { + this.javaVersion = javaVersion; + } + + public String getJavaTestVersion() { + return javaTestVersion; + } + + public void setJavaTestVersion(String javaTestVersion) { + this.javaTestVersion = javaTestVersion; + } + @Override public String toString() { return "TestStartRequest [profile=" + profile + ", testHandle=" diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java index b2c61f03c5bf5f170894141848c89fc26129115a..89a7941935b036aa3420ffe4adc06e24dae91a4c 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java @@ -114,6 +114,9 @@ public void run() { testConfiguration.setPatch(startRequest.getPatchURL()); testConfiguration.setJiraName(startRequest.getJiraName()); testConfiguration.setClearLibraryCache(startRequest.isClearLibraryCache()); + testConfiguration.resolveJavaHomes(startRequest.getJavaVersion(), + startRequest.getJavaTestVersion()); + LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger); PTest ptest = mPTestBuilder.build(testConfiguration, mExecutionContext, test.getStartRequest().getTestHandle(), logDir, diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 1cdfdb309acd8282e593abd7ed10c87721926c60..716e920b3b77ab959bab1bfc04533804e11e1379 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -72,7 +72,6 @@ private static final Logger LOG = LoggerFactory .getLogger(PTest.class); - private final TestConfiguration mConfiguration; private final ListeningExecutorService mExecutor; private final Set mAddedTests; @@ -266,8 +265,8 @@ public PTest build(TestConfiguration configuration, ExecutionContext executionCo private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME; private static final String BRANCH = TestConfiguration.BRANCH; private static final String PATCH = "patch"; - private static final String JAVA_HOME = TestConfiguration.JAVA_HOME; - private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST; + private static final String JAVA_VERSION = TestConfiguration.JAVA_VERSION; + private static final String JAVA_VERSION_FOR_TEST = TestConfiguration.JAVA_VERSION_FOR_TEST; private static final String ANT_TEST_ARGS = TestConfiguration.ANT_TEST_ARGS; private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS; private static final String ANT_TEST_TARGET = TestConfiguration.ANT_TEST_TARGET; @@ -287,8 +286,11 @@ public static void main(String[] args) throws Exception { options.addOption(null, BRANCH, true, "Overrides git branch in properties file"); options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://"); options.addOption(ANT_ARG, null, true, "Supplemental ant arguments"); - options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)"); - options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)"); + options.addOption(null, JAVA_VERSION, true, "Java version (eg. 7,8) for compiling and running tests (unless " + + JAVA_VERSION_FOR_TEST + " is specified). We expect a matching environment property " + + "(JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set"); + options.addOption(null, JAVA_VERSION_FOR_TEST, true, "Java version (eg. 7,8) for running tests." + + " We expect a matching environment property (JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set. (optional)"); options.addOption(null, ANT_TEST_ARGS, true, "Arguments to ant test on slave nodes only"); options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environment variable setting"); CommandLine commandLine = parser.parse(options, args); @@ -308,6 +310,10 @@ public static void main(String[] args) throws Exception { cleaner.setDaemon(true); cleaner.start(); TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG); + + conf.resolveJavaHomes(commandLine.getOptionValue(JAVA_VERSION).trim(), + commandLine.getOptionValue(JAVA_VERSION_FOR_TEST).trim()); + String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim(); if(!repository.isEmpty()) { conf.setRepository(repository); @@ -324,14 +330,6 @@ public static void main(String[] args) throws Exception { if(!patch.isEmpty()) { conf.setPatch(patch); } - String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim(); - if(!javaHome.isEmpty()) { - conf.setJavaHome(javaHome); - } - String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim(); - if(!javaHomeForTests.isEmpty()) { - conf.setJavaHomeForTests(javaHomeForTests); - } String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim(); if(!antTestArgs.isEmpty()) { conf.setAntTestArgs(antTestArgs); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java index e584f9c105fa134e3e267d6c6817d441b4c6b249..ecd42501b6ba3aedb1eccd66530f56fe09184d20 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java @@ -22,8 +22,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import java.util.Properties; +import com.google.common.base.Optional; +import org.apache.hive.ptest.execution.PTest; import org.slf4j.Logger; import com.google.common.annotations.VisibleForTesting; @@ -32,11 +35,13 @@ import com.google.common.collect.Maps; public class TestConfiguration { + public static final String JAVA_HOME_WITH_VERSION_PATTERN = "JAVA%s_HOME"; + public static final String JAVA_HOME = "JAVA_HOME"; public static final String REPOSITORY = "repository"; public static final String REPOSITORY_NAME = "repositoryName"; public static final String BRANCH = "branch"; - public static final String JAVA_HOME = "javaHome"; - public static final String JAVA_HOME_TEST = "javaHomeForTests"; + public static final String JAVA_VERSION = "javaVersion"; + public static final String JAVA_VERSION_FOR_TEST = "javaVersionForTest"; public static final String ANT_ENV_OPTS = "antEnvOpts"; public static final String ANT_TEST_ARGS = "antTestArgs"; public static final String ANT_TEST_TARGET = "antTestTarget"; @@ -128,8 +133,6 @@ public TestConfiguration(Context context, Logger logger) mavenTestArgs = context.getString(MAVEN_TEST_ARGS, "").trim(); mavenEnvOpts = context.getString(MAVEN_ENV_OPTS, "").trim(); additionalProfiles = context.getString(ADDITIONAL_PROFILES, "").trim(); - javaHome = context.getString(JAVA_HOME, "").trim(); - javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim(); patch = Strings.nullToEmpty(null); jiraName = Strings.nullToEmpty(null); jiraUrl = context.getString(JIRA_URL, "").trim(); @@ -288,6 +291,37 @@ public void setMavenEnvOpts(String mavenEnvOpts) { this.mavenEnvOpts = Strings.nullToEmpty(mavenEnvOpts); } public void setAdditionalProfiles(String additionalProfiles) { this.additionalProfiles = additionalProfiles; } + + /** + * Select the java home for building and running tests from the enviroment variables. + * If no environment variable is set for the provided version numbers try to set the default JAVA_HOME. + * If JAVA_HOME is also not set throw an exception. + * @param javaVersion java version to use when building the project. + * We expect a matching environment property (eg. JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set. + * @param javaVersionForTests java version to use when running tests. + * We expect a matching environment property (eg. JAVA7_HOME, JAVA8_HOME or JAVA_HOME) to be set. + * @throws IllegalArgumentException if no JAVA_HOME can be found + */ + public void resolveJavaHomes(String javaVersion, String javaVersionForTests) { + Map env = System.getenv(); + String javaHomeVersion = + String.format(JAVA_HOME_WITH_VERSION_PATTERN, javaVersion); + String javaTestVersion = + String.format(JAVA_HOME_WITH_VERSION_PATTERN, javaVersionForTests); + Optional javaHomeOpt = + Optional.fromNullable(env.get(javaHomeVersion)).or(Optional.of(env.get(JAVA_HOME))); + Optional javaTestHomeOpt = + Optional.fromNullable(env.get(javaTestVersion)).or(javaHomeOpt); + + + String javaHome = javaHomeOpt.get(); + Preconditions.checkArgument(javaHome != null, "JAVA_HOME is not set"); + String javaTestHome = javaTestHomeOpt.get(); + + this.setJavaHome(javaHome); + this.setJavaHomeForTests(javaTestHome); + } + @Override public String toString() { return "TestConfiguration [antArgs=" + antArgs + ", antTestArgs=" diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java index 848faf27af1ed8945d7013b6562bab544605e4bc..dddbb5a0799a9678c6ebb32d760cd18d64aee170 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java @@ -76,6 +76,10 @@ public void testGettersSetters() throws Exception { conf.setAntArgs("AntArgs"); Assert.assertEquals("AntArgs", conf.getAntArgs()); + + conf.resolveJavaHomes("", ""); + Assert.assertEquals(System.getenv("JAVA_HOME"), conf.getJavaHome()); + Assert.assertEquals(System.getenv("JAVA_HOME"), conf.getJavaHomeForTests()); } @Test public void testContext() throws Exception { @@ -94,6 +98,7 @@ public void testPTest() throws Exception { TestConfiguration conf = TestConfiguration.fromInputStream( Resources.getResource("test-configuration.properties").openStream(), LOG); + conf.resolveJavaHomes("", ""); ExecutionContext execContext = new ExecutionContext(null, testHosts, "test", null); PTest.Builder mPTestBuilder = new PTest.Builder(); PTest ptest = mPTestBuilder.build(conf, execContext, "1234", baseDir.newFolder(), null, null, null, null);