diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 1cdfdb309acd8282e593abd7ed10c87721926c60..fc6787b1c2e15b4963ce03795638c70862ac5d1d 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -20,6 +20,7 @@ import java.io.File; import java.net.URL; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -102,9 +103,12 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu mRsyncCommandExecutor = rsyncCommandExecutor; mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool( new ThreadFactoryBuilder().setDaemon(true).setNameFormat("HostExecutor %d").build())); + + String branch = configuration.getBranch(); + Path workingDirectory = mExecutionContext.getLocalWorkingDirectoryWrapper().getWorkingDirPath(branch); final File failedLogDir = Dirs.create(new File(logDir, "failed")); final File succeededLogDir = Dirs.create(new File(logDir, "succeeded")); - final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch")); + final File scratchDir = Dirs.createEmpty(new File(workingDirectory.toFile(), "scratch")); File patchDir = Dirs.createEmpty(new File(logDir, "patches")); File patchFile = null; if(!configuration.getPatch().isEmpty()) { @@ -117,9 +121,9 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu put("repositoryName", configuration.getRepositoryName()). put("repositoryType", configuration.getRepositoryType()). put("buildTool", configuration.getBuildTool()). - put("branch", configuration.getBranch()). + put("branch", branch). put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache())). - put("workingDir", mExecutionContext.getLocalWorkingDirectory()). + put("workingDir", workingDirectory.toString()). put("buildTag", buildTag). put("logDir", logDir.getAbsolutePath()). put("javaHome", configuration.getJavaHome()). @@ -137,7 +141,7 @@ public PTest(final TestConfiguration configuration, final ExecutionContext execu } templateDefaults = templateDefaultsBuilder.build(); TestParser testParser = new TestParser(configuration.getContext(), new AtomicInteger(1), configuration.getTestCasePropertyName(), - new File(mExecutionContext.getLocalWorkingDirectory(), configuration.getRepositoryName() + "-source"), + new File(workingDirectory.toFile(), configuration.getRepositoryName() + "-source"), logger); HostExecutorBuilder hostExecutorBuilder = new HostExecutorBuilder() { diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java index 8b82497bdaf43694e0e1552e125b5ffdce40f56c..cb4ca7b09f24be9fb14f3c2f429d87b64d3b7f7f 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java @@ -93,13 +93,13 @@ private final Map mTerminatedHosts; private final Map mLiveHosts; private final ExecutorService mTerminationExecutor; - private final File mWorkingDir; + private final WorkingDirWrapper mWorkingDirWrapper; private final SSHCommandExecutor mSSHCommandExecutor; @VisibleForTesting CloudExecutionContextProvider(String dataDir, int numHosts, CloudComputeService cloudComputeService, SSHCommandExecutor sshCommandExecutor, - String workingDirectory, String privateKey, String user, String[] slaveLocalDirs, int numThreads, + String workingDirectoryBase, String privateKey, String user, String[] slaveLocalDirs, int numThreads, long retrySleepInterval, int maxHostsPerCreateRequest) throws IOException { mNumHosts = numHosts; mMaxHostsPerCreateRequest = maxHostsPerCreateRequest; @@ -110,7 +110,7 @@ mNumThreads = numThreads; mRetrySleepInterval = retrySleepInterval; mSSHCommandExecutor = sshCommandExecutor; - mWorkingDir = Dirs.create(new File(workingDirectory, "working")); + mWorkingDirWrapper = new WorkingDirWrapper(workingDirectoryBase); mLiveHosts = Collections.synchronizedMap(new HashMap()); mTerminatedHosts = Collections .synchronizedMap(new LinkedHashMap() { @@ -200,7 +200,7 @@ public synchronized ExecutionContext createExecutionContext() hosts.add(new Host(publicIp(node), mUser, mSlaveLocalDirs, mNumThreads)); } - return new ExecutionContext(this, hosts, mWorkingDir.getAbsolutePath(), + return new ExecutionContext(this, hosts, mWorkingDirWrapper, mPrivateKey); } finally { syncLog(); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java index b09de1d4d930cf2d4d26b500f3457cea3fffa9ce..b5386ca717a2cd73e6c73dddeb79b5862a5e2740 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java @@ -27,16 +27,16 @@ public class ExecutionContext { private final Set mHosts; - private final String mLocalWorkingDirectory; + private final WorkingDirWrapper mLocalWorkingDirectoryWrapper; private final String mPrivateKey; private final ExecutionContextProvider mExecutionContextProvider; private final Set mBadHosts; public ExecutionContext(ExecutionContextProvider executionContextProvider, - Set hosts, String localWorkingDirectory, String privateKey) { + Set hosts, WorkingDirWrapper localWorkingDirectoryWrapper, String privateKey) { super(); mExecutionContextProvider = executionContextProvider; mHosts = hosts; - mLocalWorkingDirectory = localWorkingDirectory; + mLocalWorkingDirectoryWrapper = localWorkingDirectoryWrapper; mPrivateKey = privateKey; mBadHosts = Sets.newHashSet(); } @@ -58,8 +58,8 @@ boolean removeHost(Host host) { public ImmutableSet getHosts() { return ImmutableSet.copyOf(mHosts); } - public String getLocalWorkingDirectory() { - return mLocalWorkingDirectory; + public WorkingDirWrapper getLocalWorkingDirectoryWrapper() { + return mLocalWorkingDirectoryWrapper; } public String getPrivateKey() { return mPrivateKey; diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java index f7b50d6a61962d2727b5181215be9de2e64b05b7..f61951a91e7bac5d76d943c6addc82a2662e134d 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/FixedExecutionContextProvider.java @@ -38,12 +38,12 @@ private static final Logger LOG = LoggerFactory .getLogger(FixedExecutionContextProvider.class); private final ExecutionContext mExecutionContext; - private final File mWorkingDir; + private final WorkingDirWrapper mWorkingDirWrapper; - private FixedExecutionContextProvider(Set hosts, String workingDirectory, String privateKey) + private FixedExecutionContextProvider(Set hosts, String workingDirectoryBase, String privateKey) throws IOException { - mWorkingDir = Dirs.create(new File(workingDirectory, "working")); - mExecutionContext = new ExecutionContext(this, hosts, mWorkingDir.getAbsolutePath(), privateKey); + mWorkingDirWrapper = new WorkingDirWrapper(workingDirectoryBase); + mExecutionContext = new ExecutionContext(this, hosts, mWorkingDirWrapper, privateKey); } @Override public ExecutionContext createExecutionContext() diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/WorkingDirWrapper.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/WorkingDirWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..d8bdaab187958f5282bf27cc98ad61d90560b4cb --- /dev/null +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/WorkingDirWrapper.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.ptest.execution.context; + +import com.google.common.base.Strings; +import org.apache.hive.ptest.execution.Dirs; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +/** + * The working directory is built from a common base name and the branch name + * of source control branch to test. + * While the base name is static and supplied at startup, the branch name may differ + * between testing runs. + * The WorkingDirWrapper contains the static portion of the working directory and can supply + * the final string given the scm branch name. + */ +public class WorkingDirWrapper { + + private final String workingDirectoryBaseName; + + public WorkingDirWrapper(String baseName) { + this.workingDirectoryBaseName = baseName; + } + + /** + * Generate the full working dir path for the given scm branch name + * @param branchName + * @return absolute path of the working dir + * @throws IOException when working dir creation fails + */ + public Path getWorkingDirPath(String branchName) throws IOException { + return Dirs.create(new File(workingDirectoryBaseName, + Strings.isNullOrEmpty(branchName) ? "working" : branchName + "-working")).toPath(); + } +} diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestExecutor.java testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestExecutor.java index a4a789b579305d9ed573d8c1fd0b6ce75787d50f..81ff97d6027c10fbd0200c070430c59fcb1bf802 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestExecutor.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestExecutor.java @@ -40,6 +40,7 @@ import org.apache.hive.ptest.execution.conf.TestConfiguration; import org.apache.hive.ptest.execution.context.ExecutionContext; import org.apache.hive.ptest.execution.context.ExecutionContextProvider; +import org.apache.hive.ptest.execution.context.WorkingDirWrapper; import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor; import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor; import org.junit.After; @@ -81,7 +82,7 @@ public void setup() throws Exception { ptest = mock(PTest.class); Set hosts = Sets.newHashSet(); String baseDirPath = baseDir.getRoot().getAbsolutePath(); - executionContext = new ExecutionContext(executionContextProvider, hosts, baseDirPath, PRIVATE_KEY); + executionContext = new ExecutionContext(executionContextProvider, hosts, new WorkingDirWrapper(baseDirPath), PRIVATE_KEY); profileProperties = new File(baseDirPath, PROFILE + ".properties"); when(executionContextConfiguration.getProfileDirectory()).thenReturn(baseDirPath); when(executionContextConfiguration.getGlobalLogDirectory()).thenReturn(baseDirPath); diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java index 848faf27af1ed8945d7013b6562bab544605e4bc..d2fa0830fdfba442018382788e4c14f6cda94a62 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java @@ -24,11 +24,13 @@ import junit.framework.Assert; import org.apache.hive.ptest.execution.PTest; import org.apache.hive.ptest.execution.context.ExecutionContext; +import org.apache.hive.ptest.execution.context.WorkingDirWrapper; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.file.Paths; import java.util.HashSet; import java.util.Map; import java.util.Properties; @@ -56,7 +58,9 @@ public void testGettersSetters() throws Exception { Assert.assertEquals("git://github.com/apache/hive.git", conf.getRepository()); Assert.assertEquals("apache-github", conf.getRepositoryName()); Assert.assertEquals("trunk", conf.getBranch()); - Assert.assertEquals("/tmp/hive-ptest-units/working/dir/working", executionContext.getLocalWorkingDirectory()); + final WorkingDirWrapper wrapper = executionContext.getLocalWorkingDirectoryWrapper(); + Assert.assertEquals(Paths.get("/tmp/hive-ptest-units/working/dir/trunk-working"), + wrapper.getWorkingDirPath(conf.getBranch())); Assert.assertEquals("-Dtest.continue.on.failure=true -Dtest.silent=false", conf.getAntArgs()); Assert.assertEquals("hadoop-1,hadoop-2", conf.getAdditionalProfiles()); Assert.assertNotNull(conf.toString()); @@ -94,7 +98,8 @@ public void testPTest() throws Exception { TestConfiguration conf = TestConfiguration.fromInputStream( Resources.getResource("test-configuration.properties").openStream(), LOG); - ExecutionContext execContext = new ExecutionContext(null, testHosts, "test", null); + ExecutionContext execContext = new ExecutionContext(null, testHosts, + new WorkingDirWrapper("test"), null); PTest.Builder mPTestBuilder = new PTest.Builder(); PTest ptest = mPTestBuilder.build(conf, execContext, "1234", baseDir.newFolder(), null, null, null, null); Map templateDefaults = ptest.getTemplateDefaults();