diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java index 2f96ad03023e9f51d44d203f34edd04183605a22..29789c75ef53ac45028d63cb17f7d2a0701fe8d6 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/ExecutionController.java @@ -40,6 +40,7 @@ import org.apache.hive.ptest.api.response.TestStatusResponse; import org.apache.hive.ptest.api.response.TestStopResponse; import org.apache.hive.ptest.execution.PTest; +import org.apache.hive.ptest.execution.conf.Context; import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration; import org.apache.hive.ptest.execution.context.ExecutionContextProvider; import org.slf4j.Logger; @@ -82,7 +83,9 @@ public ExecutionController() String executionContextConfigurationFile = System.getProperty(CONF_PROPERTY, "").trim(); Preconditions.checkArgument(!executionContextConfigurationFile.isEmpty(), CONF_PROPERTY + " is required"); LOG.info("Reading configuration from file: " + executionContextConfigurationFile); - mExecutionContextConfiguration = ExecutionContextConfiguration.fromFile(executionContextConfigurationFile); + mExecutionContextConfiguration = ExecutionContextConfiguration.withContext( + Context.fromFile(executionContextConfigurationFile) + ); LOG.info("ExecutionContext is [{}]", mExecutionContextConfiguration); mExecutionContextProvider = mExecutionContextConfiguration.getExecutionContextProvider(); mTests = Collections.synchronizedMap(new LinkedHashMap() { diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java index b2c61f03c5bf5f170894141848c89fc26129115a..d4d28c06bac5c807b411d8690cf7ab7d7bc415aa 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java @@ -30,6 +30,7 @@ import org.apache.hive.ptest.execution.LocalCommandFactory; import org.apache.hive.ptest.execution.LogDirectoryCleaner; import org.apache.hive.ptest.execution.PTest; +import org.apache.hive.ptest.execution.conf.Context; import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration; import org.apache.hive.ptest.execution.conf.TestConfiguration; import org.apache.hive.ptest.execution.context.CreateHostsFailedException; @@ -48,6 +49,8 @@ public class TestExecutor extends Thread { private static final Logger LOG = LoggerFactory .getLogger(TestExecutor.class); + private static final String SERVER_ENV_PROPERTIES = "hive.ptest.server.env.properties"; + private final ExecutionContextConfiguration mExecutionContextConfiguration; private final ExecutionContextProvider mExecutionContextProvider; private final BlockingQueue mTestQueue; @@ -110,7 +113,18 @@ public void run() { test.setOutputFile(logFile); logStream = new PrintStream(logFile); logger = new TestLogger(logStream, TestLogger.LEVEL.DEBUG); - TestConfiguration testConfiguration = TestConfiguration.fromFile(profileConfFile, logger); + + Context.ContextBuilder builder = new Context.ContextBuilder(); + builder.addPropertiesFile(profileConfFile); + + String environmentConfigurationFile = System.getProperty(SERVER_ENV_PROPERTIES, null); + if (environmentConfigurationFile != null) { + builder.addPropertiesFile(environmentConfigurationFile); + } + + Context ctx = builder.build(); + + TestConfiguration testConfiguration = TestConfiguration.withContext(ctx, logger); testConfiguration.setPatch(startRequest.getPatchURL()); testConfiguration.setJiraName(startRequest.getJiraName()); testConfiguration.setClearLibraryCache(startRequest.isClearLibraryCache()); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 1cdfdb309acd8282e593abd7ed10c87721926c60..65a8216f6a076b0ee7baee11ca557f5e9f746316 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -41,6 +41,7 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.Options; +import org.apache.hive.ptest.execution.conf.Context; import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration; import org.apache.hive.ptest.execution.conf.Host; import org.apache.hive.ptest.execution.conf.TestConfiguration; @@ -262,6 +263,7 @@ public PTest build(TestConfiguration configuration, ExecutionContext executionCo } private static final String PROPERTIES = "properties"; + private static final String SERVER_ENV_PROPERTIES = "hive.ptest.server.env.properties"; private static final String REPOSITORY = TestConfiguration.REPOSITORY; private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME; private static final String BRANCH = TestConfiguration.BRANCH; @@ -282,6 +284,7 @@ public static void main(String[] args) throws Exception { CommandLineParser parser = new GnuParser(); Options options = new Options(); options.addOption(null, PROPERTIES, true, "properties file"); + options.addOption(null, SERVER_ENV_PROPERTIES, true, "optional properties file with environment properties"); options.addOption(null, REPOSITORY, true, "Overrides git repository in properties file"); options.addOption(null, REPOSITORY_NAME, true, "Overrides git repository *name* in properties file"); options.addOption(null, BRANCH, true, "Overrides git branch in properties file"); @@ -297,8 +300,19 @@ public static void main(String[] args) throws Exception { join(PTest.class.getName(), "--" + PROPERTIES,"config.properties")); } String testConfigurationFile = commandLine.getOptionValue(PROPERTIES); + String environmentConfigurationFile = commandLine.getOptionValue(SERVER_ENV_PROPERTIES); + + Context.ContextBuilder builder = new Context.ContextBuilder(); + builder.addPropertiesFile(testConfigurationFile); + + if (environmentConfigurationFile != null) { + builder.addPropertiesFile(environmentConfigurationFile); + } + + Context ctx = builder.build(); + ExecutionContextConfiguration executionContextConfiguration = ExecutionContextConfiguration. - fromFile(testConfigurationFile); + withContext(ctx); String buildTag = System.getenv("BUILD_TAG") == null ? "undefined-" + System.currentTimeMillis() : System.getenv("BUILD_TAG"); File logDir = Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag)); @@ -307,7 +321,7 @@ public static void main(String[] args) throws Exception { cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory()); cleaner.setDaemon(true); cleaner.start(); - TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG); + TestConfiguration conf = TestConfiguration.withContext(ctx, LOG); String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim(); if(!repository.isEmpty()) { conf.setRepository(repository); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java index 14984bafdd18fb7636e729cc7fbbfa349b0f043e..f34cdba6b6c5634f6f683ba10c48dff5c9849e6b 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Context.java @@ -19,9 +19,14 @@ package org.apache.hive.ptest.execution.conf; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -258,4 +263,83 @@ private String get(String key) { public String toString() { return "{ parameters:" + parameters + " }"; } + + /** + * Build a context with the properties read from an input stream. + * @param inputStream + * @return context + * @throws IOException + */ + public static Context fromInputStream(InputStream inputStream) + throws IOException { + Properties properties = new Properties(); + properties.load(inputStream); + return new Context(Maps.fromProperties(properties)); + } + + /** + * Build a context with the properties read from a file + * @param file + * @return + * @throws IOException + */ + public static Context fromFile(String file) throws IOException { + return fromFile(new File(file)); + } + + /** + * Build a context with the properties read from a file + * @param file + * @return + * @throws IOException + */ + public static Context fromFile(File file) throws IOException { + try (InputStream in = new FileInputStream(file)){ + return fromInputStream(in); + } + } + + /** + * Builder that can aggregate properties from several files + * when building a context. If the same key is present in more + * than one file, the last one will be used. + */ + public static class ContextBuilder { + + private Context context = new Context(); + + /** + * Add properties from a file to the context + * @param file + * @return + * @throws IOException + */ + public ContextBuilder addPropertiesFile(File file) throws IOException { + try(InputStream is = new FileInputStream(file)) { + Properties properties = new Properties(); + properties.load(is); + context.putAll(Maps.fromProperties(properties)); + } + return this; + } + + /** + * Add properties from a file to the context + * @param file + * @return + * @throws IOException + */ + public ContextBuilder addPropertiesFile(String file) throws IOException { + return addPropertiesFile(new File(file)); + } + + /** + * Build the context using the aggregated properties + * @return + * @throws IOException + */ + public Context build() throws IOException { + return context; + } + } } \ No newline at end of file diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java index 35ddd44accf34be1f5957c6df31802ee8c8022b5..8783e43ac395e8042fc302bce41f5509cec9613e 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/ExecutionContextConfiguration.java @@ -19,10 +19,7 @@ package org.apache.hive.ptest.execution.conf; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; import org.apache.hive.ptest.execution.Dirs; import org.apache.hive.ptest.execution.context.ExecutionContextProvider; @@ -88,22 +85,11 @@ public String getProfileDirectory() { public ExecutionContextProvider getExecutionContextProvider() { return mExecutionContextProvider; } - public static ExecutionContextConfiguration fromInputStream(InputStream inputStream) + + public static ExecutionContextConfiguration withContext(Context ctx) throws IOException { - Properties properties = new Properties(); - properties.load(inputStream); - Context context = new Context(Maps.fromProperties(properties)); - return new ExecutionContextConfiguration(context); + return new ExecutionContextConfiguration(ctx); } - public static ExecutionContextConfiguration fromFile(String file) throws IOException { - InputStream in = new FileInputStream(file); - try { - return fromInputStream(in); - } finally { - in.close(); - } - } - @Override public String toString() { return "ExecutionContextConfiguration{" + diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java index e584f9c105fa134e3e267d6c6817d441b4c6b249..4e3d03c351a47cbb073d027b7c5473d71a9a2029 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java @@ -18,11 +18,7 @@ */ package org.apache.hive.ptest.execution.conf; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; import org.slf4j.Logger; @@ -303,22 +299,8 @@ public String toString() { + buildTool + ", jiraName=" + jiraName + ", clearLibraryCache=" + clearLibraryCache + ", additionalProfiles=" + additionalProfiles + "]"; } - public static TestConfiguration fromInputStream(InputStream inputStream, Logger logger) + public static TestConfiguration withContext(Context context, Logger logger) throws IOException { - Properties properties = new Properties(); - properties.load(inputStream); - Context context = new Context(Maps.fromProperties(properties)); return new TestConfiguration(context, logger); } - public static TestConfiguration fromFile(String file, Logger logger) throws IOException { - return fromFile(new File(file), logger); - } - public static TestConfiguration fromFile(File file, Logger logger) throws IOException { - InputStream in = new FileInputStream(file); - try { - return fromInputStream(in, logger); - } finally { - in.close(); - } - } } diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java index a243774e52f3f5fda4a082bb99387cf5808c307b..ad6dad4ae87679f2f3568869c5cf22b0cead504c 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java @@ -276,7 +276,8 @@ public static void main(String[] args) throws Exception { .getLogger(TestParser.class); File workingDir = new File("../.."); File testConfigurationFile = new File(args[0]); - TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, log); + final Context ctx = Context.fromFile(testConfigurationFile); + TestConfiguration conf = TestConfiguration.withContext(ctx, log); TestParser testParser = new TestParser(conf.getContext(), new AtomicInteger(1), "test", workingDir, log); List testBatches = testParser.parse().get(); for (TestBatch testBatch : testBatches) { diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java index b97b890dfe855539de2696788327ba9b4a841ff3..0ea51b7add3ff973f060ec63794c7a75fc8a318e 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java @@ -31,6 +31,7 @@ import junit.framework.Assert; import org.apache.hive.ptest.execution.JIRAService.BuildInfo; +import org.apache.hive.ptest.execution.conf.Context; import org.apache.hive.ptest.execution.conf.TestConfiguration; import org.approvaltests.Approvals; import org.approvaltests.reporters.JunitReporter; @@ -50,8 +51,9 @@ @Before public void setup() throws Exception { initialize(getClass().getSimpleName()); - conf = TestConfiguration.fromInputStream( - Resources.getResource("test-configuration.properties").openStream(), logger); + conf = TestConfiguration.withContext( + Context.fromInputStream( + Resources.getResource("test-configuration.properties").openStream()), logger); conf.setPatch("https://HIVE-10000.patch"); jiraService = new JIRAService(logger, conf, "tag-10"); } diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestContext.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestContext.java new file mode 100644 index 0000000000000000000000000000000000000000..e2a6c2fd6a5093970ad6b36f08e8afcc76b0828f --- /dev/null +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestContext.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.ptest.execution.conf; + +import com.google.common.io.Resources; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class TestContext { + + @Test + public void testContextFromStream() throws IOException{ + + Context ctx = Context.fromInputStream( + Resources.getResource("test-configuration.properties").openStream()); + Assert.assertNotNull(ctx.getString("executionContextProvider")); + } + + @Test + public void testContextFromFile() throws IOException{ + + Context ctx = Context.fromFile( + Resources.getResource("test-configuration.properties").getFile()); + Assert.assertNotNull(ctx.getString("executionContextProvider")); + } + + @Test + public void testContextFromString() throws IOException{ + + Context ctx = Context.fromFile( + Resources.getResource("test-configuration.properties").getPath()); + Assert.assertNotNull(ctx.getString("executionContextProvider")); + } + + @Test + public void testContextBuilderPropertyOverride() throws IOException{ + + Context.ContextBuilder builder = new Context.ContextBuilder(); + builder.addPropertiesFile( + Resources.getResource("test-configuration.properties").getFile()); + + Context ctx = builder.build(); + Assert.assertEquals(ctx.getString("branch"), "trunk"); + + builder.addPropertiesFile( + Resources.getResource("test-configuration2.properties").getFile()); + ctx = builder.build(); + Assert.assertEquals(ctx.getString("branch"), "master"); + } +} diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java index 848faf27af1ed8945d7013b6562bab544605e4bc..360f868c3e88f7c17743afe80962abeb99ff7d54 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestConfiguration.java @@ -42,10 +42,10 @@ @Test public void testGettersSetters() throws Exception { - ExecutionContextConfiguration execConf = ExecutionContextConfiguration.fromInputStream( + Context ctx = Context.fromInputStream( Resources.getResource("test-configuration.properties").openStream()); - TestConfiguration conf = TestConfiguration.fromInputStream( - Resources.getResource("test-configuration.properties").openStream(), LOG); + ExecutionContextConfiguration execConf = ExecutionContextConfiguration.withContext(ctx); + TestConfiguration conf = TestConfiguration.withContext(ctx, LOG); Set expectedHosts = Sets.newHashSet(new Host("localhost", "hiveptest", new String[]{"/home/hiveptest"}, 2)); ExecutionContext executionContext = execConf.getExecutionContextProvider().createExecutionContext(); Assert.assertEquals(expectedHosts, executionContext.getHosts()); @@ -92,8 +92,8 @@ public void testPTest() throws Exception { Set testHosts = new HashSet(); testHosts.add(testHost); - TestConfiguration conf = TestConfiguration.fromInputStream( - Resources.getResource("test-configuration.properties").openStream(), LOG); + Context ctx = Context.fromInputStream(Resources.getResource("test-configuration.properties").openStream()); + TestConfiguration conf = TestConfiguration.withContext(ctx, LOG); ExecutionContext execContext = new ExecutionContext(null, testHosts, "test", null); PTest.Builder mPTestBuilder = new PTest.Builder(); PTest ptest = mPTestBuilder.build(conf, execContext, "1234", baseDir.newFolder(), null, null, null, null);