diff --git dev-support/jenkins-common.sh dev-support/jenkins-common.sh index 64f486fc2bd181cbf1882c6bba98e23a6abf9f8f..0467d119cc99f5f5a5b8e0bedce82282ae20da4d 100644 --- dev-support/jenkins-common.sh +++ dev-support/jenkins-common.sh @@ -15,8 +15,6 @@ # limitations under the License. JIRA_ROOT_URL="https://issues.apache.org" -JENKINS_URL="https://builds.apache.org" -JENKINS_QUEUE_QUERY="/queue/api/json?tree=items[task[name],inQueueSince,actions[parameters[name,value]],why]" fail() { echo "$@" 1>&2 diff --git dev-support/jenkins-execute-build.sh dev-support/jenkins-execute-build.sh index 35392dd565861aa2e4a62fd91749001289e688c5..f660fcb0a35775d5661e3f826cc5121cda51b762 100644 --- dev-support/jenkins-execute-build.sh +++ dev-support/jenkins-execute-build.sh @@ -51,8 +51,7 @@ call_ptest_server() { local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-3.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*" java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \ - --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" \ - --jenkinsQueueUrl "$JENKINS_URL$JENKINS_QUEUE_QUERY" "$@" + --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@" } # Unpack all test results diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java deleted file mode 100644 index f33516447ec8decd34c578732584d15e1f1a6129..0000000000000000000000000000000000000000 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.ptest.api.client; - -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.http.HttpResponse; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.ssl.SSLContexts; -import org.apache.http.util.EntityUtils; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; - -/** - * Utility class for the Precommit test job queue on Jenkins - */ -public class JenkinsQueueUtil { - - private static final String JSON_ITEMS_FIELD = "items"; - private static final String JSON_TASK_FIELD = "task"; - private static final String JSON_TASK_NAME_FIELD = "name"; - private static final String JSON_PARAMETERS_FIELD = "parameters"; - private static final String JSON_PARAMETER_NAME_FIELD = "name"; - private static final String JSON_PARAMETER_VALUE_FIELD = "value"; - - private static final String JOB_NAME = "PreCommit-HIVE-Build"; - private static final String ISSUE_FIELD_KEY = "ISSUE_NUM"; - private static final String JIRA_KEY_PREFIX = "HIVE-"; - - /** - * Looks up the current queue of the precommit job on a jenkins instance (specified by - * PTestClient.JENKINS_QUEUE_URL), and checks if current Jira is standing in queue already (i.e. - * will be executed in the future too) - * - * @param commandLine PTestClient's command line option values' list - * @return whether or not the Jira specified in the command line can be found in the job queue - */ - public static boolean isJiraAlreadyInQueue(CommandLine commandLine) { - if (!(commandLine.hasOption(PTestClient.JENKINS_QUEUE_URL) && - commandLine.hasOption(PTestClient.JIRA))) { - return false; - } - try { - System.out.println("Checking " + JOB_NAME + " queue..."); - String queueJson = httpGet(commandLine.getOptionValue(PTestClient.JENKINS_QUEUE_URL)); - List jirasInQueue = parseJiras(queueJson); - if (jirasInQueue.size() > 0) { - System.out.println(JOB_NAME + " has the following jira(s) in queue: " + jirasInQueue); - } else { - return false; - } - - String jira = commandLine.getOptionValue(PTestClient.JIRA).replaceAll(JIRA_KEY_PREFIX,""); - if (jirasInQueue.contains(jira)) { - return true; - } - - } catch (IOException e) { - System.err.println("Error checking " + JOB_NAME + " build queue: " + e); - } - return false; - } - - /** - * Parses raw json to produce a list of Jira number strings. - * @param queueJson - * @return - * @throws IOException - */ - private static List parseJiras(String queueJson) throws IOException { - List jirasInQueue = new ArrayList<>(); - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode rootNode = objectMapper.readTree(queueJson); - List items = Lists.newArrayList(rootNode.findValue(JSON_ITEMS_FIELD).iterator()); - for (JsonNode item : items) { - String taskName = item.path(JSON_TASK_FIELD).path(JSON_TASK_NAME_FIELD).asText(); - if (JOB_NAME.equals(taskName)) { - List parameters = Lists.newArrayList(item.findValue(JSON_PARAMETERS_FIELD)); - for (JsonNode parameter : parameters) { - if (ISSUE_FIELD_KEY.equals(parameter.path(JSON_PARAMETER_NAME_FIELD).asText())) { - jirasInQueue.add(parameter.path(JSON_PARAMETER_VALUE_FIELD).asText()); - } - } - } - } - return jirasInQueue; - } - - private static String httpGet(String url) - throws IOException { - - HttpGet request = new HttpGet(url); - try { - CloseableHttpClient httpClient = HttpClientBuilder - .create() - .setSslcontext(SSLContexts.custom().useProtocol("TLSv1.2").build()) - .setRetryHandler(new PTestClient.PTestHttpRequestRetryHandler()) - .build(); - request.addHeader("content-type", "application/json"); - HttpResponse httpResponse = httpClient.execute(request); - StatusLine statusLine = httpResponse.getStatusLine(); - if (statusLine.getStatusCode() != 200) { - throw new IllegalStateException(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase()); - } - String response = EntityUtils.toString(httpResponse.getEntity(), "UTF-8"); - return response; - } catch (NoSuchAlgorithmException | KeyManagementException e) { - e.printStackTrace(); - throw new IOException(e.getMessage()); - } finally { - request.abort(); - } - } - - -} diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java index 9970c360542b9f52420a81536a5e90554e2f1790..e878e1834aabd29d42c2095b79712bc50325b9b0 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/PTestClient.java @@ -81,11 +81,10 @@ private static final String PASSWORD = "password"; private static final String PROFILE = "profile"; private static final String PATCH = "patch"; - public static final String JIRA = "jira"; + private static final String JIRA = "jira"; private static final String OUTPUT_DIR = "outputDir"; private static final String TEST_HANDLE = "testHandle"; private static final String CLEAR_LIBRARY_CACHE = "clearLibraryCache"; - public static final String JENKINS_QUEUE_URL = "jenkinsQueueUrl"; private static final int MAX_RETRIES = 10; private final String mApiEndPoint; private final String mLogsEndpoint; @@ -299,7 +298,6 @@ public static void main(String[] args) throws Exception { options.addOption(null, OUTPUT_DIR, true, "Directory to download and save test-results.tar.gz to. (Optional for testStart)"); options.addOption(null, CLEAR_LIBRARY_CACHE, false, "Before starting the test, delete the ivy and maven directories (Optional for testStart)"); options.addOption(null, LOGS_ENDPOINT, true, "URL to get the logs"); - options.addOption(null, JENKINS_QUEUE_URL, true, "URL for quering Jenkins job queue"); CommandLine commandLine = parser.parse(options, args); @@ -322,13 +320,6 @@ public static void main(String[] args) throws Exception { TEST_HANDLE }); - boolean jiraAlreadyInQueue = JenkinsQueueUtil.isJiraAlreadyInQueue(commandLine); - if (jiraAlreadyInQueue) { - System.out.println("Skipping ptest execution, as " + commandLine.getOptionValue(JIRA) + - " is scheduled in " + "queue in " + "the future too."); - System.exit(0); - } - result = client.testStart(commandLine.getOptionValue(PROFILE), commandLine.getOptionValue(TEST_HANDLE), commandLine.getOptionValue(JIRA), commandLine.getOptionValue(PATCH), commandLine.hasOption(CLEAR_LIBRARY_CACHE)); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 8df5162440cab6bf91e3d6efa259dbfa055c953e..4c58d5da9e17adca1e9506e04a789360e7e988e5 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -156,7 +156,8 @@ public HostExecutor build(Host host) { } mHostExecutors = new CopyOnWriteArrayList(hostExecutors); mPhases = Lists.newArrayList(); - mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, patchFile, logger, mAddedTests)); + mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, + configuration.getPatch(), patchFile, logger, mAddedTests)); mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger)); mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults, mExecutionContext.getLocalWorkingDirectory(), scratchDir, logger, logDir, patchFile)); diff --git testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java index 1107dcd70d27b108efb953b92d0d36f9172401e3..831a9099e27fb8c3d1ffc1fb2e8055d46c490516 100644 --- testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java +++ testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/TestCheckPhase.java @@ -18,6 +18,8 @@ */ package org.apache.hive.ptest.execution; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; @@ -26,26 +28,40 @@ import java.io.FileReader; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; public class TestCheckPhase extends Phase { private final File mPatchFile; + private final String mPatchURL; private Set modifiedTestFiles; + private static Cache patchUrls = CacheBuilder.newBuilder().expireAfterWrite + (7, TimeUnit.DAYS).maximumSize(10000).build(); private static final Pattern fileNameFromDiff = Pattern.compile("[/][^\\s]*"); private static final Pattern javaTest = Pattern.compile("Test.*java"); public TestCheckPhase(List hostExecutors, - LocalCommandFactory localCommandFactory, - ImmutableMap templateDefaults, - File patchFile, Logger logger, Set modifiedTestFiles) { + LocalCommandFactory localCommandFactory, + ImmutableMap templateDefaults, + String patchUrl, File patchFile, Logger logger, Set modifiedTestFiles) { super(hostExecutors, localCommandFactory, templateDefaults, logger); this.mPatchFile = patchFile; + this.mPatchURL = patchUrl; this.modifiedTestFiles = modifiedTestFiles; } @Override public void execute() throws Exception { + if (mPatchURL != null) { + boolean patchUrlWasSeen = patchUrls.asMap().containsKey(mPatchURL); + if (!patchUrlWasSeen) { + patchUrls.put(mPatchURL, true); + } else { + throw new Exception("Patch URL " + mPatchURL + " was found in seen patch url's cache and " + + "a test was probably run already on it. Aborting..."); + } + } if(mPatchFile != null) { logger.info("Reading patchfile " + mPatchFile.getAbsolutePath()); FileReader fr = null; diff --git testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestTestCheckPhase.java testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestTestCheckPhase.java index 7183ee3e367796fab687528c8b32faf2cedb1ba2..de3386abcb5adb8cdde1273de19087336fb49642 100644 --- testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestTestCheckPhase.java +++ testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestTestCheckPhase.java @@ -18,7 +18,6 @@ */ package org.apache.hive.ptest.execution; -import org.approvaltests.Approvals; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -28,6 +27,10 @@ import java.util.HashSet; import java.util.Set; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class TestTestCheckPhase extends AbstractTestPhase { private TestCheckPhase phase; @@ -42,7 +45,7 @@ public void testNoTests() throws Exception { File patchFile = new File(url.getFile()); Set addedTests = new HashSet(); phase = new TestCheckPhase(hostExecutors, localCommandFactory, - templateDefaults, patchFile, logger, addedTests); + templateDefaults, url.toString(), patchFile, logger, addedTests); phase.execute(); Assert.assertEquals(addedTests.size(), 0); @@ -55,7 +58,7 @@ public void testJavaTests() throws Exception { File patchFile = new File(url.getFile()); Set addedTests = new HashSet(); phase = new TestCheckPhase(hostExecutors, localCommandFactory, - templateDefaults, patchFile, logger, addedTests); + templateDefaults, url.toString(), patchFile, logger, addedTests); phase.execute(); Assert.assertEquals(addedTests.size(), 3); @@ -70,7 +73,7 @@ public void testQTests() throws Exception { File patchFile = new File(url.getFile()); Set addedTests = new HashSet(); phase = new TestCheckPhase(hostExecutors, localCommandFactory, - templateDefaults, patchFile, logger, addedTests); + templateDefaults, url.toString(), patchFile, logger, addedTests); phase.execute(); Assert.assertEquals(addedTests.size(), 1); @@ -83,9 +86,32 @@ public void testRemoveTest() throws Exception { File patchFile = new File(url.getFile()); Set addedTests = new HashSet(); phase = new TestCheckPhase(hostExecutors, localCommandFactory, - templateDefaults, patchFile, logger, addedTests); + templateDefaults, url.toString(), patchFile, logger, addedTests); phase.execute(); Assert.assertEquals(addedTests.size(), 0); } + + @Test + public void testSamePatchMultipleTimes() throws Exception { + int executions = 0; + try { + URL url = this.getClass().getResource("/HIVE-19077.1.patch"); + File patchFile = new File(url.getFile()); + Set addedTests = new HashSet(); + phase = new TestCheckPhase(hostExecutors, localCommandFactory, + templateDefaults, url.toString(), patchFile, logger, addedTests); + phase.execute(); + executions++; + phase = new TestCheckPhase(hostExecutors, localCommandFactory, + templateDefaults, url.toString(), patchFile, logger, addedTests); + phase.execute(); + executions++; + fail("Should've thrown exception"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("HIVE-19077.1.patch was found in seen patch url's cache " + + "and a test was probably run already on it. Aborting...")); + } + assertEquals(1, executions); + } } diff --git testutils/ptest2/src/test/resources/HIVE-19077.1.patch testutils/ptest2/src/test/resources/HIVE-19077.1.patch new file mode 100644 index 0000000000000000000000000000000000000000..cd7b1331add89fcc1ec04c7124f99cb09597a0a9 --- /dev/null +++ testutils/ptest2/src/test/resources/HIVE-19077.1.patch @@ -0,0 +1,14 @@ +diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java +index c076d4e112a7edab2106f11fe6224247887313cf..8bcb464de540eda7c14a8c6783bb19a09071af7b 100644 +--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java ++++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java +@@ -25,7 +25,9 @@ + + import org.apache.hadoop.hive.ql.exec.ColumnInfo; + import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; ++import org.apache.hadoop.hive.ql.exec.FilterOperator; + import org.apache.hadoop.hive.ql.exec.Operator; ++import org.apache.hadoop.hive.ql.exec.OperatorFactory; + import org.apache.hadoop.hive.ql.exec.RowSchema; + import org.apache.hadoop.hive.ql.exec.SelectOperator; + import org.apache.hadoop.hive.ql.exec.UnionOperator;