diff --git a/dev-support/hive-personality.sh b/dev-support/hive-personality.sh index f3247aac6284b8dd863691b4819a10c3a896d50c..c6570d4e2439fb5c5b3a880aca24833bd32ab7f8 100644 --- a/dev-support/hive-personality.sh +++ b/dev-support/hive-personality.sh @@ -16,7 +16,7 @@ # Override these to match Apache Hadoop's requirements -personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml" +personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml,jira" ## @description Globals specific to this personality ## @audience private diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 65a8216f6a076b0ee7baee11ca557f5e9f746316..80bee45ff1acb7e1e300add03f61b9b74a69a9a0 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -158,6 +158,8 @@ public HostExecutor build(Host host) { mPhases = Lists.newArrayList(); mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, patchFile, logger, mAddedTests)); mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger)); + mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults, + mExecutionContext.getLocalWorkingDirectory(), logger, logDir, patchFile)); mPhases.add(new ExecutionPhase(mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults, succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger)); mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger)); diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java new file mode 100644 index 0000000000000000000000000000000000000000..da43d4d0b435051f06fcb7b398b1a05c7e855067 --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.ptest.execution; + +import java.io.File; +import java.util.List; + +import org.apache.hive.ptest.execution.conf.TestConfiguration; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; + +import org.slf4j.Logger; + +/** + * Wrapper phase for Yetus check execution kick-off. It will invoke ./dev-support/test-patch.sh + * with the proper arguments and run the check itself in a asynchronous fashion. + */ +public class YetusPhase extends Phase { + + private static final String YETUS_LOG_FILE = "yetus.txt"; + private static final String YETUS_OUTPUT_FOLDER = "yetus"; + + private final File mPatchFile; + private final File mWorkingDir; + private final File mLogFile; + private final File mOutputDir; + private final String buildUrl; + private final TestConfiguration conf; + + + public YetusPhase(TestConfiguration configuration, List hostExecutors, + LocalCommandFactory localCommandFactory, ImmutableMap templateDefaults, + String workingDir, Logger logger, File logDir, File patchFile) { + + super(hostExecutors, localCommandFactory, templateDefaults, logger); + this.mPatchFile = patchFile; + this.mWorkingDir = new File(workingDir, YETUS_OUTPUT_FOLDER); + this.mLogFile = new File(logDir, YETUS_LOG_FILE); + this.mOutputDir = new File(logDir, YETUS_OUTPUT_FOLDER); + this.conf = configuration; + this.buildUrl = conf.getLogsURL() + "/" + templateDefaults.get("buildTag") + "/"; + } + + @Override + public void execute() throws Exception { + + Thread t = new Thread(new Runnable() { + @Override + public void run() { + + if (!checkDependencies()) { + return; + } + + String command = String.format( + "pushd %s\n" + + "export JIRA_ISSUE=%s\n" + + "export JAVA_HOME=%s\n" + + "./dev-support/test-patch.sh %s --jenkins --jira-base-url=%s --jira-user=%s " + + "--jira-password=%s --patch-dir=%s --build-url=%s --build-url-console=%s " + + "--build-url-artifacts=%s 2>&1 | tee %s\n" + + "popd", + mWorkingDir.getAbsolutePath(), + conf.getJiraName(), + conf.getJavaHome(), + mPatchFile.getAbsolutePath(), conf.getJiraUrl(), conf.getJiraUser(), + conf.getJiraPassword(), mOutputDir.getAbsolutePath(), buildUrl, YETUS_LOG_FILE, + YETUS_OUTPUT_FOLDER, mLogFile.getAbsolutePath()); + + try { + Process process = new ProcessBuilder().command("bash", "-c", command).start(); + int exitCode = process.waitFor(); + if (exitCode == 0) { + logger.info("Finished processing Yetus check"); + } + } catch (Exception e) { + logger.error("Error processing Yetus check"); + } + } + }); + t.start(); + logger.info("Started Yetus check.."); + } + + private boolean checkDependencies(){ + + if (mPatchFile == null || !mPatchFile.canRead()) { + logger.error("Cannot run Yetus check - patch file is null or not readable."); + return false; + } + + if (!((mWorkingDir.isDirectory() && mWorkingDir.canWrite()) && + (mOutputDir.isDirectory() && mOutputDir.canWrite()))) { + logger.error("Cannot run Yetus check - output directories not present and writable: " + + "workingDir:%s, outputDir:%s", mWorkingDir.getAbsolutePath(), mOutputDir.getAbsolutePath()); + return false; + } + + if (Strings.isNullOrEmpty(conf.getJiraUrl()) || + Strings.isNullOrEmpty(conf.getJiraName()) || + Strings.isNullOrEmpty(conf.getJiraPassword()) || + Strings.isNullOrEmpty(conf.getJiraUser())) { + logger.error("Cannot run Yetus check - credentials for Jira not provided."); + return false; + } + + return true; + } +} diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm index 7ad50248af02dfaeb6524a61d4895f1a8efba211..58207312fc45430822aa2be73c53b9523aa02ef5 100644 --- a/testutils/ptest2/src/main/resources/source-prep.vm +++ b/testutils/ptest2/src/main/resources/source-prep.vm @@ -78,6 +78,10 @@ cd $workingDir/ echo "Unknown repository type '${repositoryType}'" exit 1 fi + rm -rf ../yetus + mkdir ../yetus + cp -R . ../yetus + mkdir $logDir/yetus patchCommandPath=$workingDir/scratch/smart-apply-patch.sh patchFilePath=$workingDir/scratch/build.patch if [[ -f $patchFilePath ]]