diff --git a/dev-support/hive-personality.sh b/dev-support/hive-personality.sh index f3247aac6284b8dd863691b4819a10c3a896d50c..c6570d4e2439fb5c5b3a880aca24833bd32ab7f8 100644 --- a/dev-support/hive-personality.sh +++ b/dev-support/hive-personality.sh @@ -16,7 +16,7 @@ # Override these to match Apache Hadoop's requirements -personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml" +personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml,jira" ## @description Globals specific to this personality ## @audience private diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 65a8216f6a076b0ee7baee11ca557f5e9f746316..8df5162440cab6bf91e3d6efa259dbfa055c953e 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -158,6 +158,8 @@ public HostExecutor build(Host host) { mPhases = Lists.newArrayList(); mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, patchFile, logger, mAddedTests)); mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger)); + mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults, + mExecutionContext.getLocalWorkingDirectory(), scratchDir, logger, logDir, patchFile)); mPhases.add(new ExecutionPhase(mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults, succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger)); mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger)); diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java new file mode 100644 index 0000000000000000000000000000000000000000..021646b4890da3e7564d4bbe11b688d21e71a622 --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.ptest.execution; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hive.ptest.execution.conf.TestConfiguration; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; + +import org.slf4j.Logger; + +/** + * Wrapper phase for Yetus check execution kick-off. It will invoke ./dev-support/test-patch.sh + * with the proper arguments and run the check itself in a asynchronous fashion. + */ +public class YetusPhase extends Phase { + + private static final String YETUS_LOG_FILE = "yetus.txt"; + private static final String YETUS_OUTPUT_FOLDER = "yetus"; + private static final String YETUS_EXEC_SCRIPT = "yetus-exec.sh"; + private static final String YETUS_EXEC_VM = "yetus-exec.vm"; + + private final File mPatchFile; + private final File mWorkingDir; + private final File mLogFile; + private final File mOutputDir; + private final File mScratchDir; + private final String buildUrl; + private final TestConfiguration conf; + + + public YetusPhase(TestConfiguration configuration, List hostExecutors, + LocalCommandFactory localCommandFactory, ImmutableMap templateDefaults, + String workingDir, File scratchDir, Logger logger, File logDir, File patchFile) { + + super(hostExecutors, localCommandFactory, templateDefaults, logger); + this.mPatchFile = patchFile; + this.mWorkingDir = new File(workingDir, YETUS_OUTPUT_FOLDER); + this.mLogFile = new File(logDir, YETUS_LOG_FILE); + this.mOutputDir = new File(logDir, YETUS_OUTPUT_FOLDER); + this.mScratchDir = scratchDir; + this.conf = configuration; + this.buildUrl = conf.getLogsURL() + "/" + templateDefaults.get("buildTag") + "/"; + } + + @Override + public void execute() throws Exception { + + Thread t = new Thread(new Runnable() { + @Override + public void run() { + + if (!checkDependencies()) { + return; + } + + File yetusExecScript = new File(mScratchDir, YETUS_EXEC_SCRIPT); + Map templateVars = new HashMap<>(); + templateVars.putAll(getTemplateDefaults()); + templateVars.put("workingDir", mWorkingDir.getAbsolutePath()); + templateVars.put("jiraName", conf.getJiraName()); + templateVars.put("patchFile", mPatchFile.getAbsolutePath()); + templateVars.put("jiraUrl", conf.getJiraUrl()); + templateVars.put("jiraUser", conf.getJiraUser()); + templateVars.put("jiraPass", conf.getJiraPassword()); + templateVars.put("outputDir", mOutputDir.getAbsolutePath()); + templateVars.put("buildUrl", buildUrl); + templateVars.put("buildUrlLog", YETUS_LOG_FILE); + templateVars.put("buildUrlOutputDir", YETUS_OUTPUT_FOLDER); + templateVars.put("logFile", mLogFile.getAbsolutePath()); + + try { + logger.info("Writing {} from template", yetusExecScript); + + Templates.writeTemplateResult(YETUS_EXEC_VM, yetusExecScript, templateVars); + Process proc = new ProcessBuilder().command("bash", yetusExecScript.getPath()).start(); + int exitCode = proc.waitFor(); + + if (exitCode == 0) { + logger.info("Finished processing Yetus check successfully"); + } + } catch (Exception e) { + logger.error("Error processing Yetus check", e); + } finally { + logger.debug("Deleting " + yetusExecScript + ": " + yetusExecScript.delete()); + } + } + }); + t.start(); + logger.info("Started Yetus check.."); + } + + private boolean checkDependencies(){ + + if (mPatchFile == null || !mPatchFile.canRead()) { + logger.error("Cannot run Yetus check - patch file is null or not readable."); + return false; + } + + if (!((mWorkingDir.isDirectory() && mWorkingDir.canWrite()) && + (mOutputDir.isDirectory() && mOutputDir.canWrite()))) { + logger.error("Cannot run Yetus check - output directories not present and writable: " + + "workingDir:%s, outputDir:%s", mWorkingDir.getAbsolutePath(), mOutputDir.getAbsolutePath()); + return false; + } + + if (Strings.isNullOrEmpty(conf.getJiraUrl()) || + Strings.isNullOrEmpty(conf.getJiraName()) || + Strings.isNullOrEmpty(conf.getJiraPassword()) || + Strings.isNullOrEmpty(conf.getJiraUser())) { + logger.error("Cannot run Yetus check - credentials for Jira not provided."); + return false; + } + + return true; + } +} diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm index 7ad50248af02dfaeb6524a61d4895f1a8efba211..58207312fc45430822aa2be73c53b9523aa02ef5 100644 --- a/testutils/ptest2/src/main/resources/source-prep.vm +++ b/testutils/ptest2/src/main/resources/source-prep.vm @@ -78,6 +78,10 @@ cd $workingDir/ echo "Unknown repository type '${repositoryType}'" exit 1 fi + rm -rf ../yetus + mkdir ../yetus + cp -R . ../yetus + mkdir $logDir/yetus patchCommandPath=$workingDir/scratch/smart-apply-patch.sh patchFilePath=$workingDir/scratch/build.patch if [[ -f $patchFilePath ]] diff --git a/testutils/ptest2/src/main/resources/yetus-exec.vm b/testutils/ptest2/src/main/resources/yetus-exec.vm new file mode 100644 index 0000000000000000000000000000000000000000..9dea59e5660f88c420eeaaffc77d5702d646131f --- /dev/null +++ b/testutils/ptest2/src/main/resources/yetus-exec.vm @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### Remember this is a velocity template +set -e +set -x +date +"%Y-%m-%d %T.%3N" + +pushd ${workingDir} +export JIRA_ISSUE=${jiraName} +export JAVA_HOME=${javaHome} +./dev-support/test-patch.sh ${patchFile} --jenkins --jira-base-url=${jiraUrl} --jira-user=${jiraUser} \ + --jira-password=${jiraPass} --patch-dir=${outputDir} --build-url=${buildUrl} --build-url-console=${buildUrlLog} \ + --build-url-artifacts=${buildUrlOutputDir} 2>&1 | tee ${logFile} +popd +