From 5a81566becde7fa73953c8e7d1dc65a804ddf0ba Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 3 Sep 2019 09:06:30 +0800 Subject: [PATCH] HBASE-22967 Delete all jenkins files on branch-2.0 --- dev-support/Jenkinsfile_GitHub | 220 ------------------ .../flaky-tests/flaky-reporting.Jenkinsfile | 66 ------ .../flaky-tests/run-flaky-tests.Jenkinsfile | 79 ------- 3 files changed, 365 deletions(-) delete mode 100644 dev-support/Jenkinsfile_GitHub delete mode 100644 dev-support/flaky-tests/flaky-reporting.Jenkinsfile delete mode 100644 dev-support/flaky-tests/run-flaky-tests.Jenkinsfile diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub deleted file mode 100644 index 63cadeb2bd..0000000000 --- a/dev-support/Jenkinsfile_GitHub +++ /dev/null @@ -1,220 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pipeline { - - agent { - label 'Hadoop' - } - - options { - // N.B. this is per-branch, which means per PR - disableConcurrentBuilds() - buildDiscarder(logRotator(numToKeepStr: '15')) - timeout (time: 7, unit: 'HOURS') - timestamps() - checkoutToSubdirectory('src') - } - - environment { - SOURCEDIR = 'src' - // will also need to change notification section below - PATCHDIR = 'out' - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" - YETUS='yetus' - // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION='rel/0.10.0' - } - - parameters { - booleanParam(name: 'DEBUG', - defaultValue: false, - description: 'Print extra outputs for debugging the jenkins job and yetus') - } - - stages { - stage ('install yetus') { - steps { - dir("${WORKSPACE}/${YETUS}") { - checkout([ - $class: 'GitSCM', - branches: [[name: "${env.YETUS_VERSION}"]], - userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]] - ) - } - } - } - - stage ('precommit-run') { - steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hbase-at-github.com', - passwordVariable: 'GITHUB_PASSWORD', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hbaseqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash - set -e - TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" - # this must be clean for every run - if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then - rm -rf "${WORKSPACE}/${PATCHDIR}" - fi - mkdir -p "${WORKSPACE}/${PATCHDIR}" - - ## Checking on H* machine nonsense - echo "JAVA_HOME: ${JAVA_HOME}" - ls -l "${JAVA_HOME}" || true - echo "MAVEN_HOME: ${MAVEN_HOME}" - echo "maven version:" - mvn --offline --version || true - echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/" - mkdir "${PATCHDIR}/machine" - cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true - cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true - cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true - cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true - df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true - ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true - ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true - lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true - lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true - cat /proc/loadavg >"${output}/loadavg" 2>&1 || true - ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true - ## /H* - - # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it. - # Otherwise exit, because we don't want HBase to do a - # full build. We wouldn't normally do this check for smaller - # projects. :) - if [[ -z "${CHANGE_URL}" ]]; then - echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html" - exit 0 - fi - # enable debug output for yetus - if [[ "true" = "${DEBUG}" ]]; then - YETUS_ARGS+=("--debug") - fi - YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") - # where the source is located - YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") - # our project defaults come from a personality file - # which will get loaded automatically by setting the project name - YETUS_ARGS+=("--project=hbase") - # lots of different output formats - YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") - YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") - YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") - # enable writing back to Github - YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}") - YETUS_ARGS+=(--github-user=${GITHUB_USER}) - # enable writing back to ASF JIRA - YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}") - YETUS_ARGS+=(--jira-user="${JIRA_USER}") - # auto-kill any surefire stragglers during unit test runs - YETUS_ARGS+=("--reapermode=kill") - YETUS_ARGS+=("--multijdktests=compile") - # set relatively high limits for ASF machines - # changing these to higher values may cause problems - # with other jobs on systemd-enabled machines - YETUS_ARGS+=("--proclimit=10000") - YETUS_ARGS+=("--dockermemlimit=20g") - # -1 findbugs issues that show up prior to the patch being applied - YETUS_ARGS+=("--findbugs-strict-precheck") - # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=rat.txt") - # URL for user-side presentation in reports and such to our artifacts - # (needs to match the archive bits below) - YETUS_ARGS+=("--build-url-artifacts=artifact/out") - # plugins to enable - YETUS_ARGS+=("--plugins=all") - # don't let these tests cause -1s because we aren't really paying that - # much attention to them - YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests") - # run in docker mode and specifically point to our - # Dockerfile since we don't want to use the auto-pulled version. - YETUS_ARGS+=("--docker") - YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") - YETUS_ARGS+=("--mvn-custom-repos") - YETUS_ARGS+=("--jenkins") - YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64") - YETUS_ARGS+=("--findbugs-home=/usr") - YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*") - YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*") - YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh") - YETUS_ARGS+=("--quick-hadoopcheck") - YETUS_ARGS+=("--skip-errorprone") - # effectively treat dev-support as a custom maven module - YETUS_ARGS+=("--skip-dir=dev-support") - # help keep the ASF boxes clean - YETUS_ARGS+=("--sentinel") - "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" - ''' - } - } - } - - } - - post { - always { - script { - // Yetus output - archiveArtifacts "${env.PATCHDIR}/**" - // Publish the HTML report so that it can be looked at - // Has to be relative to WORKSPACE. - publishHTML (target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: "${env.PATCHDIR}", - reportFiles: 'report.html', - reportName: 'Yetus Report' - ]) - // Publish JUnit results - try { - junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" - } catch(e) { - echo 'junit processing: ' + e.toString() - } - } - } - - // Jenkins pipeline jobs fill slaves on PRs without this :( - cleanup() { - script { - sh ''' - # See YETUS-764 - if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then - echo "test-patch process appears to still be running: killing" - kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true - sleep 10 - fi - if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then - echo "test-patch container appears to still be running: killing" - docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true - fi - # See HADOOP-13951 - chmod -R u+rxw "${WORKSPACE}" - ''' - deleteDir() - } - } - } -} diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile deleted file mode 100644 index 75354e8591..0000000000 --- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -pipeline { - agent { - node { - label 'Hadoop' - } - } - triggers { - cron('@daily') - } - options { - buildDiscarder(logRotator(numToKeepStr: '50')) - timeout (time: 15, unit: 'MINUTES') - timestamps() - } - parameters { - booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') - } - stages { - stage ('build flaky report') { - steps { - sh '''#!/usr/bin/env bash - set -e - if [ "${DEBUG}" = "true" ]; then - set -x - fi - declare -a flaky_args - flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5) - flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 30) - docker build -t hbase-dev-support dev-support - docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}" -''' - } - } - } - post { - always { - // Has to be relative to WORKSPACE. - archive "includes,excludes,dashboard.html" - publishHTML target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: ".", - reportFiles: 'dashboard.html', - reportName: 'Flaky Test Report' - ] - } - } -} diff --git a/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile b/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile deleted file mode 100644 index bca95ddb18..0000000000 --- a/dev-support/flaky-tests/run-flaky-tests.Jenkinsfile +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -pipeline { - agent { - node { - label 'Hadoop' - } - } - triggers { - cron('H */4 * * *') // Every four hours. See https://jenkins.io/doc/book/pipeline/syntax/#cron-syntax - } - options { - // this should roughly match how long we tell the flaky dashboard to look at - buildDiscarder(logRotator(numToKeepStr: '30')) - timeout (time: 2, unit: 'HOURS') - timestamps() - } - parameters { - booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') - } - tools { - // this should match what the yetus nightly job for the branch will use - maven 'Maven (latest)' - jdk "JDK 1.8 (latest)" - } - stages { - stage ('run flaky tests') { - steps { - sh '''#!/usr/bin/env bash - set -e - declare -a curl_args=(--fail) - declare -a mvn_args=(--batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dmaven.repo.local="${WORKSPACE}/local-repository") - if [ "${DEBUG}" = "true" ]; then - curl_args=("${curl_args[@]}" -v) - mvn_args=("${mvn_args[@]}" -X) - set -x - fi - curl "${curl_args[@]}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes" - if [ -s includes.txt ]; then - rm -rf local-repository/org/apache/hbase - mvn clean "${mvn_args[@]}" - rm -rf "target/machine" && mkdir -p "target/machine" - if [ -x dev-support/gather_machine_environment.sh ]; then - "./dev-support/gather_machine_environment.sh" "target/machine" - echo "got the following saved stats in 'target/machine'" - ls -lh "target/machine" - else - echo "Skipped gathering machine environment because we couldn't read the script to do so." - fi - mvn package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3 - else - echo "set of flaky tests is currently empty." - fi -''' - } - } - } - post { - always { - junit testResults: "**/surefire-reports/*.xml", allowEmptyResults: true - // TODO compress these logs - archive 'includes.txt,**/surefire-reports/*,**/test-data/*,target/machine/*' - } - } -} -- 2.17.1