| #!/bin/bash |
| |
| # Copyright (C) 2016 Red Hat, Inc |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| set -e |
| |
| set -x |
| |
| [ -n "$DEBUG" ] && set -x |
| |
| source $(dirname $BASH_SOURCE)/rpm-koji-gating-lib.rc |
| |
| target_label="$1" |
| koji_cmd="${2:-koji}" |
| if [ "$koji_cmd" = "koji" ]; then |
| koji_conf="/etc/koji.conf" |
| elif [ "$koji_cmd" = "cbs" ]; then |
| koji_conf="/etc/koji.conf.d/cbs-koji.conf" |
| else |
| echo "Un-supported koji command !" |
| exit 1 |
| fi |
| |
| if [ ! -f "$koji_conf" ]; then |
| echo "Unable to find koji/cbs configuration file !" |
| exit 1 |
| fi |
| |
| koji_server=$(awk -F "/" '/^server =/ {print $3}' $koji_conf) |
| koji_weburl=$(awk -F " = " '/^weburl = / {print $2}' $koji_conf) |
| koji_topurl=$(awk -F " = " '/^topurl = / {print $2}' $koji_conf) |
| if [ -z "$koji_server" ]; then |
| echo "Unable to grab koji/cbs server from configuration file !" |
| exit 1 |
| fi |
| if [ -z "$koji_weburl" ]; then |
| echo "Unable to grab koji/cbs weburl from configuration file !" |
| exit 1 |
| fi |
| if [ -z "$koji_topurl" ]; then |
| echo "Unable to grab koji/cbs topurl from configuration file !" |
| exit 1 |
| fi |
| |
| # Read from .rc file |
| rpmfactory_server="${SF_HOST}" |
| |
| currentdir=$(dirname "$0") |
| koji_ui_tasks_uri="${koji_weburl}/taskinfo?taskID=" |
| rpmfactory_clone_url="http://${rpmfactory_server}/r" |
| rpmbuild="${HOME}/rpmbuild" |
| |
| zuul_ref=$(echo $ZUUL_REF |awk -F/ '{print $NF}') |
| validatedurl="${koji_topurl}/repos/${target_label}-build/latest/x86_64/" |
| |
| [ -z "${WORKSPACE}" ] && { |
| echo "Please set WORKSPACE environment variable" |
| exit 1 |
| } |
| |
| workdir="${WORKSPACE}/build_env/" |
| temprepopath="${workdir}/${zuul_ref}/x86_64/" |
| url="file://$temprepopath" |
| |
| # ZUUL_CHANGES=p2:master:refs/changes/24/24/1^p1:master:refs/changes/23/23/1 |
| # ZUUL_CHANGES contains changes that must be included in that job |
| CHANGES=(${ZUUL_CHANGES//^/ }) |
| |
| function sanitize() { |
| [ -d "${rpmbuild}" ] && rm -Rf "${rpmbuild}" |
| rpmdev-setuptree |
| [ -d "${workdir}" ] && rm -Rf "${workdir}" |
| mkdir -p $workdir |
| [ -d ${temprepopath} ] && rm -Rf ${temprepopath} |
| mkdir -p $temprepopath |
| } |
| |
| function check_dlrn_placeholder() { |
| if [[ $new_nvr =~ .*XXX.* ]]; then |
| echo "No XXX placeholders in <release>-rdo branches. Please fix your change." |
| return 1 |
| fi |
| } |
| |
| function build_srpm() { |
| # Install openstack-macros which includes compatibility macros that can be required to |
| # build srpm. |
| if [ $(rpm -qa|grep -c openstack-macros) -eq 0 ]; then |
| source /etc/os-release |
| DIST_VER=${VERSION_ID} |
| if [ "${DIST_VER}" == "8" ]; then |
| sudo dnf install -y https://cbs.centos.org/kojifiles/packages/openstack-macros/2020.1.2/1.el8/noarch/openstack-macros-2020.1.2-1.el8.noarch.rpm |
| elif [ "${DIST_VER}" == "9" ]; then |
| sudo dnf install -y https://cbs.centos.org/kojifiles/packages/openstack-macros/2020.1.2/1.el9s/noarch/openstack-macros-2020.1.2-1.el9s.noarch.rpm |
| else |
| sudo yum install -y https://cbs.centos.org/kojifiles/packages/openstack-macros/2019.1.1/0.el7/noarch/openstack-macros-2019.1.1-0.el7.noarch.rpm |
| fi |
| fi |
| # |
| spectool -g ./*.spec -C ${rpmbuild}/SOURCES/ |
| rsync --exclude="*.spec" ./* ${rpmbuild}/SOURCES/ |
| TARGET=${1:-${DIST_VER}} |
| if [ "$1" == "8" ] && [ $(grep -c "BuildRequires:.*python" ./*.spec) -ne 0 ]; then |
| sed -i '1i %{?python_enable_dependency_generator}' ./*.spec |
| fi |
| # Add openstack-macros as BR to include Fedora macros not existing en CentOS |
| if [ "$1" != "7" ] && [ $(grep -c "BuildRequires:.*openstack-macros" ./*.spec) -eq 0 ]; then |
| awk '/^BuildRequires:.*$/ && !x {print "BuildRequires: openstack-macros"; x=1} 1' ./*.spec > newspec.spec.temp |
| mv -f newspec.spec.temp ./*.spec |
| fi |
| # Add ansible-macros as BR to include Ansible macros not existing en CentOS |
| if [ $(grep -c "Name:.*ansible-collection-" ./*.spec) -ne 0 ]; then |
| awk '/^BuildRequires:.*$/ && !x {print "BuildRequires: /usr/lib/rpm/macros.d/macros.ansible"; x=1} 1' ./*.spec > newspec.spec.temp |
| mv -f newspec.spec.temp ./*.spec |
| fi |
| # Check sources integrity running %prep from specs |
| if [ $(grep -c gpgverify ./*.spec) -ne 0 ]; then |
| $(rpmspec -P *spec|grep gpgverify| tr -d \') |
| if [ $? -ne 0 ]; then |
| exit 1 |
| fi |
| fi |
| # TARGET (and therefore distribution) is defined by the host OS. |
| # In theory, TARGET can be passed to this function, but it isn't. |
| # This causes a problem with building el9 packages on CentOS 8 |
| rpmbuild_output=$(rpmbuild --define "dist .el${TARGET}" --define "rhel ${TARGET:0:1}" -bs ./*.spec) |
| } |
| |
| function check_existing_build_on_koji() { |
| local workdir=$1 |
| local nvr=$2 |
| $koji_cmd buildinfo $nvr &> $workdir/check_existing_build_output |
| tid=$(awk '/Task:/ { print $2 }' $workdir/check_existing_build_output) |
| state=$(awk '/State:/ { print $2 }' $workdir/check_existing_build_output) |
| if [ "${state,,}" = "complete" ]; then |
| echo "$tid" |
| else |
| echo "KO" |
| fi |
| } |
| |
| function check_rpmspec_parse(){ |
| if [ -f ${workdir%'_meta'}/*.spec ]; then |
| SPEC_PATH=${workdir%'_meta'}/*.spec |
| else |
| SPEC_PATH=${rpmbuild}/SPECS/*.spec |
| fi |
| rpmspec --parse ${SPEC_PATH} > /dev/null |
| if [ $? != 0 ]; then |
| echo "rpmspec --parse failed. Please fix your change." |
| return 2 |
| fi |
| } |
| |
| |
| function start_build_on_koji() { |
| local srpm=$1 |
| local project=$2 |
| local workdir="$workdir/${2}_meta" |
| [ ! -d ${workdir} ] && mkdir -p ${workdir} |
| local mode="$3" |
| if [[ "$srpm" =~ .*.src.rpm ]]; then |
| local nvr=$(basename $srpm .src.rpm) |
| # Install srpm to make it available for check_rpmspec_parse |
| rpm -ivh $srpm |
| check_rpmspec_parse || return 2 |
| else |
| # if $srpm is not ending in .src.rpm it's building from distgit url |
| get_nvr_from_distgit $srpm |
| local nvr=$nvr_distgit |
| fi |
| local tid=$(check_existing_build_on_koji $workdir $nvr) |
| check_dlrn_placeholder $new_nvr || return 1 |
| if [[ ! $mode =~ .*--scratch.* ]]; then |
| pkg_name=$(echo $nvr|rev|cut -d- -f3-|rev) |
| tag_candidate=$(echo $target_label|cut -d- -f -3)-candidate |
| # When not building in scratch mode, if package is not in the tag, let's add. |
| set +e |
| $koji_cmd list-pkgs --quiet --package=${pkg_name} --tag=${tag_candidate} |
| RET_CODE=$? |
| set -e |
| if [ $RET_CODE -ne 0 ]; then |
| echo "Adding package ${pkg_name} to ${tag_candidate} tag" |
| $koji_cmd add-pkg ${tag_candidate} ${pkg_name} --owner=rdobuilder |
| fi |
| if [[ $tid != "KO" ]]; then |
| echo "Package $nvr already built by task $tid" |
| echo "$tid" > $workdir/tid |
| set +e |
| # When not building in scratch mode, if build is not already tagged, tag it. |
| $koji_cmd list-tagged ${tag_candidate} $pkg_name | grep -w $nvr |
| RET_CODE=$? |
| set -e |
| if [ $RET_CODE -ne 0 ]; then |
| echo "Build $nvr not already tagged to ${tag_candidate}, tag it" |
| $koji_cmd tag-build ${tag_candidate} $nvr |
| fi |
| return 0 |
| fi |
| fi |
| set +e |
| echo "Start build of: $srpm" |
| $koji_cmd build $mode "$target_label" "$srpm" &> $workdir/task_build_output |
| set -e |
| tid=$(grep 'Created' $workdir/task_build_output | awk -F': ' '{print $2}') |
| echo "$tid" > $workdir/tid |
| echo "$tid" > /tmp/tid_report |
| echo "Task id is: $tid" |
| echo "Task console is: ${koji_ui_tasks_uri}${tid}" |
| } |
| |
| function check_build_on_koji() { |
| local project=$1 |
| local workdir="$workdir/${project}_meta" |
| local tid=$(cat $workdir/tid) |
| $koji_cmd taskinfo -vr "$tid" &> $workdir/task_status_output |
| state=$(egrep "^State:" $workdir/task_status_output | awk -F': ' '{print $2}') |
| nvr=$(sed -n 's@Source:.*/\(.*\).src.rpm@\1@p' $workdir/task_status_output) |
| |
| case "${state,,}" in |
| "failed") |
| echo "Task $tid failed with status: $state" |
| touch $workdir/failed |
| ;; |
| "open" | "free") |
| echo "Task $tid is processing: $state ..." |
| ;; |
| "closed" | "complete") |
| echo "Task $tid is done: $state ..." |
| touch $workdir/built |
| ;; |
| *) |
| echo "Unknown status for task $tid" |
| ;; |
| esac |
| } |
| |
| function fetch_projects() { |
| echo -e "\n--- Fetch $project at the right revision ---" |
| |
| if [[ -d $ZUUL3_HOME ]]; then |
| mkdir -p $workdir |
| pushd ~/src/review.rdoproject.org/ |
| cp -pr * $workdir |
| popd |
| else |
| for chg in ${CHANGES[@]}; do |
| project=$(echo $chg | cut -d':' -f1) |
| zuul-cloner --workspace $workdir $rpmfactory_clone_url $project |
| done |
| fi |
| } |
| |
| function build_srpms() { |
| for chg in ${CHANGES[@]}; do |
| project=$(echo $chg | cut -d':' -f1) |
| pushd "${workdir}/${project}" > /dev/null |
| if [ -f ./*.spec ]; then |
| if [ $(grep -c "Version:.*XXX" ./*.spec) -eq 0 ]; then |
| echo -e "\n--- Build SRPM for $project ---" |
| git log --simplify-merges -n1 |
| build_srpm $TARGET |
| fpname=$(echo $rpmbuild_output | awk -F'/' '{print $NF}') |
| pname=$(echo $fpname|rev|cut -d "-" -f3-|rev) |
| mkdir -p "${workdir}/${project}_meta/" |
| echo $pname > "${workdir}/${project}_meta/pname" |
| else |
| echo "WARNING: Project ${project} has XXX Version" |
| exit 0 |
| fi |
| else |
| echo "WARNING: Project ${project} has no spec file and will be skipped" |
| fi |
| popd > /dev/null |
| done |
| } |
| |
| function build_all_on_koji() { |
| mode="--scratch" |
| for chg in ${CHANGES[@]}; do |
| project=$(echo $chg | cut -d':' -f1) |
| if [ -d ${workdir}/${project}_meta ]; then |
| echo -e "\n--- Start koji build for $project ---" |
| pname=$(cat ${workdir}/${project}_meta/pname) |
| srpm=$(ls ${rpmbuild}/SRPMS/${pname}*.src.rpm) |
| start_build_on_koji $srpm $project $mode |
| fi |
| done |
| } |
| |
| function wait_for_all_built_on_koji() { |
| while true; do |
| allbuilt=true |
| for chg in ${CHANGES[@]}; do |
| project=$(echo $chg | cut -d':' -f1) |
| if [ -d ${workdir}/${project}_meta ]; then |
| echo -e "\n--- Check koji build for $project ---" |
| if [ ! -f "$workdir/${project}_meta/built" ]; then |
| allbuilt=false |
| check_build_on_koji $project |
| if [ -f "$workdir/${project}_meta/failed" ]; then |
| echo "Build of project $project failed. Abort !" |
| exit 1 |
| fi |
| else |
| echo "Already built. Skip." |
| fi |
| fi |
| done |
| if $allbuilt; then |
| echo "All packages have been built" |
| break |
| fi |
| sleep 10 |
| done |
| } |
| |
| function fetch_rpms() { |
| pushd $temprepopath > /dev/null |
| for chg in ${CHANGES[@]}; do |
| project=$(echo $chg | cut -d':' -f1) |
| if [ -d ${workdir}/${project}_meta ]; then |
| echo -e "\n--- Fetchs RPMs for $project ---" |
| tid=$(cat $workdir/${project}_meta/tid) |
| taskinfo=$(cbs taskinfo -r -v $tid |grep -e Task: -e Type: -e "Build Arch:"|paste -sd ' '|sed 's/Task: /\n/g') |
| if [[ `echo -e "$taskinfo" | grep -c buildArch` -gt 1 ]]; then |
| tid=$(echo -e "$taskinfo" | grep -e x86_64 | awk '{print $1}') |
| fi |
| $koji_cmd download-task $tid |
| fi |
| done |
| popd |
| } |
| |
| function create_local_repo() { |
| echo -e "\n--- Create a local repository ---" |
| find $temprepopath \( -name "*aarch64.rpm" -o -name "*ppc64le.rpm" \) -exec rm {} \; |
| createrepo $temprepopath |
| find $temprepopath |
| # Build release RPM with temporary repo included |
| $currentdir/build-release-rpm.sh $url $validatedurl &> /dev/null |
| rm rpmfactory-temp-release.spec |
| } |
| |
| function get_nvr_from_distgit(){ |
| # In CBS distgit url to build is format like: |
| # git+https://git.centos.org/rpms/centos-release-openstack.git#de53a1a48a1c8854735b58f00ddfe8771c315c6e |
| local url=$1 |
| local tempdir=$(mktemp -d) |
| local repo_url=$(echo $url|awk -F'#' '{print $1}'|sed 's/git+//') |
| local repo_hash=$(echo $url|awk -F'#' '{print $2}') |
| git clone $repo_url $tempdir/c_distgit |
| pushd $tempdir/c_distgit |
| git checkout $repo_hash |
| local centos_version=$(basename $(git branch -a --contains $repo_hash|grep remotes|head -1)| cut -d- -f1 |cut -c 2-) |
| nvr_distgit=$(rpmspec -D "%dist .el${centos_version}" -q SPECS/*spec --queryformat="%{NAME}-%{VERSION}-%{RELEASE}\n"|head -1) |
| popd |
| rm -rf $tempdir |
| } |