rpm-koji-gating-lib.common

#!/bin/bash

# Copyright (C) 2016 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

set -e

set -x

[ -n "$DEBUG" ] && set -x

source $(dirname $BASH_SOURCE)/rpm-koji-gating-lib.rc

target_label="$1"
koji_cmd="${2:-koji}"
if [ "$koji_cmd" = "koji" ]; then
    koji_conf="/etc/koji.conf"
elif [ "$koji_cmd" = "cbs" ]; then
    koji_conf="/etc/koji.conf.d/cbs-koji.conf"
else
    echo "Un-supported koji command !"
    exit 1
fi

if [ ! -f "$koji_conf" ]; then
    echo "Unable to find koji/cbs configuration file !"
    exit 1
fi

koji_server=$(awk -F "/" '/^server =/ {print $3}' $koji_conf)
koji_weburl=$(awk -F " = " '/^weburl = / {print $2}' $koji_conf)
koji_topurl=$(awk -F " = " '/^topurl = / {print $2}' $koji_conf)
if [ -z "$koji_server" ]; then
    echo "Unable to grab koji/cbs server from configuration file !"
    exit 1
fi
if [ -z "$koji_weburl" ]; then
    echo "Unable to grab koji/cbs weburl from configuration file !"
    exit 1
fi
if [ -z "$koji_topurl" ]; then
    echo "Unable to grab koji/cbs topurl from configuration file !"
    exit 1
fi

# Read from .rc file
rpmfactory_server="${SF_HOST}"

currentdir=$(dirname "$0")
koji_ui_tasks_uri="${koji_weburl}/taskinfo?taskID="
rpmfactory_clone_url="http://${rpmfactory_server}/r"
rpmbuild="${HOME}/rpmbuild"

zuul_ref=$(echo $ZUUL_REF |awk -F/ '{print $NF}')
validatedurl="${koji_topurl}/repos/${target_label}-build/latest/x86_64/"

[ -z "${WORKSPACE}" ] && {
  echo "Please set WORKSPACE environment variable"
  exit 1
}

workdir="${WORKSPACE}/build_env/"
temprepopath="${workdir}/${zuul_ref}/x86_64/"
url="file://$temprepopath" 

# ZUUL_CHANGES=p2:master:refs/changes/24/24/1^p1:master:refs/changes/23/23/1
# ZUUL_CHANGES contains changes that must be included in that job
CHANGES=(${ZUUL_CHANGES//^/ })

function sanitize() {
  [ -d "${rpmbuild}" ] && rm -Rf "${rpmbuild}"
  rpmdev-setuptree
  [ -d "${workdir}" ] && rm -Rf "${workdir}"
  mkdir -p $workdir
  [ -d ${temprepopath} ] && rm -Rf ${temprepopath}
  mkdir -p $temprepopath
}

function check_dlrn_placeholder() {
  if [[ $new_nvr =~ .*XXX.* ]]; then
    echo "No XXX placeholders in <release>-rdo branches. Please fix your change."
    return 1
  fi
}

function build_srpm() {
  # Install openstack-macros which includes compatibility macros that can be required to
  # build srpm.
  if [ $(rpm -qa|grep -c openstack-macros) -eq 0 ]; then
    source /etc/os-release
    DIST_VER=${VERSION_ID}
    if [ "${DIST_VER}" == "8" ]; then
    sudo dnf install -y https://cbs.centos.org/kojifiles/packages/openstack-macros/2020.1.2/1.el8/noarch/openstack-macros-2020.1.2-1.el8.noarch.rpm
    else
    sudo yum install -y https://cbs.centos.org/kojifiles/packages/openstack-macros/2019.1.1/0.el7/noarch/openstack-macros-2019.1.1-0.el7.noarch.rpm
    fi
  fi
  #
  spectool -g ./*.spec -C ${rpmbuild}/SOURCES/
  rsync --exclude="*.spec" ./* ${rpmbuild}/SOURCES/
  TARGET=${1:-${DIST_VER}}
  if [ "$1" == "8" ] && [ $(grep -c "BuildRequires:.*python" ./*.spec) -ne 0 ]; then
    sed -i '1i %{?python_enable_dependency_generator}' ./*.spec
  fi
  # Add openstack-macros as BR to include Fedora macros not existing en CentOS
  if [ "$1" == "8" ] && [ $(grep -c "BuildRequires:.*openstack-macros" ./*.spec) -eq 0 ]; then
    awk '/^BuildRequires:.*$/ && !x {print "BuildRequires: openstack-macros"; x=1} 1' ./*.spec > newspec.spec.temp
    mv -f newspec.spec.temp ./*.spec
  fi
  # Check sources integrity running %prep from specs
  if [ $(grep -c gpgverify ./*.spec) -ne 0 ]; then
    $(rpmspec -P *spec|grep gpgverify| tr -d \')
    if [ $? -ne 0 ]; then
      exit 1
    fi
  fi
  #
  rpmbuild_output=$(rpmbuild --define "dist .el${TARGET}" -bs ./*.spec)
}

function check_existing_build_on_koji() {
  local workdir=$1
  local nvr=$2
  $koji_cmd buildinfo $nvr &> $workdir/check_existing_build_output
  tid=$(awk '/Task:/ { print $2 }' $workdir/check_existing_build_output)
  state=$(awk '/State:/ { print $2 }' $workdir/check_existing_build_output)
  if [ "${state,,}" = "complete" ]; then
    echo "$tid"
  else
    echo "KO"
  fi
}

function check_rpmspec_parse(){
  if [ -f ${workdir%'_meta'}/*.spec ]; then
    SPEC_PATH=${workdir%'_meta'}/*.spec
  else
    SPEC_PATH=${rpmbuild}/SPECS/*.spec
  fi
  rpmspec --parse ${SPEC_PATH} > /dev/null
  if [ $? != 0 ]; then
    echo "rpmspec --parse failed.  Please fix your change."
    return 2
  fi
}


function start_build_on_koji() {
  local srpm=$1
  local project=$2
  local workdir="$workdir/${2}_meta"
  [ ! -d ${workdir} ] && mkdir -p ${workdir}
  local mode="$3"
  local nvr=$(basename $srpm .src.rpm)
  local tid=$(check_existing_build_on_koji $workdir $nvr)
  check_dlrn_placeholder $new_nvr || return 1
  check_rpmspec_parse || return 2
  if [[ ! $mode =~ ".*--scratch.*" ]] && [[ $tid != "KO" ]]; then
    echo "Package $nvr already built by task $tid"
    echo "$tid" > $workdir/tid
    return 0
  fi
  # When not building in scratch mode, if package is not in the tag, let's add.
  set +e
  if [[ ! $mode =~ ".*--scratch.*" ]]; then
    set -x
    pkg_name=$(rpm -q --qf %{NAME} -p $srpm)
    tag_candidate=$(echo $target_label|cut -d- -f -3)-candidate
    $koji_cmd list-pkgs --quiet --package=${pkg_name} --tag=${tag_candidate}
    if [ $? -ne 0 ]; then
      echo "Adding package ${pkg_name} to ${tag_candidate} tag"
      $koji_cmd add-pkg ${tag_candidate} ${pkg_name} --owner=rdobuilder
    fi
    set +x
  fi
  echo "Start build of: $srpm"
  $koji_cmd build $mode "$target_label" "$srpm" &> $workdir/task_build_output
  set -e
  tid=$(grep 'Created' $workdir/task_build_output | awk -F': ' '{print $2}')
  echo "$tid" > $workdir/tid
  echo "Task id is: $tid"
  echo "Task console is: ${koji_ui_tasks_uri}${tid}"
}

function check_build_on_koji() {
  local project=$1
  local workdir="$workdir/${project}_meta"
  local tid=$(cat $workdir/tid)
  $koji_cmd taskinfo -vr "$tid" &> $workdir/task_status_output
  state=$(egrep "^State:" $workdir/task_status_output | awk -F': ' '{print $2}')
  nvr=$(sed -n 's@Source:.*/\(.*\).src.rpm@\1@p' $workdir/task_status_output)

  case "${state,,}" in
    "failed")
      echo "Task $tid failed with status: $state"
      touch $workdir/failed
      ;;
    "open" | "free")
      echo "Task $tid is processing: $state ..."
      ;;
    "closed" | "complete")
      echo "Task $tid is done: $state ..."
      touch $workdir/built
      ;;
    *)
      echo "Unknown status for task $tid"
      ;;
  esac
}

function fetch_projects() {
  echo -e "\n--- Fetch $project at the right revision ---"

  if [[ -d /home/zuul ]]; then
    mkdir -p $workdir
    pushd ~/src/review.rdoproject.org/
    cp -pr * $workdir
    popd
  else
    for chg in ${CHANGES[@]}; do
      project=$(echo $chg | cut -d':' -f1)
      zuul-cloner --workspace $workdir $rpmfactory_clone_url $project
    done
  fi
}

function build_srpms() {
  for chg in ${CHANGES[@]}; do
    project=$(echo $chg | cut -d':' -f1)
    pushd "${workdir}/${project}" > /dev/null
    if [ -f ./*.spec ]; then
        echo -e "\n--- Build SRPM for $project ---"
        git log --simplify-merges -n1
        build_srpm
        fpname=$(echo $rpmbuild_output | awk -F'/' '{print $NF}')
        pname=$(echo $fpname|rev|cut -d "-" -f3-|rev)
        mkdir -p "${workdir}/${project}_meta/"
        echo $pname > "${workdir}/${project}_meta/pname"
    else
        echo "WARNING: Project ${project} has no spec file and will be skipped"
    fi
    popd > /dev/null
  done
}

function build_all_on_koji() {
  mode="--scratch"
  for chg in ${CHANGES[@]}; do
    project=$(echo $chg | cut -d':' -f1)
    if [ -d ${workdir}/${project}_meta ]; then
        echo -e "\n--- Start koji build for $project ---"
        pname=$(cat ${workdir}/${project}_meta/pname)
        srpm=$(ls ${rpmbuild}/SRPMS/${pname}*.src.rpm)
        start_build_on_koji $srpm $project $mode
    fi
  done
}

function wait_for_all_built_on_koji() {
  while true; do
    allbuilt=true
    for chg in ${CHANGES[@]}; do
      project=$(echo $chg | cut -d':' -f1)
      if [ -d ${workdir}/${project}_meta ]; then
        echo -e "\n--- Check koji build for $project ---"
        if [ ! -f "$workdir/${project}_meta/built" ]; then
            allbuilt=false
            check_build_on_koji $project
            if [ -f "$workdir/${project}_meta/failed" ]; then
            echo "Build of project $project failed. Abort !"
            exit 1
            fi
        else
            echo "Already built. Skip."
        fi
      fi
    done
    if $allbuilt; then
      echo "All packages have been built"
      break
    fi
    sleep 10
  done
}

function fetch_rpms() {
  pushd $temprepopath > /dev/null
  for chg in ${CHANGES[@]}; do
    project=$(echo $chg | cut -d':' -f1)
    if [ -d ${workdir}/${project}_meta ]; then
        echo -e "\n--- Fetchs RPMs for $project ---"
        tid=$(cat $workdir/${project}_meta/tid)
        $koji_cmd download-task $tid
    fi
  done
  popd 
}

function create_local_repo() {
  echo -e "\n--- Create a local repository ---"
  createrepo $temprepopath
  find $temprepopath
  # Build release RPM with temporary repo included
  $currentdir/build-release-rpm.sh $url $validatedurl &> /dev/null
  rm rpmfactory-temp-release.spec
}