osmocom
/
osmo-ci
mirror of https://gerrit.osmocom.org/osmo-ci
10
0
Fork 1
Code Releases Activity

jobs/gerrit-binpkgs: don't hardcode distros 

Make it possible to set a list of distributions to check in
gerrit-verifications.yml instead of only having one boolean for testing
two hardcoded rpm and deb distributions.

Change-Id: I59487e3dc2f55057de1b6a322f088fff0d18654c
This commit is contained in:
Oliver Smith 2023-07-26 14:34:42 +02:00 committed by osmith
parent bd763747ad
commit 1dcb49a811
3 changed files with 96 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
jobs/gerrit-binpkgs.yml
View File

@ -5,13 +5,10 @@
- project:
name: gerrit-binpkgs
jobs:
- 'gerrit-binpkgs-{type}'
type:
- 'deb'
- 'rpm'
- 'gerrit-binpkgs'
- job-template:
name: 'gerrit-binpkgs-{type}'
- job:
name: 'gerrit-binpkgs'
project-type: freestyle
node: osmocom-gerrit
concurrent: true
@ -21,7 +18,7 @@
days-to-keep: 30
num-to-keep: 1000
description: |
{type} package job of CI for patches sent to
binary packages job (deb/rpm) of CI for patches sent to
<a href="https://gerrit.osmocom.org">gerrit</a>
</br></br>
Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a>
@ -44,6 +41,9 @@
- string:
name: PROJECT_NAME
description: project to test, e.g. 'libosmocore' (set by pipeline)
- string:
name: DISTRO
description: distribution to build for, e.g. 'debian:12' (set by pipeline)
scm:
- git:
@ -62,14 +62,13 @@
builders:
- shell: |-
# Skip rpm build if .spec.in file is missing
if [ "{type}" = "rpm" ] && [ -z "$(find code-from-gerrit -name '*.spec.in')" ]; then
set +x
echo
echo "No *.spec.in file found, skipping rpm build."
echo
exit 0
fi
# Make obvious which distro we build for, for humans and for parsing
# by scripts/jenkins-gerrit/comment_generate.py
set +x
echo
echo "Building binary packages for distro: '$DISTRO'"
echo
set -x
# Clone osmo-ci.git
rm -rf osmo-ci
@ -80,13 +79,6 @@
osmo-ci
git -C osmo-ci log --oneline
# Get distro from type
case "{type}" in
deb) distro="debian:11" ;;
rpm) distro="almalinux:8" ;;
*) echo "unknown package format: '{type}'"; exit 1 ;;
esac
# Move code from gerrit to build_srcpkg.py's git cache
cache_dir=osmo-ci/scripts/obs/_cache
mkdir -p $cache_dir
@ -106,7 +98,7 @@
# Build binary package
./build_binpkg.py \
--docker "$distro" \
--docker "$DISTRO" \
"$PROJECT_NAME"
wrappers:
- ansicolor:

91
jobs/gerrit-verifications.yml
View File

@ -92,20 +92,21 @@
# Which jobs to run in the pipeline
pipeline_build: true
pipeline_lint: true
pipeline_binpkgs: true
pipeline_binpkgs: "almalinux:8 debian:11"
pipeline_endianness: true
# in alphabetical order
repos:
- asn1c:
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_lint: false
- cellmgr-ng
- cellmgr-ng:
pipeline_binpkgs: "debian:11"
- docker-playground:
pipeline_build: false
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_endianness: false
- libasn1c
@ -125,7 +126,7 @@
- libsmpp34
- libtelnet:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- libusrp
@ -143,7 +144,7 @@
-e SMPP="$SMPP" \
-e MGCP="$MGCP" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
pipeline_binpkgs: false
pipeline_binpkgs: ""
- openggsn:
a1_name: GTP
@ -190,6 +191,7 @@
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
pipeline_endianness: false
pipeline_binpkgs: "debian:11"
- osmo-e1-hardware:
a2_name: JOB_TYPE
@ -206,11 +208,11 @@
-e JOB_TYPE="$JOB_TYPE" \
"$DOCKER_IMG" \
{timeout_cmd} /build/contrib/jenkins.sh
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-ci:
pipeline_build: false
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-ggsn:
@ -229,12 +231,13 @@
- osmo-gsm-manuals:
pipeline_endianness: false
pipeline_binpkgs: "debian:11"
- osmo_gsup:
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run_ccache} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-hlr
- osmo-iuh
@ -290,6 +293,7 @@
repos_url: '{gerrit_url}/python/{repos}'
gerrit_project: 'python/{repos}'
pipeline_endianness: false
pipeline_binpkgs: "debian:11"
- osmo-sgsn:
a1_name: IU
@ -309,7 +313,7 @@
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh'
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-sip-connector
@ -336,39 +340,41 @@
"$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
- osmocom-bb:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-tetra:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-sysmon:
pipeline_binpkgs: "debian:11"
- osmo-sysmon
- osmo-remsim
- simtrace2
- osmo-opencm3-projects:
cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh'
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-asf4-dfu:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-ccid-firmware:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-e1d
- osmo-cbc
- osmo-e1-recorder:
docker_img: '$USER/debian-bullseye-jenkins' # needs python2 (OS#5950)
pipeline_binpkgs: false
pipeline_binpkgs: ""
- gapk:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-uecups
- osmo-el2tpd:
pipeline_binpkgs: false
pipeline_binpkgs: ""
- osmo-smlc:
docker_img: '$USER/debian-bullseye-jenkins' # needs python2 (OS#5950)
@ -413,7 +419,7 @@
{timeout_cmd} /build/contrib/jenkins.sh
;;
esac
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-ttcn3-hacks:
@ -431,12 +437,12 @@
su build -c 'make clean';
su build -c 'make compile';
"
pipeline_binpkgs: false
pipeline_binpkgs: ""
pipeline_endianness: false
- dahdi-tools:
pipeline_lint: false
pipeline_binpkgs: false
pipeline_binpkgs: ""
jobs:
- 'gerrit-{repos}'
@ -459,21 +465,26 @@
name: REPO_URL
description: Clone URL, to be passed to jobs started in the pipeline
default: '{obj:repos_url}'
- bool:
- string:
name: PIPELINE_BINPKGS
description: Enable the binpkgs job (build deb/rpm packages)
description: |
Enable the binpkgs job (build deb/rpm packages) for the given space
separated distributions. Empty list disables the binpkgs job. The
distributions must have binary packages in the osmocom:master
repository.
default: '{obj:pipeline_binpkgs}'
- bool:
name: PIPELINE_ENDIANNESS
description: Run struct_endianness.py from libosmocore.git
default: '{obj:pipeline_endianness}'
dsl: |
def run_job(display_name, job_name, comment_type=null) {{
def run_job(display_name, job_name, comment_type=null, distro=null) {{
try {{
echo "pipeline_${{display_name}}: running job ${{job_name}}"
build job: job_name, parameters: [
string(name: "COMMENT_TYPE", value: comment_type),
string(name: "DISTRO", value: distro),
string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
string(name: "GERRIT_CHANGE_NUMBER", value: "${{env.GERRIT_CHANGE_NUMBER}}"),
string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
@ -530,24 +541,42 @@
}}
}}
stage("deb") {{
// Generating binpkgs stages is not possible if we want to run
// them in parallel and not have another job that just waits
// until they complete.
stage("binpkgs-0") {{
when {{
expression {{ params.PIPELINE_BINPKGS }}
expression {{ params.PIPELINE_BINPKGS.split().size() > 0 }}
}}
steps {{
script {{
run_job("deb", "gerrit-binpkgs-deb")
distro = params.PIPELINE_BINPKGS.split()[0]
run_job(distro, "gerrit-binpkgs", null, distro)
}}
}}
}}
stage("rpm") {{
stage("binpkgs-1") {{
when {{
expression {{ params.PIPELINE_BINPKGS }}
expression {{ params.PIPELINE_BINPKGS.split().size() > 1 }}
}}
steps {{
script {{
run_job("rpm", "gerrit-binpkgs-rpm")
distro = params.PIPELINE_BINPKGS.split()[1]
run_job(distro, "gerrit-binpkgs", null, distro)
}}
}}
}}
stage("binpkgs-2") {{
when {{
expression {{ params.PIPELINE_BINPKGS.split().size() > 2 }}
}}
steps {{
script {{
distro = params.PIPELINE_BINPKGS.split()[2]
run_job(distro, "gerrit-binpkgs", null, distro)
}}
}}
}}

28
scripts/jenkins-gerrit/comment_generate.py
View File

@ -9,8 +9,9 @@ import urllib.request
jenkins_url = "https://jenkins.osmocom.org"
re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*")
re_result = re.compile("^pipeline_([a-zA-Z-_0-9]*): (SUCCESS|FAILED)$")
re_result = re.compile("^pipeline_([a-zA-Z-_0-9:]*): (SUCCESS|FAILED)$")
re_job_type = re.compile("JOB_TYPE=([a-zA-Z-_0-9]*),")
re_distro = re.compile("Building binary packages for distro: '([a-zA-Z0-9:].*)'")
def parse_args():
@ -31,16 +32,29 @@ def parse_args():
return parser.parse_args()
def stage_from_job_name(job_name):
def stage_binpkgs_from_url(job_url):
""" Multiple gerrit-binpkgs jobs may be started to build binary packages
for multiple distributions. It is not clear from the job name / URL of
a job for which distro it is building, so read it from the log output.
:returns: a distro like "debian:12" """
global re_distro
url = f"{job_url}/consoleText"
with urllib.request.urlopen(url) as response:
content = response.read().decode("utf-8")
match = re_distro.search(content)
assert match, f"couldn't find distro name in log: {url}"
return match.group(1)
def stage_from_job_name(job_name, job_url):
if job_name == "gerrit-verifications-comment":
# The job that runs this script. Don't include it in the summary.
return None
if job_name == "gerrit-lint":
return "lint"
if job_name == "gerrit-binpkgs-deb":
return "deb"
if job_name == "gerrit-binpkgs-rpm":
return "rpm"
if job_name == "gerrit-binpkgs":
return stage_binpkgs_from_url(job_url)
if job_name == "gerrit-pipeline-endianness":
return "endianness"
if job_name.endswith("-build"):
@ -69,7 +83,7 @@ def parse_pipeline(build_url):
job_name = match.split(" ")[2]
job_id = int(match.split(" ")[3].replace("#", ""))
job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}"
stage = stage_from_job_name(job_name)
stage = stage_from_job_name(job_name, job_url)
if stage:
ret[stage] = {"url": job_url, "name": job_name, "id": job_id}