osmo-ci/scripts/obs/lib/debian.py

110 lines
3.5 KiB
Python
Raw Normal View History

scripts/obs: rewrite pushing source pkgs to OBS Harald requested that the OBS scripts should not stop if building one specific source package fails, instead it should keep going and report at the end a non-success exit code. Given that the shell script code has historically grown and became hard to maintain, I decided to rewrite the scripts for implementing this feature. This rewrite solves additional problems: * No full checkout of an OBS project like network:osmocom:latest anymore, with lots of packages that won't get updated (e.g. the uhd package has a uhd-images_3.14.1.1.tar.xz file that is 108 MB). With the old code, developers had to wait minutes during the checkout before the script reaches code that is currently being developed. Now only single packages get checked out right before they get updated. * No need to clone git repositories over and over. With the new code, git repos only get cloned if needed (for latest it is not needed if the remote git tag is the same as the version in OBS). During development, the cloned git repositories are cached. * Output from commands like "git tag -l" is not written to the log unless they failed. This makes the log more readable, which is especially important when a package fails to build, we keep going and need to spot the build error in the middle of the log later on. * No more duplicated code for nightly and latest scripts that worked similar but had slight differences. Also the list of packages is not duplicated for nightly and latest anymore; nightly uses all packages and latest uses packages that have at least one git tag. * Building source packages is decoupled from uploading them. A separate script build_srcpkg.py can be used to just build the deb + rpm spec source packages, without interacting with the OBS server. * The scripts can optionally run in docker with a command-line switch, and this is used by jenkins. This way we don't need to install more dependencies on the host such as rebar3 which is now needed for erlang/osmo_dia2gsup. * Add erlang/osmo_dia2gsup and run its generate_build_dep.sh (SYS#6006) I have done the new implementation in python to make use of argparse and to be able to use try/except and print a trace when building one package fails. Example output: * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_nightly_obs.osmocom.org/48/console * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_latest_obs.osmocom.org/46/console Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9
2022-07-13 10:50:21 +00:00
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2022 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
import datetime
import os
import shlex
import lib
import lib.git
def control_add_depend(project, pkgname, version):
""" :param pkgname: of the meta-package to depend on (e.g. osmocom-nightly)
:param version: of the meta-pkgname to depend on or None """
repo_path = lib.git.get_repo_path(project)
if version:
depend = f"{pkgname} (= {version})"
else:
depend = pkgname
cmd = ["sed", f"s/^Depends: /Depends: {depend}, /", "-i", "debian/control"]
lib.run_cmd(cmd, cwd=repo_path)
def changelog_add_entry(project, feed, version):
""" :param version: for the new changelog entry """
repo_path = lib.git.get_repo_path(project)
changelog_path = f"{repo_path}/debian/changelog"
changelog_old = open(changelog_path).read()
# Package name might be different from project name, read it from changelog
pkgname = changelog_old.split(" ", 1)[0]
assert pkgname
# Debian doesn't allow '-' in version
version = version.replace("-", ".")
# Debian changelog requires this specific date format
date = datetime.datetime.now(datetime.timezone.utc)
date_str = date.strftime("%a, %d %b %Y %H:%M:%S %z")
# Add new changelog entry
with open(changelog_path, "w") as f:
f.write(f"{pkgname} ({version}) unstable; urgency=medium\n")
f.write("\n")
f.write(" * Automatically generated changelog entry for building the"
f" Osmocom {feed} feed\n")
f.write("\n")
f.write(f" -- Osmocom OBS scripts <info@osmocom.org> {date_str}\n")
f.write("\n")
f.write(changelog_old)
def fix_source_format(project):
""" Always use format "3.0 (native)" (e.g. limesuite has "3.0 (quilt)")."""
repo_path = lib.git.get_repo_path(project)
format_path = f"{repo_path}/debian/source/format"
if not os.path.exists(format_path):
return
expected = "3.0 (native)\n"
current = open(format_path, "r").read()
if current == expected:
return
print(f"{project}: fixing debian/source/format ({current.rstrip()} =>"
f" {expected.rstrip()})")
open(format_path, "w").write(expected)
def get_last_version_from_changelog(project):
repo_path = lib.git.get_repo_path(project)
changelog_path = f"{repo_path}/debian/changelog"
if not os.path.exists(changelog_path):
return None
changelog = open(changelog_path).read()
if not changelog:
return None
return changelog.split("(", 1)[1].split(")", 1)[0]
def changelog_add_entry_if_needed(project, feed, version):
""" Adjust the changelog if the version in the changelog is different from
the given version. """
version_changelog = get_last_version_from_changelog(project)
if version_changelog == version:
return
print(f"{project}: adding debian/changelog entry ({version_changelog} =>"
f" {version})")
changelog_add_entry(project, feed, version)
def build_source_package(project):
fix_source_format(project)
print(f"{project}: building debian source package")
lib.run_cmd(["dpkg-buildpackage", "-S", "-uc", "-us", "-d"],
cwd=lib.git.get_repo_path(project))
def move_files_to_output(project):
path_output = lib.get_output_path(project)
lib.run_cmd(f"mv *.tar* *.dsc {shlex.quote(path_output)}", shell=True,
cwd=lib.config.path_cache)