diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml new file mode 100644 index 00000000..f13d4b77 --- /dev/null +++ b/jobs/osmocom-depcheck.yml @@ -0,0 +1,72 @@ +--- +- project: + name: Osmocom-depcheck + jobs: + - Osmocom-depcheck + +- job-template: + name: 'Osmocom-depcheck' + project-type: freestyle + defaults: global + description: | + Verifies that Osmocom programs really build with the dependency + versions they claim to support in configure.ac. + (Generated by job-builder) + node: osmocom-master-debian9 + parameters: + - string: + name: PROJECTS + description: | + Which Osmocom projects and revisions to build, leave + empty to default to all projects (!), + default revision is "master". + Examples: "osmo-hlr", "osmo-hlr:0.2.1 osmo-bts:0.8.1" + default: 'osmo-hlr:0.2.1' + - string: + name: GIT_URL_PREFIX + description: | + Where to clone the sources from + default: 'git://git.osmocom.org/' + - bool: + name: BUILD + description: | + Attempt to build the project with the minimum dependency + versions found in the configure.ac files. If this is unchecked, + this job will only clone the git repositories and parse the + configure.ac files. + default: true + - bool: + name: PRINT_OLD_DEPENDS + description: | + Report dependencies on old releases (printed after the other + parsing output, before the build starts) + default: false + - string: + name: BRANCH + description: | + Branch where the osmo-depcheck.py script gets pulled from. + Only modify this if you are hacking on osmo-depcheck.py. + default: '*/master' + builders: + - shell: | + # Build the arguments + args="$PROJECTS" + args="$args -j 5" + args="$args -g $PWD/DEPCHECK_GITDIR" + args="$args -u $GIT_URL_PREFIX" + [ "$BUILD" = "true" ] && args="$args -b" + [ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o" + + # Run osmo-depcheck + mkdir DEPCHECK_GITDIR + export PYTHONUNBUFFERED=1 + scripts/osmo-depcheck/osmo-depcheck.py $args + scm: + - git: + branches: + - '$BRANCH' + url: git://git.osmocom.org/osmo-ci + git-config-name: 'Jenkins Builder' + git-config-email: 'jenkins@osmocom.org' + +# vim: expandtab tabstop=2 shiftwidth=2 diff --git a/scripts/osmo-depcheck/buildstack.py b/scripts/osmo-depcheck/buildstack.py new file mode 100644 index 00000000..87210abd --- /dev/null +++ b/scripts/osmo-depcheck/buildstack.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2018 sysmocom - s.f.m.c. GmbH + +import atexit +import collections +import sys +import os +import shutil +import subprocess +import tempfile + + +def next_buildable(depends, done): + """ Find the next program that can be built, because it has all + dependencies satisfied. Initially this would be libosmocore, as it has + no dependencies, then the only library that depends on libosmocore and + so on. + + :param depends: return value of dependencies.generate() + :param done: ordered dict of programs that would already have been + built at this point. + Example: {"lib-a": "0.11.0", "lib-b": "0.5.0"} + """ + # Iterate over dependencies + for program, data in depends.items(): + # Skip what's already done + if program in done: + continue + + # Check for missing dependencies + depends_done = True + for depend in data["depends"]: + if depend not in done: + depends_done = False + break + + # All dependencies satisfied: we have a winner! + if depends_done: + return program, data["version"] + + # Impossible to build the dependency tree + print_dict(done) + print("ERROR: can't figure out how to build the rest!") + sys.exit(1) + + +def generate(depends): + """ Generate an ordered dictionary with the right build order. + + :param depends: return value of dependencies.generate() + :returns: an ordered dict like the following: + {"libosmocore": "0.11.0", + "libosmo-abis": "0.5.0", + "osmo-bts": "master"} """ + # Iterate over dependencies + ret = collections.OrderedDict() + count = len(depends.keys()) + while len(ret) != count: + # Continue with the one without unsatisfied dependencies + program, version = next_buildable(depends, ret) + ret[program] = version + return ret + + +def print_dict(stack): + """ Print the whole build stack. + :param stack: return value from generate() above """ + print("Build order:") + for program, version in stack.items(): + print(" * " + program + ":" + version) + + +def temp_install_folder(): + """ Generate a temporary installation folder + + It will be used as configure prefix, so when running 'make install', + the files will get copied in there instead of "/usr/local/". The folder + will get removed when the script has finished. + + :returns: the path to the temporary folder """ + ret = tempfile.mkdtemp(prefix="depcheck_") + atexit.register(shutil.rmtree, ret) + print("Temporary install folder: " + ret) + return ret + + +def set_environment(jobs, tempdir): + """ Configure the environment variables before running configure, make etc. + + :param jobs: parallel build jobs (for make) + :param tempdir: temporary installation dir (see temp_install_folder()) + """ + # Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH + extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig", + "LD_LIBRARY_PATH": tempdir + "/lib"} + for env_var, folder in extend.items(): + old = os.environ[env_var] if env_var in os.environ else "" + os.environ[env_var] = old + ":" + folder + + # Set JOBS for make + os.environ["JOBS"] = str(jobs) + + +def build(gitdir, jobs, stack): + """ Build one program with all its dependencies. + + :param gitdir: folder to which the sources will be cloned + :param jobs: parallel build jobs (for make) + :param stack: the build stack as returned by generate() above + + The dependencies.clone() function has already downloaded missing + sources and checked out the right version tags. So in this function we + can directly enter the source folder and run the build commands. + + Notes about the usage of 'make clean' and 'make distclean': + * Without 'make clean' we might have files in the build directory with + a different prefix hardcoded (e.g. from a previous run of + osmo-depcheck): + + * 'make distclean' gets used to remove everything that mentioned the + prefix set by osmo-depcheck. That way the user won't have it set + anymore in case they decide to compile the code again manually from + the source folder. """ + # Prepare the install folder and environment + tempdir = temp_install_folder() + unitdir = tempdir + "/lib/systemd/system/" + set_environment(jobs, tempdir) + + # Iterate over stack + for program, version in stack.items(): + print("Building " + program + ":" + version) + os.chdir(gitdir + "/" + program) + + # Run the build commands + commands = [["autoreconf", "-fi"], + ["./configure", "--prefix", tempdir, + "--with-systemdsystemunitdir=" + unitdir], + ["make", "clean"], + ["make"], + ["make", "install"], + ["make", "distclean"]] + for command in commands: + print("+ " + " ".join(command)) + subprocess.run(command, check=True) diff --git a/scripts/osmo-depcheck/config.py b/scripts/osmo-depcheck/config.py new file mode 100644 index 00000000..3e993bf7 --- /dev/null +++ b/scripts/osmo-depcheck/config.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2018 sysmocom - s.f.m.c. GmbH + +# Where to clone sources from (with trailing slash) +git_url_prefix = "git://git.osmocom.org/" + +# Default projects to build when none are specified on the command line +projects = ("osmo-bts", + "osmo-pcu", + "osmo-hlr", + "osmo-mgw", + "osmo-msc", + "osmo-sgsn", + "osmo-ggsn") + +# Libraries coming from Osmocom repositories (glob patterns) +# All other libraries (e.g. libsystemd) are ignored by this script, even if +# they are mentioned with PKG_CHECK_MODULES in configure.ac. +relevant_library_patterns = ("libasn1c", + "libgtp", + "libosmo*") + + +# Library locations in the git repositories +# Libraries that have the same name as the git repository don't need to be +# listed here. Left: repository name, right: libraries +repos = {"libosmocore": ("libosmocodec", + "libosmocoding", + "libosmoctrl", + "libosmogb", + "libosmogsm", + "libosmosim", + "libosmovty"), + "libosmo-abis": ("libosmoabis", + "libosmotrau"), + "libosmo-sccp": ("libosmo-mtp", + "libosmo-sigtran", + "libosmo-xua"), + "osmo-ggsn": ("libgtp"), + "osmo-hlr": ("libosmo-gsup-client"), + "osmo-iuh": ("libosmo-ranap"), + "osmo-mgw": ("libosmo-mgcp-client", + "libosmo-legacy-mgcp")} diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py new file mode 100644 index 00000000..78cf4a00 --- /dev/null +++ b/scripts/osmo-depcheck/dependencies.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2018 sysmocom - s.f.m.c. GmbH + +import collections +import os +import subprocess +import sys + +# Same folder +import parse + + +def git_clone(gitdir, prefix, repository, version): + """ Clone a missing git repository and checkout a specific version tag. + + :param gitdir: folder to which the sources will be cloned + :param prefix: git url prefix (e.g. "git://git.osmocom.org/") + :param repository: Osmocom git repository name (e.g. "libosmo-abis") + :param version: "master" or a version tag like "0.11.0" """ + # Clone when needed + if not os.path.exists(gitdir + "/" + repository): + url = prefix + repository + print("Cloning git repo: " + url) + try: + subprocess.run(["git", "-C", gitdir, "clone", "-q", url], + check=True) + except subprocess.CalledProcessError: + print("NOTE: if '" + repository + "' is part of a git repository" + " with a different name, please add it to the mapping in" + " 'config.py' and try again.") + sys.exit(1) + + # Checkout the version tag + subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout", + version, "-q"], check=True) + + +def generate(gitdir, prefix, initial, rev): + """ Generate the dependency graph of an Osmocom program by cloning the git + repository, parsing the "configure.ac" file, and recursing. + + :param gitdir: folder to which the sources will be cloned + :param prefix: git url prefix (e.g. "git://git.osmocom.org/") + :param initial: the first program to look at (e.g. "osmo-bts") + :param rev: the git revision to check out ("master", "0.1.0", ...) + :returns: a dictionary like the following: + {"osmo-bts": {"version": "master", + "depends": {"libosmocore": "0.11.0", + "libosmo-abis": "0.5.0"}}, + "libosmocore": {"version": "0.11.0", + "depends": {}}, + "libosmo-abis": {"version": "0.5.0", + "depends": {"libosmocore": "0.11.0"}} """ + # Iterate over stack + stack = collections.OrderedDict({initial: rev}) + ret = collections.OrderedDict() + while len(stack): + # Pop program from stack + program, version = next(iter(stack.items())) + del stack[program] + + # Skip when already parsed + if program in ret: + continue + + # Add the programs dependencies to the stack + print("Looking at " + program + ":" + version) + git_clone(gitdir, prefix, program, version) + depends = parse.configure_ac(gitdir, program) + stack.update(depends) + + # Add the program to the ret + ret[program] = {"version": version, "depends": depends} + + return ret + + +def print_dict(depends): + """ Print the whole dependency graph. + :param depends: return value from generate() above """ + print("Dependency graph:") + + for program, data in depends.items(): + version = data["version"] + depends = data["depends"] + print(" * " + program + ":" + version + " depends: " + str(depends)) + + +def git_latest_tag(gitdir, repository): + """ Get the last release string by asking git for the latest tag. + + :param gitdir: folder to which the sources will be cloned + :param repository: Osmocom git repository name (e.g. "libosmo-abis") + :returns: the latest git tag (e.g. "1.0.2") """ + dir = gitdir + "/" + repository + complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0", + "master"], check=True, stdout=subprocess.PIPE) + return complete.stdout.decode().rstrip() + + +def print_old(gitdir, depends): + """ Print dependencies tied to an old release tag + + :param gitdir: folder to which the sources will be cloned + :param depends: return value from generate() above """ + print("Dependencies on old releases:") + + for program, data in depends.items(): + for depend, version in data["depends"].items(): + latest = git_latest_tag(gitdir, depend) + if latest == version: + continue + print(" * " + program + ":" + data["version"] + " -> " + + depend + ":" + version + " (latest: " + latest + ")") diff --git a/scripts/osmo-depcheck/osmo-depcheck.py b/scripts/osmo-depcheck/osmo-depcheck.py new file mode 100755 index 00000000..92c0ce64 --- /dev/null +++ b/scripts/osmo-depcheck/osmo-depcheck.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2018 sysmocom - s.f.m.c. GmbH + +import argparse +import os +import sys + +# Same folder +import config +import dependencies +import buildstack + + +def parse_arguments(): + # Create argparser + description = ("This script verifies that Osmocom programs really build" + " with the dependency versions they claim to support in" + " configure.ac. In order to do that, it clones the" + " dependency repositories if they don't exist in gitdir" + " already, and checks out the minimum version tag. This" + " happens recursively for their dependencies as well.") + parser = argparse.ArgumentParser(description=description) + + # Git sources folder + gitdir_default = os.path.expanduser("~") + "/code" + parser.add_argument("-g", "--gitdir", default=gitdir_default, + help="folder to which the sources will be cloned" + " (default: " + gitdir_default + ")") + + # Build switch + parser.add_argument("-b", "--build", action="store_true", + help="don't only parse the dependencies, but also try" + " to build the program") + + # Build switch + parser.add_argument("-o", "--old", action="store_true", + help="report dependencies on old releases") + + # Job count + parser.add_argument("-j", "--jobs", type=int, + help="parallel build jobs (for make)") + + # Git URL prefix + parser.add_argument("-u", "--git-url-prefix", dest="prefix", + default=config.git_url_prefix, + help="where to clone the sources from (default: " + + config.git_url_prefix + ")") + + # Projects + parser.add_argument("projects_revs", nargs="*", default=config.projects, + help="which Osmocom projects to look at" + " (e.g. 'osmo-hlr:0.2.1', 'osmo-bts', defaults to" + " all projects defined in config.py, default" + " revision is 'master')", + metavar="project[:revision]") + + # Gitdir must exist + ret = parser.parse_args() + if not os.path.exists(ret.gitdir): + print("ERROR: gitdir does not exist: " + ret.gitdir) + sys.exit(1) + return ret + + +def main(): + # Iterate over projects + args = parse_arguments() + for project_rev in args.projects_revs: + # Split the git revision from the project name + project = project_rev + rev = "master" + if ":" in project_rev: + project, rev = project_rev.split(":", 1) + + # Clone and parse the repositories + depends = dependencies.generate(args.gitdir, args.prefix, project, rev) + print("---") + dependencies.print_dict(depends) + stack = buildstack.generate(depends) + print("---") + buildstack.print_dict(stack) + + # Old versions + if args.old: + print("---") + dependencies.print_old(args.gitdir, depends) + + # Build + if args.build: + print("---") + buildstack.build(args.gitdir, args.jobs, stack) + + # Success + print("---") + print("Success for " + project + ":" + rev + "!") + print("---") + + +if __name__ == '__main__': + main() diff --git a/scripts/osmo-depcheck/parse.py b/scripts/osmo-depcheck/parse.py new file mode 100644 index 00000000..c6297d63 --- /dev/null +++ b/scripts/osmo-depcheck/parse.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2018 sysmocom - s.f.m.c. GmbH + +import sys +import fnmatch + +# Same folder +import config + + +def error(line_i, message): + """ Print a configure.ac error message with the line number. + :param line_i: the zero based line counter """ + print("ERROR: configure.ac line " + str(line_i+1) + ": " + message) + sys.exit(1) + + +def repository(library, version): + """ Find the git repository that contains a certain library. Based on the + information in config.py. + + :param library: the name as referenced in the PKG_CHECK_MODULES + statement. For example: "libosmoabis" + :param version: for example "0.5.0" + :returns: the repository name, e.g. "libosmo-abis" """ + for repo, libraries in config.repos.items(): + if library in libraries: + print(" * " + library + ":" + version + " (part of " + repo + ")") + return repo + + print(" * " + library + ":" + version) + return library + + +def library_is_relevant(library): + """ :returns: True when we would build the library in question from source, + False otherwise. """ + for pattern in config.relevant_library_patterns: + if fnmatch.fnmatch(library, pattern): + return True + return False + + +def parse_condition(line): + """ Find the PKG_CHECK_MODULES conditions in any line from a configure.ac. + + Example lines: + PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 0.10.0) + PKG_CHECK_MODULES(LIBSYSTEMD, libsystemd) + + :returns: * None when there's no condition in that line + * a string like "libosmocore >= 0.1.0" """ + # Only look at PKG_CHECK_MODULES lines + if "PKG_CHECK_MODULES" not in line: + return + + # Extract the condition + ret = line.split(",")[1].split(")")[0].strip() + + # Only look at Osmocom libraries + library = ret.split(" ")[0] + if library_is_relevant(library): + return ret + + +def library_version(line_i, condition): + """ Get the library and version strings from a condition. + :param line_i: the zero based line counter + :param condition: a condition like "libosmocore >= 0.1.0" """ + # Split by space and remove empty list elements + split = list(filter(None, condition.split(" "))) + if len(split) != 3: + error(line_i, "invalid condition format, expected something" + " like 'libosmocore >= 0.10.0' but got: '" + + condition + "'") + library, operator, version = split + + # Right operator + if operator == ">=": + return (library, version) + + # Wrong operator + error(line_i, "invalid operator, expected '>=' but got: '" + + operator + "'") + + +def configure_ac(gitdir, repo): + """ Parse the PKG_CHECK_MODULES statements of a configure.ac file. + + :param gitdir: parent folder of all locally cloned git repositories + :param repo: the repository to look at (e.g. "osmo-bts") + :returns: a dictionary like the following: + {"libosmocore": "0.11.0", + "libosmo-abis": "0.5.0"} """ + # Read configure.ac + path = gitdir + "/" + repo + "/configure.ac" + with open(path) as handle: + lines = handle.readlines() + + # Parse the file into ret + ret = {} + for i in range(0, len(lines)): + # Parse the line + condition = parse_condition(lines[i]) + if not condition: + continue + (library, version) = library_version(i, condition) + + # Add to ret (with duplicate check) + repo_dependency = repository(library, version) + if repo_dependency in ret and version != ret[repo_dependency]: + error(i, "found multiple PKG_CHECK_MODULES statements for " + + repo_dependency + ".git, and they have different" + " versions!") + ret[repo_dependency] = version + return ret