osmo-depcheck: script to verify PKG_CHECK_MODULES

This script verifies that Osomcom programs really build with the
dependency versions they claim to support in configure.ac. In order to
do that, it clones the dependency repositories if they don't exist
already, and checks out the minimum version tag. This happens
recursively for their dependencies as well. See 'osmo-depcheck.py -h'
for the full usage instructions.

There's also a new jenkins job in jobs/osmocom-depcheck.yml.

Change-Id: I8f495dbe030775f66ac125e60ded95c5d7660b65
Relates: OS#2642
This commit is contained in:
Oliver Smith 2018-09-13 15:39:44 +02:00
parent 7fab6f5412
commit 85c2effd89
6 changed files with 590 additions and 0 deletions

72
jobs/osmocom-depcheck.yml Normal file
View File

@ -0,0 +1,72 @@
---
- project:
name: Osmocom-depcheck
jobs:
- Osmocom-depcheck
- job-template:
name: 'Osmocom-depcheck'
project-type: freestyle
defaults: global
description: |
Verifies that Osmocom programs really build with the dependency
versions they claim to support in configure.ac.
(Generated by job-builder)
node: osmocom-master-debian9
parameters:
- string:
name: PROJECTS
description: |
Which Osmocom projects and revisions to build, leave
empty to default to all projects (!),
default revision is "master".
Examples: "osmo-hlr", "osmo-hlr:0.2.1 osmo-bts:0.8.1"
default: 'osmo-hlr:0.2.1'
- string:
name: GIT_URL_PREFIX
description: |
Where to clone the sources from
default: 'git://git.osmocom.org/'
- bool:
name: BUILD
description: |
Attempt to build the project with the minimum dependency
versions found in the configure.ac files. If this is unchecked,
this job will only clone the git repositories and parse the
configure.ac files.
default: true
- bool:
name: PRINT_OLD_DEPENDS
description: |
Report dependencies on old releases (printed after the other
parsing output, before the build starts)
default: false
- string:
name: BRANCH
description: |
Branch where the osmo-depcheck.py script gets pulled from.
Only modify this if you are hacking on osmo-depcheck.py.
default: '*/master'
builders:
- shell: |
# Build the arguments
args="$PROJECTS"
args="$args -j 5"
args="$args -g $PWD/DEPCHECK_GITDIR"
args="$args -u $GIT_URL_PREFIX"
[ "$BUILD" = "true" ] && args="$args -b"
[ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
# Run osmo-depcheck
mkdir DEPCHECK_GITDIR
export PYTHONUNBUFFERED=1
scripts/osmo-depcheck/osmo-depcheck.py $args
scm:
- git:
branches:
- '$BRANCH'
url: git://git.osmocom.org/osmo-ci
git-config-name: 'Jenkins Builder'
git-config-email: 'jenkins@osmocom.org'
# vim: expandtab tabstop=2 shiftwidth=2

View File

@ -0,0 +1,144 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
import atexit
import collections
import sys
import os
import shutil
import subprocess
import tempfile
def next_buildable(depends, done):
""" Find the next program that can be built, because it has all
dependencies satisfied. Initially this would be libosmocore, as it has
no dependencies, then the only library that depends on libosmocore and
so on.
:param depends: return value of dependencies.generate()
:param done: ordered dict of programs that would already have been
built at this point.
Example: {"lib-a": "0.11.0", "lib-b": "0.5.0"}
"""
# Iterate over dependencies
for program, data in depends.items():
# Skip what's already done
if program in done:
continue
# Check for missing dependencies
depends_done = True
for depend in data["depends"]:
if depend not in done:
depends_done = False
break
# All dependencies satisfied: we have a winner!
if depends_done:
return program, data["version"]
# Impossible to build the dependency tree
print_dict(done)
print("ERROR: can't figure out how to build the rest!")
sys.exit(1)
def generate(depends):
""" Generate an ordered dictionary with the right build order.
:param depends: return value of dependencies.generate()
:returns: an ordered dict like the following:
{"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0",
"osmo-bts": "master"} """
# Iterate over dependencies
ret = collections.OrderedDict()
count = len(depends.keys())
while len(ret) != count:
# Continue with the one without unsatisfied dependencies
program, version = next_buildable(depends, ret)
ret[program] = version
return ret
def print_dict(stack):
""" Print the whole build stack.
:param stack: return value from generate() above """
print("Build order:")
for program, version in stack.items():
print(" * " + program + ":" + version)
def temp_install_folder():
""" Generate a temporary installation folder
It will be used as configure prefix, so when running 'make install',
the files will get copied in there instead of "/usr/local/". The folder
will get removed when the script has finished.
:returns: the path to the temporary folder """
ret = tempfile.mkdtemp(prefix="depcheck_")
atexit.register(shutil.rmtree, ret)
print("Temporary install folder: " + ret)
return ret
def set_environment(jobs, tempdir):
""" Configure the environment variables before running configure, make etc.
:param jobs: parallel build jobs (for make)
:param tempdir: temporary installation dir (see temp_install_folder())
"""
# Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH
extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig",
"LD_LIBRARY_PATH": tempdir + "/lib"}
for env_var, folder in extend.items():
old = os.environ[env_var] if env_var in os.environ else ""
os.environ[env_var] = old + ":" + folder
# Set JOBS for make
os.environ["JOBS"] = str(jobs)
def build(gitdir, jobs, stack):
""" Build one program with all its dependencies.
:param gitdir: folder to which the sources will be cloned
:param jobs: parallel build jobs (for make)
:param stack: the build stack as returned by generate() above
The dependencies.clone() function has already downloaded missing
sources and checked out the right version tags. So in this function we
can directly enter the source folder and run the build commands.
Notes about the usage of 'make clean' and 'make distclean':
* Without 'make clean' we might have files in the build directory with
a different prefix hardcoded (e.g. from a previous run of
osmo-depcheck):
<https://lists.gnu.org/archive/html/libtool/2006-12/msg00011.html>
* 'make distclean' gets used to remove everything that mentioned the
prefix set by osmo-depcheck. That way the user won't have it set
anymore in case they decide to compile the code again manually from
the source folder. """
# Prepare the install folder and environment
tempdir = temp_install_folder()
unitdir = tempdir + "/lib/systemd/system/"
set_environment(jobs, tempdir)
# Iterate over stack
for program, version in stack.items():
print("Building " + program + ":" + version)
os.chdir(gitdir + "/" + program)
# Run the build commands
commands = [["autoreconf", "-fi"],
["./configure", "--prefix", tempdir,
"--with-systemdsystemunitdir=" + unitdir],
["make", "clean"],
["make"],
["make", "install"],
["make", "distclean"]]
for command in commands:
print("+ " + " ".join(command))
subprocess.run(command, check=True)

View File

@ -0,0 +1,43 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
# Where to clone sources from (with trailing slash)
git_url_prefix = "git://git.osmocom.org/"
# Default projects to build when none are specified on the command line
projects = ("osmo-bts",
"osmo-pcu",
"osmo-hlr",
"osmo-mgw",
"osmo-msc",
"osmo-sgsn",
"osmo-ggsn")
# Libraries coming from Osmocom repositories (glob patterns)
# All other libraries (e.g. libsystemd) are ignored by this script, even if
# they are mentioned with PKG_CHECK_MODULES in configure.ac.
relevant_library_patterns = ("libasn1c",
"libgtp",
"libosmo*")
# Library locations in the git repositories
# Libraries that have the same name as the git repository don't need to be
# listed here. Left: repository name, right: libraries
repos = {"libosmocore": ("libosmocodec",
"libosmocoding",
"libosmoctrl",
"libosmogb",
"libosmogsm",
"libosmosim",
"libosmovty"),
"libosmo-abis": ("libosmoabis",
"libosmotrau"),
"libosmo-sccp": ("libosmo-mtp",
"libosmo-sigtran",
"libosmo-xua"),
"osmo-ggsn": ("libgtp"),
"osmo-hlr": ("libosmo-gsup-client"),
"osmo-iuh": ("libosmo-ranap"),
"osmo-mgw": ("libosmo-mgcp-client",
"libosmo-legacy-mgcp")}

View File

@ -0,0 +1,114 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
import collections
import os
import subprocess
import sys
# Same folder
import parse
def git_clone(gitdir, prefix, repository, version):
""" Clone a missing git repository and checkout a specific version tag.
:param gitdir: folder to which the sources will be cloned
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:param version: "master" or a version tag like "0.11.0" """
# Clone when needed
if not os.path.exists(gitdir + "/" + repository):
url = prefix + repository
print("Cloning git repo: " + url)
try:
subprocess.run(["git", "-C", gitdir, "clone", "-q", url],
check=True)
except subprocess.CalledProcessError:
print("NOTE: if '" + repository + "' is part of a git repository"
" with a different name, please add it to the mapping in"
" 'config.py' and try again.")
sys.exit(1)
# Checkout the version tag
subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout",
version, "-q"], check=True)
def generate(gitdir, prefix, initial, rev):
""" Generate the dependency graph of an Osmocom program by cloning the git
repository, parsing the "configure.ac" file, and recursing.
:param gitdir: folder to which the sources will be cloned
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param initial: the first program to look at (e.g. "osmo-bts")
:param rev: the git revision to check out ("master", "0.1.0", ...)
:returns: a dictionary like the following:
{"osmo-bts": {"version": "master",
"depends": {"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0"}},
"libosmocore": {"version": "0.11.0",
"depends": {}},
"libosmo-abis": {"version": "0.5.0",
"depends": {"libosmocore": "0.11.0"}} """
# Iterate over stack
stack = collections.OrderedDict({initial: rev})
ret = collections.OrderedDict()
while len(stack):
# Pop program from stack
program, version = next(iter(stack.items()))
del stack[program]
# Skip when already parsed
if program in ret:
continue
# Add the programs dependencies to the stack
print("Looking at " + program + ":" + version)
git_clone(gitdir, prefix, program, version)
depends = parse.configure_ac(gitdir, program)
stack.update(depends)
# Add the program to the ret
ret[program] = {"version": version, "depends": depends}
return ret
def print_dict(depends):
""" Print the whole dependency graph.
:param depends: return value from generate() above """
print("Dependency graph:")
for program, data in depends.items():
version = data["version"]
depends = data["depends"]
print(" * " + program + ":" + version + " depends: " + str(depends))
def git_latest_tag(gitdir, repository):
""" Get the last release string by asking git for the latest tag.
:param gitdir: folder to which the sources will be cloned
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:returns: the latest git tag (e.g. "1.0.2") """
dir = gitdir + "/" + repository
complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
"master"], check=True, stdout=subprocess.PIPE)
return complete.stdout.decode().rstrip()
def print_old(gitdir, depends):
""" Print dependencies tied to an old release tag
:param gitdir: folder to which the sources will be cloned
:param depends: return value from generate() above """
print("Dependencies on old releases:")
for program, data in depends.items():
for depend, version in data["depends"].items():
latest = git_latest_tag(gitdir, depend)
if latest == version:
continue
print(" * " + program + ":" + data["version"] + " -> " +
depend + ":" + version + " (latest: " + latest + ")")

View File

@ -0,0 +1,101 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
import argparse
import os
import sys
# Same folder
import config
import dependencies
import buildstack
def parse_arguments():
# Create argparser
description = ("This script verifies that Osmocom programs really build"
" with the dependency versions they claim to support in"
" configure.ac. In order to do that, it clones the"
" dependency repositories if they don't exist in gitdir"
" already, and checks out the minimum version tag. This"
" happens recursively for their dependencies as well.")
parser = argparse.ArgumentParser(description=description)
# Git sources folder
gitdir_default = os.path.expanduser("~") + "/code"
parser.add_argument("-g", "--gitdir", default=gitdir_default,
help="folder to which the sources will be cloned"
" (default: " + gitdir_default + ")")
# Build switch
parser.add_argument("-b", "--build", action="store_true",
help="don't only parse the dependencies, but also try"
" to build the program")
# Build switch
parser.add_argument("-o", "--old", action="store_true",
help="report dependencies on old releases")
# Job count
parser.add_argument("-j", "--jobs", type=int,
help="parallel build jobs (for make)")
# Git URL prefix
parser.add_argument("-u", "--git-url-prefix", dest="prefix",
default=config.git_url_prefix,
help="where to clone the sources from (default: " +
config.git_url_prefix + ")")
# Projects
parser.add_argument("projects_revs", nargs="*", default=config.projects,
help="which Osmocom projects to look at"
" (e.g. 'osmo-hlr:0.2.1', 'osmo-bts', defaults to"
" all projects defined in config.py, default"
" revision is 'master')",
metavar="project[:revision]")
# Gitdir must exist
ret = parser.parse_args()
if not os.path.exists(ret.gitdir):
print("ERROR: gitdir does not exist: " + ret.gitdir)
sys.exit(1)
return ret
def main():
# Iterate over projects
args = parse_arguments()
for project_rev in args.projects_revs:
# Split the git revision from the project name
project = project_rev
rev = "master"
if ":" in project_rev:
project, rev = project_rev.split(":", 1)
# Clone and parse the repositories
depends = dependencies.generate(args.gitdir, args.prefix, project, rev)
print("---")
dependencies.print_dict(depends)
stack = buildstack.generate(depends)
print("---")
buildstack.print_dict(stack)
# Old versions
if args.old:
print("---")
dependencies.print_old(args.gitdir, depends)
# Build
if args.build:
print("---")
buildstack.build(args.gitdir, args.jobs, stack)
# Success
print("---")
print("Success for " + project + ":" + rev + "!")
print("---")
if __name__ == '__main__':
main()

View File

@ -0,0 +1,116 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright 2018 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
import sys
import fnmatch
# Same folder
import config
def error(line_i, message):
""" Print a configure.ac error message with the line number.
:param line_i: the zero based line counter """
print("ERROR: configure.ac line " + str(line_i+1) + ": " + message)
sys.exit(1)
def repository(library, version):
""" Find the git repository that contains a certain library. Based on the
information in config.py.
:param library: the name as referenced in the PKG_CHECK_MODULES
statement. For example: "libosmoabis"
:param version: for example "0.5.0"
:returns: the repository name, e.g. "libosmo-abis" """
for repo, libraries in config.repos.items():
if library in libraries:
print(" * " + library + ":" + version + " (part of " + repo + ")")
return repo
print(" * " + library + ":" + version)
return library
def library_is_relevant(library):
""" :returns: True when we would build the library in question from source,
False otherwise. """
for pattern in config.relevant_library_patterns:
if fnmatch.fnmatch(library, pattern):
return True
return False
def parse_condition(line):
""" Find the PKG_CHECK_MODULES conditions in any line from a configure.ac.
Example lines:
PKG_CHECK_MODULES(LIBOSMOCORE, libosmocore >= 0.10.0)
PKG_CHECK_MODULES(LIBSYSTEMD, libsystemd)
:returns: * None when there's no condition in that line
* a string like "libosmocore >= 0.1.0" """
# Only look at PKG_CHECK_MODULES lines
if "PKG_CHECK_MODULES" not in line:
return
# Extract the condition
ret = line.split(",")[1].split(")")[0].strip()
# Only look at Osmocom libraries
library = ret.split(" ")[0]
if library_is_relevant(library):
return ret
def library_version(line_i, condition):
""" Get the library and version strings from a condition.
:param line_i: the zero based line counter
:param condition: a condition like "libosmocore >= 0.1.0" """
# Split by space and remove empty list elements
split = list(filter(None, condition.split(" ")))
if len(split) != 3:
error(line_i, "invalid condition format, expected something"
" like 'libosmocore >= 0.10.0' but got: '" +
condition + "'")
library, operator, version = split
# Right operator
if operator == ">=":
return (library, version)
# Wrong operator
error(line_i, "invalid operator, expected '>=' but got: '" +
operator + "'")
def configure_ac(gitdir, repo):
""" Parse the PKG_CHECK_MODULES statements of a configure.ac file.
:param gitdir: parent folder of all locally cloned git repositories
:param repo: the repository to look at (e.g. "osmo-bts")
:returns: a dictionary like the following:
{"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0"} """
# Read configure.ac
path = gitdir + "/" + repo + "/configure.ac"
with open(path) as handle:
lines = handle.readlines()
# Parse the file into ret
ret = {}
for i in range(0, len(lines)):
# Parse the line
condition = parse_condition(lines[i])
if not condition:
continue
(library, version) = library_version(i, condition)
# Add to ret (with duplicate check)
repo_dependency = repository(library, version)
if repo_dependency in ret and version != ret[repo_dependency]:
error(i, "found multiple PKG_CHECK_MODULES statements for " +
repo_dependency + ".git, and they have different"
" versions!")
ret[repo_dependency] = version
return ret