osmocom
/
osmo-ci
mirror of https://gerrit.osmocom.org/osmo-ci
10
0
Fork 1
Code Releases Activity

osmo-depcheck: don't use /tmp, better git code 

* replace --gitdir with --workdir and give it a new folder structure:
  * git/$repo: downloaded source code
  * build/$repo: files created during the build process
  * install/: installation prefix
* adjust the jenkins job to use --workdir
* fetch --tags when source exists already
* readable error message for failed git checkout

Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058
Relates: OS#2642
This commit is contained in:
Oliver Smith 2018-09-21 10:29:51 +02:00 committed by osmith
parent 8768ad510a
commit 6cced05c01
5 changed files with 96 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
jobs/osmocom-depcheck.yml
View File

@ -52,13 +52,13 @@
# Build the arguments # Build the arguments
args="$PROJECTS" args="$PROJECTS"
args="$args -j 5" args="$args -j 5"
args="$args -g $PWD/DEPCHECK_GITDIR" args="$args -w $PWD/DEPCHECK_WORKDIR"
args="$args -u $GIT_URL_PREFIX" args="$args -u $GIT_URL_PREFIX"
[ "$BUILD" = "true" ] && args="$args -b" [ "$BUILD" = "true" ] && args="$args -b"
[ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o" [ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
# Run osmo-depcheck # Run osmo-depcheck
mkdir DEPCHECK_GITDIR mkdir DEPCHECK_WORKDIR
export PYTHONUNBUFFERED=1 export PYTHONUNBUFFERED=1
scripts/osmo-depcheck/osmo-depcheck.py $args scripts/osmo-depcheck/osmo-depcheck.py $args
scm: scm:

45
scripts/osmo-depcheck/buildstack.py
View File

@ -70,29 +70,15 @@ def print_dict(stack):
print(" * " + program + ":" + version) print(" * " + program + ":" + version)
def temp_install_folder(): def set_environment(jobs, prefix):
""" Generate a temporary installation folder
It will be used as configure prefix, so when running 'make install',
the files will get copied in there instead of "/usr/local/". The folder
will get removed when the script has finished.
:returns: the path to the temporary folder """
ret = tempfile.mkdtemp(prefix="depcheck_")
atexit.register(shutil.rmtree, ret)
print("Temporary install folder: " + ret)
return ret
def set_environment(jobs, tempdir):
""" Configure the environment variables before running configure, make etc. """ Configure the environment variables before running configure, make etc.
:param jobs: parallel build jobs (for make) :param jobs: parallel build jobs (for make)
:param tempdir: temporary installation dir (see temp_install_folder()) :param prefix: installation folder
""" """
# Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH # Add prefix to PKG_CONFIG_PATH and LD_LIBRARY_PATH
extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig", extend = {"PKG_CONFIG_PATH": prefix + "/lib/pkgconfig",
"LD_LIBRARY_PATH": tempdir + "/lib"} "LD_LIBRARY_PATH": prefix + "/lib"}
for env_var, folder in extend.items(): for env_var, folder in extend.items():
old = os.environ[env_var] if env_var in os.environ else "" old = os.environ[env_var] if env_var in os.environ else ""
os.environ[env_var] = old + ":" + folder os.environ[env_var] = old + ":" + folder
@ -101,10 +87,10 @@ def set_environment(jobs, tempdir):
os.environ["JOBS"] = str(jobs) os.environ["JOBS"] = str(jobs)
def build(gitdir, jobs, stack): def build(workdir, jobs, stack):
""" Build one program with all its dependencies. """ Build one program with all its dependencies.
:param gitdir: folder to which the sources will be cloned :param workdir: path to where all data (git, build, install) is stored
:param jobs: parallel build jobs (for make) :param jobs: parallel build jobs (for make)
:param stack: the build stack as returned by generate() above :param stack: the build stack as returned by generate() above
@ -122,18 +108,23 @@ def build(gitdir, jobs, stack):
anymore in case they decide to compile the code again manually from anymore in case they decide to compile the code again manually from
the source folder. """ the source folder. """
# Prepare the install folder and environment # Prepare the install folder and environment
tempdir = temp_install_folder() prefix = workdir + "/install"
unitdir = tempdir + "/lib/systemd/system/" unitdir = prefix + "/lib/systemd/system/"
set_environment(jobs, tempdir) set_environment(jobs, prefix)
# Iterate over stack # Iterate over stack
for program, version in stack.items(): for program, version in stack.items():
print("Building " + program + ":" + version) print("Building " + program + ":" + version)
os.chdir(gitdir + "/" + program)
# Create and enter the build folder
builddir = workdir + "/build/" + program
os.mkdir(builddir)
os.chdir(builddir)
# Run the build commands # Run the build commands
commands = [["autoreconf", "-fi"], gitdir = workdir + "/git/" + program
["./configure", "--prefix", tempdir, commands = [["autoreconf", "-fi", gitdir],
[gitdir + "/configure", "--prefix", prefix,
"--with-systemdsystemunitdir=" + unitdir], "--with-systemdsystemunitdir=" + unitdir],
["make", "clean"], ["make", "clean"],
["make"], ["make"],

60
scripts/osmo-depcheck/dependencies.py
View File

@ -10,37 +10,55 @@ import sys
import parse import parse
def git_clone(gitdir, prefix, repository, version): def git_clone(workdir, prefix, cache_git_fetch, repository, version):
""" Clone a missing git repository and checkout a specific version tag. """ Clone a missing git repository and checkout a specific version tag.
:param gitdir: folder to which the sources will be cloned :param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/") :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param repository: Osmocom git repository name (e.g. "libosmo-abis") :param repository: Osmocom git repository name (e.g. "libosmo-abis")
:param version: "master" or a version tag like "0.11.0" """ :param version: "master" or a version tag like "0.11.0" """
# Clone when needed repodir = workdir + "/git/" + repository
if not os.path.exists(gitdir + "/" + repository): if repository not in cache_git_fetch:
if os.path.exists(repodir):
# Fetch tags for existing source
print("Fetching tags...")
subprocess.run(["git", "-C", repodir, "fetch", "--tags", "-q"],
check=True)
else:
# Clone the source
url = prefix + repository url = prefix + repository
print("Cloning git repo: " + url) print("Cloning git repo: " + url)
try: try:
subprocess.run(["git", "-C", gitdir, "clone", "-q", url], subprocess.run(["git", "-C", workdir + "/git", "clone", "-q",
check=True) url], check=True)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print("NOTE: if '" + repository + "' is part of a git repository" print("NOTE: if '" + repository + "' is part of a git"
" with a different name, please add it to the mapping in" " repository with a different name, please add it to the"
" 'config.py' and try again.") " mapping in 'config.py' and try again.")
sys.exit(1) sys.exit(1)
# Only fetch the same repository once per session
cache_git_fetch.append(repository)
# Checkout the version tag # Checkout the version tag
subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout", try:
version, "-q"], check=True) subprocess.run(["git", "-C", repodir, "checkout", version, "-q"],
check=True)
except subprocess.CalledProcessError:
print("ERROR: git checkout failed! Invalid version specified?")
sys.exit(1)
def generate(gitdir, prefix, initial, rev): def generate(workdir, prefix, cache_git_fetch, initial, rev):
""" Generate the dependency graph of an Osmocom program by cloning the git """ Generate the dependency graph of an Osmocom program by cloning the git
repository, parsing the "configure.ac" file, and recursing. repository, parsing the "configure.ac" file, and recursing.
:param gitdir: folder to which the sources will be cloned :param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/") :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param initial: the first program to look at (e.g. "osmo-bts") :param initial: the first program to look at (e.g. "osmo-bts")
:param rev: the git revision to check out ("master", "0.1.0", ...) :param rev: the git revision to check out ("master", "0.1.0", ...)
:returns: a dictionary like the following: :returns: a dictionary like the following:
@ -65,8 +83,8 @@ def generate(gitdir, prefix, initial, rev):
# Add the programs dependencies to the stack # Add the programs dependencies to the stack
print("Looking at " + program + ":" + version) print("Looking at " + program + ":" + version)
git_clone(gitdir, prefix, program, version) git_clone(workdir, prefix, cache_git_fetch, program, version)
depends = parse.configure_ac(gitdir, program) depends = parse.configure_ac(workdir, program)
stack.update(depends) stack.update(depends)
# Add the program to the ret # Add the program to the ret
@ -86,28 +104,28 @@ def print_dict(depends):
print(" * " + program + ":" + version + " depends: " + str(depends)) print(" * " + program + ":" + version + " depends: " + str(depends))
def git_latest_tag(gitdir, repository): def git_latest_tag(workdir, repository):
""" Get the last release string by asking git for the latest tag. """ Get the last release string by asking git for the latest tag.
:param gitdir: folder to which the sources will be cloned :param workdir: path to where all data (git, build, install) is stored
:param repository: Osmocom git repository name (e.g. "libosmo-abis") :param repository: Osmocom git repository name (e.g. "libosmo-abis")
:returns: the latest git tag (e.g. "1.0.2") """ :returns: the latest git tag (e.g. "1.0.2") """
dir = gitdir + "/" + repository dir = workdir + "/git/" + repository
complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0", complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
"master"], check=True, stdout=subprocess.PIPE) "master"], check=True, stdout=subprocess.PIPE)
return complete.stdout.decode().rstrip() return complete.stdout.decode().rstrip()
def print_old(gitdir, depends): def print_old(workdir, depends):
""" Print dependencies tied to an old release tag """ Print dependencies tied to an old release tag
:param gitdir: folder to which the sources will be cloned :param workdir: path to where all data (git, build, install) is stored
:param depends: return value from generate() above """ :param depends: return value from generate() above """
print("Dependencies on old releases:") print("Dependencies on old releases:")
for program, data in depends.items(): for program, data in depends.items():
for depend, version in data["depends"].items(): for depend, version in data["depends"].items():
latest = git_latest_tag(gitdir, depend) latest = git_latest_tag(workdir, depend)
if latest == version: if latest == version:
continue continue
print(" * " + program + ":" + data["version"] + " -> " + print(" * " + program + ":" + data["version"] + " -> " +

41
scripts/osmo-depcheck/osmo-depcheck.py
View File

@ -4,6 +4,7 @@
import argparse import argparse
import os import os
import shutil
import sys import sys
# Same folder # Same folder
@ -17,16 +18,16 @@ def parse_arguments():
description = ("This script verifies that Osmocom programs really build" description = ("This script verifies that Osmocom programs really build"
" with the dependency versions they claim to support in" " with the dependency versions they claim to support in"
" configure.ac. In order to do that, it clones the" " configure.ac. In order to do that, it clones the"
" dependency repositories if they don't exist in gitdir" " dependency repositories if they don't exist in workdir"
" already, and checks out the minimum version tag. This" " already, and checks out the minimum version tag. This"
" happens recursively for their dependencies as well.") " happens recursively for their dependencies as well.")
parser = argparse.ArgumentParser(description=description) parser = argparse.ArgumentParser(description=description)
# Git sources folder # Git sources folder
gitdir_default = os.path.expanduser("~") + "/code" workdir_default = os.path.expanduser("~") + "/osmo-depcheck-work"
parser.add_argument("-g", "--gitdir", default=gitdir_default, parser.add_argument("-w", "--workdir", default=workdir_default,
help="folder to which the sources will be cloned" help="folder to which the sources will be cloned"
" (default: " + gitdir_default + ")") " (default: " + workdir_default + ")")
# Build switch # Build switch
parser.add_argument("-b", "--build", action="store_true", parser.add_argument("-b", "--build", action="store_true",
@ -55,17 +56,33 @@ def parse_arguments():
" revision is 'master')", " revision is 'master')",
metavar="project[:revision]") metavar="project[:revision]")
# Gitdir must exist # Workdir must exist
ret = parser.parse_args() ret = parser.parse_args()
if not os.path.exists(ret.gitdir): if not os.path.exists(ret.workdir):
print("ERROR: gitdir does not exist: " + ret.gitdir) print("ERROR: workdir does not exist: " + ret.workdir)
sys.exit(1) sys.exit(1)
return ret return ret
def workdir_prepare(workdir):
""" Delete old binaries and create the subfolders in workdir
:param workdir: path to where all data is stored """
# Delete folders with binaries from previous runs
for subfolder in ("build", "install"):
full = workdir + "/" + subfolder
if os.path.exists(full):
shutil.rmtree(full)
# Create all subfolders
for subfolder in ("build", "install", "git"):
os.makedirs(workdir + "/" + subfolder, exist_ok=True)
def main(): def main():
# Iterate over projects
args = parse_arguments() args = parse_arguments()
# Iterate over projects
cache_git_fetch = []
for project_rev in args.projects_revs: for project_rev in args.projects_revs:
# Split the git revision from the project name # Split the git revision from the project name
project = project_rev project = project_rev
@ -74,7 +91,9 @@ def main():
project, rev = project_rev.split(":", 1) project, rev = project_rev.split(":", 1)
# Clone and parse the repositories # Clone and parse the repositories
depends = dependencies.generate(args.gitdir, args.prefix, project, rev) workdir_prepare(args.workdir)
depends = dependencies.generate(args.workdir, args.prefix,
cache_git_fetch, project, rev)
print("---") print("---")
dependencies.print_dict(depends) dependencies.print_dict(depends)
stack = buildstack.generate(depends) stack = buildstack.generate(depends)
@ -84,12 +103,12 @@ def main():
# Old versions # Old versions
if args.old: if args.old:
print("---") print("---")
dependencies.print_old(args.gitdir, depends) dependencies.print_old(args.workdir, depends)
# Build # Build
if args.build: if args.build:
print("---") print("---")
buildstack.build(args.gitdir, args.jobs, stack) buildstack.build(args.workdir, args.jobs, stack)
# Success # Success
print("---") print("---")

6
scripts/osmo-depcheck/parse.py
View File

@ -84,16 +84,16 @@ def library_version(line_i, condition):
operator + "'") operator + "'")
def configure_ac(gitdir, repo): def configure_ac(workdir, repo):
""" Parse the PKG_CHECK_MODULES statements of a configure.ac file. """ Parse the PKG_CHECK_MODULES statements of a configure.ac file.
:param gitdir: parent folder of all locally cloned git repositories :param workdir: path to where all data (git, build, install) is stored
:param repo: the repository to look at (e.g. "osmo-bts") :param repo: the repository to look at (e.g. "osmo-bts")
:returns: a dictionary like the following: :returns: a dictionary like the following:
{"libosmocore": "0.11.0", {"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0"} """ "libosmo-abis": "0.5.0"} """
# Read configure.ac # Read configure.ac
path = gitdir + "/" + repo + "/configure.ac" path = workdir + "/git/" + repo + "/configure.ac"
with open(path) as handle: with open(path) as handle:
lines = handle.readlines() lines = handle.readlines()