osmocom
/
osmo-ci
mirror of https://gerrit.osmocom.org/osmo-ci
10
0
Fork 1
Code Releases Activity

osmo-depcheck: don't use /tmp, better git code 

* replace --gitdir with --workdir and give it a new folder structure:
  * git/$repo: downloaded source code
  * build/$repo: files created during the build process
  * install/: installation prefix
* adjust the jenkins job to use --workdir
* fetch --tags when source exists already
* readable error message for failed git checkout

Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058
Relates: OS#2642
This commit is contained in:
Oliver Smith 2018-09-21 10:29:51 +02:00
parent 85c2effd89
commit a99a4ef541
5 changed files with 96 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
jobs/osmocom-depcheck.yml
View File

@ -52,13 +52,13 @@
# Build the arguments
args="$PROJECTS"
args="$args -j 5"
args="$args -g $PWD/DEPCHECK_GITDIR"
args="$args -w $PWD/DEPCHECK_WORKDIR"
args="$args -u $GIT_URL_PREFIX"
[ "$BUILD" = "true" ] && args="$args -b"
[ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
# Run osmo-depcheck
mkdir DEPCHECK_GITDIR
mkdir DEPCHECK_WORKDIR
export PYTHONUNBUFFERED=1
scripts/osmo-depcheck/osmo-depcheck.py $args
scm:

45
scripts/osmo-depcheck/buildstack.py
View File

@ -70,29 +70,15 @@ def print_dict(stack):
print(" * " + program + ":" + version)
def temp_install_folder():
""" Generate a temporary installation folder
It will be used as configure prefix, so when running 'make install',
the files will get copied in there instead of "/usr/local/". The folder
will get removed when the script has finished.
:returns: the path to the temporary folder """
ret = tempfile.mkdtemp(prefix="depcheck_")
atexit.register(shutil.rmtree, ret)
print("Temporary install folder: " + ret)
return ret
def set_environment(jobs, tempdir):
def set_environment(jobs, prefix):
""" Configure the environment variables before running configure, make etc.
:param jobs: parallel build jobs (for make)
:param tempdir: temporary installation dir (see temp_install_folder())
:param prefix: installation folder
"""
# Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH
extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig",
"LD_LIBRARY_PATH": tempdir + "/lib"}
# Add prefix to PKG_CONFIG_PATH and LD_LIBRARY_PATH
extend = {"PKG_CONFIG_PATH": prefix + "/lib/pkgconfig",
"LD_LIBRARY_PATH": prefix + "/lib"}
for env_var, folder in extend.items():
old = os.environ[env_var] if env_var in os.environ else ""
os.environ[env_var] = old + ":" + folder
@ -101,10 +87,10 @@ def set_environment(jobs, tempdir):
os.environ["JOBS"] = str(jobs)
def build(gitdir, jobs, stack):
def build(workdir, jobs, stack):
""" Build one program with all its dependencies.
:param gitdir: folder to which the sources will be cloned
:param workdir: path to where all data (git, build, install) is stored
:param jobs: parallel build jobs (for make)
:param stack: the build stack as returned by generate() above
@ -122,18 +108,23 @@ def build(gitdir, jobs, stack):
anymore in case they decide to compile the code again manually from
the source folder. """
# Prepare the install folder and environment
tempdir = temp_install_folder()
unitdir = tempdir + "/lib/systemd/system/"
set_environment(jobs, tempdir)
prefix = workdir + "/install"
unitdir = prefix + "/lib/systemd/system/"
set_environment(jobs, prefix)
# Iterate over stack
for program, version in stack.items():
print("Building " + program + ":" + version)
os.chdir(gitdir + "/" + program)
# Create and enter the build folder
builddir = workdir + "/build/" + program
os.mkdir(builddir)
os.chdir(builddir)
# Run the build commands
commands = [["autoreconf", "-fi"],
["./configure", "--prefix", tempdir,
gitdir = workdir + "/git/" + program
commands = [["autoreconf", "-fi", gitdir],
[gitdir + "/configure", "--prefix", prefix,
"--with-systemdsystemunitdir=" + unitdir],
["make", "clean"],
["make"],

68
scripts/osmo-depcheck/dependencies.py
View File

@ -10,37 +10,55 @@ import sys
import parse
def git_clone(gitdir, prefix, repository, version):
def git_clone(workdir, prefix, cache_git_fetch, repository, version):
""" Clone a missing git repository and checkout a specific version tag.
:param gitdir: folder to which the sources will be cloned
:param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:param version: "master" or a version tag like "0.11.0" """
# Clone when needed
if not os.path.exists(gitdir + "/" + repository):
url = prefix + repository
print("Cloning git repo: " + url)
try:
subprocess.run(["git", "-C", gitdir, "clone", "-q", url],
repodir = workdir + "/git/" + repository
if repository not in cache_git_fetch:
if os.path.exists(repodir):
# Fetch tags for existing source
print("Fetching tags...")
subprocess.run(["git", "-C", repodir, "fetch", "--tags", "-q"],
check=True)
except subprocess.CalledProcessError:
print("NOTE: if '" + repository + "' is part of a git repository"
" with a different name, please add it to the mapping in"
" 'config.py' and try again.")
sys.exit(1)
else:
# Clone the source
url = prefix + repository
print("Cloning git repo: " + url)
try:
subprocess.run(["git", "-C", workdir + "/git", "clone", "-q",
url], check=True)
except subprocess.CalledProcessError:
print("NOTE: if '" + repository + "' is part of a git"
" repository with a different name, please add it to the"
" mapping in 'config.py' and try again.")
sys.exit(1)
# Only fetch the same repository once per session
cache_git_fetch.append(repository)
# Checkout the version tag
subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout",
version, "-q"], check=True)
try:
subprocess.run(["git", "-C", repodir, "checkout", version, "-q"],
check=True)
except subprocess.CalledProcessError:
print("ERROR: git checkout failed! Invalid version specified?")
sys.exit(1)
def generate(gitdir, prefix, initial, rev):
def generate(workdir, prefix, cache_git_fetch, initial, rev):
""" Generate the dependency graph of an Osmocom program by cloning the git
repository, parsing the "configure.ac" file, and recursing.
:param gitdir: folder to which the sources will be cloned
:param workdir: path to where all data (git, build, install) is stored
:param prefix: git url prefix (e.g. "git://git.osmocom.org/")
:param cache_git_fetch: list of repositories that have already been
fetched in this run of osmo-depcheck
:param initial: the first program to look at (e.g. "osmo-bts")
:param rev: the git revision to check out ("master", "0.1.0", ...)
:returns: a dictionary like the following:
@ -65,8 +83,8 @@ def generate(gitdir, prefix, initial, rev):
# Add the programs dependencies to the stack
print("Looking at " + program + ":" + version)
git_clone(gitdir, prefix, program, version)
depends = parse.configure_ac(gitdir, program)
git_clone(workdir, prefix, cache_git_fetch, program, version)
depends = parse.configure_ac(workdir, program)
stack.update(depends)
# Add the program to the ret
@ -86,28 +104,28 @@ def print_dict(depends):
print(" * " + program + ":" + version + " depends: " + str(depends))
def git_latest_tag(gitdir, repository):
def git_latest_tag(workdir, repository):
""" Get the last release string by asking git for the latest tag.
:param gitdir: folder to which the sources will be cloned
:param workdir: path to where all data (git, build, install) is stored
:param repository: Osmocom git repository name (e.g. "libosmo-abis")
:returns: the latest git tag (e.g. "1.0.2") """
dir = gitdir + "/" + repository
dir = workdir + "/git/" + repository
complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
"master"], check=True, stdout=subprocess.PIPE)
return complete.stdout.decode().rstrip()
def print_old(gitdir, depends):
def print_old(workdir, depends):
""" Print dependencies tied to an old release tag
:param gitdir: folder to which the sources will be cloned
:param workdir: path to where all data (git, build, install) is stored
:param depends: return value from generate() above """
print("Dependencies on old releases:")
for program, data in depends.items():
for depend, version in data["depends"].items():
latest = git_latest_tag(gitdir, depend)
latest = git_latest_tag(workdir, depend)
if latest == version:
continue
print(" * " + program + ":" + data["version"] + " -> " +

41
scripts/osmo-depcheck/osmo-depcheck.py
View File

@ -4,6 +4,7 @@
import argparse
import os
import shutil
import sys
# Same folder
@ -17,16 +18,16 @@ def parse_arguments():
description = ("This script verifies that Osmocom programs really build"
" with the dependency versions they claim to support in"
" configure.ac. In order to do that, it clones the"
" dependency repositories if they don't exist in gitdir"
" dependency repositories if they don't exist in workdir"
" already, and checks out the minimum version tag. This"
" happens recursively for their dependencies as well.")
parser = argparse.ArgumentParser(description=description)
# Git sources folder
gitdir_default = os.path.expanduser("~") + "/code"
parser.add_argument("-g", "--gitdir", default=gitdir_default,
workdir_default = os.path.expanduser("~") + "/osmo-depcheck-work"
parser.add_argument("-w", "--workdir", default=workdir_default,
help="folder to which the sources will be cloned"
" (default: " + gitdir_default + ")")
" (default: " + workdir_default + ")")
# Build switch
parser.add_argument("-b", "--build", action="store_true",
@ -55,17 +56,33 @@ def parse_arguments():
" revision is 'master')",
metavar="project[:revision]")
# Gitdir must exist
# Workdir must exist
ret = parser.parse_args()
if not os.path.exists(ret.gitdir):
print("ERROR: gitdir does not exist: " + ret.gitdir)
if not os.path.exists(ret.workdir):
print("ERROR: workdir does not exist: " + ret.workdir)
sys.exit(1)
return ret
def workdir_prepare(workdir):
""" Delete old binaries and create the subfolders in workdir
:param workdir: path to where all data is stored """
# Delete folders with binaries from previous runs
for subfolder in ("build", "install"):
full = workdir + "/" + subfolder
if os.path.exists(full):
shutil.rmtree(full)
# Create all subfolders
for subfolder in ("build", "install", "git"):
os.makedirs(workdir + "/" + subfolder, exist_ok=True)
def main():
# Iterate over projects
args = parse_arguments()
# Iterate over projects
cache_git_fetch = []
for project_rev in args.projects_revs:
# Split the git revision from the project name
project = project_rev
@ -74,7 +91,9 @@ def main():
project, rev = project_rev.split(":", 1)
# Clone and parse the repositories
depends = dependencies.generate(args.gitdir, args.prefix, project, rev)
workdir_prepare(args.workdir)
depends = dependencies.generate(args.workdir, args.prefix,
cache_git_fetch, project, rev)
print("---")
dependencies.print_dict(depends)
stack = buildstack.generate(depends)
@ -84,12 +103,12 @@ def main():
# Old versions
if args.old:
print("---")
dependencies.print_old(args.gitdir, depends)
dependencies.print_old(args.workdir, depends)
# Build
if args.build:
print("---")
buildstack.build(args.gitdir, args.jobs, stack)
buildstack.build(args.workdir, args.jobs, stack)
# Success
print("---")

6
scripts/osmo-depcheck/parse.py
View File

@ -84,16 +84,16 @@ def library_version(line_i, condition):
operator + "'")
def configure_ac(gitdir, repo):
def configure_ac(workdir, repo):
""" Parse the PKG_CHECK_MODULES statements of a configure.ac file.
:param gitdir: parent folder of all locally cloned git repositories
:param workdir: path to where all data (git, build, install) is stored
:param repo: the repository to look at (e.g. "osmo-bts")
:returns: a dictionary like the following:
{"libosmocore": "0.11.0",
"libosmo-abis": "0.5.0"} """
# Read configure.ac
path = gitdir + "/" + repo + "/configure.ac"
path = workdir + "/git/" + repo + "/configure.ac"
with open(path) as handle:
lines = handle.readlines()