mirror of https://gerrit.osmocom.org/osmo-ci
scripts/osmo-build.sh / ARTIFACT_STORE: remove
This script was supposed to cache already built Osmocom libraries. But it was only used with openbsc, didn't work as expected anymore (looking at openbsc-master, it just builds all deps from source without caching) and will be replaced with logic that allows using ccache in a future patch. Using ccache has the big advantage that we can cache all build objects where the source files did not change, and not just do it on a dependency level. Oftentimes only few source files change in our patches, so we can really cache everything else that is not affected within any git repository and not just a dependency that didn't change. ccache also does automatic cache size management to ensure a cache stays within a given size limit. Related: OS#5848 Depends: openbsc Ib3272feec76b30412ca60dec204255b64e33831b Change-Id: Id94d6126b476077d57839e4a884621b8c034f0c6changes/41/31041/2
parent
113b295075
commit
989c47bc91
|
|
@ -48,11 +48,8 @@
|
|||
# Documentation for variables (keep in sync!):
|
||||
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
|
||||
docker_run: |
|
||||
ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
|
||||
mkdir -p "$ARTIFACT_STORE"
|
||||
docker run --rm=true \
|
||||
--cap-add SYS_PTRACE \
|
||||
-e ARTIFACT_STORE=/artifact_store \
|
||||
-e ASCIIDOC_WARNINGS_CHECK="1" \
|
||||
-e HOME=/build \
|
||||
-e JOB_NAME="$JOB_NAME" \
|
||||
|
|
@ -63,7 +60,6 @@
|
|||
-e WITH_MANUALS="1" \
|
||||
-w /build -i -u build \
|
||||
-v "$PWD:/build" \
|
||||
-v "$ARTIFACT_STORE:/artifact_store" \
|
||||
docker_img: '$USER/debian-buster-jenkins'
|
||||
docker_img_erlang: '$USER/debian-bullseye-erlang'
|
||||
timeout_cmd: '/usr/bin/timeout 30m'
|
||||
|
|
|
|||
|
|
@ -22,11 +22,8 @@
|
|||
# Documentation for variables (keep in sync!):
|
||||
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
|
||||
docker_run: |
|
||||
ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
|
||||
mkdir -p "$ARTIFACT_STORE"
|
||||
docker run --rm=true \
|
||||
--cap-add SYS_PTRACE \
|
||||
-e ARTIFACT_STORE=/artifact_store \
|
||||
-e ASCIIDOC_WARNINGS_CHECK="1" \
|
||||
-e HOME=/build \
|
||||
-e JOB_NAME="$JOB_NAME" \
|
||||
|
|
@ -40,7 +37,6 @@
|
|||
-w /build -i -u build \
|
||||
-v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
|
||||
-v "$PWD:/build" \
|
||||
-v "$ARTIFACT_STORE:/artifact_store" \
|
||||
-v "$HOME/.ssh:/home/build/.ssh:ro" \
|
||||
docker_img: '$USER/debian-buster-jenkins'
|
||||
timeout_cmd: '/usr/bin/timeout 30m'
|
||||
|
|
|
|||
|
|
@ -1,230 +0,0 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# This script enables artifacts holding dependencies on a jenkins job level to
|
||||
# speed up builds. Basically, it holds logic to check whether the necessary artifact
|
||||
# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded
|
||||
# it triggers the actual build.
|
||||
#
|
||||
# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh
|
||||
# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built
|
||||
# dependencies are detrmined after dependencies are built to ensure catching new
|
||||
# changes in dep_n+1 meanwhile building dep_n.
|
||||
#
|
||||
# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves.
|
||||
# The JOB_NAME variable will be injected to each jenkins' job by jenkins itself.
|
||||
# When using script within a docker container one must inject jenkins' JOB_NAME variable
|
||||
# to the container and ensure that ARTIFACT_STORE is mounted to the container's
|
||||
# internal ARTIFACT_STORE.
|
||||
#
|
||||
# Artifacts will be stored as follows:
|
||||
#
|
||||
# $ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
|
||||
# ..._<dep_n>.<tag_n>.tar.gz
|
||||
#
|
||||
# Note: each matrix-build has its own directory inside ARTIFACT_STORE.
|
||||
#
|
||||
# In order to make use of osmo-build.sh one needs to source it, e.g. from
|
||||
# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of
|
||||
# the project to be built in the workspace root. Following functions needs to be
|
||||
# declared within a build script that sources osmo-build.sh:
|
||||
#
|
||||
# - artifact_name()
|
||||
# - build_deps()
|
||||
# - build_project()
|
||||
#
|
||||
# This is an example for building "libosmo-netif" which depends on "libosmocore"
|
||||
# and "libosmo-abis".
|
||||
#
|
||||
# #!/bin/sh
|
||||
#
|
||||
# artifact_deps() {
|
||||
# # $1 will be one of folllowing functions:
|
||||
# # - artifact_name_by_local_repo()
|
||||
# # - artifact_name_by_remote_repo()
|
||||
# # osmo-build.sh takes care about which function to use
|
||||
#
|
||||
# x="$($1 libosmocore)"
|
||||
# x="${x}_$($1 libosmo-abis)"
|
||||
#
|
||||
# echo "${x}.tar.gz"
|
||||
# }
|
||||
#
|
||||
# build_deps() {
|
||||
# # all commands to build necessary dependencies
|
||||
# osmo-build-dep.sh libosmocore master ac_cv_path_DOXYGEN=false
|
||||
# "$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
|
||||
# osmo-build-dep.sh libosmo-abis
|
||||
# }
|
||||
#
|
||||
# build_project() {
|
||||
# # Necessary commands to build the project, expecting all dependencies have
|
||||
# # been built or fetched. Commands within build_project() will be executed
|
||||
# # in jenkins' $WORKSPACE.
|
||||
#
|
||||
# autoreconf --install --force
|
||||
# ./configure --enable-sanitize
|
||||
# $MAKE $PARALLEL_MAKE
|
||||
# $MAKE distcheck || cat-testlogs.sh
|
||||
# }
|
||||
#
|
||||
##
|
||||
# # source osmo-build.sh to fire the build
|
||||
# . osmo-build.sh
|
||||
|
||||
log() {
|
||||
set +x
|
||||
echo
|
||||
echo "[INFO] $1"
|
||||
echo
|
||||
set -x
|
||||
}
|
||||
|
||||
# SOURCING SANITY
|
||||
log "source sanity check to ensure that sourcing script holds necessary functions"
|
||||
type artifact_deps
|
||||
type build_deps
|
||||
type build_project
|
||||
log "check whether necessary dependency build scripts are in PATH"
|
||||
type osmo-build-dep.sh
|
||||
type osmo-deps.sh
|
||||
|
||||
# BUILD FUNCTIONS
|
||||
init_build() {
|
||||
|
||||
if [ -z "$JOB_NAME" ]; then
|
||||
log "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$ARTIFACT_STORE" ]; then
|
||||
log "[ERROR] ARTIFACT_STORE variable is not set on this build slave"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
base="$(pwd)"
|
||||
deps="$base/deps"
|
||||
inst="$deps/install"
|
||||
rm -rf "$deps" || true
|
||||
|
||||
# obtain the project name from the git clone found in the workspace root
|
||||
project=$(git config --get --local remote.origin.url \
|
||||
| cut -d '/' -f4 | cut -d '.' -f1)
|
||||
|
||||
# replace invalid char for dirs in $JOB_NAME (jenkins variable)
|
||||
# ( '/' separates job name and matrix-axis)
|
||||
job_name="$( echo "$JOB_NAME" | tr '/' '_')"
|
||||
|
||||
export base deps inst project job_name
|
||||
export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
|
||||
export LD_LIBRARY_PATH="$inst/lib"
|
||||
|
||||
log "$project build initialized"
|
||||
}
|
||||
|
||||
build() {
|
||||
|
||||
init_build
|
||||
|
||||
artifact_name="$(artifact_name)"
|
||||
|
||||
if [ -f "$ARTIFACT_STORE/$job_name/$artifact_name" ]; then
|
||||
fetch_artifact "$ARTIFACT_STORE/$job_name" "$artifact_name"
|
||||
else
|
||||
log "Compile $project dependencies from source."
|
||||
mkdir -p "$deps"
|
||||
rm -rf "$inst"
|
||||
|
||||
build_deps
|
||||
archive_artifact
|
||||
fi
|
||||
|
||||
log "building $project"
|
||||
build_project
|
||||
}
|
||||
|
||||
# ARTIFACT FUNCTIONS
|
||||
artifact_name() {
|
||||
# in case deps is empty or does not exist we
|
||||
if [ -d "$deps" ]; then
|
||||
artifact_deps "branch_and_rev_of_local_repo"
|
||||
cd "$base"
|
||||
else
|
||||
artifact_deps "branch_and_rev_of_remote_repo"
|
||||
fi
|
||||
}
|
||||
|
||||
branch_and_rev_of_local_repo() {
|
||||
cd "$deps/$1"
|
||||
rev="$(git rev-parse --short HEAD)"
|
||||
branch="$(git rev-parse --abbrev-ref HEAD)"
|
||||
|
||||
# check whether it is a tag
|
||||
if [ "$branch" = "HEAD" ]; then
|
||||
tag="$(git describe --tags HEAD)"
|
||||
tag="$(echo "$tag" | tr '/' '_')"
|
||||
echo "$1.$tag"
|
||||
else
|
||||
branch="$( echo "$branch" | tr '/' '_')"
|
||||
echo "$1.$branch.$rev"
|
||||
fi
|
||||
}
|
||||
|
||||
branch_and_rev_of_remote_repo() {
|
||||
if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
|
||||
branch="$( echo "$branch" | tr '/' '_')"
|
||||
rev="$(git ls-remote "https://gerrit.osmocom.org/$1" "refs/heads/$branch")"
|
||||
|
||||
# check whether branch is a tag
|
||||
if [ "$rev" = "" ]; then
|
||||
echo "$1.$branch"
|
||||
else
|
||||
rev="$(echo "$rev" | cut -c 1-7)"
|
||||
echo "$1.$branch.$rev"
|
||||
fi
|
||||
}
|
||||
|
||||
archive_artifact() {
|
||||
log "Archiving artifact to artifactStore."
|
||||
|
||||
cd "$base"
|
||||
artifact="$(artifact_name)"
|
||||
# temp_job_store is necessary to atomically move it to production.
|
||||
temp_job_store="$ARTIFACT_STORE/tmp/$job_name/"
|
||||
job_store="$ARTIFACT_STORE/$job_name/"
|
||||
|
||||
if [ ! -f "$temp_job_store/$artifact" ]; then
|
||||
mkdir -p "$job_store" "$temp_job_store"
|
||||
# remove outdated artifact first to avoid temporarily
|
||||
# doubling of artifact storage consumption
|
||||
rm -f "$job_store/*"
|
||||
tar czf "$temp_job_store/$artifact" "deps"
|
||||
mv -n "$temp_job_store/$artifact" "$job_store/$artifact"
|
||||
rm -rf "$temp_job_store"
|
||||
|
||||
log_artifact_hash "$job_store/$artifact"
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_artifact() {
|
||||
log "Fetching artifact from artifactStore."
|
||||
|
||||
log_artifact_hash "$1/$2"
|
||||
cp "$1/$2" .
|
||||
log_artifact_hash "$2"
|
||||
tar xzf "$2"
|
||||
|
||||
if [ $? -gt 0 ]; then
|
||||
log "Artifact could not be fetched, triggering build_deps()"
|
||||
build_deps
|
||||
else
|
||||
log "Artifact successfully fetched, triggering $project compilation"
|
||||
fi
|
||||
}
|
||||
|
||||
# checksum is not used by script itself,
|
||||
# but might be handy in logs when debugging.
|
||||
log_artifact_hash() {
|
||||
log "name: $1 \n sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
|
||||
}
|
||||
|
||||
build
|
||||
Loading…
Reference in New Issue