diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh new file mode 100644 index 00000000..199cf1dd --- /dev/null +++ b/scripts/osmo-build.sh @@ -0,0 +1,234 @@ +#!/bin/bash +# +# This script enables artifacts holding dependencies on a jenkins job level to +# speed up builds. Basically, it holds logic to check whether the necessary artifact +# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded +# it triggers the actual build. +# +# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh +# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built +# dependencies are detrmined after dependencies are built to ensure catching new +# changes in dep_n+1 meanwhile dep_n building. +# +# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves. +# The JOB_NAME variables will be injected to each jenkins' job by jenkins itself. +# When using script within a docker container one must inject jenkins' JOB_NAME variable +# to the container and ensure that ARTIFACT_STORE is mounted to the container's +# internal ARTIFACT_STORE. +# +# Artifacts will be stored as follows: +# +# $ARTIFACT_STORE/$JOB_NAME/.._... +# ..._...tar.gz +# +# Note: each matrix-build has its own directory inside ARTIFACT_STORE. +# +# In order to make use of osmo-build.sh one needs to source it, e.g. from +# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of +# the project to be built in the workspace root. Following functions needs to be +# declared within a build script that sources osmo-build.sh: +# +# - generic_deps() +# - build_project() +# +# This is an example for building "libosmo-netif" which depends on "libosmocore" +# and "libosmo-abis". +# +# #!/bin/bash +# +# # osmo-build.sh must be in PATH +# source osmo-build.sh +# +# generic_deps() { +# # Holds all required dependencies in the following form: +# # +# # x="$( $1 )" +# # +# # $1 will be one of these script/functions: +# # - osmo-build-dep.sh: build a source tree +# # - artifact_name_by_local_repos() +# # - artifact_name_by_remote_repos() +# # +# # The arguments are: +# # +# # - PARALLEL_MAKE: override global PARALLEL_MAKE used for make, +# # (e.g. PARALLEL_MAKE="-j1" to disable mutlithreading) +# # - dep_project: the git repository name, gets places in a git.osmocom.org URL. +# # - branch: branch (optional: default = master) +# # - arg 3: configuration (optional: $cfg used in osmo-build-dep.sh) +# +# x="$($1 libosmocore master ac_cv_path_DOXYGEN=false)" +# "$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]") +# x="${x}_$($1 libosmo-abis)" +# +# echo "${x}.tar.gz" +# } +# +# build_project() { +# # Necessary commands to build the project, expecting all dependencies have +# # been build or fetched. Commands within build_project() will be executed +# # in jenkins' $WORKSPACE. +# +# autoreconf --install --force +# ./configure --enable-sanitize +# $MAKE $PARALLEL_MAKE +# $MAKE distcheck || cat-testlogs.sh +# } +# +# build +# + + +# BUILD FUNCTIONS +init_build() { + + if [ -z "$JOB_NAME" ]; then + set +x + echo + echo "[ERROR] JOB_NAME variable is not set, running in Jenkins?" + echo + set -x + exit 1 + fi + + if [ -z "$ARTIFACT_STORE" ]; then + set +x + echo + echo "[ERROR] ARTIFACT_STORE variable is not set on this build slave" + echo + set -x + exit 1 + fi + + base="$(pwd)" + deps="$base/deps" + inst="$deps/install" + + # obtain the project name from the git clone found in the workspace root + project=$(git config --get --local remote.origin.url \ + | cut -d '/' -f4 | cut -d '.' -f1) + + # replace invalid char for dirs in jenkins variable + # ( '/' separates job name and matrix-axis) + job_name="${JOB_NAME//\//__}" + + export base deps inst project job_name + export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH" + export LD_LIBRARY_PATH="$inst/lib" +} + +build_deps() { + set +x + echo + echo "[INFO] Compile $project dependencies from source." + echo + set -x + + mkdir -p "$deps" + rm -rf "$inst" + + generic_deps "osmo-build-dep.sh" +} + +build() { + + init_build + + needed_artifact="$(artifact_name_by_remote_repos)" + + if [ -f "$ARTIFACT_STORE/$job_name/$needed_artifact" ]; then + fetch_artifact "$ARTIFACT_STORE/$job_name" "$needed_artifact" + else + build_deps + archive_artifact + fi + + set +x + echo + echo " ============================= $project ==========================" + echo + set -x + + build_project +} + +# ARTIFACT FUNCTIONS +artifact_name_by_local_repos() { + generic_deps "branch_and_rev_of_local_repo" + cd "$base" +} + +artifact_name_by_remote_repos() { + generic_deps "branch_and_rev_of_remote_repo" +} + +branch_and_rev_of_local_repo() { + cd "$deps/$1" + rev=$(git rev-parse --short HEAD) + branch=$(git rev-parse --abbrev-ref HEAD) + echo "$1.${branch//\//__}.${rev}" +} + +branch_and_rev_of_remote_repo() { + if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi + rev=$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch") + echo "$1.${branch//\//__}.${rev:0:7}" +} + +archive_artifact() { + set +x + echo + echo "[INFO] Archiving artifact to artifactStore." + echo + set -x + + cd "$base" + artifact="$(artifact_name_by_local_repos)" + # temp_job_store is necessary to atomically move it to production. + temp_job_store="$ARTIFACT_STORE/tmp/$job_name/" + job_store="$ARTIFACT_STORE/$job_name/" + + if [ ! -f "$temp_job_store/$artifact" ]; then + mkdir -p "$job_store" "$temp_job_store" + # remove outdated artifact first to avoid temporarily + # doubling of artifact storage consumption + rm -f "$job_store/*" + tar czf "$temp_job_store/$artifact" "deps" + mv -n "$temp_job_store/$artifact" "$job_store/$artifact" + rm -rf "$temp_job_store" + + log_artifact_hashes "$job_store/$artifact" + fi +} + +fetch_artifact() { + set +x + echo + echo "[INFO] Fetching artifact from artifactStore." + echo + set -x + + log_artifact_hashes "$1/$2" + cp "$1/$2" . + log_artifact_hashes "$2" + tar xzf "$2" + + if [ $? -gt 0 ]; then + set +x + echo + echo "[INFO] Artifact could not be fetched, triggering build_deps()" + set -x + build_deps() + fi +} + +# checksum is not used by this script itself, +# but might be handy in logs when debugging. +log_artifact_hashes() { + set +x + echo + echo "[INFO] name: $1" + echo "[INFO] sha256: $(sha256sum "$1" | cut -d ' ' -f1)" + echo + set -x +}