osmo-ci/scripts/osmo-build.sh

#!/bin/bash
#
# This script enables artifacts holding dependencies on a jenkins job level to
# speed up builds. Basically, it holds logic to check whether the necessary artifact
# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded
# it triggers the actual build.
#
# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh
# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built
# dependencies are detrmined after dependencies are built to ensure catching new
# changes in dep_n+1 meanwhile dep_n building.
#
# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves.
# The JOB_NAME variables will be injected to each jenkins' job by jenkins itself.
# When using script within a docker container one must inject jenkins' JOB_NAME variable
# to the container and ensure that ARTIFACT_STORE is mounted to the container's
# internal ARTIFACT_STORE.
#
# Artifacts will be stored as follows:
#
#		$ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
#			..._<dep_n>.<branch_n>.<rev_n>.tar.gz
#
# Note: each matrix-build has its own directory inside ARTIFACT_STORE.
#
# In order to make use of osmo-build.sh one needs to source it, e.g. from
# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of
# the project to be built in the workspace root. Following functions needs to be
# declared within a build script that sources osmo-build.sh:
#
#	- generic_deps()
#	- build_project()
#
# This is an example for building "libosmo-netif" which depends on "libosmocore"
# and "libosmo-abis".
#
# #!/bin/bash
#
# # osmo-build.sh must be in PATH
#	source osmo-build.sh
#
#	generic_deps() {
#   # Holds all required dependencies in the following form:
#   #
#   #  x="$(<PARALLEL_MAKE> $1 <dep_project> <branch> <config>)"
#   #
#   # $1 will be one of these script/functions:
#   #     - osmo-build-dep.sh: build a source tree
#   #     - artifact_name_by_local_repos()
#   #     - artifact_name_by_remote_repos()
#   #
#   # The arguments are:
#   #
#   # - PARALLEL_MAKE: override global PARALLEL_MAKE used for make,
#   #                  (e.g. PARALLEL_MAKE="-j1" to disable mutlithreading)
#   # - dep_project: the git repository name, gets places in a git.osmocom.org URL.
#   # - branch: branch (optional: default = master)
#   # - arg 3: configuration (optional: $cfg used in osmo-build-dep.sh)
#
#		x="$($1 libosmocore master ac_cv_path_DOXYGEN=false)"
#		"$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
#		x="${x}_$($1 libosmo-abis)"
#
#		echo "${x}.tar.gz"
#	}
#
#	build_project() {
#		# Necessary commands to build the project, expecting all dependencies have
#		# been build or fetched. Commands within build_project() will be executed
#		# in jenkins' $WORKSPACE.
#
# 	autoreconf --install --force
# 	./configure --enable-sanitize
# 	$MAKE $PARALLEL_MAKE
# 	$MAKE distcheck || cat-testlogs.sh
#	}
#
#	build
#


# BUILD FUNCTIONS
init_build() {

	if [ -z "$JOB_NAME" ]; then
		set +x
		echo
		echo "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
		echo
		set -x
		exit 1
	fi

	if [ -z "$ARTIFACT_STORE" ]; then
		set +x
		echo
		echo "[ERROR] ARTIFACT_STORE variable is not set on this build slave"
		echo
		set -x
		exit 1
	fi

	base="$(pwd)"
	deps="$base/deps"
	inst="$deps/install"

	# obtain the project name from the git clone found in the workspace root
	project=$(git config --get --local remote.origin.url \
		| cut -d '/' -f4 | cut -d '.' -f1)

	# replace invalid char for dirs in jenkins variable
	# ( '/' separates job name and matrix-axis)
	job_name="${JOB_NAME//\//__}"

	export base deps inst project job_name
	export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
	export LD_LIBRARY_PATH="$inst/lib"
}

build_deps() {
	set +x
	echo
	echo "[INFO] Compile $project dependencies from source."
	echo
	set -x

	mkdir -p "$deps"
	rm -rf "$inst"

	generic_deps "osmo-build-dep.sh"
}

build() {

	init_build

	needed_artifact="$(artifact_name_by_remote_repos)"

	if [ -f "$ARTIFACT_STORE/$job_name/$needed_artifact" ]; then
		fetch_artifact "$ARTIFACT_STORE/$job_name" "$needed_artifact"
	else
		build_deps
		archive_artifact
	fi

	set +x
	echo
	echo " ============================= $project =========================="
	echo
	set -x

	build_project
}

# ARTIFACT FUNCTIONS
artifact_name_by_local_repos() {
	generic_deps "branch_and_rev_of_local_repo"
	cd "$base"
}

artifact_name_by_remote_repos() {
	generic_deps "branch_and_rev_of_remote_repo"
}

branch_and_rev_of_local_repo() {
	cd "$deps/$1"
	rev=$(git rev-parse --short HEAD)
	branch=$(git rev-parse --abbrev-ref HEAD)
	echo "$1.${branch//\//__}.${rev}"
}

branch_and_rev_of_remote_repo() {
	if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
	rev=$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch")
	echo "$1.${branch//\//__}.${rev:0:7}"
}

archive_artifact() {
	set +x
	echo
	echo "[INFO] Archiving artifact to artifactStore."
	echo
	set -x

	cd "$base"
	artifact="$(artifact_name_by_local_repos)"
	# temp_job_store is necessary to atomically move it to production.
	temp_job_store="$ARTIFACT_STORE/tmp/$job_name/"
	job_store="$ARTIFACT_STORE/$job_name/"

	if [ ! -f "$temp_job_store/$artifact" ]; then
		mkdir -p "$job_store" "$temp_job_store"
		# remove outdated artifact first to avoid temporarily
		# doubling of artifact storage consumption
		rm -f "$job_store/*"
		tar czf "$temp_job_store/$artifact" "deps"
		mv -n "$temp_job_store/$artifact" "$job_store/$artifact"
		rm -rf "$temp_job_store"

		log_artifact_hashes "$job_store/$artifact"
	fi
}

fetch_artifact() {
	set +x
	echo
	echo "[INFO] Fetching artifact from artifactStore."
	echo
	set -x

	log_artifact_hashes "$1/$2"
	cp "$1/$2" .
	log_artifact_hashes "$2"
	tar xzf "$2"

	if [ $? -gt 0 ]; then
		set +x
		echo
		echo "[INFO] Artifact could not be fetched, triggering build_deps()"
		set -x
		build_deps()
	fi
}

# checksum is not used by this script itself,
# but might be handy in logs when debugging.
log_artifact_hashes() {
	set +x
	echo
	echo "[INFO] name: $1"
	echo "[INFO] sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
	echo
	set -x
}