jenkins: extract common parts of build scripts to separate file

Have all complexity in one common shell script, greatly simplify the individual
scripts.

This allows to provide a specific branch or git hash to build instead of
current master. Some scripts allowed to provide branch names before, this now
also allows using git hashes directly.

Environment variables can be used to override the git hash/branch to use for
specific repositories.

Motivation for this patch: we need this to investigate failure causes more
easily.

Change-Id: I5ac2f90d006a1b2f6c246976346d852a70c89089
This commit is contained in:
Neels Hofmeyr 2017-05-29 22:53:34 +02:00
parent e62364e7fc
commit 53e758ae4c
6 changed files with 190 additions and 364 deletions

View File

@ -0,0 +1,152 @@
#!source_this_file
# Common parts for osmo-gsm-tester jenkins build scripts. Use like in below example:
#
#--------------
# #!/bin/sh
# set -e -x
# base="$PWD"
# name="osmo-name"
# . "$(dirname "$0")/jenkins-build-common.sh"
#
# build_repo libosmocore --configure --opts
# build_repo libosmo-foo special_branch --configure --opts
# build_repo osmo-bar
#
# create_bin_tgz
#--------------
#
# Some explanations:
#
# To allow calling from arbitrary working directories, other scripts should
# source this file like shown above.
#
# Sourcing scripts must provide some variables/functions, see above.
# In addition, these values can optionally be passed to override:
# git_url, prefix, prefix_real, BUILD_NUMBER
#
# CONFIGURE_FLAGS may contain flags that should be passed to all builds'
# ./configure steps (useful e.g. for building in the sysmobts SDK).
#
# For each built repository, a specific git branch or hash can be provided by
# environment variable: OSMO_GSM_TESTER_BUILD_$repo="<git-hash>"
# NOTE: convert $repo's dashes to underscore. For example:
# OSMO_GSM_TESTER_BUILD_osmo_hlr="f001234abc"
# OSMO_GSM_TESTER_BUILD_libosmocore="my/branch"
# ("origin/" is prepended to branch names automatically)
if [ -z "$name" -o -z "$base" ]; then
set +x
echo "Some environment variables are not provided as required by jenkins-build-common.sh. Error."
exit 1
fi
git_url="${git_url-"git://git.osmocom.org"}"
prefix="${prefix-"$base/inst-$name"}"
# prefix_real is usually identical with prefix, except when installing to a
# different $DESTDIR than /, which is the case for example when building
# osmo-bts within the sysmoBTS SDK
prefix_real="${prefix_real-"$prefix"}"
export PKG_CONFIG_PATH="$prefix_real/lib/pkgconfig:$PKG_CONFIG_PATH"
export LD_LIBRARY_PATH="$prefix_real/lib:$LD_LIBRARY_PATH"
# Show current environment. Sometimes the LESS_ vars have ansi colors making a
# mess, so exclude those.
env | grep -v "^LESS" | sort
# clean the workspace
rm -f "$base/${name}"*.tgz rm -f "$base/${name}"*.md5
rm -rf "$prefix_real"
mkdir -p "$prefix_real"
have_repo() {
repo="$1"
branch="${2-master}"
# Evaluate environment for instructions to build a specific git hash.
# Using a hash as $branch above unfortunately doesn't work.
branch_override_var="$(echo "OSMO_GSM_TESTER_BUILD_$repo" | sed 's/-/_/g')"
branch_override="$(eval "echo \$$branch_override_var")"
if [ -n "$branch_override" ]; then
branch="$branch_override"
fi
cd "$base"
if [ ! -d "$repo" ]; then
git clone "$git_url/$repo" "$repo"
fi
cd "$repo"
rm -rf *
git fetch origin
# Figure out whether we need to prepend origin/ to find branches in upstream
if ! git rev-parse "$branch"; then
branch="origin/$branch"
fi
git reset --hard "$branch"
git rev-parse HEAD
cd "$base"
}
build_repo() {
# usage: build_repo <name> [<branch>] [--configure-opts [...]]
dep="$1"
branch="master"
if [ -z "$(echo "$2" | grep '^-')" ]; then
# second arg does not start with a dash, it's empty or a branch
branch="$2"
if [ -n "$branch" ]; then
# we had a branch arg, need to shift once more to get config options
shift
else
branch="master"
fi
fi
shift
configure_opts="$@"
set +x; echo "
====================== $dep
"; set -x
have_repo "$dep" "$branch"
cd "$dep"
echo "$(git rev-parse HEAD) $dep" >> "$prefix_real/${name}_git_hashes.txt"
# special shim: we know the openbsc.git needs to be built in the openbsc/ subdir.
if [ "$dep" = "openbsc" ]; then
cd openbsc
fi
set +x; echo; echo; set -x
autoreconf -fi
set +x; echo; echo; set -x
./configure --prefix="$prefix" $CONFIGURE_FLAGS $configure_opts
set +x; echo; echo; set -x
make -j8 || make # libsmpp34 can't build in parallel
set +x; echo; echo; set -x
make install
}
create_bin_tgz() {
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix_real/share/doc/{libosmocore,libosmo-sccp}" || true
# build the archive that is going to be copied to the tester
cd "$prefix_real"
this="$name.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
}

View File

@ -1,92 +1,23 @@
#!/bin/sh
set -e -x
deps="
libosmocore
libosmo-abis
osmo-bts
"
poky="/opt/poky/1.5.4"
. "$poky/environment-setup-armv5te-poky-linux-gnueabi"
# Cross-compilation: all installations need to be put in the sysmo SDK sysroot
export DESTDIR="$poky/sysroots/armv5te-poky-linux-gnueabi"
base="$PWD"
rm -f "$base/osmo-bts-sysmo.*.tgz"
have_repo() {
repo="$1"
cd "$base"
if [ ! -d "$repo" ]; then
git clone "git://git.osmocom.org/$repo" "$repo"
fi
cd "$repo"
git clean -dxf
git fetch origin
git reset --hard origin/master
git rev-parse HEAD
cd "$base"
}
for dep in $deps; do
have_repo "$dep"
done
name="osmo-bts-sysmo"
prefix="/usr/local/jenkins-build/inst-$name"
prefix_real="$DESTDIR$prefix"
. "$(dirname "$0")/jenkins-build-common.sh"
# for gsm_data_shared.h
have_repo openbsc
. /opt/poky/1.5.4/environment-setup-armv5te-poky-linux-gnueabi
build_repo libosmocore --disable-pcsc
build_repo libosmo-abis
build_repo osmo-bts --enable-sysmocom-bts --with-openbsc=$base/openbsc/openbsc/include
# Cross-compilation: all installations need to be put in the sysmo SDK sysroot
export DESTDIR="/opt/poky/1.5.4/sysroots/armv5te-poky-linux-gnueabi"
prefix_base="/usr/local/jenkins-build"
prefix_base_real="$DESTDIR$prefix_base"
rm -rf "$prefix_base_real"
prefix="$prefix_base/inst-osmo-bts-sysmo"
prefix_real="$DESTDIR$prefix"
mkdir -p "$prefix_real"
# Installation in non-system dir, but keep the PKG_CONFIG_PATH from the SDK:
export PKG_CONFIG_PATH="$prefix_real/lib/pkgconfig:$PKG_CONFIG_PATH"
env
for dep in $deps; do
set +x; echo "
====================== $dep
"; set -x
cd "$base/$dep"
rm -rf *
git checkout .
echo "$(git rev-parse HEAD) $dep" >> "$prefix_real/osmo-bts-sysmo_git_hashes.txt"
autoreconf -fi
config_opts=""
case "$dep" in
'libosmocore') config_opts="--disable-pcsc" ;;
'osmo-bts') config_opts="--enable-sysmocom-bts --with-openbsc=$base/openbsc/openbsc/include" ;;
esac
set +x; echo; echo; set -x
./configure --prefix="$prefix" $CONFIGURE_FLAGS $config_opts
set +x; echo; echo; set -x
make -j8
set +x; echo; echo; set -x
make install
done
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix_real/share/doc"
# build the archive that is going to be copied to the tester and then to the BTS
rm "$base"/*.tgz "$base"/*.md5 || true
cd "$prefix_real"
this="osmo-bts-sysmo.build-${BUILD_NUMBER}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
create_bin_tgz

View File

@ -1,82 +1,15 @@
#!/bin/sh
set -x -e
set -e -x
base="$PWD"
prefix="$base/inst-osmo-bts-trx"
rm -f "$base/osmo-bts-trx*.tgz"
deps="
libosmocore
libosmo-abis
osmo-trx
osmo-bts
"
have_repo() {
repo="$1"
cd "$base"
if [ ! -d "$repo" ]; then
git clone "git://git.osmocom.org/$repo" "$repo"
fi
cd "$repo"
git clean -dxf
git fetch origin
git reset --hard origin/master
git rev-parse HEAD
cd "$base"
}
name="osmo-bts-trx"
. "$(dirname "$0")/jenkins-build-common.sh"
# for gsm_data_shared.*
have_repo openbsc
build_repo libosmocore
build_repo libosmo-abis
build_repo osmo-trx --without-sse
build_repo osmo-bts --enable-trx --with-openbsc=$base/openbsc/openbsc/include
rm -rf "$prefix"
mkdir -p "$prefix"
export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
export LD_LIBRARY_PATH="$prefix/lib"
for dep in $deps; do
set +x; echo "
====================== $dep
"; set -x
have_repo "$dep"
cd "$dep"
rm -rf *
git checkout .
echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-bts-trx_osmo-trx_git_hashes.txt"
autoreconf -fi
config_opts=""
case "$repo" in
'osmo-bts') config_opts="--enable-trx --with-openbsc=$base/openbsc/openbsc/include" ;;
'osmo-trx') config_opts="--without-sse" ;;
esac
set +x; echo; echo; set -x
./configure --prefix="$prefix" $config_opts
set +x; echo; echo; set -x
make -j8
set +x; echo; echo; set -x
make install
done
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix/share/doc"
# build the archive that is going to be copied to the tester
rm "$base"/*.tgz "$base"/*.md5 || true
cd "$prefix"
this="osmo-bts-trx.build-${BUILD_NUMBER}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
create_bin_tgz

View File

@ -1,69 +1,11 @@
#!/bin/sh
set -e -x
base="$PWD"
prefix="$base/inst-osmo-hlr"
rm -f "$base/osmo-hlr*.tgz"
git_url="git://git.osmocom.org"
have_repo() {
repo="$1"
branch="${2-master}"
cd "$base"
if [ ! -d "$repo" ]; then
git clone "$git_url/$repo" -b "$branch" "$repo"
fi
cd "$repo"
rm -rf *
git fetch origin
git checkout .
git checkout "$branch"
git reset --hard origin/"$branch"
git rev-parse HEAD
cd "$base"
}
build_repo() {
dep="$1"
branch="${2-master}"
have_repo "$dep" "$branch"
cd "$dep"
echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-hlr_git_hashes.txt"
config_opts=""
autoreconf -fi
./configure --prefix="$prefix" $config_opts
make -j8
make install
}
rm -rf "$prefix"
mkdir -p "$prefix"
export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
export LD_LIBRARY_PATH="$prefix/lib"
name="osmo-hlr"
. "$(dirname "$0")/jenkins-build-common.sh"
build_repo libosmocore
build_repo libosmo-abis
build_repo osmo-hlr
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix/share/doc/libosmocore"
# build the archive that is going to be copied to the tester
rm "$base"/*.tgz "$base"/*.md5 || true
cd "$prefix"
this="osmo-hlr.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
create_bin_tgz

View File

@ -1,62 +1,8 @@
#!/bin/sh
set -e -x
base="$PWD"
prefix="$base/inst-osmo-msc"
rm -f "$base/osmo-msc*.tgz"
git_url="git://git.osmocom.org"
have_repo() {
repo="$1"
branch="${2-master}"
cd "$base"
if [ ! -d "$repo" ]; then
git clone "$git_url/$repo" -b "$branch" "$repo"
fi
cd "$repo"
rm -rf *
git fetch origin
git checkout .
git checkout "$branch"
git reset --hard origin/"$branch"
git rev-parse HEAD
cd "$base"
}
build_repo() {
dep="$1"
branch="${2-master}"
have_repo "$dep" "$branch"
cd "$dep"
echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-msc_git_hashes.txt"
config_opts=""
case "$dep" in
'openbsc')
config_opts="$config_opts --enable-smpp --enable-osmo-bsc --enable-nat --enable-iu"
cd openbsc/
;;
esac
autoreconf -fi
./configure --prefix="$prefix" $config_opts
make -j8 || make # libsmpp34 can't build in parallel
make install
}
rm -rf "$prefix"
mkdir -p "$prefix"
export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
export LD_LIBRARY_PATH="$prefix/lib"
name="osmo-msc"
. "$(dirname "$0")/jenkins-build-common.sh"
build_repo libosmocore
build_repo libosmo-abis
@ -66,17 +12,6 @@ build_repo libsmpp34
build_repo libosmo-sccp neels/aoip # TEMPORARY BRANCH
build_repo libasn1c
build_repo osmo-iuh neels/sigtran # TEMPORARY BRANCH
build_repo openbsc aoip
build_repo openbsc aoip --enable-smpp --enable-osmo-bsc --enable-nat --enable-iu
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix/share/doc"
# build the archive that is going to be copied to the tester
rm "$base"/*.tgz "$base"/*.md5 || true
cd "$prefix"
this="osmo-msc.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
create_bin_tgz

View File

@ -1,82 +1,15 @@
#!/bin/sh
set -e -x
base="$PWD"
prefix="$base/inst-osmo-nitb"
name="osmo-nitb"
. "$(dirname "$0")/jenkins-build-common.sh"
rm -f "$base/osmo-nitb*.tgz"
build_repo libosmocore
build_repo libosmo-abis
build_repo libosmo-netif
build_repo openggsn
build_repo libsmpp34
build_repo libosmo-sccp
build_repo openbsc --enable-smpp --enable-osmo-bsc --enable-nat
deps="
libosmocore
libosmo-abis
libosmo-netif
openggsn
libsmpp34
libosmo-sccp
openbsc
"
have_repo() {
repo="$1"
cd "$base"
if [ ! -d "$repo" ]; then
git clone "git://git.osmocom.org/$repo" "$repo"
fi
cd "$repo"
git clean -dxf
git fetch origin
git reset --hard origin/master
git rev-parse HEAD
cd "$base"
}
rm -rf "$prefix"
mkdir -p "$prefix"
export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
export LD_LIBRARY_PATH="$prefix/lib"
for dep in $deps; do
set +x; echo "
====================== $dep
"; set -x
have_repo "$dep"
cd "$dep"
rm -rf *
git checkout .
echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-nitb_git_hashes.txt"
config_opts=""
case "$dep" in
'openbsc')
config_opts="$config_opts --enable-smpp --enable-osmo-bsc --enable-nat"
cd openbsc/
;;
esac
autoreconf -fi
set +x; echo; echo; set -x
./configure --prefix="$prefix" $config_opts
set +x; echo; echo; set -x
make -j8 || make # libsmpp34 can't build in parallel
set +x; echo; echo; set -x
make install
done
# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
# a significant amount compared to the binaries
rm -rf "$prefix/share/doc"
# build the archive that is going to be copied to the tester
rm "$base"/*.tgz "$base"/*.md5 || true
cd "$prefix"
this="osmo-nitb.build-${BUILD_NUMBER}"
tar="${this}.tgz"
tar czf "$base/$tar" *
cd "$base"
md5sum "$tar" > "${this}.md5"
create_bin_tgz