jenkins-common.sh: add common clean up trap

Add set_clean_up_trap() in jenkins-common.sh and run it at the beginning
of the jenkins.sh files. Move the common clean up code from the end of
every jenkins.sh file into clean_up_common(), which gets called by the
trap. Add a custom clean_up() function to those jenkins.sh files that
need additional clean up.

Replace explicit container stop commands (for containers attached to the
docker network) with one call to network_clean() in clean_up_common(). It
kills all containers attached to the docker network.

The motivation for this change is the upcoming optional build of initrd
and kernel during ttcn3-ggsn-test/jenkins.sh. After building these, a
short smoke test will be performed to make sure we can boot the kernel
and initrd, before continuing to run the entire testsuite against it. If
building or the smoke test fails, we must do a proper clean up of the
network and fix permissions.

Related: OS#3208
Change-Id: I807c6221d22fec31db23e2516e477a3eb43347fb
This commit is contained in:
Oliver Smith 2021-02-19 15:14:42 +01:00
parent c63a131a60
commit 12b961942a
25 changed files with 100 additions and 177 deletions

View File

@ -160,6 +160,36 @@ collect_logs() {
cat "$VOL_BASE_DIR"/*/junit-*.log || true
}
clean_up_common() {
set +e
set +x
echo
echo "### Clean up ###"
echo
set -x
# Clear trap
trap - EXIT INT TERM 0
# Run clean_up() from ttcn3-*/jenkins.sh, if defined
if type clean_up >/dev/null; then
clean_up
fi
network_clean
network_remove
rm -rf "$VOL_BASE_DIR"/unix
collect_logs
}
# Run clean up code when the script stops (either by failing command, by ^C, or
# after running through successfully). The caller can define a custom clean_up
# function.
set_clean_up_trap() {
trap clean_up_common EXIT INT TERM 0
}
set -x
# non-jenkins execution: assume local user name

View File

@ -9,6 +9,8 @@ docker_images_require \
"debian-stretch-titan" \
"nplab-m3ua-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/m3ua-tester
cp m3ua-param-testtool.scm all-sgp-tests.txt $VOL_BASE_DIR/m3ua-tester/
@ -34,8 +36,3 @@ docker run --rm \
-v $VOL_BASE_DIR/m3ua-tester:/data \
--name ${BUILD_TAG}-m3ua-test \
$REPO_USER/nplab-m3ua-test > $VOL_BASE_DIR/junit-xml-m3ua.log
docker container stop -t 1 ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -9,6 +9,8 @@ docker_images_require \
"sigtran-tests" \
"nplab-sua-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/sua-tester
cp sua-param-testtool-sgp.scm some-sua-sgp-tests.txt $VOL_BASE_DIR/sua-tester/
@ -34,8 +36,3 @@ docker run --rm \
-v $VOL_BASE_DIR/sua-tester:/data \
--name ${BUILD_TAG}-sua-test \
$REPO_USER/nplab-sua-test > $VOL_BASE_DIR/junit-xml-sua.log
docker container stop -t 1 ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -58,6 +58,8 @@ docker_images_require \
"debian-stretch-jenkins" \
"osmo-gsm-tester"
set_clean_up_trap
SUBNET=50
network_create $SUBNET
@ -101,12 +103,6 @@ docker run --rm \
$REPO_USER/osmo-gsm-tester
rc=$?
echo Stopping containers
docker container kill ${BUILD_TAG}-ogt-slave
network_remove
collect_logs
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color

View File

@ -7,6 +7,8 @@ docker_images_require \
"osmo-bts-$IMAGE_SUFFIX" \
"ttcn3-bsc-test"
set_clean_up_trap
#Make sure NET_NAME doesn't clash with the AoIP BSC test
NET_NAME=ttcn3-bsc_sccplite-test
@ -48,12 +50,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-bsc-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-bsc-test
echo Stopping containers
for i in `seq 0 2`; do
docker container kill ${BUILD_TAG}-bts$i
done
docker container kill ${BUILD_TAG}-bsc
network_remove
collect_logs

View File

@ -8,6 +8,8 @@ docker_images_require \
"osmo-bts-$IMAGE_SUFFIX" \
"ttcn3-bsc-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/bsc-tester
cp BSC_Tests.cfg $VOL_BASE_DIR/bsc-tester/
@ -62,13 +64,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-bsc-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-bsc-test
echo Stopping containers
for i in `seq 0 2`; do
docker container kill ${BUILD_TAG}-bts$i
done
docker container kill ${BUILD_TAG}-bsc
docker container kill ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -6,6 +6,7 @@ docker_images_require \
"osmo-nitb-$IMAGE_SUFFIX" \
"ttcn3-bscnat-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/bscnat-tester
cp BSCNAT_Tests.cfg $VOL_BASE_DIR/bscnat-tester/
@ -36,9 +37,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-bscnat-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-bscnat-test
echo Stopping containers
docker container kill ${BUILD_TAG}-bscnat
network_remove
collect_logs

View File

@ -9,6 +9,17 @@ docker_images_require \
"osmocom-bb-host-master" \
"ttcn3-bts-test"
set_clean_up_trap
clean_up() {
# append ':hopping' to the classnames,
# e.g. "classname='BTS_Tests'" => "classname='BTS_Tests:hopping'"
# e.g. "classname='BTS_Tests_SMSCB'" => "classname='BTS_Tests_SMSCB:hopping'"
# so the hopping test cases would not interfere with non-hopping ones in Jenkins
sed -i "s/classname='\([^']\+\)'/classname='\1:hopping'/g" \
$VOL_BASE_DIR/bts-tester-hopping/junit-xml-hopping-*.log
}
start_bsc() {
echo Starting container with BSC
docker run --rm \
@ -170,20 +181,3 @@ docker container kill ${BUILD_TAG}-bts
start_bsc
start_bts trx 1
start_testsuite hopping
# append ':hopping' to the classnames,
# e.g. "classname='BTS_Tests'" => "classname='BTS_Tests:hopping'"
# e.g. "classname='BTS_Tests_SMSCB'" => "classname='BTS_Tests_SMSCB:hopping'"
# so the hopping test cases would not interfere with non-hopping ones in Jenkins
sed -i "s/classname='\([^']\+\)'/classname='\1:hopping'/g" \
$VOL_BASE_DIR/bts-tester-hopping/junit-xml-hopping-*.log
echo Stopping containers
docker container kill ${BUILD_TAG}-trxcon
docker container kill ${BUILD_TAG}-fake_trx
docker container kill ${BUILD_TAG}-bsc
docker container kill ${BUILD_TAG}-bts
docker container kill ${BUILD_TAG}-bsc
network_remove
rm -rf $VOL_BASE_DIR/unix
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-cbc-$IMAGE_SUFFIX" \
"ttcn3-cbc-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/cbc-tester
cp CBC_Tests.cfg $VOL_BASE_DIR/cbc-tester/
@ -36,9 +38,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-cbc-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-cbc-test
echo Stopping containers
docker container kill ${BUILD_TAG}-cbc
network_remove
collect_logs

View File

@ -5,6 +5,8 @@ IMAGE_SUFFIX="${IMAGE_SUFFIX:-master}"
docker_images_require \
"ttcn3-fr-test"
set_clean_up_trap
SUBNET=26
network_create $SUBNET
@ -59,10 +61,3 @@ done
# emulate running container in foreground, which is no longer possible as we
# must shift the net-devices into the container _after_ it is started
docker logs -f ${BUILD_TAG}-ttcn3-fr-test
echo Stopping containers
docker container kill ${BUILD_TAG}-frnet
network_remove
collect_logs

View File

@ -19,6 +19,8 @@ docker_images_require \
"osmo-gbproxy-$IMAGE_SUFFIX" \
"ttcn3-gbproxy-test"
set_clean_up_trap
SUBNET=25
network_create $SUBNET
@ -72,9 +74,3 @@ done
# emulate runnign container in foreground, which is no longer possible as we
# must shift the net-devices into the container _after_ it is started
docker logs -f ${BUILD_TAG}-ttcn3-gbproxy-test
echo Stopping containers
docker container kill ${BUILD_TAG}-gbproxy
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-gbproxy-$IMAGE_SUFFIX" \
"ttcn3-gbproxy-test"
set_clean_up_trap
SUBNET=24
network_create $SUBNET
@ -39,9 +41,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-gbproxy-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-gbproxy-test $@
echo Stopping containers
docker container kill ${BUILD_TAG}-gbproxy
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-ggsn-$IMAGE_SUFFIX" \
"ttcn3-ggsn-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/ggsn-tester
cp GGSN_Tests.cfg $VOL_BASE_DIR/ggsn-tester/
@ -38,9 +40,3 @@ docker run --rm \
--name ${BUILD_TAG}-ggsn-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-ggsn-test
# stop GGSN after test has completed
docker container stop ${BUILD_TAG}-ggsn
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-hlr-$IMAGE_SUFFIX" \
"ttcn3-hlr-test"
set_clean_up_trap
SUBNET=10
network_create $SUBNET
@ -34,9 +36,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-hlr-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-hlr-test
echo Stopping containers
docker container kill ${BUILD_TAG}-hlr
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-mgw-$IMAGE_SUFFIX" \
"ttcn3-mgw-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/mgw-tester
cp MGCP_Test.cfg $VOL_BASE_DIR/mgw-tester/
@ -33,9 +35,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-mgw-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-mgw-test
# stop mgw after test has completed
docker container stop ${BUILD_TAG}-mgw
network_remove
collect_logs

View File

@ -7,6 +7,8 @@ docker_images_require \
"osmo-msc-$IMAGE_SUFFIX" \
"ttcn3-msc-test"
set_clean_up_trap
SUBNET=20
network_create $SUBNET
@ -52,11 +54,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-msc-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-msc-test
echo Stopping containers
docker container kill ${BUILD_TAG}-msc
docker container kill ${BUILD_TAG}-stp
network_remove
rm -rf $VOL_BASE_DIR/unix
collect_logs

View File

@ -2,6 +2,22 @@
. ../jenkins-common.sh
set_clean_up_trap
clean_up() {
# start some stupid helper container so we can access the volume
docker run --rm \
-v ttcn3-nitb-sysinfo-vol:/ttcn3-nitb-sysinfo \
-v nitb-vol:/nitb \
-v bts-vol:/bts \
--name ${BUILD_TAG}-sysinfo-helper -d \
busybox /bin/sh -c 'sleep 1000 & wait'
docker cp ${BUILD_TAG}-sysinfo-helper:/ttcn3-nitb-sysinfo $VOL_BASE_DIR
docker cp ${BUILD_TAG}-sysinfo-helper:/nitb $VOL_BASE_DIR
docker cp ${BUILD_TAG}-sysinfo-helper:/bts $VOL_BASE_DIR
docker container stop -t 0 ${BUILD_TAG}-sysinfo-helper
}
SUBNET=5
network_create $SUBNET
@ -35,22 +51,3 @@ docker run --rm \
-v ttcn3-nitb-sysinfo-vol:/data \
--name ${BUILD_TAG}-ttcn3-nitb-sysinfo \
$REPO_USER/ttcn3-nitb-sysinfo
# stop bts + nitb after test has completed
docker container stop ${BUILD_TAG}-bts
docker container stop ${BUILD_TAG}-nitb
# start some stupid helper container so we can access the volume
docker run --rm \
-v ttcn3-nitb-sysinfo-vol:/ttcn3-nitb-sysinfo \
-v nitb-vol:/nitb \
-v bts-vol:/bts \
--name ${BUILD_TAG}-sysinfo-helper -d \
busybox /bin/sh -c 'sleep 1000 & wait'
docker cp ${BUILD_TAG}-sysinfo-helper:/ttcn3-nitb-sysinfo $VOL_BASE_DIR
docker cp ${BUILD_TAG}-sysinfo-helper:/nitb $VOL_BASE_DIR
docker cp ${BUILD_TAG}-sysinfo-helper:/bts $VOL_BASE_DIR
docker container stop -t 0 ${BUILD_TAG}-sysinfo-helper
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-pcu-$IMAGE_SUFFIX" \
"ttcn3-pcu-test"
set_clean_up_trap
SUBNET=14
network_create $SUBNET
@ -40,10 +42,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-pcu-test-sns \
$DOCKER_ARGS \
$REPO_USER/ttcn3-pcu-test
echo Stopping containers
docker container kill ${BUILD_TAG}-pcu-sns
network_remove
rm -rf $VOL_BASE_DIR/unix
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-pcu-$IMAGE_SUFFIX" \
"ttcn3-pcu-test"
set_clean_up_trap
SUBNET=13
network_create $SUBNET
@ -40,10 +42,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-pcu-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-pcu-test
echo Stopping containers
docker container kill ${BUILD_TAG}-pcu
network_remove
rm -rf $VOL_BASE_DIR/unix
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-remsim-$IMAGE_SUFFIX" \
"ttcn3-remsim-test"
set_clean_up_trap
start_server() {
echo Starting container with osmo-remsim-server
docker run --rm \
@ -87,7 +89,3 @@ echo "Changing to client configuration"
start_client
cp client/REMSIM_Tests.cfg $VOL_BASE_DIR/remsim-tester/
start_testsuite
docker container kill ${BUILD_TAG}-client
network_remove
collect_logs

View File

@ -7,6 +7,8 @@ docker_images_require \
"osmo-stp-master" \
"ttcn3-sccp-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/sccp-tester
cp SCCP_Tests.cfg $VOL_BASE_DIR/sccp-tester/
@ -36,8 +38,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-sccp-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-sccp-test
docker container kill ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -7,6 +7,8 @@ docker_images_require \
"osmo-sgsn-$IMAGE_SUFFIX" \
"ttcn3-sgsn-test"
set_clean_up_trap
SUBNET=8
network_create $SUBNET
@ -49,10 +51,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-sgsn-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-sgsn-test
echo Stopping containers
docker container kill ${BUILD_TAG}-sgsn
docker container kill ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-sip-$IMAGE_SUFFIX" \
"ttcn3-sip-test"
set_clean_up_trap
SUBNET=11
network_create $SUBNET
@ -40,10 +42,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-sip-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-sip-test
echo Stopping containers
docker container kill ${BUILD_TAG}-sip-connector
network_remove
rm -rf $VOL_BASE_DIR/unix
collect_logs

View File

@ -7,6 +7,8 @@ docker_images_require \
"osmo-smlc-$IMAGE_SUFFIX" \
"ttcn3-smlc-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/smlc-tester
cp SMLC_Tests.cfg $VOL_BASE_DIR/smlc-tester/
@ -47,10 +49,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-smlc-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-smlc-test
echo Stopping containers
docker container kill ${BUILD_TAG}-smlc
docker container kill ${BUILD_TAG}-stp
network_remove
collect_logs

View File

@ -6,6 +6,8 @@ docker_images_require \
"osmo-stp-$IMAGE_SUFFIX" \
"ttcn3-stp-test"
set_clean_up_trap
mkdir $VOL_BASE_DIR/stp-tester
cp STP_Tests.cfg $VOL_BASE_DIR/stp-tester/
@ -33,8 +35,3 @@ docker run --rm \
--name ${BUILD_TAG}-ttcn3-stp-test \
$DOCKER_ARGS \
$REPO_USER/ttcn3-stp-test
docker container kill ${BUILD_TAG}-stp
network_remove
collect_logs