jenkins-common.sh: add common clean up trap

Add set_clean_up_trap() in jenkins-common.sh and run it at the beginning
of the jenkins.sh files. Move the common clean up code from the end of
every jenkins.sh file into clean_up_common(), which gets called by the
trap. Add a custom clean_up() function to those jenkins.sh files that
need additional clean up.

Replace explicit container stop commands (for containers attached to the
docker network) with one call to network_clean() in clean_up_common(). It
kills all containers attached to the docker network.

The motivation for this change is the upcoming optional build of initrd
and kernel during ttcn3-ggsn-test/jenkins.sh. After building these, a
short smoke test will be performed to make sure we can boot the kernel
and initrd, before continuing to run the entire testsuite against it. If
building or the smoke test fails, we must do a proper clean up of the
network and fix permissions.

Related: OS#3208
Change-Id: I807c6221d22fec31db23e2516e477a3eb43347fb

jenkins-common.sh

@@ -160,6 +160,36 @@ collect_logs() {
 	cat "$VOL_BASE_DIR"/*/junit-*.log || true
 }
 
+clean_up_common() {
+	set +e
+
+	set +x
+	echo
+	echo "### Clean up ###"
+	echo
+	set -x
+
+	# Clear trap
+	trap - EXIT INT TERM 0
+
+	# Run clean_up() from ttcn3-*/jenkins.sh, if defined
+	if type clean_up >/dev/null; then
+		clean_up
+	fi
+
+	network_clean
+	network_remove
+	rm -rf "$VOL_BASE_DIR"/unix
+	collect_logs
+}
+
+# Run clean up code when the script stops (either by failing command, by ^C, or
+# after running through successfully). The caller can define a custom clean_up
+# function.
+set_clean_up_trap() {
+	trap clean_up_common EXIT INT TERM 0
+}
+
 set -x
 
 # non-jenkins execution: assume local user name

nplab-m3ua-test/jenkins.sh

@@ -9,6 +9,8 @@ docker_images_require \
 	"debian-stretch-titan" \
 	"nplab-m3ua-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/m3ua-tester
 cp m3ua-param-testtool.scm all-sgp-tests.txt $VOL_BASE_DIR/m3ua-tester/
 
@@ -34,8 +36,3 @@ docker run	--rm \
 		-v $VOL_BASE_DIR/m3ua-tester:/data \
 		--name ${BUILD_TAG}-m3ua-test \
 		$REPO_USER/nplab-m3ua-test > $VOL_BASE_DIR/junit-xml-m3ua.log
-
-docker container stop -t 1 ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

nplab-sua-test/jenkins.sh

@@ -9,6 +9,8 @@ docker_images_require \
 	"sigtran-tests" \
 	"nplab-sua-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/sua-tester
 cp sua-param-testtool-sgp.scm some-sua-sgp-tests.txt $VOL_BASE_DIR/sua-tester/
 
@@ -34,8 +36,3 @@ docker run	--rm \
 		-v $VOL_BASE_DIR/sua-tester:/data \
 		--name ${BUILD_TAG}-sua-test \
 		$REPO_USER/nplab-sua-test > $VOL_BASE_DIR/junit-xml-sua.log
-
-docker container stop -t 1 ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

osmo-gsm-tester/jenkins.sh

@@ -58,6 +58,8 @@ docker_images_require \
 	"debian-stretch-jenkins" \
 	"osmo-gsm-tester"
 
+set_clean_up_trap
+
 SUBNET=50
 network_create $SUBNET
 
@@ -101,12 +103,6 @@ docker run	--rm \
 		$REPO_USER/osmo-gsm-tester
 rc=$?
 
-echo Stopping containers
-docker container kill ${BUILD_TAG}-ogt-slave
-
-network_remove
-collect_logs
-
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 NC='\033[0m' # No Color

ttcn3-bsc-test/jenkins-sccplite.sh

@@ -7,6 +7,8 @@ docker_images_require \
 	"osmo-bts-$IMAGE_SUFFIX" \
 	"ttcn3-bsc-test"
 
+set_clean_up_trap
+
 #Make sure NET_NAME doesn't clash with the AoIP BSC test
 NET_NAME=ttcn3-bsc_sccplite-test
 
@@ -48,12 +50,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-bsc-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-bsc-test
-
-echo Stopping containers
-for i in `seq 0 2`; do
-	docker container kill ${BUILD_TAG}-bts$i
-done
-docker container kill ${BUILD_TAG}-bsc
-
-network_remove
-collect_logs

ttcn3-bsc-test/jenkins.sh

@@ -8,6 +8,8 @@ docker_images_require \
 	"osmo-bts-$IMAGE_SUFFIX" \
 	"ttcn3-bsc-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/bsc-tester
 cp BSC_Tests.cfg $VOL_BASE_DIR/bsc-tester/
 
@@ -62,13 +64,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-bsc-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-bsc-test
-
-echo Stopping containers
-for i in `seq 0 2`; do
-	docker container kill ${BUILD_TAG}-bts$i
-done
-docker container kill ${BUILD_TAG}-bsc
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

ttcn3-bscnat-test/jenkins.sh

@@ -6,6 +6,7 @@ docker_images_require \
 	"osmo-nitb-$IMAGE_SUFFIX" \
 	"ttcn3-bscnat-test"
 
+set_clean_up_trap
 
 mkdir $VOL_BASE_DIR/bscnat-tester
 cp BSCNAT_Tests.cfg $VOL_BASE_DIR/bscnat-tester/
@@ -36,9 +37,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-bscnat-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-bscnat-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-bscnat
-
-network_remove
-collect_logs

ttcn3-bts-test/jenkins.sh

@@ -9,6 +9,17 @@ docker_images_require \
 	"osmocom-bb-host-master" \
 	"ttcn3-bts-test"
 
+set_clean_up_trap
+
+clean_up() {
+	# append ':hopping' to the classnames,
+	# e.g. "classname='BTS_Tests'" => "classname='BTS_Tests:hopping'"
+	# e.g. "classname='BTS_Tests_SMSCB'" => "classname='BTS_Tests_SMSCB:hopping'"
+	# so the hopping test cases would not interfere with non-hopping ones in Jenkins
+	sed -i "s/classname='\([^']\+\)'/classname='\1:hopping'/g" \
+		$VOL_BASE_DIR/bts-tester-hopping/junit-xml-hopping-*.log
+}
+
 start_bsc() {
 	echo Starting container with BSC
 	docker run	--rm \
@@ -170,20 +181,3 @@ docker container kill ${BUILD_TAG}-bts
 start_bsc
 start_bts trx 1
 start_testsuite hopping
-# append ':hopping' to the classnames,
-# e.g. "classname='BTS_Tests'" => "classname='BTS_Tests:hopping'"
-# e.g. "classname='BTS_Tests_SMSCB'" => "classname='BTS_Tests_SMSCB:hopping'"
-# so the hopping test cases would not interfere with non-hopping ones in Jenkins
-sed -i "s/classname='\([^']\+\)'/classname='\1:hopping'/g" \
-	$VOL_BASE_DIR/bts-tester-hopping/junit-xml-hopping-*.log
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-trxcon
-docker container kill ${BUILD_TAG}-fake_trx
-docker container kill ${BUILD_TAG}-bsc
-docker container kill ${BUILD_TAG}-bts
-docker container kill ${BUILD_TAG}-bsc
-
-network_remove
-rm -rf $VOL_BASE_DIR/unix
-collect_logs

ttcn3-cbc-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-cbc-$IMAGE_SUFFIX" \
 	"ttcn3-cbc-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/cbc-tester
 cp CBC_Tests.cfg $VOL_BASE_DIR/cbc-tester/
 
@@ -36,9 +38,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-cbc-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-cbc-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-cbc
-
-network_remove
-collect_logs

ttcn3-fr-test/jenkins.sh

@@ -5,6 +5,8 @@ IMAGE_SUFFIX="${IMAGE_SUFFIX:-master}"
 docker_images_require \
 	"ttcn3-fr-test"
 
+set_clean_up_trap
+
 SUBNET=26
 network_create $SUBNET
 
@@ -59,10 +61,3 @@ done
 # emulate running container in foreground, which is no longer possible as we
 # must shift the net-devices into the container _after_ it is started
 docker logs	-f ${BUILD_TAG}-ttcn3-fr-test
-
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-frnet
-
-network_remove
-collect_logs

ttcn3-gbproxy-test-fr/jenkins.sh

@@ -19,6 +19,8 @@ docker_images_require \
 	"osmo-gbproxy-$IMAGE_SUFFIX" \
 	"ttcn3-gbproxy-test"
 
+set_clean_up_trap
+
 SUBNET=25
 network_create $SUBNET
 
@@ -72,9 +74,3 @@ done
 # emulate runnign container in foreground, which is no longer possible as we
 # must shift the net-devices into the container _after_ it is started
 docker logs	-f ${BUILD_TAG}-ttcn3-gbproxy-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-gbproxy
-
-network_remove
-collect_logs

ttcn3-gbproxy-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-gbproxy-$IMAGE_SUFFIX" \
 	"ttcn3-gbproxy-test"
 
+set_clean_up_trap
+
 SUBNET=24
 network_create $SUBNET
 
@@ -39,9 +41,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-gbproxy-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-gbproxy-test $@
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-gbproxy
-
-network_remove
-collect_logs

ttcn3-ggsn-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-ggsn-$IMAGE_SUFFIX" \
 	"ttcn3-ggsn-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/ggsn-tester
 cp GGSN_Tests.cfg $VOL_BASE_DIR/ggsn-tester/
 
@@ -38,9 +40,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ggsn-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-ggsn-test
-
-# stop GGSN after test has completed
-docker container stop ${BUILD_TAG}-ggsn
-
-network_remove
-collect_logs

ttcn3-hlr-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-hlr-$IMAGE_SUFFIX" \
 	"ttcn3-hlr-test"
 
+set_clean_up_trap
+
 SUBNET=10
 network_create $SUBNET
 
@@ -34,9 +36,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-hlr-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-hlr-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-hlr
-
-network_remove
-collect_logs

ttcn3-mgw-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-mgw-$IMAGE_SUFFIX" \
 	"ttcn3-mgw-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/mgw-tester
 cp MGCP_Test.cfg $VOL_BASE_DIR/mgw-tester/
 
@@ -33,9 +35,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-mgw-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-mgw-test
-
-# stop mgw after test has completed
-docker container stop ${BUILD_TAG}-mgw
-
-network_remove
-collect_logs

ttcn3-msc-test/jenkins.sh

@@ -7,6 +7,8 @@ docker_images_require \
 	"osmo-msc-$IMAGE_SUFFIX" \
 	"ttcn3-msc-test"
 
+set_clean_up_trap
+
 SUBNET=20
 network_create $SUBNET
 
@@ -52,11 +54,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-msc-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-msc-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-msc
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-rm -rf $VOL_BASE_DIR/unix
-collect_logs

ttcn3-nitb-sysinfo/jenkins.sh

@@ -2,6 +2,22 @@
 
 . ../jenkins-common.sh
 
+set_clean_up_trap
+
+clean_up() {
+	# start some stupid helper container so we can access the volume
+	docker run	--rm \
+		-v ttcn3-nitb-sysinfo-vol:/ttcn3-nitb-sysinfo \
+		-v nitb-vol:/nitb \
+		-v bts-vol:/bts \
+		--name ${BUILD_TAG}-sysinfo-helper -d \
+		busybox /bin/sh -c 'sleep 1000 & wait'
+	docker cp ${BUILD_TAG}-sysinfo-helper:/ttcn3-nitb-sysinfo $VOL_BASE_DIR
+	docker cp ${BUILD_TAG}-sysinfo-helper:/nitb $VOL_BASE_DIR
+	docker cp ${BUILD_TAG}-sysinfo-helper:/bts $VOL_BASE_DIR
+	docker container stop -t 0 ${BUILD_TAG}-sysinfo-helper
+}
+
 SUBNET=5
 network_create $SUBNET
 
@@ -35,22 +51,3 @@ docker run	--rm \
 		-v ttcn3-nitb-sysinfo-vol:/data \
 		--name ${BUILD_TAG}-ttcn3-nitb-sysinfo \
 		$REPO_USER/ttcn3-nitb-sysinfo
-
-# stop bts + nitb after test has completed
-docker container stop ${BUILD_TAG}-bts
-docker container stop ${BUILD_TAG}-nitb
-
-# start some stupid helper container so we can access the volume
-docker run	--rm \
-		-v ttcn3-nitb-sysinfo-vol:/ttcn3-nitb-sysinfo \
-		-v nitb-vol:/nitb \
-		-v bts-vol:/bts \
-		--name ${BUILD_TAG}-sysinfo-helper -d \
-		busybox /bin/sh -c 'sleep 1000 & wait'
-docker cp ${BUILD_TAG}-sysinfo-helper:/ttcn3-nitb-sysinfo $VOL_BASE_DIR
-docker cp ${BUILD_TAG}-sysinfo-helper:/nitb $VOL_BASE_DIR
-docker cp ${BUILD_TAG}-sysinfo-helper:/bts $VOL_BASE_DIR
-docker container stop -t 0 ${BUILD_TAG}-sysinfo-helper
-
-network_remove
-collect_logs

ttcn3-pcu-test/jenkins-sns.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-pcu-$IMAGE_SUFFIX" \
 	"ttcn3-pcu-test"
 
+set_clean_up_trap
+
 SUBNET=14
 network_create $SUBNET
 
@@ -40,10 +42,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-pcu-test-sns \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-pcu-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-pcu-sns
-
-network_remove
-rm -rf $VOL_BASE_DIR/unix
-collect_logs

ttcn3-pcu-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-pcu-$IMAGE_SUFFIX" \
 	"ttcn3-pcu-test"
 
+set_clean_up_trap
+
 SUBNET=13
 network_create $SUBNET
 
@@ -40,10 +42,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-pcu-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-pcu-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-pcu
-
-network_remove
-rm -rf $VOL_BASE_DIR/unix
-collect_logs

ttcn3-remsim-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-remsim-$IMAGE_SUFFIX" \
 	"ttcn3-remsim-test"
 
+set_clean_up_trap
+
 start_server() {
 	echo Starting container with osmo-remsim-server
 	docker run	--rm \
@@ -87,7 +89,3 @@ echo "Changing to client configuration"
 start_client
 cp client/REMSIM_Tests.cfg $VOL_BASE_DIR/remsim-tester/
 start_testsuite
-docker container kill ${BUILD_TAG}-client
-
-network_remove
-collect_logs

ttcn3-sccp-test/jenkins.sh

@@ -7,6 +7,8 @@ docker_images_require \
 	"osmo-stp-master" \
 	"ttcn3-sccp-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/sccp-tester
 cp SCCP_Tests.cfg $VOL_BASE_DIR/sccp-tester/
 
@@ -36,8 +38,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-sccp-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-sccp-test
-
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

ttcn3-sgsn-test/jenkins.sh

@@ -7,6 +7,8 @@ docker_images_require \
 	"osmo-sgsn-$IMAGE_SUFFIX" \
 	"ttcn3-sgsn-test"
 
+set_clean_up_trap
+
 SUBNET=8
 network_create $SUBNET
 
@@ -49,10 +51,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-sgsn-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-sgsn-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-sgsn
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

ttcn3-sip-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-sip-$IMAGE_SUFFIX" \
 	"ttcn3-sip-test"
 
+set_clean_up_trap
+
 SUBNET=11
 network_create $SUBNET
 
@@ -40,10 +42,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-sip-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-sip-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-sip-connector
-
-network_remove
-rm -rf $VOL_BASE_DIR/unix
-collect_logs

ttcn3-smlc-test/jenkins.sh

@@ -7,6 +7,8 @@ docker_images_require \
 	"osmo-smlc-$IMAGE_SUFFIX" \
 	"ttcn3-smlc-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/smlc-tester
 cp SMLC_Tests.cfg $VOL_BASE_DIR/smlc-tester/
 
@@ -47,10 +49,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-smlc-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-smlc-test
-
-echo Stopping containers
-docker container kill ${BUILD_TAG}-smlc
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-collect_logs

ttcn3-stp-test/jenkins.sh

@@ -6,6 +6,8 @@ docker_images_require \
 	"osmo-stp-$IMAGE_SUFFIX" \
 	"ttcn3-stp-test"
 
+set_clean_up_trap
+
 mkdir $VOL_BASE_DIR/stp-tester
 cp STP_Tests.cfg $VOL_BASE_DIR/stp-tester/
 
@@ -33,8 +35,3 @@ docker run	--rm \
 		--name ${BUILD_TAG}-ttcn3-stp-test \
 		$DOCKER_ARGS \
 		$REPO_USER/ttcn3-stp-test
-
-docker container kill ${BUILD_TAG}-stp
-
-network_remove
-collect_logs