docker-playground/jenkins-common.sh

CACHE_DIR="$(realpath ../_cache)"
KERNEL_TEST_DIR="$(realpath ../scripts/kernel-test)"
DEBIAN_DEFAULT="bookworm"

docker_image_exists() {
	test -n "$(docker images -q "$REPO_USER/$1")"
}

docker_depends() {
	local feed

	case "$1" in
	osmo-*-20*q*-centos8)
		# e.g. osmo-mgw-2021q1-centos8 -> centos8-obs-2021q1
		feed="$(echo "$1" | grep -o -P -- "20\d\dq.*$")"  # e.g. "2021q1-centos8"
		feed="$(echo "$feed" | sed 's/\-centos8$//')" # e.g. "2021q1"
		echo "centos8-obs-$feed"
		;;
	osmo-*-latest-centos7) echo "centos7-obs-latest" ;;
	osmo-*-latest-centos8) echo "centos8-obs-latest" ;;
	osmo-*-centos7) echo "centos7-build" ;;
	osmo-*-centos8) echo "centos8-build" ;;
	osmo-*-latest) echo "debian-$DEBIAN_DEFAULT-obs-latest" ;;
	osmo_dia2gsup-*) echo "debian-$DEBIAN_DEFAULT-erlang" ;;
	osmo-epdg-*) echo "debian-$DEBIAN_DEFAULT-erlang" ;;
	osmo-*) echo "debian-$DEBIAN_DEFAULT-build" ;;
	open5gs-master) echo "debian-$DEBIAN_DEFAULT-build" ;;
	ttcn3-*) echo "debian-bookworm-titan" ;;
	esac
}

docker_distro_from_image_name() {
	case "$1" in
	osmo-*-centos7) echo "centos7" ;;
	osmo-*-centos8) echo "centos8" ;;
	centos7-*) echo "centos7" ;;
	centos8-*) echo "centos8" ;;
	debian-buster-*) echo "debian-buster" ;;
	debian-bullseye-*) echo "debian-bullseye" ;;
	debian-bookworm-*) echo "debian-bookworm" ;;
	*) echo "debian-$DEBIAN_DEFAULT" ;;
	esac
}

docker_upstream_distro_from_image_name() {
	case "$1" in
	osmo-*-centos7) echo "centos:centos7"; ;;
	osmo-*-centos8) echo "almalinux:8"; ;;
	centos7-*) echo "centos:centos7" ;;
	centos8-*) echo "almalinux:8" ;;
	debian9-*) echo "debian:stretch" ;;
	debian10-*) echo "debian:buster" ;;
	debian11-*) echo "debian:bullseye" ;;
	debian12-*) echo "debian:bookworm" ;;
	debian-stretch-*) echo "debian:stretch" ;;
	debian-buster-*) echo "debian:buster" ;;
	debian-bullseye-*) echo "debian:bullseye" ;;
	debian-bookworm-*) echo "debian:bookworm" ;;
	*) echo "debian:$DEBIAN_DEFAULT" ;;
	esac
}

docker_dir_from_image_name() {
	case "$1" in
	osmo-*-20*q*-centos8)
		# e.g. osmo-mgw-2021q1-centos8 -> osmo-mgw-latest
		echo "$1" | sed 's/20[0-9][0-9]q.*\-centos8$/latest/'
		;;
	osmo-*-centos7)
		# e.g. osmo-mgw-latest-centos7 -> osmo-mgw-latest
		echo "$1" | sed 's/\-centos7$//'
		;;
	osmo-*-centos8)
		# e.g. osmo-mgw-latest-centos8 -> osmo-mgw-latest
		echo "$1" | sed 's/\-centos8$//'
		;;
	centos8-obs-20*q*)
		# e.g. centos8-obs-2021q1 -> centos8-obs-latest
		echo "$1" | sed 's/20[0-9][0-9]q.*$/latest/'
		;;
	*)
		echo "$1"
		;;
	esac
}

# $1: distro name, from docker_distro_from_image_name()
# $2: docker image name (without $REPO_USER/ prefix)
list_osmo_packages() {
	local distro="$1"
	local image="$2"
	local docker_run_sh="docker run --rm --entrypoint=/bin/sh $REPO_USER/$image -c"

	if [ -n "$NO_LIST_OSMO_PACKAGES" ]; then
		return
	fi

	# Don't run on all images
	case "$image" in
	osmo-*) ;;
	*) return ;;
	esac

	set +x
	echo
	echo "### Installed Osmocom packages in: $image ###"
	echo

	case "$distro" in
	centos*)
		$docker_run_sh "rpm -qa | grep osmo"
		;;
	debian*)
		$docker_run_sh "dpkg -l | grep osmo"
		;;
	*)
		echo "ERROR: don't know how to list installed packages for distro=$distro"
		;;
	esac

	echo
	set -x
}

# Get the osmo-ttcn3-hacks branch to use, based on the IMAGE_SUFFIX we are
# testing. This allows e.g. running 2023q1 SUTs against the 2023q1 branch of
# osmo-ttcn3-hacks.git (SYS#6638). The OSMO_TTCN3_BRANCH env var can be used to
# override it in any case.
# $IMAGE_SUFFIX: e.g. 2023q1-centos8
docker_osmo_ttcn3_branch() {
	if [ -n "$OSMO_TTCN3_BRANCH" ]; then
		echo "$OSMO_TTCN3_BRANCH"
		return
	fi

	case "$IMAGE_SUFFIX" in
	20*q*)
		echo "$IMAGE_SUFFIX" | cut -d- -f 1
		;;
	*)
		echo "master"
		;;
	esac
}

# Make sure required images are available and build them if necessary.
# $*: image names (e.g. "debian-bullseye-build", "osmo-mgw-master", "osmo-mgw-master-centos8")
#	The images are automatically built from the Dockerfile of the subdir of
#	the same name. If there is a distribution name at the end of the image
#	name (e.g. osmo-mgw-master-centos8), it gets removed from the subdir
#	where the Dockerfile is taken from (e.g. osmo-mgw-master/Dockerfile)
#	and UPSTREAM_DISTRO and DISTRO are passed accordingly (e.g.
#	UPSTREAM_DISTRO=almalinux:8 DISTRO=centos8). This allows one
#	Dockerfile for multiple distributions, without duplicating configs for
#	each distribution. Dependencies listed in docker_depends() are built
#	automatically too.
IMAGE_DIR_PREFIX=".."
docker_images_require() {
	local i
	local from_line
	local pull_arg
	local upstream_distro_arg
	local distro_arg
	local depends
	local dir

	for i in $@; do
		# Don't build images that are available on the private
		# registry, if using it. Instead, pull the images to make sure
		# they are up-to-date.
		if [ "$REGISTRY_HOST" = "registry.osmocom.org" ]; then
			case "$i" in
			debian-bookworm-titan)
				docker pull "$REGISTRY_HOST/$USER/$i"
				continue
				;;
			esac
		fi

		# Build dependencies first
		depends="$(docker_depends "$i")"
		if [ -n "$depends" ]; then
			docker_images_require $depends
		fi

		distro_arg="$(docker_distro_from_image_name "$i")"

		# Trigger image build (cache will be used when up-to-date)
		if [ -z "$NO_DOCKER_IMAGE_BUILD" ]; then
			upstream_distro_arg="$(docker_upstream_distro_from_image_name "$i")"
			dir="$(docker_dir_from_image_name "$i")"

			# Pull upstream base images
			pull_arg="--pull"
			from_line="$(grep '^FROM' ${IMAGE_DIR_PREFIX}/${dir}/Dockerfile)"
			if echo "$from_line" | grep -q '$USER'; then
				pull_arg=""
			fi

			set +x
			echo "Building image: $i (export NO_DOCKER_IMAGE_BUILD=1 to prevent this)"
			set -x
			make -C "${IMAGE_DIR_PREFIX}/${dir}" \
				BUILD_ARGS="$pull_arg" \
				UPSTREAM_DISTRO="$upstream_distro_arg" \
				DISTRO="$distro_arg" \
				IMAGE="$REPO_USER/$i" \
				OSMO_TTCN3_BRANCH="$(docker_osmo_ttcn3_branch)" \
				|| exit 1
		fi

		# Detect missing images (build skipped)
		if ! docker_image_exists "$i"; then
			set +x
			echo "ERROR: missing image: $i"
			exit 1
		fi

		list_osmo_packages "$distro_arg" "$i"
	done
}

# Kill a docker container and ensure it has actually stopped (OS#5928)
docker_kill_wait() {
	docker kill "$@"
	docker wait "$@" || true
}

#kills all containers attached to network
network_clean() {
	local containers="$(docker network inspect $NET_NAME | grep Name | cut -d : -f2 | awk -F\" 'NR>1{print $2}')"

	if [ -n "$containers" ]; then
		docker_kill_wait $containers
	fi
}

# Create network and find a free subnet automatically. The global variables
# SUBNET (subnet number) and NET_NAME (name of the docker network) get set.
network_create() {
	SUBNET="$PPID"
	for i in $(seq 1 30); do
		SUBNET="$(echo "($SUBNET + 1) % 256" | bc)"
		NET_NAME="$SUITE_NAME-$SUBNET"
		SUB4="172.18.$SUBNET.0/24"
		SUB6="fd02:db8:$SUBNET::/64"
		set +x
		echo "Creating network $NET_NAME, trying SUBNET=$SUBNET..."
		set -x
		if docker network create \
				--internal \
				--subnet "$SUB4" \
				--ipv6 \
				--subnet "$SUB6" \
				"$NET_NAME"; then
			set +x
			echo
			echo "### Network $NET_NAME created (SUBNET=$SUBNET) ###"
			echo
			set -x
			return
		fi
	done

	set +x
	echo "ERROR: failed to create docker network"
	exit 1
}

network_bridge_create() {
	if [ -z "$NET_NAME" ]; then
		set +x
		echo "ERROR: network_bridge_create: NET_NAME needs to be set"
		exit 1
	fi

	NET=$1
	if docker network ls | grep -q $NET_NAME; then
		set +x
		echo "Removing stale network and containers..."
		set -x
		network_clean
		network_remove
	fi
	SUB4="172.18.$NET.0/24"
	SUB6="fd02:db8:$NET::/64"
	set +x
	echo "Creating network $NET_NAME"
	set -x
	docker network create \
		--driver=bridge \
		--subnet $SUB4 \
		--ipv6 --subnet $SUB6 \
		-o "com.docker.network.bridge.host_binding_ipv4"="172.18.$NET.1" \
		$NET_NAME
}

network_remove() {
	set +x
	echo "Removing network $NET_NAME"
	set -x
	docker network remove $NET_NAME
}

# Clean and remove all docker networks related to ttcn3 testing. This prevents
# running testsuites in parallel, so usually we don't want this and rely on the
# clean_up_trap to clean the network. But on e.g. gtp0-deb10fr this gets used
# as we only run one testsuite at once and it has dahdi netdevices attached.
# Due connection loss, it may not be cleaned up there and so another testsuite
# cannot start.
network_clean_remove_all_ttcn3() {
	local networks
	local i

	networks="$(docker network ls --format '{{.Name}}' | grep ^ttcn3- || true)"

	if [ -z "$networks" ]; then
		return
	fi

	set +x
	echo "Removing stale network and containers..."
	set -x

	for i in $networks; do
		NET_NAME="$i"
		network_clean
		network_remove
	done

	unset NET_NAME
}

network_replace_subnet_in_configs() {
	set +x

	local i
	local files="$(find \
		"$VOL_BASE_DIR" \
		-name '*.cfg' -o \
		-name '*.conf' -o \
		-name '*.config' -o \
		-name '*.confmerge' -o \
		-name '*.inc' -o \
		-name '*.scm' -o \
		-name '*.sh' -o \
		-name '*.txt' -o \
		-name '*.yaml' -o \
		-name 'Makefile' \
	)"

	if [ -z "$files" ]; then
		echo "ERROR: network_replace_subnet_in_configs:" \
			"no config files found!"
		exit 1
	fi

	for i in $files; do
		echo "Applying SUBNET=$SUBNET to: $i"
		sed \
			-i \
			-E \
			-e "s/172\.18\.[0-9]{1,3}\./172.18.$SUBNET./g" \
			-e "s/fd02:db8:[0-9]{1,3}:/fd02:db8:$SUBNET:/g" \
			"$i"
	done

	set -x
}

# Generates list of params to pass to "docker run" to configure IP addresses
# $1: SUBNET to use, same as set by network_create()
# $2: Address suffix from SUBNET to apply to the container
docker_network_params() {
	NET=$1
	ADDR_SUFIX=$2
	echo --network $NET_NAME --ip "172.18.$NET.$ADDR_SUFIX" --ip6 "fd02:db8:$NET::$ADDR_SUFIX"
}

fix_perms() {
	set +x
	echo "Fixing permissions"
	set -x
	docker run 	--rm \
			-v $VOL_BASE_DIR:/data \
			-v $CACHE_DIR:/cache \
			--name ${BUILD_TAG}-cleaner \
			"debian:$DEBIAN_DEFAULT" \
			sh -e -x -c "
				chmod -R a+rX /data/ /cache/
				chown -R $(id -u):$(id -g) /data /cache
			"
}

collect_logs() {
	cat "$VOL_BASE_DIR"/*/junit-*.log || true
}

clean_up_common() {
	set +e

	set +x
	echo
	echo "### Clean up ###"
	echo
	set -x

	# Clear trap
	trap - EXIT INT TERM 0

	# Run clean_up() from ttcn3-*/jenkins.sh, if defined
	if type clean_up >/dev/null; then
		clean_up
	fi

	network_clean
	network_remove
	rm -rf "$VOL_BASE_DIR"/unix
	fix_perms
	collect_logs
}

# Run clean up code when the script stops (either by failing command, by ^C, or
# after running through successfully). The caller can define a custom clean_up
# function.
set_clean_up_trap() {
	trap clean_up_common EXIT INT TERM 0
}

docker_kvm_param() {
	if [ "$KERNEL_TEST_KVM" != 0 ]; then
		echo "--device /dev/kvm:/dev/kvm"
	fi
}

# Generate the initrd, and optionally build a kernel, for tests that involve
# kernel modules. Boot the kernel once in QEMU inside docker to verify that it
# works. See README.md for description of the KERNEL_* environment variables.
# $1: kernel config base (e.g. defconfig, tinyconfig, allnoconfig)
# $2: path to kernel config fragment
# $3: path to project specific initrd build script, which adds the osmo
#     program, kernel modules etc. to the initrd (gets sourced by
#     scripts/kernel-test/initrd-build.sh)
# $4: docker image name
# $5-n: (optional) additional arguments to "docker run", like a volume
#       containing a config file
kernel_test_prepare() {
	local kernel_config_base="$1"
	local kernel_config_fragment="$2"
	local initrd_project_script="$3"
	local docker_image="$4"
	shift 4

	# Store KVM availibility in global KERNEL_TEST_KVM
	if [ -z "$KERNEL_TEST_KVM" ]; then
		if [ -e "/dev/kvm" ]; then
			KERNEL_TEST_KVM=1
		else
			KERNEL_TEST_KVM=0
		fi
	fi

	mkdir -p "$CACHE_DIR/kernel-test"

	cp "$kernel_config_fragment" \
		"$CACHE_DIR/kernel-test/fragment.config"
	cp "$initrd_project_script" \
		"$CACHE_DIR/kernel-test/initrd-project-script.sh"

	fix_perms

	# Build kernel and initramfs
	docker run \
		--rm \
		--user "build" \
		-v "$CACHE_DIR:/cache" \
		-v "$KERNEL_TEST_DIR:/kernel-test:ro" \
		-e "KERNEL_BRANCH=$KERNEL_BRANCH" \
		-e "KERNEL_BUILD=$KERNEL_BUILD" \
		-e "KERNEL_CONFIG_BASE=$kernel_config_base" \
		-e "KERNEL_REMOTE_NAME=$KERNEL_REMOTE_NAME" \
		-e "KERNEL_URL=$KERNEL_URL" \
		-e "KERNEL_SKIP_REBUILD=$KERNEL_SKIP_REBUILD" \
		$DOCKER_ARGS \
		"$@" \
		"$docker_image" \
		"/kernel-test/prepare.sh"

	# Smoke test
	if [ "$KERNEL_SKIP_SMOKE_TEST" != 1 ]; then
		docker run \
			--rm \
			--cap-add=NET_ADMIN \
			$(docker_kvm_param) \
			--device /dev/net/tun:/dev/net/tun \
			-v "$CACHE_DIR:/cache" \
			-v "$KERNEL_TEST_DIR:/kernel-test:ro" \
			$DOCKER_ARGS \
			"$@" \
			"$docker_image" \
			"/kernel-test/smoke-test.sh"
	fi
}

# Wait until the linux kernel is booted inside QEMU inside docker, and the
# initrd is right before running the project-specific commands (e.g. starting
# osmo-ggsn). This may take a few seconds if running without KVM.
# $1: path to the VM's log file
kernel_test_wait_for_vm() {
	local log="$1"
	local i

	if [ "$KERNEL_TEST" != 1 ]; then
		return
	fi

	for i in $(seq 1 15); do
		sleep 1

		if grep -q KERNEL_TEST_VM_IS_READY "$log"; then
			return
		fi
	done

	# Let clean_up_common kill the VM
	set +x
	echo "Timeout while waiting for kernel test VM"
	exit 1
}

# Check if IMAGE_SUFFIX starts with "latest" (e.g. "latest-centos8")
image_suffix_is_latest() {
	case "$IMAGE_SUFFIX" in
	latest*) return 0 ;;
	*) return 1 ;;
	esac
}

# Check if IMAGE_SUFFIX starts with "master" (e.g. "master-centos8")
image_suffix_is_master() {
	case "$IMAGE_SUFFIX" in
	master*) return 0 ;;
	*) return 1 ;;
	esac
}

# Check if IMAGE_SUFFIX starts with "2023q1" (e.g. "2023q1-centos8")
image_suffix_is_2023q1() {
	case "$IMAGE_SUFFIX" in
	2023q1*) return 0 ;;
	*) return 1 ;;
	esac
}

# Write the Osmocom repository to the TTCN3 config file, so the tests may take
# different code paths (OS#5327)
# $1: path to TTCN3 config file (e.g. BSC_Tests.cfg)
write_mp_osmo_repo() {
	local repo="nightly"
	local config="$1"
	local line

	if ! [ -e "$config" ]; then
		set +x
		echo
		echo "ERROR: TTCN3 config file '$config' not found in $PWD"
		echo
		exit 1
	fi

	case "$IMAGE_SUFFIX" in
	latest*)
		repo="latest"
		;;
	20*q*-*)  # e.g. 2021q1-centos8
		repo="$(echo "$IMAGE_SUFFIX" | cut -d- -f 1)"  # e.g. 2021q1
		;;
	*)
		;;
	esac

	line="Misc_Helpers.mp_osmo_repo := \"$repo\""

	sed \
		-i \
		"s/\[MODULE_PARAMETERS\]/\[MODULE_PARAMETERS\]\n$line/g" \
		"$config"
}

# Output the name of the test config and check if it is enabled. Use this
# function in jenkins.sh, after setting TEST_CONFIGS_ALL, e.g.:
#   TEST_CONFIGS_ALL="generic oml hopping"
# The user can then set TEST_CONFIGS to only run one of the test
# configurations.
# $1: one of TEST_CONFIGS_ALL, e.g. "classic"
test_config_enabled() {
	local config="$1"
	local i
	local valid=0

	for i in $TEST_CONFIGS_ALL; do
		if [ "$config" = "$i" ]; then
			valid=1
			break
		fi
	done

	if [ "$valid" != "1" ]; then
		set +x
		echo "ERROR: config name '$config' is not one of '$TEST_CONFIGS_ALL'"
		exit 1
	fi

	if [ -z "$TEST_CONFIGS" ]; then
		return 0
	fi

	for i in $TEST_CONFIGS; do
		if [ "$config" = "$i" ]; then
			return 0
		fi
	done

	return 1
}

set -x

# non-jenkins execution: assume local user name
if [ "x$REPO_USER" = "x" ]; then
	REPO_USER=$USER
fi

if [ "x$WORKSPACE" = "x" ]; then
	# non-jenkins execution: put logs in /tmp
	VOL_BASE_DIR="$(mktemp -d)"

	# point /tmp/logs to the last ttcn3 run
	rm /tmp/logs || true
	ln -s "$VOL_BASE_DIR" /tmp/logs || true
else
	# jenkins execution: put logs in workspace
	VOL_BASE_DIR="$WORKSPACE/logs"
	rm -rf "$VOL_BASE_DIR"
	mkdir -p "$VOL_BASE_DIR"
fi

if [ ! -d "$VOL_BASE_DIR" ]; then
	set +x
	echo "ERROR: \$VOL_BASE_DIR does not exist: '$VOL_BASE_DIR'"
	exit 1
fi

# non-jenkins execution: set a unique BUILD_TAG to avoid collisions (OS#5358)
if [ "x$BUILD_TAG" = "x" ]; then
	BUILD_TAG="nonjenkins-$(date +%N)"
fi

SUITE_NAME=`basename $PWD`