tools/testing/selftests/net/ipv6_route_update_soft_lockup.sh - linux - Git at Google

 #!/bin/bash
 # SPDX-License-Identifier: GPL-2.0
 #
 # Testing for potential kernel soft lockup during IPv6 routing table
 # refresh under heavy outgoing IPv6 traffic. If a kernel soft lockup
 # occurs, a kernel panic will be triggered to prevent associated issues.
 #
 #
 #                            Test Environment Layout
 #
 # ┌----------------┐                                         ┌----------------┐
 # |     SOURCE_NS  |                                         |     SINK_NS    |
 # |    NAMESPACE   |                                         |    NAMESPACE   |
 # |(iperf3 clients)|                                         |(iperf3 servers)|
 # |                |                                         |                |
 # |                |                                         |                |
 # |    ┌-----------|                             nexthops    |---------┐      |
 # |    |veth_source|<--------------------------------------->|veth_sink|<┐    |
 # |    └-----------|2001:0DB8:1::0:1/96  2001:0DB8:1::1:1/96 |---------┘ |    |
 # |                |         ^           2001:0DB8:1::1:2/96 |           |    |
 # |                |         .                   .           |       fwd |    |
 # |  ┌---------┐   |         .                   .           |           |    |
 # |  |   IPv6  |   |         .                   .           |           V    |
 # |  | routing |   |         .           2001:0DB8:1::1:80/96|        ┌-----┐ |
 # |  |  table  |   |         .                               |        | lo  | |
 # |  | nexthop |   |         .                               └--------┴-----┴-┘
 # |  | update  |   |         ............................> 2001:0DB8:2::1:1/128
 # |  └-------- ┘   |
 # └----------------┘
 #
 # The test script sets up two network namespaces, source_ns and sink_ns,
 # connected via a veth link. Within source_ns, it continuously updates the
 # IPv6 routing table by flushing and inserting IPV6_NEXTHOP_ADDR_COUNT nexthop
 # IPs destined for SINK_LOOPBACK_IP_ADDR in sink_ns. This refresh occurs at a
 # rate of 1/ROUTING_TABLE_REFRESH_PERIOD per second for TEST_DURATION seconds.
 #
 # Simultaneously, multiple iperf3 clients within source_ns generate heavy
 # outgoing IPv6 traffic. Each client is assigned a unique port number starting
 # at 5000 and incrementing sequentially. Each client targets a unique iperf3
 # server running in sink_ns, connected to the SINK_LOOPBACK_IFACE interface
 # using the same port number.
 #
 # The number of iperf3 servers and clients is set to half of the total
 # available cores on each machine.
 #
 # NOTE: We have tested this script on machines with various CPU specifications,
 # ranging from lower to higher performance as listed below. The test script
 # effectively triggered a kernel soft lockup on machines running an unpatched
 # kernel in under a minute:
 #
 # - 1x Intel Xeon E-2278G 8-Core Processor @ 3.40GHz
 # - 1x Intel Xeon E-2378G Processor 8-Core @ 2.80GHz
 # - 1x AMD EPYC 7401P 24-Core Processor @ 2.00GHz
 # - 1x AMD EPYC 7402P 24-Core Processor @ 2.80GHz
 # - 2x Intel Xeon Gold 5120 14-Core Processor @ 2.20GHz
 # - 1x Ampere Altra Q80-30 80-Core Processor @ 3.00GHz
 # - 2x Intel Xeon Gold 5120 14-Core Processor @ 2.20GHz
 # - 2x Intel Xeon Silver 4214 24-Core Processor @ 2.20GHz
 # - 1x AMD EPYC 7502P 32-Core @ 2.50GHz
 # - 1x Intel Xeon Gold 6314U 32-Core Processor @ 2.30GHz
 # - 2x Intel Xeon Gold 6338 32-Core Processor @ 2.00GHz
 #
 # On less performant machines, you may need to increase the TEST_DURATION
 # parameter to enhance the likelihood of encountering a race condition leading
 # to a kernel soft lockup and avoid a false negative result.
 #
 # NOTE: The test may not produce the expected result in virtualized
 # environments (e.g., qemu) due to differences in timing and CPU handling,
 # which can affect the conditions needed to trigger a soft lockup.

 source lib.sh

 TEST_DURATION=300
 ROUTING_TABLE_REFRESH_PERIOD=0.01

 IPERF3_BITRATE="300m"


 IPV6_NEXTHOP_ADDR_COUNT="128"
 IPV6_NEXTHOP_ADDR_MASK="96"
 IPV6_NEXTHOP_PREFIX="2001:0DB8:1"


 SOURCE_TEST_IFACE="veth_source"
 SOURCE_TEST_IP_ADDR="2001:0DB8:1::0:1/96"

 SINK_TEST_IFACE="veth_sink"
 # ${SINK_TEST_IFACE} is populated with the following range of IPv6 addresses:
 # 2001:0DB8:1::1:1  to 2001:0DB8:1::1:${IPV6_NEXTHOP_ADDR_COUNT}
 SINK_LOOPBACK_IFACE="lo"
 SINK_LOOPBACK_IP_MASK="128"
 SINK_LOOPBACK_IP_ADDR="2001:0DB8:2::1:1"

 nexthop_ip_list=""
 termination_signal=""
 kernel_softlokup_panic_prev_val=""

 terminate_ns_processes_by_pattern() {
 	local ns=$1
 	local pattern=$2

 	for pid in $(ip netns pids ${ns}); do
 		[ -e /proc/$pid/cmdline ] && grep -qe "${pattern}" /proc/$pid/cmdline && kill -9 $pid
 	done
 }

 cleanup() {
 	echo "info: cleaning up namespaces and terminating all processes within them..."


 	# Terminate iperf3 instances running in the source_ns. To avoid race
 	# conditions, first iterate over the PIDs and terminate those
 	# associated with the bash shells running the
 	# `while true; do iperf3 -c ...; done` loops. In a second iteration,
 	# terminate the individual `iperf3 -c ...` instances.
 	terminate_ns_processes_by_pattern ${source_ns} while
 	terminate_ns_processes_by_pattern ${source_ns} iperf3

 	# Repeat the same process for sink_ns
 	terminate_ns_processes_by_pattern ${sink_ns} while
 	terminate_ns_processes_by_pattern ${sink_ns} iperf3

 	# Check if any iperf3 instances are still running. This could happen
 	# if a core has entered an infinite loop and the timeout for detecting
 	# the soft lockup has not expired, but either the test interval has
 	# already elapsed or the test was terminated manually (e.g., with ^C)
 	for pid in $(ip netns pids ${source_ns}); do
 		if [ -e /proc/$pid/cmdline ] && grep -qe 'iperf3' /proc/$pid/cmdline; then
 			echo "FAIL: unable to terminate some iperf3 instances. Soft lockup is underway. A kernel panic is on the way!"
 			exit ${ksft_fail}
 		fi
 	done

 	if [ "$termination_signal" == "SIGINT" ]; then
 		echo "SKIP: Termination due to ^C (SIGINT)"
 	elif [ "$termination_signal" == "SIGALRM" ]; then
 		echo "PASS: No kernel soft lockup occurred during this ${TEST_DURATION} second test"
 	fi

 	cleanup_ns ${source_ns} ${sink_ns}

 	sysctl -qw kernel.softlockup_panic=${kernel_softlokup_panic_prev_val}
 }

 setup_prepare() {
 	setup_ns source_ns sink_ns

 	ip -n ${source_ns} link add name ${SOURCE_TEST_IFACE} type veth peer name ${SINK_TEST_IFACE} netns ${sink_ns}

 	# Setting up the Source namespace
 	ip -n ${source_ns} addr add ${SOURCE_TEST_IP_ADDR} dev ${SOURCE_TEST_IFACE}
 	ip -n ${source_ns} link set dev ${SOURCE_TEST_IFACE} qlen 10000
 	ip -n ${source_ns} link set dev ${SOURCE_TEST_IFACE} up
 	ip netns exec ${source_ns} sysctl -qw net.ipv6.fib_multipath_hash_policy=1

 	# Setting up the Sink namespace
 	ip -n ${sink_ns} addr add ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK} dev ${SINK_LOOPBACK_IFACE}
 	ip -n ${sink_ns} link set dev ${SINK_LOOPBACK_IFACE} up
 	ip netns exec ${sink_ns} sysctl -qw net.ipv6.conf.${SINK_LOOPBACK_IFACE}.forwarding=1

 	ip -n ${sink_ns} link set ${SINK_TEST_IFACE} up
 	ip netns exec ${sink_ns} sysctl -qw net.ipv6.conf.${SINK_TEST_IFACE}.forwarding=1


 	# Populate nexthop IPv6 addresses on the test interface in the sink_ns
 	echo "info: populating ${IPV6_NEXTHOP_ADDR_COUNT} IPv6 addresses on the ${SINK_TEST_IFACE} interface ..."
 	for IP in $(seq 1 ${IPV6_NEXTHOP_ADDR_COUNT}); do
 		ip -n ${sink_ns} addr add ${IPV6_NEXTHOP_PREFIX}::$(printf "1:%x" "${IP}")/${IPV6_NEXTHOP_ADDR_MASK} dev ${SINK_TEST_IFACE};
 	done

 	# Preparing list of nexthops
 	for IP in $(seq 1 ${IPV6_NEXTHOP_ADDR_COUNT}); do
 		nexthop_ip_list=$nexthop_ip_list" nexthop via ${IPV6_NEXTHOP_PREFIX}::$(printf "1:%x" $IP) dev ${SOURCE_TEST_IFACE} weight 1"
 	done
 }


 test_soft_lockup_during_routing_table_refresh() {
 	# Start num_of_iperf_servers iperf3 servers in the sink_ns namespace,
 	# each listening on ports starting at 5001 and incrementing
 	# sequentially. Since iperf3 instances may terminate unexpectedly, a
 	# while loop is used to automatically restart them in such cases.
 	echo "info: starting ${num_of_iperf_servers} iperf3 servers in the sink_ns namespace ..."
 	for i in $(seq 1 ${num_of_iperf_servers}); do
 		cmd="iperf3 --bind ${SINK_LOOPBACK_IP_ADDR} -s -p $(printf '5%03d' ${i}) --rcv-timeout 200 &>/dev/null"
 		ip netns exec ${sink_ns} bash -c "while true; do ${cmd}; done &" &>/dev/null
 	done

 	# Wait for the iperf3 servers to be ready
 	for i in $(seq ${num_of_iperf_servers}); do
 		port=$(printf '5%03d' ${i});
 		wait_local_port_listen ${sink_ns} ${port} tcp
 	done

 	# Continuously refresh the routing table in the background within
 	# the source_ns namespace
 	ip netns exec ${source_ns} bash -c "
 		while \$(ip netns list | grep -q ${source_ns}); do
 			ip -6 route add ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK} ${nexthop_ip_list};
 			sleep ${ROUTING_TABLE_REFRESH_PERIOD};
 			ip -6 route delete ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK};
 		done &"

 	# Start num_of_iperf_servers iperf3 clients in the source_ns namespace,
 	# each sending TCP traffic on sequential ports starting at 5001.
 	# Since iperf3 instances may terminate unexpectedly (e.g., if the route
 	# to the server is deleted in the background during a route refresh), a
 	# while loop is used to automatically restart them in such cases.
 	echo "info: starting ${num_of_iperf_servers} iperf3 clients in the source_ns namespace ..."
 	for i in $(seq 1 ${num_of_iperf_servers}); do
 		cmd="iperf3 -c ${SINK_LOOPBACK_IP_ADDR} -p $(printf '5%03d' ${i}) --length 64 --bitrate ${IPERF3_BITRATE} -t 0 --connect-timeout 150 &>/dev/null"
 		ip netns exec ${source_ns} bash -c "while true; do ${cmd}; done &" &>/dev/null
 	done

 	echo "info: IPv6 routing table is being updated at the rate of $(echo "1/${ROUTING_TABLE_REFRESH_PERIOD}" | bc)/s for ${TEST_DURATION} seconds ..."
 	echo "info: A kernel soft lockup, if detected, results in a kernel panic!"

 	wait
 }

 # Make sure 'iperf3' is installed, skip the test otherwise
 if [ ! -x "$(command -v "iperf3")" ]; then
 	echo "SKIP: 'iperf3' is not installed. Skipping the test."
 	exit ${ksft_skip}
 fi

 # Determine the number of cores on the machine
 num_of_iperf_servers=$(( $(nproc)/2 ))

 # Check if we are running on a multi-core machine, skip the test otherwise
 if [ "${num_of_iperf_servers}" -eq 0 ]; then
 	echo "SKIP: This test is not valid on a single core machine!"
 	exit ${ksft_skip}
 fi

 # Since the kernel soft lockup we're testing causes at least one core to enter
 # an infinite loop, destabilizing the host and likely affecting subsequent
 # tests, we trigger a kernel panic instead of reporting a failure and
 # continuing
 kernel_softlokup_panic_prev_val=$(sysctl -n kernel.softlockup_panic)
 sysctl -qw kernel.softlockup_panic=1

 handle_sigint() {
 	termination_signal="SIGINT"
 	cleanup
 	exit ${ksft_skip}
 }

 handle_sigalrm() {
 	termination_signal="SIGALRM"
 	cleanup
 	exit ${ksft_pass}
 }

 trap handle_sigint SIGINT
 trap handle_sigalrm SIGALRM

 (sleep ${TEST_DURATION} && kill -s SIGALRM $$)&

 setup_prepare
 test_soft_lockup_during_routing_table_refresh
	#!/bin/bash
	# SPDX-License-Identifier: GPL-2.0
	#
	# Testing for potential kernel soft lockup during IPv6 routing table
	# refresh under heavy outgoing IPv6 traffic. If a kernel soft lockup
	# occurs, a kernel panic will be triggered to prevent associated issues.
	#
	#
	# Test Environment Layout
	#
	# ┌----------------┐ ┌----------------┐
	# \| SOURCE_NS \| \| SINK_NS \|
	# \| NAMESPACE \| \| NAMESPACE \|
	# \|(iperf3 clients)\| \|(iperf3 servers)\|
	# \| \| \| \|
	# \| \| \| \|
	# \| ┌-----------\| nexthops \|---------┐ \|
	# \| \|veth_source\|<--------------------------------------->\|veth_sink\|<┐ \|
	# \| └-----------\|2001:0DB8:1::0:1/96 2001:0DB8:1::1:1/96 \|---------┘ \| \|
	# \| \| ^ 2001:0DB8:1::1:2/96 \| \| \|
	# \| \| . . \| fwd \| \|
	# \| ┌---------┐ \| . . \| \| \|
	# \| \| IPv6 \| \| . . \| V \|
	# \| \| routing \| \| . 2001:0DB8:1::1:80/96\| ┌-----┐ \|
	# \| \| table \| \| . \| \| lo \| \|
	# \| \| nexthop \| \| . └--------┴-----┴-┘
	# \| \| update \| \| ............................> 2001:0DB8:2::1:1/128
	# \| └-------- ┘ \|
	# └----------------┘
	#
	# The test script sets up two network namespaces, source_ns and sink_ns,
	# connected via a veth link. Within source_ns, it continuously updates the
	# IPv6 routing table by flushing and inserting IPV6_NEXTHOP_ADDR_COUNT nexthop
	# IPs destined for SINK_LOOPBACK_IP_ADDR in sink_ns. This refresh occurs at a
	# rate of 1/ROUTING_TABLE_REFRESH_PERIOD per second for TEST_DURATION seconds.
	#
	# Simultaneously, multiple iperf3 clients within source_ns generate heavy
	# outgoing IPv6 traffic. Each client is assigned a unique port number starting
	# at 5000 and incrementing sequentially. Each client targets a unique iperf3
	# server running in sink_ns, connected to the SINK_LOOPBACK_IFACE interface
	# using the same port number.
	#
	# The number of iperf3 servers and clients is set to half of the total
	# available cores on each machine.
	#
	# NOTE: We have tested this script on machines with various CPU specifications,
	# ranging from lower to higher performance as listed below. The test script
	# effectively triggered a kernel soft lockup on machines running an unpatched
	# kernel in under a minute:
	#
	# - 1x Intel Xeon E-2278G 8-Core Processor @ 3.40GHz
	# - 1x Intel Xeon E-2378G Processor 8-Core @ 2.80GHz
	# - 1x AMD EPYC 7401P 24-Core Processor @ 2.00GHz
	# - 1x AMD EPYC 7402P 24-Core Processor @ 2.80GHz
	# - 2x Intel Xeon Gold 5120 14-Core Processor @ 2.20GHz
	# - 1x Ampere Altra Q80-30 80-Core Processor @ 3.00GHz
	# - 2x Intel Xeon Gold 5120 14-Core Processor @ 2.20GHz
	# - 2x Intel Xeon Silver 4214 24-Core Processor @ 2.20GHz
	# - 1x AMD EPYC 7502P 32-Core @ 2.50GHz
	# - 1x Intel Xeon Gold 6314U 32-Core Processor @ 2.30GHz
	# - 2x Intel Xeon Gold 6338 32-Core Processor @ 2.00GHz
	#
	# On less performant machines, you may need to increase the TEST_DURATION
	# parameter to enhance the likelihood of encountering a race condition leading
	# to a kernel soft lockup and avoid a false negative result.
	#
	# NOTE: The test may not produce the expected result in virtualized
	# environments (e.g., qemu) due to differences in timing and CPU handling,
	# which can affect the conditions needed to trigger a soft lockup.

	source lib.sh

	TEST_DURATION=300
	ROUTING_TABLE_REFRESH_PERIOD=0.01

	IPERF3_BITRATE="300m"


	IPV6_NEXTHOP_ADDR_COUNT="128"
	IPV6_NEXTHOP_ADDR_MASK="96"
	IPV6_NEXTHOP_PREFIX="2001:0DB8:1"


	SOURCE_TEST_IFACE="veth_source"
	SOURCE_TEST_IP_ADDR="2001:0DB8:1::0:1/96"

	SINK_TEST_IFACE="veth_sink"
	# ${SINK_TEST_IFACE} is populated with the following range of IPv6 addresses:
	# 2001:0DB8:1::1:1 to 2001:0DB8:1::1:${IPV6_NEXTHOP_ADDR_COUNT}
	SINK_LOOPBACK_IFACE="lo"
	SINK_LOOPBACK_IP_MASK="128"
	SINK_LOOPBACK_IP_ADDR="2001:0DB8:2::1:1"

	nexthop_ip_list=""
	termination_signal=""
	kernel_softlokup_panic_prev_val=""

	terminate_ns_processes_by_pattern() {
	local ns=$1
	local pattern=$2

	for pid in $(ip netns pids ${ns}); do
	[ -e /proc/$pid/cmdline ] && grep -qe "${pattern}" /proc/$pid/cmdline && kill -9 $pid
	done
	}

	cleanup() {
	echo "info: cleaning up namespaces and terminating all processes within them..."


	# Terminate iperf3 instances running in the source_ns. To avoid race
	# conditions, first iterate over the PIDs and terminate those
	# associated with the bash shells running the
	# `while true; do iperf3 -c ...; done` loops. In a second iteration,
	# terminate the individual `iperf3 -c ...` instances.
	terminate_ns_processes_by_pattern ${source_ns} while
	terminate_ns_processes_by_pattern ${source_ns} iperf3

	# Repeat the same process for sink_ns
	terminate_ns_processes_by_pattern ${sink_ns} while
	terminate_ns_processes_by_pattern ${sink_ns} iperf3

	# Check if any iperf3 instances are still running. This could happen
	# if a core has entered an infinite loop and the timeout for detecting
	# the soft lockup has not expired, but either the test interval has
	# already elapsed or the test was terminated manually (e.g., with ^C)
	for pid in $(ip netns pids ${source_ns}); do
	if [ -e /proc/$pid/cmdline ] && grep -qe 'iperf3' /proc/$pid/cmdline; then
	echo "FAIL: unable to terminate some iperf3 instances. Soft lockup is underway. A kernel panic is on the way!"
	exit ${ksft_fail}
	fi
	done

	if [ "$termination_signal" == "SIGINT" ]; then
	echo "SKIP: Termination due to ^C (SIGINT)"
	elif [ "$termination_signal" == "SIGALRM" ]; then
	echo "PASS: No kernel soft lockup occurred during this ${TEST_DURATION} second test"
	fi

	cleanup_ns ${source_ns} ${sink_ns}

	sysctl -qw kernel.softlockup_panic=${kernel_softlokup_panic_prev_val}
	}

	setup_prepare() {
	setup_ns source_ns sink_ns

	ip -n ${source_ns} link add name ${SOURCE_TEST_IFACE} type veth peer name ${SINK_TEST_IFACE} netns ${sink_ns}

	# Setting up the Source namespace
	ip -n ${source_ns} addr add ${SOURCE_TEST_IP_ADDR} dev ${SOURCE_TEST_IFACE}
	ip -n ${source_ns} link set dev ${SOURCE_TEST_IFACE} qlen 10000
	ip -n ${source_ns} link set dev ${SOURCE_TEST_IFACE} up
	ip netns exec ${source_ns} sysctl -qw net.ipv6.fib_multipath_hash_policy=1

	# Setting up the Sink namespace
	ip -n ${sink_ns} addr add ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK} dev ${SINK_LOOPBACK_IFACE}
	ip -n ${sink_ns} link set dev ${SINK_LOOPBACK_IFACE} up
	ip netns exec ${sink_ns} sysctl -qw net.ipv6.conf.${SINK_LOOPBACK_IFACE}.forwarding=1

	ip -n ${sink_ns} link set ${SINK_TEST_IFACE} up
	ip netns exec ${sink_ns} sysctl -qw net.ipv6.conf.${SINK_TEST_IFACE}.forwarding=1


	# Populate nexthop IPv6 addresses on the test interface in the sink_ns
	echo "info: populating ${IPV6_NEXTHOP_ADDR_COUNT} IPv6 addresses on the ${SINK_TEST_IFACE} interface ..."
	for IP in $(seq 1 ${IPV6_NEXTHOP_ADDR_COUNT}); do
	ip -n ${sink_ns} addr add ${IPV6_NEXTHOP_PREFIX}::$(printf "1:%x" "${IP}")/${IPV6_NEXTHOP_ADDR_MASK} dev ${SINK_TEST_IFACE};
	done

	# Preparing list of nexthops
	for IP in $(seq 1 ${IPV6_NEXTHOP_ADDR_COUNT}); do
	nexthop_ip_list=$nexthop_ip_list" nexthop via ${IPV6_NEXTHOP_PREFIX}::$(printf "1:%x" $IP) dev ${SOURCE_TEST_IFACE} weight 1"
	done
	}


	test_soft_lockup_during_routing_table_refresh() {
	# Start num_of_iperf_servers iperf3 servers in the sink_ns namespace,
	# each listening on ports starting at 5001 and incrementing
	# sequentially. Since iperf3 instances may terminate unexpectedly, a
	# while loop is used to automatically restart them in such cases.
	echo "info: starting ${num_of_iperf_servers} iperf3 servers in the sink_ns namespace ..."
	for i in $(seq 1 ${num_of_iperf_servers}); do
	cmd="iperf3 --bind ${SINK_LOOPBACK_IP_ADDR} -s -p $(printf '5%03d' ${i}) --rcv-timeout 200 &>/dev/null"
	ip netns exec ${sink_ns} bash -c "while true; do ${cmd}; done &" &>/dev/null
	done

	# Wait for the iperf3 servers to be ready
	for i in $(seq ${num_of_iperf_servers}); do
	port=$(printf '5%03d' ${i});
	wait_local_port_listen ${sink_ns} ${port} tcp
	done

	# Continuously refresh the routing table in the background within
	# the source_ns namespace
	ip netns exec ${source_ns} bash -c "
	while \$(ip netns list \| grep -q ${source_ns}); do
	ip -6 route add ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK} ${nexthop_ip_list};
	sleep ${ROUTING_TABLE_REFRESH_PERIOD};
	ip -6 route delete ${SINK_LOOPBACK_IP_ADDR}/${SINK_LOOPBACK_IP_MASK};
	done &"

	# Start num_of_iperf_servers iperf3 clients in the source_ns namespace,
	# each sending TCP traffic on sequential ports starting at 5001.
	# Since iperf3 instances may terminate unexpectedly (e.g., if the route
	# to the server is deleted in the background during a route refresh), a
	# while loop is used to automatically restart them in such cases.
	echo "info: starting ${num_of_iperf_servers} iperf3 clients in the source_ns namespace ..."
	for i in $(seq 1 ${num_of_iperf_servers}); do
	cmd="iperf3 -c ${SINK_LOOPBACK_IP_ADDR} -p $(printf '5%03d' ${i}) --length 64 --bitrate ${IPERF3_BITRATE} -t 0 --connect-timeout 150 &>/dev/null"
	ip netns exec ${source_ns} bash -c "while true; do ${cmd}; done &" &>/dev/null
	done

	echo "info: IPv6 routing table is being updated at the rate of $(echo "1/${ROUTING_TABLE_REFRESH_PERIOD}" \| bc)/s for ${TEST_DURATION} seconds ..."
	echo "info: A kernel soft lockup, if detected, results in a kernel panic!"

	wait
	}

	# Make sure 'iperf3' is installed, skip the test otherwise
	if [ ! -x "$(command -v "iperf3")" ]; then
	echo "SKIP: 'iperf3' is not installed. Skipping the test."
	exit ${ksft_skip}
	fi

	# Determine the number of cores on the machine
	num_of_iperf_servers=$(( $(nproc)/2 ))

	# Check if we are running on a multi-core machine, skip the test otherwise
	if [ "${num_of_iperf_servers}" -eq 0 ]; then
	echo "SKIP: This test is not valid on a single core machine!"
	exit ${ksft_skip}
	fi

	# Since the kernel soft lockup we're testing causes at least one core to enter
	# an infinite loop, destabilizing the host and likely affecting subsequent
	# tests, we trigger a kernel panic instead of reporting a failure and
	# continuing
	kernel_softlokup_panic_prev_val=$(sysctl -n kernel.softlockup_panic)
	sysctl -qw kernel.softlockup_panic=1

	handle_sigint() {
	termination_signal="SIGINT"
	cleanup
	exit ${ksft_skip}
	}

	handle_sigalrm() {
	termination_signal="SIGALRM"
	cleanup
	exit ${ksft_pass}
	}

	trap handle_sigint SIGINT
	trap handle_sigalrm SIGALRM

	(sleep ${TEST_DURATION} && kill -s SIGALRM $$)&

	setup_prepare
	test_soft_lockup_during_routing_table_refresh