samples/bpf/do_hbm_test.sh - linux - Git at Google

 #!/bin/bash
 # SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2019 Facebook
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of version 2 of the GNU General Public
 # License as published by the Free Software Foundation.

 Usage() {
   echo "Script for testing HBM (Host Bandwidth Manager) framework."
   echo "It creates a cgroup to use for testing and load a BPF program to limit"
   echo "egress or ingress bandwidth. It then uses iperf3 or netperf to create"
   echo "loads. The output is the goodput in Mbps (unless -D was used)."
   echo ""
   echo "USAGE: $name [out] [-b=<prog>|--bpf=<prog>] [-c=<cc>|--cc=<cc>]"
   echo "             [-D] [-d=<delay>|--delay=<delay>] [--debug] [-E] [--edt]"
   echo "             [-f=<#flows>|--flows=<#flows>] [-h] [-i=<id>|--id=<id >]"
   echo "             [-l] [-N] [--no_cn] [-p=<port>|--port=<port>] [-P]"
   echo "             [-q=<qdisc>] [-R] [-s=<server>|--server=<server]"
   echo "             [-S|--stats] -t=<time>|--time=<time>] [-w] [cubic|dctcp]"
   echo "  Where:"
   echo "    out               egress (default)"
   echo "    -b or --bpf       BPF program filename to load and attach."
   echo "                      Default is hbm_out_kern.o for egress,"
   echo "    -c or -cc         TCP congestion control (cubic or dctcp)"
   echo "    --debug           print BPF trace buffer"
   echo "    -d or --delay     add a delay in ms using netem"
   echo "    -D                In addition to the goodput in Mbps, it also outputs"
   echo "                      other detailed information. This information is"
   echo "                      test dependent (i.e. iperf3 or netperf)."
   echo "    -E                enable ECN (not required for dctcp)"
   echo "    --edt             use fq's Earliest Departure Time (requires fq)"
   echo "    -f or --flows     number of concurrent flows (default=1)"
   echo "    -i or --id        cgroup id (an integer, default is 1)"
   echo "    -N                use netperf instead of iperf3"
   echo "    --no_cn           Do not return CN notifications"
   echo "    -l                do not limit flows using loopback"
   echo "    -h                Help"
   echo "    -p or --port      iperf3 port (default is 5201)"
   echo "    -P                use an iperf3 instance for each flow"
   echo "    -q                use the specified qdisc"
   echo "    -r or --rate      rate in Mbps (default 1s 1Gbps)"
   echo "    -R                Use TCP_RR for netperf. 1st flow has req"
   echo "                      size of 10KB, rest of 1MB. Reply in all"
   echo "                      cases is 1 byte."
   echo "                      More detailed output for each flow can be found"
   echo "                      in the files netperf.<cg>.<flow>, where <cg> is the"
   echo "                      cgroup id as specified with the -i flag, and <flow>"
   echo "                      is the flow id starting at 1 and increasing by 1 for"
   echo "                      flow (as specified by -f)."
   echo "    -s or --server    hostname of netperf server. Used to create netperf"
   echo "                      test traffic between to hosts (default is within host)"
   echo "                      netserver must be running on the host."
   echo "    -S or --stats     whether to update hbm stats (default is yes)."
   echo "    -t or --time      duration of iperf3 in seconds (default=5)"
   echo "    -w                Work conserving flag. cgroup can increase its"
   echo "                      bandwidth beyond the rate limit specified"
   echo "                      while there is available bandwidth. Current"
   echo "                      implementation assumes there is only one NIC"
   echo "                      (eth0), but can be extended to support multiple"
   echo "                       NICs."
   echo "    cubic or dctcp    specify which TCP CC to use"
   echo " "
   exit
 }

 #set -x

 debug_flag=0
 args="$@"
 name="$0"
 netem=0
 cc=x
 dir="-o"
 dir_name="out"
 dur=5
 flows=1
 id=1
 prog=""
 port=5201
 rate=1000
 multi_iperf=0
 flow_cnt=1
 use_netperf=0
 rr=0
 ecn=0
 details=0
 server=""
 qdisc=""
 flags=""
 do_stats=0

 BPFFS=/sys/fs/bpf
 function config_bpffs () {
 	if mount | grep $BPFFS > /dev/null; then
 		echo "bpffs already mounted"
 	else
 		echo "bpffs not mounted. Mounting..."
 		mount -t bpf none $BPFFS
 	fi
 }

 function start_hbm () {
   rm -f hbm.out
   echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
   echo " " >> hbm.out
   ./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog >> hbm.out 2>&1  &
   echo $!
 }

 processArgs () {
   for i in $args ; do
     case $i in
     # Support for upcomming ingress rate limiting
     #in)         # support for upcoming ingress rate limiting
     #  dir="-i"
     #  dir_name="in"
     #  ;;
     out)
       dir="-o"
       dir_name="out"
       ;;
     -b=*|--bpf=*)
       prog="${i#*=}"
       ;;
     -c=*|--cc=*)
       cc="${i#*=}"
       ;;
     --no_cn)
       flags="$flags --no_cn"
       ;;
     --debug)
       flags="$flags -d"
       debug_flag=1
       ;;
     -d=*|--delay=*)
       netem="${i#*=}"
       ;;
     -D)
       details=1
       ;;
     -E)
       ecn=1
       ;;
     --edt)
       flags="$flags --edt"
       qdisc="fq"
      ;;
     -f=*|--flows=*)
       flows="${i#*=}"
       ;;
     -i=*|--id=*)
       id="${i#*=}"
       ;;
     -l)
       flags="$flags -l"
       ;;
     -N)
       use_netperf=1
       ;;
     -p=*|--port=*)
       port="${i#*=}"
       ;;
     -P)
       multi_iperf=1
       ;;
     -q=*)
       qdisc="${i#*=}"
       ;;
     -r=*|--rate=*)
       rate="${i#*=}"
       ;;
     -R)
       rr=1
       ;;
     -s=*|--server=*)
       server="${i#*=}"
       ;;
     -S|--stats)
       flags="$flags -s"
       do_stats=1
       ;;
     -t=*|--time=*)
       dur="${i#*=}"
       ;;
     -w)
       flags="$flags -w"
       ;;
     cubic)
       cc=cubic
       ;;
     dctcp)
       cc=dctcp
       ;;
     *)
       echo "Unknown arg:$i"
       Usage
       ;;
     esac
   done
 }

 processArgs
 config_bpffs

 if [ $debug_flag -eq 1 ] ; then
   rm -f hbm_out.log
 fi

 hbm_pid=$(start_hbm)
 usleep 100000

 host=`hostname`
 cg_base_dir=/sys/fs/cgroup/unified
 cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"

 echo $$ >> $cg_dir/cgroup.procs

 ulimit -l unlimited

 rm -f ss.out
 rm -f hbm.[0-9]*.$dir_name
 if [ $ecn -ne 0 ] ; then
   sysctl -w -q -n net.ipv4.tcp_ecn=1
 fi

 if [ $use_netperf -eq 0 ] ; then
   cur_cc=`sysctl -n net.ipv4.tcp_congestion_control`
   if [ "$cc" != "x" ] ; then
     sysctl -w -q -n net.ipv4.tcp_congestion_control=$cc
   fi
 fi

 if [ "$netem" -ne "0" ] ; then
   if [ "$qdisc" != "" ] ; then
     echo "WARNING: Ignoring -q options because -d option used"
   fi
   tc qdisc del dev lo root > /dev/null 2>&1
   tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1
 elif [ "$qdisc" != "" ] ; then
   tc qdisc del dev eth0 root > /dev/null 2>&1
   tc qdisc add dev eth0 root $qdisc > /dev/null 2>&1
 fi

 n=0
 m=$[$dur * 5]
 hn="::1"
 if [ $use_netperf -ne 0 ] ; then
   if [ "$server" != "" ] ; then
     hn=$server
   fi
 fi

 ( ping6 -i 0.2 -c $m $hn > ping.out 2>&1 ) &

 if [ $use_netperf -ne 0 ] ; then
   begNetserverPid=`ps ax | grep netserver | grep --invert-match "grep" | \
                    awk '{ print $1 }'`
   if [ "$begNetserverPid" == "" ] ; then
     if [ "$server" == "" ] ; then
       ( ./netserver > /dev/null 2>&1) &
       usleep 100000
     fi
   fi
   flow_cnt=1
   if [ "$server" == "" ] ; then
     np_server=$host
   else
     np_server=$server
   fi
   if [ "$cc" == "x" ] ; then
     np_cc=""
   else
     np_cc="-K $cc,$cc"
   fi
   replySize=1
   while [ $flow_cnt -le $flows ] ; do
     if [ $rr -ne 0 ] ; then
       reqSize=1M
       if [ $flow_cnt -eq 1 ] ; then
         reqSize=10K
       fi
       if [ "$dir" == "-i" ] ; then
         replySize=$reqSize
         reqSize=1
       fi
       ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR  -- -r $reqSize,$replySize $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,REMOTE_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,LOCAL_RECV_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
     else
       if [ "$dir" == "-i" ] ; then
         ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r 1,10M $np_cc -k P50_LATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REMOTE_TRANSPORT_RETRANS,REMOTE_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
       else
         ( ./netperf -H $np_server -l $dur -f m -j -t TCP_STREAM -- $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
       fi
     fi
     flow_cnt=$[flow_cnt+1]
   done

 # sleep for duration of test (plus some buffer)
   n=$[dur+2]
   sleep $n

 # force graceful termination of netperf
   pids=`pgrep netperf`
   for p in $pids ; do
     kill -SIGALRM $p
   done

   flow_cnt=1
   rate=0
   if [ $details -ne 0 ] ; then
     echo ""
     echo "Details for HBM in cgroup $id"
     if [ $do_stats -eq 1 ] ; then
       if [ -e hbm.$id.$dir_name ] ; then
         cat hbm.$id.$dir_name
       fi
     fi
   fi
   while [ $flow_cnt -le $flows ] ; do
     if [ "$dir" == "-i" ] ; then
       r=`cat netperf.$id.$flow_cnt | grep -o "REMOTE_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
     else
       r=`cat netperf.$id.$flow_cnt | grep -o "LOCAL_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
     fi
     echo "rate for flow $flow_cnt: $r"
     rate=$[rate+r]
     if [ $details -ne 0 ] ; then
       echo "-----"
       echo "Details for cgroup $id, flow $flow_cnt"
       cat netperf.$id.$flow_cnt
     fi
     flow_cnt=$[flow_cnt+1]
   done
   if [ $details -ne 0 ] ; then
     echo ""
     delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
     echo "PING AVG DELAY:$delay"
     echo "AGGREGATE_GOODPUT:$rate"
   else
     echo $rate
   fi
 elif [ $multi_iperf -eq 0 ] ; then
   (iperf3 -s -p $port -1 > /dev/null 2>&1) &
   usleep 100000
   iperf3 -c $host -p $port -i 0 -P $flows -f m -t $dur > iperf.$id
   rates=`grep receiver iperf.$id | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*"`
   rate=`echo $rates | grep -o "[0-9]*$"`

   if [ $details -ne 0 ] ; then
     echo ""
     echo "Details for HBM in cgroup $id"
     if [ $do_stats -eq 1 ] ; then
       if [ -e hbm.$id.$dir_name ] ; then
         cat hbm.$id.$dir_name
       fi
     fi
     delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
     echo "PING AVG DELAY:$delay"
     echo "AGGREGATE_GOODPUT:$rate"
   else
     echo $rate
   fi
 else
   flow_cnt=1
   while [ $flow_cnt -le $flows ] ; do
     (iperf3 -s -p $port -1 > /dev/null 2>&1) &
     ( iperf3 -c $host -p $port -i 0 -P 1 -f m -t $dur | grep receiver | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*" | grep -o "[0-9]*$" > iperf3.$id.$flow_cnt ) &
     port=$[port+1]
     flow_cnt=$[flow_cnt+1]
   done
   n=$[dur+1]
   sleep $n
   flow_cnt=1
   rate=0
   if [ $details -ne 0 ] ; then
     echo ""
     echo "Details for HBM in cgroup $id"
     if [ $do_stats -eq 1 ] ; then
       if [ -e hbm.$id.$dir_name ] ; then
         cat hbm.$id.$dir_name
       fi
     fi
   fi

   while [ $flow_cnt -le $flows ] ; do
     r=`cat iperf3.$id.$flow_cnt`
 #    echo "rate for flow $flow_cnt: $r"
   if [ $details -ne 0 ] ; then
     echo "Rate for cgroup $id, flow $flow_cnt LOCAL_SEND_THROUGHPUT=$r"
   fi
     rate=$[rate+r]
     flow_cnt=$[flow_cnt+1]
   done
   if [ $details -ne 0 ] ; then
     delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
     echo "PING AVG DELAY:$delay"
     echo "AGGREGATE_GOODPUT:$rate"
   else
     echo $rate
   fi
 fi

 if [ $use_netperf -eq 0 ] ; then
   sysctl -w -q -n net.ipv4.tcp_congestion_control=$cur_cc
 fi
 if [ $ecn -ne 0 ] ; then
   sysctl -w -q -n net.ipv4.tcp_ecn=0
 fi
 if [ "$netem" -ne "0" ] ; then
   tc qdisc del dev lo root > /dev/null 2>&1
 fi
 if [ "$qdisc" != "" ] ; then
   tc qdisc del dev eth0 root > /dev/null 2>&1
 fi
 sleep 2

 hbmPid=`ps ax | grep "hbm " | grep --invert-match "grep" | awk '{ print $1 }'`
 if [ "$hbmPid" == "$hbm_pid" ] ; then
   kill $hbm_pid
 fi

 sleep 1

 # Detach any pinned BPF programs that may have lingered
 rm -rf $BPFFS/hbm*

 if [ $use_netperf -ne 0 ] ; then
   if [ "$server" == "" ] ; then
     if [ "$begNetserverPid" == "" ] ; then
       netserverPid=`ps ax | grep netserver | grep --invert-match "grep" | awk '{ print $1 }'`
       if [ "$netserverPid" != "" ] ; then
         kill $netserverPid
       fi
     fi
   fi
 fi
 exit
	#!/bin/bash
	# SPDX-License-Identifier: GPL-2.0
	#
	# Copyright (c) 2019 Facebook
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of version 2 of the GNU General Public
	# License as published by the Free Software Foundation.

	Usage() {
	echo "Script for testing HBM (Host Bandwidth Manager) framework."
	echo "It creates a cgroup to use for testing and load a BPF program to limit"
	echo "egress or ingress bandwidth. It then uses iperf3 or netperf to create"
	echo "loads. The output is the goodput in Mbps (unless -D was used)."
	echo ""
	echo "USAGE: $name [out] [-b=<prog>\|--bpf=<prog>] [-c=<cc>\|--cc=<cc>]"
	echo " [-D] [-d=<delay>\|--delay=<delay>] [--debug] [-E] [--edt]"
	echo " [-f=<#flows>\|--flows=<#flows>] [-h] [-i=<id>\|--id=<id >]"
	echo " [-l] [-N] [--no_cn] [-p=<port>\|--port=<port>] [-P]"
	echo " [-q=<qdisc>] [-R] [-s=<server>\|--server=<server]"
	echo " [-S\|--stats] -t=<time>\|--time=<time>] [-w] [cubic\|dctcp]"
	echo " Where:"
	echo " out egress (default)"
	echo " -b or --bpf BPF program filename to load and attach."
	echo " Default is hbm_out_kern.o for egress,"
	echo " -c or -cc TCP congestion control (cubic or dctcp)"
	echo " --debug print BPF trace buffer"
	echo " -d or --delay add a delay in ms using netem"
	echo " -D In addition to the goodput in Mbps, it also outputs"
	echo " other detailed information. This information is"
	echo " test dependent (i.e. iperf3 or netperf)."
	echo " -E enable ECN (not required for dctcp)"
	echo " --edt use fq's Earliest Departure Time (requires fq)"
	echo " -f or --flows number of concurrent flows (default=1)"
	echo " -i or --id cgroup id (an integer, default is 1)"
	echo " -N use netperf instead of iperf3"
	echo " --no_cn Do not return CN notifications"
	echo " -l do not limit flows using loopback"
	echo " -h Help"
	echo " -p or --port iperf3 port (default is 5201)"
	echo " -P use an iperf3 instance for each flow"
	echo " -q use the specified qdisc"
	echo " -r or --rate rate in Mbps (default 1s 1Gbps)"
	echo " -R Use TCP_RR for netperf. 1st flow has req"
	echo " size of 10KB, rest of 1MB. Reply in all"
	echo " cases is 1 byte."
	echo " More detailed output for each flow can be found"
	echo " in the files netperf.<cg>.<flow>, where <cg> is the"
	echo " cgroup id as specified with the -i flag, and <flow>"
	echo " is the flow id starting at 1 and increasing by 1 for"
	echo " flow (as specified by -f)."
	echo " -s or --server hostname of netperf server. Used to create netperf"
	echo " test traffic between to hosts (default is within host)"
	echo " netserver must be running on the host."
	echo " -S or --stats whether to update hbm stats (default is yes)."
	echo " -t or --time duration of iperf3 in seconds (default=5)"
	echo " -w Work conserving flag. cgroup can increase its"
	echo " bandwidth beyond the rate limit specified"
	echo " while there is available bandwidth. Current"
	echo " implementation assumes there is only one NIC"
	echo " (eth0), but can be extended to support multiple"
	echo " NICs."
	echo " cubic or dctcp specify which TCP CC to use"
	echo " "
	exit
	}

	#set -x

	debug_flag=0
	args="$@"
	name="$0"
	netem=0
	cc=x
	dir="-o"
	dir_name="out"
	dur=5
	flows=1
	id=1
	prog=""
	port=5201
	rate=1000
	multi_iperf=0
	flow_cnt=1
	use_netperf=0
	rr=0
	ecn=0
	details=0
	server=""
	qdisc=""
	flags=""
	do_stats=0

	BPFFS=/sys/fs/bpf
	function config_bpffs () {
	if mount \| grep $BPFFS > /dev/null; then
	echo "bpffs already mounted"
	else
	echo "bpffs not mounted. Mounting..."
	mount -t bpf none $BPFFS
	fi
	}

	function start_hbm () {
	rm -f hbm.out
	echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
	echo " " >> hbm.out
	./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog >> hbm.out 2>&1 &
	echo $!
	}

	processArgs () {
	for i in $args ; do
	case $i in
	# Support for upcomming ingress rate limiting
	#in) # support for upcoming ingress rate limiting
	# dir="-i"
	# dir_name="in"
	# ;;
	out)
	dir="-o"
	dir_name="out"
	;;
	-b=\|--bpf=)
	prog="${i#*=}"
	;;
	-c=\|--cc=)
	cc="${i#*=}"
	;;
	--no_cn)
	flags="$flags --no_cn"
	;;
	--debug)
	flags="$flags -d"
	debug_flag=1
	;;
	-d=\|--delay=)
	netem="${i#*=}"
	;;
	-D)
	details=1
	;;
	-E)
	ecn=1
	;;
	--edt)
	flags="$flags --edt"
	qdisc="fq"
	;;
	-f=\|--flows=)
	flows="${i#*=}"
	;;
	-i=\|--id=)
	id="${i#*=}"
	;;
	-l)
	flags="$flags -l"
	;;
	-N)
	use_netperf=1
	;;
	-p=\|--port=)
	port="${i#*=}"
	;;
	-P)
	multi_iperf=1
	;;
	-q=*)
	qdisc="${i#*=}"
	;;
	-r=\|--rate=)
	rate="${i#*=}"
	;;
	-R)
	rr=1
	;;
	-s=\|--server=)
	server="${i#*=}"
	;;
	-S\|--stats)
	flags="$flags -s"
	do_stats=1
	;;
	-t=\|--time=)
	dur="${i#*=}"
	;;
	-w)
	flags="$flags -w"
	;;
	cubic)
	cc=cubic
	;;
	dctcp)
	cc=dctcp
	;;
	*)
	echo "Unknown arg:$i"
	Usage
	;;
	esac
	done
	}

	processArgs
	config_bpffs

	if [ $debug_flag -eq 1 ] ; then
	rm -f hbm_out.log
	fi

	hbm_pid=$(start_hbm)
	usleep 100000

	host=`hostname`
	cg_base_dir=/sys/fs/cgroup/unified
	cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"

	echo $$ >> $cg_dir/cgroup.procs

	ulimit -l unlimited

	rm -f ss.out
	rm -f hbm.[0-9]*.$dir_name
	if [ $ecn -ne 0 ] ; then
	sysctl -w -q -n net.ipv4.tcp_ecn=1
	fi

	if [ $use_netperf -eq 0 ] ; then
	cur_cc=`sysctl -n net.ipv4.tcp_congestion_control`
	if [ "$cc" != "x" ] ; then
	sysctl -w -q -n net.ipv4.tcp_congestion_control=$cc
	fi
	fi

	if [ "$netem" -ne "0" ] ; then
	if [ "$qdisc" != "" ] ; then
	echo "WARNING: Ignoring -q options because -d option used"
	fi
	tc qdisc del dev lo root > /dev/null 2>&1
	tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1
	elif [ "$qdisc" != "" ] ; then
	tc qdisc del dev eth0 root > /dev/null 2>&1
	tc qdisc add dev eth0 root $qdisc > /dev/null 2>&1
	fi

	n=0
	m=$[$dur * 5]
	hn="::1"
	if [ $use_netperf -ne 0 ] ; then
	if [ "$server" != "" ] ; then
	hn=$server
	fi
	fi

	( ping6 -i 0.2 -c $m $hn > ping.out 2>&1 ) &

	if [ $use_netperf -ne 0 ] ; then
	begNetserverPid=`ps ax \| grep netserver \| grep --invert-match "grep" \| \
	awk '{ print $1 }'`
	if [ "$begNetserverPid" == "" ] ; then
	if [ "$server" == "" ] ; then
	( ./netserver > /dev/null 2>&1) &
	usleep 100000
	fi
	fi
	flow_cnt=1
	if [ "$server" == "" ] ; then
	np_server=$host
	else
	np_server=$server
	fi
	if [ "$cc" == "x" ] ; then
	np_cc=""
	else
	np_cc="-K $cc,$cc"
	fi
	replySize=1
	while [ $flow_cnt -le $flows ] ; do
	if [ $rr -ne 0 ] ; then
	reqSize=1M
	if [ $flow_cnt -eq 1 ] ; then
	reqSize=10K
	fi
	if [ "$dir" == "-i" ] ; then
	replySize=$reqSize
	reqSize=1
	fi
	( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r $reqSize,$replySize $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,REMOTE_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,LOCAL_RECV_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
	else
	if [ "$dir" == "-i" ] ; then
	( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r 1,10M $np_cc -k P50_LATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REMOTE_TRANSPORT_RETRANS,REMOTE_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
	else
	( ./netperf -H $np_server -l $dur -f m -j -t TCP_STREAM -- $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
	fi
	fi
	flow_cnt=$[flow_cnt+1]
	done

	# sleep for duration of test (plus some buffer)
	n=$[dur+2]
	sleep $n

	# force graceful termination of netperf
	pids=`pgrep netperf`
	for p in $pids ; do
	kill -SIGALRM $p
	done

	flow_cnt=1
	rate=0
	if [ $details -ne 0 ] ; then
	echo ""
	echo "Details for HBM in cgroup $id"
	if [ $do_stats -eq 1 ] ; then
	if [ -e hbm.$id.$dir_name ] ; then
	cat hbm.$id.$dir_name
	fi
	fi
	fi
	while [ $flow_cnt -le $flows ] ; do
	if [ "$dir" == "-i" ] ; then
	r=`cat netperf.$id.$flow_cnt \| grep -o "REMOTE_SEND_THROUGHPUT=[0-9]" \| grep -o "[0-9]"`
	else
	r=`cat netperf.$id.$flow_cnt \| grep -o "LOCAL_SEND_THROUGHPUT=[0-9]" \| grep -o "[0-9]"`
	fi
	echo "rate for flow $flow_cnt: $r"
	rate=$[rate+r]
	if [ $details -ne 0 ] ; then
	echo "-----"
	echo "Details for cgroup $id, flow $flow_cnt"
	cat netperf.$id.$flow_cnt
	fi
	flow_cnt=$[flow_cnt+1]
	done
	if [ $details -ne 0 ] ; then
	echo ""
	delay=`grep "avg" ping.out \| grep -o "= [0-9.]/[0-9.]" \| grep -o "[0-9.]*$"`
	echo "PING AVG DELAY:$delay"
	echo "AGGREGATE_GOODPUT:$rate"
	else
	echo $rate
	fi
	elif [ $multi_iperf -eq 0 ] ; then
	(iperf3 -s -p $port -1 > /dev/null 2>&1) &
	usleep 100000
	iperf3 -c $host -p $port -i 0 -P $flows -f m -t $dur > iperf.$id
	rates=`grep receiver iperf.$id \| grep -o "[0-9.]* Mbits" \| grep -o "^[0-9]*"`
	rate=`echo $rates \| grep -o "[0-9]*$"`

	if [ $details -ne 0 ] ; then
	echo ""
	echo "Details for HBM in cgroup $id"
	if [ $do_stats -eq 1 ] ; then
	if [ -e hbm.$id.$dir_name ] ; then
	cat hbm.$id.$dir_name
	fi
	fi
	delay=`grep "avg" ping.out \| grep -o "= [0-9.]/[0-9.]" \| grep -o "[0-9.]*$"`
	echo "PING AVG DELAY:$delay"
	echo "AGGREGATE_GOODPUT:$rate"
	else
	echo $rate
	fi
	else
	flow_cnt=1
	while [ $flow_cnt -le $flows ] ; do
	(iperf3 -s -p $port -1 > /dev/null 2>&1) &
	( iperf3 -c $host -p $port -i 0 -P 1 -f m -t $dur \| grep receiver \| grep -o "[0-9.]* Mbits" \| grep -o "^[0-9]" \| grep -o "[0-9]$" > iperf3.$id.$flow_cnt ) &
	port=$[port+1]
	flow_cnt=$[flow_cnt+1]
	done
	n=$[dur+1]
	sleep $n
	flow_cnt=1
	rate=0
	if [ $details -ne 0 ] ; then
	echo ""
	echo "Details for HBM in cgroup $id"
	if [ $do_stats -eq 1 ] ; then
	if [ -e hbm.$id.$dir_name ] ; then
	cat hbm.$id.$dir_name
	fi
	fi
	fi

	while [ $flow_cnt -le $flows ] ; do
	r=`cat iperf3.$id.$flow_cnt`
	# echo "rate for flow $flow_cnt: $r"
	if [ $details -ne 0 ] ; then
	echo "Rate for cgroup $id, flow $flow_cnt LOCAL_SEND_THROUGHPUT=$r"
	fi
	rate=$[rate+r]
	flow_cnt=$[flow_cnt+1]
	done
	if [ $details -ne 0 ] ; then
	delay=`grep "avg" ping.out \| grep -o "= [0-9.]/[0-9.]" \| grep -o "[0-9.]*$"`
	echo "PING AVG DELAY:$delay"
	echo "AGGREGATE_GOODPUT:$rate"
	else
	echo $rate
	fi
	fi

	if [ $use_netperf -eq 0 ] ; then
	sysctl -w -q -n net.ipv4.tcp_congestion_control=$cur_cc
	fi
	if [ $ecn -ne 0 ] ; then
	sysctl -w -q -n net.ipv4.tcp_ecn=0
	fi
	if [ "$netem" -ne "0" ] ; then
	tc qdisc del dev lo root > /dev/null 2>&1
	fi
	if [ "$qdisc" != "" ] ; then
	tc qdisc del dev eth0 root > /dev/null 2>&1
	fi
	sleep 2

	hbmPid=`ps ax \| grep "hbm " \| grep --invert-match "grep" \| awk '{ print $1 }'`
	if [ "$hbmPid" == "$hbm_pid" ] ; then
	kill $hbm_pid
	fi

	sleep 1

	# Detach any pinned BPF programs that may have lingered
	rm -rf $BPFFS/hbm*

	if [ $use_netperf -ne 0 ] ; then
	if [ "$server" == "" ] ; then
	if [ "$begNetserverPid" == "" ] ; then
	netserverPid=`ps ax \| grep netserver \| grep --invert-match "grep" \| awk '{ print $1 }'`
	if [ "$netserverPid" != "" ] ; then
	kill $netserverPid
	fi
	fi
	fi
	fi
	exit