| #!/usr/bin/env python3 | |
| # SPDX-License-Identifier: GPL-2.0 | |
| """A simple test for TSO.""" | |
| import fcntl | |
| import mmap | |
| import socket | |
| import struct | |
| import termios | |
| import time | |
| from lib.py import ksft_pr, ksft_run, ksft_exit, KsftSkipEx, KsftXfailEx | |
| from lib.py import ksft_eq, ksft_ge, ksft_lt | |
| from lib.py import EthtoolFamily, NetdevFamily, NetDrvEpEnv | |
| from lib.py import bkg, cmd, defer, ethtool, ip, rand_port, wait_port_listen | |
| MAP_HUGETLB = getattr(mmap, "MAP_HUGETLB", 0x40000) | |
| MSG_ZEROCOPY = getattr(socket, "MSG_ZEROCOPY", 0x4000000) | |
| SO_ZEROCOPY = getattr(socket, "SO_ZEROCOPY", 60) | |
| GSO_LEGACY_MAX_SIZE = 65536 | |
| # Pool of the default hugepage size, the one /proc/meminfo reports on. | |
| NR_HUGEPAGES = "/proc/sys/vm/nr_hugepages" | |
| def default_huge_page_size(): | |
| """Return the hugepage size in bytes""" | |
| try: | |
| with open("/proc/meminfo", encoding="utf-8") as meminfo: | |
| for line in meminfo: | |
| if line.startswith("Hugepagesize:"): | |
| return int(line.split()[1]) * 1024 | |
| except OSError: | |
| pass | |
| return 2 * 1024 * 1024 | |
| def hugepages_free(): | |
| """Return the number of unused hugepages of the default size.""" | |
| try: | |
| with open("/proc/meminfo", encoding="utf-8") as meminfo: | |
| for line in meminfo: | |
| if line.startswith("HugePages_Free:"): | |
| return int(line.split()[1]) | |
| except OSError: | |
| pass | |
| return 0 | |
| def set_nr_hugepages(count): | |
| with open(NR_HUGEPAGES, "w", encoding="utf-8") as sysctl: | |
| sysctl.write(f"{count}\n") | |
| def tx_dropped(ifname): | |
| with open(f"/sys/class/net/{ifname}/statistics/tx_dropped", | |
| encoding="utf-8") as counter: | |
| return int(counter.read()) | |
| def setup_hugepage(): | |
| """Reserve one hugepage, and put the pool back afterwards.""" | |
| if hugepages_free() >= 1: | |
| return | |
| try: | |
| with open(NR_HUGEPAGES, encoding="utf-8") as sysctl: | |
| old_count = int(sysctl.read()) | |
| set_nr_hugepages(old_count + 1) | |
| except OSError as error: | |
| raise KsftSkipEx(f"Unable to reserve a hugepage: {error}") from error | |
| defer(set_nr_hugepages, old_count) | |
| if hugepages_free() < 1: | |
| raise KsftSkipEx("Unable to reserve a hugepage") | |
| def mmap_large_buffer(): | |
| """Allocate a buffer backed by one huge page.""" | |
| size = default_huge_page_size() | |
| setup_hugepage() | |
| try: | |
| return mmap.mmap(-1, size, | |
| flags=mmap.MAP_PRIVATE | | |
| mmap.MAP_ANONYMOUS | | |
| MAP_HUGETLB, | |
| prot=mmap.PROT_READ) | |
| except OSError as e: | |
| raise KsftSkipEx(f"Unable to allocate a {size >> 20}MB hugepage " | |
| f"buffer: {e}") from e | |
| def sock_wait_drain(sock, max_wait=1000): | |
| """Wait for all pending write data on the socket to get ACKed.""" | |
| for _ in range(max_wait): | |
| one = b'\0' * 4 | |
| outq = fcntl.ioctl(sock.fileno(), termios.TIOCOUTQ, one) | |
| outq = struct.unpack("I", outq)[0] | |
| if outq == 0: | |
| break | |
| time.sleep(0.01) | |
| ksft_eq(outq, 0) | |
| def tcp_sock_get_retrans(sock): | |
| """Get the number of retransmissions for the TCP socket.""" | |
| info = sock.getsockopt(socket.SOL_TCP, socket.TCP_INFO, 512) | |
| return struct.unpack("I", info[100:104])[0] | |
| def setup_big_tcp(cfg): | |
| """Lift the GSO ceiling to what the device advertises for TSO.""" | |
| if cfg.dev["tso_max_size"] <= GSO_LEGACY_MAX_SIZE: | |
| raise KsftSkipEx("Device does not support BIG TCP") | |
| ip(f"link set dev {cfg.ifname} " | |
| f"gso_max_size {cfg.dev['tso_max_size']} " | |
| f"gso_ipv4_max_size {cfg.dev['tso_max_size']}") | |
| defer(ip, f"link set dev {cfg.ifname} " | |
| f"gso_max_size {cfg.dev['gso_max_size']} " | |
| f"gso_ipv4_max_size {cfg.dev['gso_ipv4_max_size']}") | |
| def sock_send_zerocopy(sock): | |
| """Send with MSG_ZEROCOPY, return the bytes queued.""" | |
| try: | |
| sock.setsockopt(socket.SOL_SOCKET, SO_ZEROCOPY, 1) | |
| except OSError as e: | |
| raise KsftSkipEx(f"SO_ZEROCOPY not supported: {e}") from e | |
| with mmap_large_buffer() as tx_buf: | |
| sock.sendall(tx_buf, MSG_ZEROCOPY) | |
| return len(tx_buf) | |
| def run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso): | |
| cfg.require_cmd("socat", local=False, remote=True) | |
| # Set recv window clamp to avoid overwhelming receiver on debug kernels | |
| # the 200k clamp should still let use reach > 15Gbps on real HW | |
| port = rand_port() | |
| listen_opts = f"{port},reuseport,tcp-window-clamp=200000" | |
| listen_cmd = f"socat -{ipver} -t 2 -u TCP-LISTEN:{listen_opts} /dev/null,ignoreeof" | |
| with bkg(listen_cmd, host=cfg.remote, exit_wait=True) as nc: | |
| wait_port_listen(port, host=cfg.remote) | |
| if ipver == "4": | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| sock.connect((remote_v4, port)) | |
| else: | |
| sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) | |
| sock.connect((remote_v6, port)) | |
| # Small send to make sure the connection is working. | |
| sock.send("ping".encode()) | |
| sock_wait_drain(sock) | |
| # Send 4MB of data, record the LSO packet count. | |
| qstat_old = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True)[0] | |
| buf = b"0" * 1024 * 1024 * 4 | |
| sock.send(buf) | |
| sock_wait_drain(sock) | |
| qstat_new = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True)[0] | |
| # Check that at least 90% of the data was sent as LSO packets. | |
| # System noise may cause false negatives. Also header overheads | |
| # will add up to 5% of extra packes... The check is best effort. | |
| total_lso_wire = len(buf) * 0.90 // cfg.dev["mtu"] | |
| total_lso_super = len(buf) * 0.90 // cfg.dev["tso_max_size"] | |
| # Make sure we have order of magnitude more LSO packets than | |
| # retransmits, in case TCP retransmitted all the LSO packets. | |
| ksft_lt(tcp_sock_get_retrans(sock), total_lso_wire / 16) | |
| sock.close() | |
| if should_lso: | |
| if cfg.have_stat_super_count: | |
| ksft_ge(qstat_new['tx-hw-gso-packets'] - | |
| qstat_old['tx-hw-gso-packets'], | |
| total_lso_super, | |
| comment="Number of LSO super-packets with LSO enabled") | |
| if cfg.have_stat_wire_count: | |
| ksft_ge(qstat_new['tx-hw-gso-wire-packets'] - | |
| qstat_old['tx-hw-gso-wire-packets'], | |
| total_lso_wire, | |
| comment="Number of LSO wire-packets with LSO enabled") | |
| else: | |
| if cfg.have_stat_super_count: | |
| ksft_lt(qstat_new['tx-hw-gso-packets'] - | |
| qstat_old['tx-hw-gso-packets'], | |
| 15, comment="Number of LSO super-packets with LSO disabled") | |
| if cfg.have_stat_wire_count: | |
| ksft_lt(qstat_new['tx-hw-gso-wire-packets'] - | |
| qstat_old['tx-hw-gso-wire-packets'], | |
| 500, comment="Number of LSO wire-packets with LSO disabled") | |
| def run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6): | |
| """Send with MSG_ZEROCOPY out of a huge page, so the frags exceed 64kB.""" | |
| cfg.require_cmd("socat", local=False, remote=True) | |
| # No clamping, as it would keep the frags under 64kB | |
| port = rand_port() | |
| listen_opts = f"{port},reuseport" | |
| listen_cmd = f"socat -{ipver} -t 2 -u TCP-LISTEN:{listen_opts} /dev/null,ignoreeof" | |
| with bkg(listen_cmd, host=cfg.remote, exit_wait=True): | |
| wait_port_listen(port, host=cfg.remote) | |
| if ipver == "4": | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |
| sock.connect((remote_v4, port)) | |
| else: | |
| sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) | |
| sock.connect((remote_v6, port)) | |
| # Small send to make sure the connection is working. | |
| sock.send("ping".encode()) | |
| sock_wait_drain(sock) | |
| retrans_old = tcp_sock_get_retrans(sock) | |
| drops_old = tx_dropped(cfg.ifname) | |
| sent = sock_send_zerocopy(sock) | |
| sock_wait_drain(sock) | |
| drops = tx_dropped(cfg.ifname) - drops_old | |
| retrans = tcp_sock_get_retrans(sock) - retrans_old | |
| sock.close() | |
| ksft_eq(drops, 0, comment="Driver TX drops during BIG TCP send") | |
| # Same best effort bound as the plain stream. | |
| total_lso_wire = sent * 0.90 // cfg.dev["mtu"] | |
| ksft_lt(retrans, total_lso_wire / 16) | |
| def build_tunnel(cfg, outer_ipver, tun_info): | |
| local_v4 = NetDrvEpEnv.nsim_v4_pfx + "1" | |
| local_v6 = NetDrvEpEnv.nsim_v6_pfx + "1" | |
| remote_v4 = NetDrvEpEnv.nsim_v4_pfx + "2" | |
| remote_v6 = NetDrvEpEnv.nsim_v6_pfx + "2" | |
| local_addr = cfg.addr_v[outer_ipver] | |
| remote_addr = cfg.remote_addr_v[outer_ipver] | |
| tun_type = tun_info[0] | |
| tun_arg = tun_info[1] | |
| ip(f"link add {tun_type}-ksft type {tun_type} {tun_arg} local {local_addr} remote {remote_addr} dev {cfg.ifname}") | |
| defer(ip, f"link del {tun_type}-ksft") | |
| ip(f"link set dev {tun_type}-ksft up") | |
| ip(f"addr add {local_v4}/24 dev {tun_type}-ksft") | |
| ip(f"addr add {local_v6}/64 dev {tun_type}-ksft") | |
| ip(f"link add {tun_type}-ksft type {tun_type} {tun_arg} local {remote_addr} remote {local_addr} dev {cfg.remote_ifname}", | |
| host=cfg.remote) | |
| defer(ip, f"link del {tun_type}-ksft", host=cfg.remote) | |
| ip(f"link set dev {tun_type}-ksft up", host=cfg.remote) | |
| ip(f"addr add {remote_v4}/24 dev {tun_type}-ksft", host=cfg.remote) | |
| ip(f"addr add {remote_v6}/64 dev {tun_type}-ksft", host=cfg.remote) | |
| return remote_v4, remote_v6 | |
| def restore_wanted_features(cfg): | |
| features_cmd = "" | |
| for feature in cfg.hw_features: | |
| setting = "on" if feature in cfg.wanted_features else "off" | |
| features_cmd += f" {feature} {setting}" | |
| try: | |
| ethtool(f"-K {cfg.ifname} {features_cmd}") | |
| except Exception as e: | |
| ksft_pr(f"WARNING: failure restoring wanted features: {e}") | |
| def test_builder(name, cfg, outer_ipver, feature, tun=None, inner_ipver=None): | |
| """Construct specific tests from the common template.""" | |
| def f(cfg): | |
| cfg.require_ipver(outer_ipver) | |
| defer(restore_wanted_features, cfg) | |
| if not cfg.have_stat_super_count and \ | |
| not cfg.have_stat_wire_count: | |
| raise KsftSkipEx(f"Device does not support LSO queue stats") | |
| if feature not in cfg.hw_features: | |
| raise KsftSkipEx(f"Device does not support {feature}") | |
| # Run non-tunnel test cases under the BIG TCP limits too. | |
| big_tcp = "big_tcp" in name | |
| if big_tcp: | |
| setup_big_tcp(cfg) | |
| ipver = outer_ipver | |
| if tun: | |
| remote_v4, remote_v6 = build_tunnel(cfg, ipver, tun) | |
| ipver = inner_ipver | |
| else: | |
| remote_v4 = cfg.remote_addr_v["4"] | |
| remote_v6 = cfg.remote_addr_v["6"] | |
| # First test without the feature enabled. | |
| ethtool(f"-K {cfg.ifname} {feature} off") | |
| run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso=False) | |
| if big_tcp: | |
| run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6) | |
| ethtool(f"-K {cfg.ifname} tx-gso-partial off") | |
| ethtool(f"-K {cfg.ifname} tx-tcp-mangleid-segmentation off") | |
| if feature in cfg.partial_features: | |
| ethtool(f"-K {cfg.ifname} tx-gso-partial on") | |
| if ipver == "4": | |
| ksft_pr("Testing with mangleid enabled") | |
| ethtool(f"-K {cfg.ifname} tx-tcp-mangleid-segmentation on") | |
| # Full feature enabled. | |
| ethtool(f"-K {cfg.ifname} {feature} on") | |
| run_one_stream(cfg, ipver, remote_v4, remote_v6, should_lso=True) | |
| if big_tcp: | |
| run_big_tcp_stream(cfg, ipver, remote_v4, remote_v6) | |
| f.__name__ = name + ((outer_ipver + "_") if tun else "") + "ipv" + inner_ipver | |
| return f | |
| def query_nic_features(cfg) -> None: | |
| """Query and cache the NIC features.""" | |
| cfg.have_stat_super_count = False | |
| cfg.have_stat_wire_count = False | |
| features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}}) | |
| cfg.wanted_features = set() | |
| for f in features["wanted"]["bits"]["bit"]: | |
| cfg.wanted_features.add(f["name"]) | |
| cfg.hw_features = set() | |
| for f in features["hw"]["bits"]["bit"]: | |
| if f.get("value", False): | |
| cfg.hw_features.add(f["name"]) | |
| # Check which features are supported via GSO partial | |
| cfg.partial_features = set() | |
| if 'tx-gso-partial' in cfg.hw_features: | |
| seg_features = {f for f in cfg.hw_features if "segmentation" in f} | |
| ethtool(f"-K {cfg.ifname} " + | |
| " ".join(f"{f} on" for f in seg_features)) | |
| ethtool(f"-K {cfg.ifname} tx-gso-partial off") | |
| no_partial = set() | |
| features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}}) | |
| for f in features["active"]["bits"]["bit"]: | |
| no_partial.add(f["name"]) | |
| cfg.partial_features = seg_features - no_partial | |
| ethtool(f"-K {cfg.ifname} tx-gso-partial on") | |
| restore_wanted_features(cfg) | |
| stats = cfg.netnl.qstats_get({"ifindex": cfg.ifindex}, dump=True) | |
| if stats: | |
| if 'tx-hw-gso-packets' in stats[0]: | |
| ksft_pr("Detected qstat for LSO super-packets") | |
| cfg.have_stat_super_count = True | |
| if 'tx-hw-gso-wire-packets' in stats[0]: | |
| ksft_pr("Detected qstat for LSO wire-packets") | |
| cfg.have_stat_wire_count = True | |
| def main() -> None: | |
| with NetDrvEpEnv(__file__, nsim_test=False) as cfg: | |
| cfg.ethnl = EthtoolFamily() | |
| cfg.netnl = NetdevFamily() | |
| query_nic_features(cfg) | |
| test_info = ( | |
| # name, v4/v6 ethtool_feature tun:(type, args, inner ip versions) | |
| ("", "4", "tx-tcp-segmentation", None), | |
| ("", "6", "tx-tcp6-segmentation", None), | |
| ("big_tcp_", "4", "tx-tcp-segmentation", None), | |
| ("big_tcp_", "6", "tx-tcp6-segmentation", None), | |
| ("vxlan", "4", "tx-udp_tnl-segmentation", ("vxlan", "id 100 dstport 4789 noudpcsum", ("4", "6"))), | |
| ("vxlan", "6", "tx-udp_tnl-segmentation", ("vxlan", "id 100 dstport 4789 udp6zerocsumtx udp6zerocsumrx", ("4", "6"))), | |
| ("vxlan_csum", "", "tx-udp_tnl-csum-segmentation", ("vxlan", "id 100 dstport 4789 udpcsum", ("4", "6"))), | |
| ("gre", "4", "tx-gre-segmentation", ("gre", "", ("4", "6"))), | |
| ("gre", "6", "tx-gre-segmentation", ("ip6gre","", ("4", "6"))), | |
| ("ip", "6", "tx-ipxip6-segmentation", ("ip6tnl","mode any", ("4", "6"))), | |
| ("ip", "4", "tx-ipxip4-segmentation", ("sit","", ("6", ))), | |
| ("ip", "4", "tx-ipxip4-segmentation", ("ipip","", ("4", ))), | |
| ) | |
| cases = [] | |
| for outer_ipver in ["4", "6"]: | |
| for info in test_info: | |
| # Skip if test which only works for a specific IP version | |
| if info[1] and outer_ipver != info[1]: | |
| continue | |
| if info[3]: | |
| cases += [ | |
| test_builder(info[0], cfg, outer_ipver, info[2], info[3], inner_ipver) | |
| for inner_ipver in info[3][2] | |
| ] | |
| else: | |
| cases.append(test_builder(info[0], cfg, outer_ipver, info[2], None, outer_ipver)) | |
| ksft_run(cases=cases, args=(cfg, )) | |
| ksft_exit() | |
| if __name__ == "__main__": | |
| main() |