| // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause | |
| /* Copyright (c) 2021, Microsoft Corporation. */ | |
| #include <linux/inetdevice.h> | |
| #include <linux/etherdevice.h> | |
| #include <linux/ethtool.h> | |
| #include <net/mana/mana.h> | |
| struct mana_stats_desc { | |
| char name[ETH_GSTRING_LEN]; | |
| u16 offset; | |
| }; | |
| static const struct mana_stats_desc mana_eth_stats[] = { | |
| {"stop_queue", offsetof(struct mana_ethtool_stats, stop_queue)}, | |
| {"wake_queue", offsetof(struct mana_ethtool_stats, wake_queue)}, | |
| {"tx_cq_err", offsetof(struct mana_ethtool_stats, tx_cqe_err)}, | |
| {"tx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, | |
| tx_cqe_unknown_type)}, | |
| {"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats, | |
| tx_linear_pkt_cnt)}, | |
| {"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, | |
| rx_cqe_unknown_type)}, | |
| }; | |
| static const struct mana_stats_desc mana_hc_stats[] = { | |
| {"hc_rx_discards_no_wqe", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_discards_no_wqe)}, | |
| {"hc_rx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_err_vport_disabled)}, | |
| {"hc_rx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_rx_bytes)}, | |
| {"hc_rx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_ucast_pkts)}, | |
| {"hc_rx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_ucast_bytes)}, | |
| {"hc_rx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_bcast_pkts)}, | |
| {"hc_rx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_bcast_bytes)}, | |
| {"hc_rx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_mcast_pkts)}, | |
| {"hc_rx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_rx_mcast_bytes)}, | |
| {"hc_tx_err_gf_disabled", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_gf_disabled)}, | |
| {"hc_tx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_vport_disabled)}, | |
| {"hc_tx_err_inval_vportoffset_pkt", | |
| offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_inval_vportoffset_pkt)}, | |
| {"hc_tx_err_vlan_enforcement", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_vlan_enforcement)}, | |
| {"hc_tx_err_eth_type_enforcement", | |
| offsetof(struct mana_ethtool_hc_stats, hc_tx_err_eth_type_enforcement)}, | |
| {"hc_tx_err_sa_enforcement", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_sa_enforcement)}, | |
| {"hc_tx_err_sqpdid_enforcement", | |
| offsetof(struct mana_ethtool_hc_stats, hc_tx_err_sqpdid_enforcement)}, | |
| {"hc_tx_err_cqpdid_enforcement", | |
| offsetof(struct mana_ethtool_hc_stats, hc_tx_err_cqpdid_enforcement)}, | |
| {"hc_tx_err_mtu_violation", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_mtu_violation)}, | |
| {"hc_tx_err_inval_oob", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_inval_oob)}, | |
| {"hc_tx_err_gdma", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_err_gdma)}, | |
| {"hc_tx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_tx_bytes)}, | |
| {"hc_tx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_ucast_pkts)}, | |
| {"hc_tx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_ucast_bytes)}, | |
| {"hc_tx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_bcast_pkts)}, | |
| {"hc_tx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_bcast_bytes)}, | |
| {"hc_tx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_mcast_pkts)}, | |
| {"hc_tx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, | |
| hc_tx_mcast_bytes)}, | |
| }; | |
| static const struct mana_stats_desc mana_phy_stats[] = { | |
| { "hc_rx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_drop_phy) }, | |
| { "hc_tx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_drop_phy) }, | |
| { "hc_tc0_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc0_phy) }, | |
| { "hc_tc0_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc0_phy) }, | |
| { "hc_tc0_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc0_phy) }, | |
| { "hc_tc0_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc0_phy) }, | |
| { "hc_tc1_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc1_phy) }, | |
| { "hc_tc1_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc1_phy) }, | |
| { "hc_tc1_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc1_phy) }, | |
| { "hc_tc1_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc1_phy) }, | |
| { "hc_tc2_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc2_phy) }, | |
| { "hc_tc2_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc2_phy) }, | |
| { "hc_tc2_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc2_phy) }, | |
| { "hc_tc2_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc2_phy) }, | |
| { "hc_tc3_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc3_phy) }, | |
| { "hc_tc3_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc3_phy) }, | |
| { "hc_tc3_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc3_phy) }, | |
| { "hc_tc3_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc3_phy) }, | |
| { "hc_tc4_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc4_phy) }, | |
| { "hc_tc4_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc4_phy) }, | |
| { "hc_tc4_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc4_phy) }, | |
| { "hc_tc4_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc4_phy) }, | |
| { "hc_tc5_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc5_phy) }, | |
| { "hc_tc5_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc5_phy) }, | |
| { "hc_tc5_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc5_phy) }, | |
| { "hc_tc5_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc5_phy) }, | |
| { "hc_tc6_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc6_phy) }, | |
| { "hc_tc6_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc6_phy) }, | |
| { "hc_tc6_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc6_phy) }, | |
| { "hc_tc6_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc6_phy) }, | |
| { "hc_tc7_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc7_phy) }, | |
| { "hc_tc7_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc7_phy) }, | |
| { "hc_tc7_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc7_phy) }, | |
| { "hc_tc7_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc7_phy) }, | |
| { "hc_tc0_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc0_phy) }, | |
| { "hc_tc0_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc0_phy) }, | |
| { "hc_tc1_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc1_phy) }, | |
| { "hc_tc1_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc1_phy) }, | |
| { "hc_tc2_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc2_phy) }, | |
| { "hc_tc2_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc2_phy) }, | |
| { "hc_tc3_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc3_phy) }, | |
| { "hc_tc3_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc3_phy) }, | |
| { "hc_tc4_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc4_phy) }, | |
| { "hc_tc4_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc4_phy) }, | |
| { "hc_tc5_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc5_phy) }, | |
| { "hc_tc5_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc5_phy) }, | |
| { "hc_tc6_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc6_phy) }, | |
| { "hc_tc6_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc6_phy) }, | |
| { "hc_tc7_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc7_phy) }, | |
| { "hc_tc7_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc7_phy) }, | |
| }; | |
| static const char mana_priv_flags[MANA_PRIV_FLAG_MAX][ETH_GSTRING_LEN] = { | |
| [MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF] = "full-page-rx" | |
| }; | |
| static int mana_get_sset_count(struct net_device *ndev, int stringset) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| unsigned int num_queues = apc->num_queues; | |
| switch (stringset) { | |
| case ETH_SS_STATS: | |
| return ARRAY_SIZE(mana_eth_stats) + | |
| ARRAY_SIZE(mana_phy_stats) + | |
| ARRAY_SIZE(mana_hc_stats) + | |
| num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT); | |
| case ETH_SS_PRIV_FLAGS: | |
| return MANA_PRIV_FLAG_MAX; | |
| default: | |
| return -EINVAL; | |
| } | |
| } | |
| static void mana_get_strings_stats(struct mana_port_context *apc, u8 **data) | |
| { | |
| unsigned int num_queues = apc->num_queues; | |
| int i, j; | |
| for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++) | |
| ethtool_puts(data, mana_eth_stats[i].name); | |
| for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++) | |
| ethtool_puts(data, mana_hc_stats[i].name); | |
| for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++) | |
| ethtool_puts(data, mana_phy_stats[i].name); | |
| for (i = 0; i < num_queues; i++) { | |
| ethtool_sprintf(data, "rx_%d_packets", i); | |
| ethtool_sprintf(data, "rx_%d_bytes", i); | |
| ethtool_sprintf(data, "rx_%d_xdp_drop", i); | |
| ethtool_sprintf(data, "rx_%d_xdp_tx", i); | |
| ethtool_sprintf(data, "rx_%d_xdp_redirect", i); | |
| ethtool_sprintf(data, "rx_%d_pkt_len0_err", i); | |
| for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) | |
| ethtool_sprintf(data, | |
| "rx_%d_coalesced_cqe_%d", | |
| i, | |
| j + 2); | |
| } | |
| for (i = 0; i < num_queues; i++) { | |
| ethtool_sprintf(data, "tx_%d_packets", i); | |
| ethtool_sprintf(data, "tx_%d_bytes", i); | |
| ethtool_sprintf(data, "tx_%d_xdp_xmit", i); | |
| ethtool_sprintf(data, "tx_%d_tso_packets", i); | |
| ethtool_sprintf(data, "tx_%d_tso_bytes", i); | |
| ethtool_sprintf(data, "tx_%d_tso_inner_packets", i); | |
| ethtool_sprintf(data, "tx_%d_tso_inner_bytes", i); | |
| ethtool_sprintf(data, "tx_%d_long_pkt_fmt", i); | |
| ethtool_sprintf(data, "tx_%d_short_pkt_fmt", i); | |
| ethtool_sprintf(data, "tx_%d_csum_partial", i); | |
| ethtool_sprintf(data, "tx_%d_mana_map_err", i); | |
| } | |
| } | |
| static void mana_get_strings_priv_flags(u8 **data) | |
| { | |
| int i; | |
| for (i = 0; i < MANA_PRIV_FLAG_MAX; i++) | |
| ethtool_puts(data, mana_priv_flags[i]); | |
| } | |
| static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| switch (stringset) { | |
| case ETH_SS_STATS: | |
| mana_get_strings_stats(apc, &data); | |
| break; | |
| case ETH_SS_PRIV_FLAGS: | |
| mana_get_strings_priv_flags(&data); | |
| break; | |
| default: | |
| break; | |
| } | |
| } | |
| static void mana_get_ethtool_stats(struct net_device *ndev, | |
| struct ethtool_stats *e_stats, u64 *data) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| unsigned int num_queues = apc->num_queues; | |
| void *eth_stats = &apc->eth_stats; | |
| void *hc_stats = &apc->ac->hc_stats; | |
| void *phy_stats = &apc->phy_stats; | |
| struct mana_stats_rx *rx_stats; | |
| struct mana_stats_tx *tx_stats; | |
| unsigned int start; | |
| u64 packets, bytes; | |
| u64 xdp_redirect; | |
| u64 xdp_xmit; | |
| u64 xdp_drop; | |
| u64 xdp_tx; | |
| u64 pkt_len0_err; | |
| u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1]; | |
| u64 tso_packets; | |
| u64 tso_bytes; | |
| u64 tso_inner_packets; | |
| u64 tso_inner_bytes; | |
| u64 long_pkt_fmt; | |
| u64 short_pkt_fmt; | |
| u64 csum_partial; | |
| u64 mana_map_err; | |
| int q, i = 0, j; | |
| if (!apc->port_is_up) | |
| return; | |
| /* We call this mana function to get the phy stats from GDMA and includes | |
| * aggregate tx/rx drop counters, Per-TC(Traffic Channel) tx/rx and pause | |
| * counters. | |
| */ | |
| mana_query_phy_stats(apc); | |
| for (q = 0; q < ARRAY_SIZE(mana_eth_stats); q++) | |
| data[i++] = *(u64 *)(eth_stats + mana_eth_stats[q].offset); | |
| for (q = 0; q < ARRAY_SIZE(mana_hc_stats); q++) | |
| data[i++] = *(u64 *)(hc_stats + mana_hc_stats[q].offset); | |
| for (q = 0; q < ARRAY_SIZE(mana_phy_stats); q++) | |
| data[i++] = *(u64 *)(phy_stats + mana_phy_stats[q].offset); | |
| for (q = 0; q < num_queues; q++) { | |
| rx_stats = &apc->rxqs[q]->stats; | |
| do { | |
| start = u64_stats_fetch_begin(&rx_stats->syncp); | |
| packets = rx_stats->packets; | |
| bytes = rx_stats->bytes; | |
| xdp_drop = rx_stats->xdp_drop; | |
| xdp_tx = rx_stats->xdp_tx; | |
| xdp_redirect = rx_stats->xdp_redirect; | |
| pkt_len0_err = rx_stats->pkt_len0_err; | |
| for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) | |
| coalesced_cqe[j] = rx_stats->coalesced_cqe[j]; | |
| } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); | |
| data[i++] = packets; | |
| data[i++] = bytes; | |
| data[i++] = xdp_drop; | |
| data[i++] = xdp_tx; | |
| data[i++] = xdp_redirect; | |
| data[i++] = pkt_len0_err; | |
| for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) | |
| data[i++] = coalesced_cqe[j]; | |
| } | |
| for (q = 0; q < num_queues; q++) { | |
| tx_stats = &apc->tx_qp[q]->txq.stats; | |
| do { | |
| start = u64_stats_fetch_begin(&tx_stats->syncp); | |
| packets = tx_stats->packets; | |
| bytes = tx_stats->bytes; | |
| xdp_xmit = tx_stats->xdp_xmit; | |
| tso_packets = tx_stats->tso_packets; | |
| tso_bytes = tx_stats->tso_bytes; | |
| tso_inner_packets = tx_stats->tso_inner_packets; | |
| tso_inner_bytes = tx_stats->tso_inner_bytes; | |
| long_pkt_fmt = tx_stats->long_pkt_fmt; | |
| short_pkt_fmt = tx_stats->short_pkt_fmt; | |
| csum_partial = tx_stats->csum_partial; | |
| mana_map_err = tx_stats->mana_map_err; | |
| } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); | |
| data[i++] = packets; | |
| data[i++] = bytes; | |
| data[i++] = xdp_xmit; | |
| data[i++] = tso_packets; | |
| data[i++] = tso_bytes; | |
| data[i++] = tso_inner_packets; | |
| data[i++] = tso_inner_bytes; | |
| data[i++] = long_pkt_fmt; | |
| data[i++] = short_pkt_fmt; | |
| data[i++] = csum_partial; | |
| data[i++] = mana_map_err; | |
| } | |
| } | |
| static u32 mana_get_rx_ring_count(struct net_device *ndev) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| return apc->num_queues; | |
| } | |
| static u32 mana_get_rxfh_key_size(struct net_device *ndev) | |
| { | |
| return MANA_HASH_KEY_SIZE; | |
| } | |
| static u32 mana_rss_indir_size(struct net_device *ndev) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| return apc->indir_table_sz; | |
| } | |
| static int mana_get_rxfh(struct net_device *ndev, | |
| struct ethtool_rxfh_param *rxfh) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| int i; | |
| rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ | |
| if (rxfh->indir) { | |
| for (i = 0; i < apc->indir_table_sz; i++) | |
| rxfh->indir[i] = apc->indir_table[i]; | |
| } | |
| if (rxfh->key) | |
| memcpy(rxfh->key, apc->hashkey, MANA_HASH_KEY_SIZE); | |
| return 0; | |
| } | |
| static int mana_set_rxfh(struct net_device *ndev, | |
| struct ethtool_rxfh_param *rxfh, | |
| struct netlink_ext_ack *extack) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| bool update_hash = false, update_table = false; | |
| u8 save_key[MANA_HASH_KEY_SIZE]; | |
| u32 *save_table; | |
| int i, err; | |
| if (!apc->port_is_up) | |
| return -EOPNOTSUPP; | |
| if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && | |
| rxfh->hfunc != ETH_RSS_HASH_TOP) | |
| return -EOPNOTSUPP; | |
| save_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); | |
| if (!save_table) | |
| return -ENOMEM; | |
| if (rxfh->indir) { | |
| for (i = 0; i < apc->indir_table_sz; i++) | |
| if (rxfh->indir[i] >= apc->num_queues) { | |
| err = -EINVAL; | |
| goto cleanup; | |
| } | |
| update_table = true; | |
| for (i = 0; i < apc->indir_table_sz; i++) { | |
| save_table[i] = apc->indir_table[i]; | |
| apc->indir_table[i] = rxfh->indir[i]; | |
| } | |
| } | |
| if (rxfh->key) { | |
| update_hash = true; | |
| memcpy(save_key, apc->hashkey, MANA_HASH_KEY_SIZE); | |
| memcpy(apc->hashkey, rxfh->key, MANA_HASH_KEY_SIZE); | |
| } | |
| err = mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); | |
| if (err) { /* recover to original values */ | |
| if (update_table) { | |
| for (i = 0; i < apc->indir_table_sz; i++) | |
| apc->indir_table[i] = save_table[i]; | |
| } | |
| if (update_hash) | |
| memcpy(apc->hashkey, save_key, MANA_HASH_KEY_SIZE); | |
| mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); | |
| } | |
| cleanup: | |
| kfree(save_table); | |
| return err; | |
| } | |
| static void mana_get_channels(struct net_device *ndev, | |
| struct ethtool_channels *channel) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| channel->max_combined = apc->max_queues; | |
| channel->combined_count = apc->num_queues; | |
| } | |
| #define MANA_RX_CQE_NSEC_DEF 2048 | |
| static int mana_get_coalesce(struct net_device *ndev, | |
| struct ethtool_coalesce *ec, | |
| struct kernel_ethtool_coalesce *kernel_coal, | |
| struct netlink_ext_ack *extack) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| kernel_coal->rx_cqe_frames = | |
| apc->cqe8_coalescing_enable ? MANA_CQE_COAL_PKTS_8 : | |
| apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1; | |
| kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns; | |
| /* Return the default timeout value for old FW not providing | |
| * this value. | |
| */ | |
| if (apc->port_is_up && apc->cqe_coalescing_enable && | |
| !kernel_coal->rx_cqe_nsecs) | |
| kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF; | |
| ec->rx_coalesce_usecs = apc->intr_modr_rx_usec; | |
| ec->rx_max_coalesced_frames = apc->intr_modr_rx_comp; | |
| ec->tx_coalesce_usecs = apc->intr_modr_tx_usec; | |
| ec->tx_max_coalesced_frames = apc->intr_modr_tx_comp; | |
| ec->use_adaptive_rx_coalesce = apc->rx_dim_enabled; | |
| ec->use_adaptive_tx_coalesce = apc->tx_dim_enabled; | |
| return 0; | |
| } | |
| static int mana_set_coalesce(struct net_device *ndev, | |
| struct ethtool_coalesce *ec, | |
| struct kernel_ethtool_coalesce *kernel_coal, | |
| struct netlink_ext_ack *extack) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| struct { | |
| u16 intr_modr_rx_usec; | |
| u16 intr_modr_rx_comp; | |
| u16 intr_modr_tx_usec; | |
| u16 intr_modr_tx_comp; | |
| u8 cqe_coalescing_enable; | |
| u8 cqe8_coalescing_enable; | |
| bool rx_dim_enabled; | |
| bool tx_dim_enabled; | |
| } saved; | |
| bool modr_changed = false; | |
| bool dim_changed = false; | |
| struct gdma_context *gc; | |
| u32 max_cqe_frames; | |
| int err; | |
| gc = apc->ac->gdma_dev->gdma_context; | |
| max_cqe_frames = gc->cqe8_coalescing_sup ? MANA_CQE_COAL_PKTS_8 : | |
| MANA_RXCOMP_OOB_NUM_PPI; | |
| /* Both static and dynamic interrupt moderation (DIM) rely on the | |
| * same HW capability advertised by the PF. | |
| */ | |
| if ((ec->use_adaptive_rx_coalesce || ec->use_adaptive_tx_coalesce || | |
| ec->rx_coalesce_usecs || ec->tx_coalesce_usecs || | |
| ec->rx_max_coalesced_frames || ec->tx_max_coalesced_frames) && | |
| !(gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)) { | |
| NL_SET_ERR_MSG(extack, | |
| "Interrupt Moderation is not supported by HW"); | |
| return -EOPNOTSUPP; | |
| } | |
| if (kernel_coal->rx_cqe_frames != 1 && | |
| kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI && | |
| kernel_coal->rx_cqe_frames != max_cqe_frames) { | |
| NL_SET_ERR_MSG_FMT(extack, | |
| "rx-frames must be 1 or %u%s, got %u", | |
| MANA_RXCOMP_OOB_NUM_PPI, | |
| gc->cqe8_coalescing_sup ? " or 8" : "", | |
| kernel_coal->rx_cqe_frames); | |
| return -EINVAL; | |
| } | |
| if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX || | |
| ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) { | |
| NL_SET_ERR_MSG_FMT(extack, | |
| "coalesce usecs must be <= %lu", | |
| MANA_INTR_MODR_USEC_MAX); | |
| return -EINVAL; | |
| } | |
| if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX || | |
| ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) { | |
| NL_SET_ERR_MSG_FMT(extack, | |
| "coalesce frames must be <= %lu", | |
| MANA_INTR_MODR_COMP_MAX); | |
| return -EINVAL; | |
| } | |
| if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec || | |
| ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp || | |
| ec->tx_coalesce_usecs != apc->intr_modr_tx_usec || | |
| ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp) | |
| modr_changed = true; | |
| saved.intr_modr_rx_usec = apc->intr_modr_rx_usec; | |
| saved.intr_modr_rx_comp = apc->intr_modr_rx_comp; | |
| saved.intr_modr_tx_usec = apc->intr_modr_tx_usec; | |
| saved.intr_modr_tx_comp = apc->intr_modr_tx_comp; | |
| apc->intr_modr_rx_usec = ec->rx_coalesce_usecs; | |
| apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames; | |
| apc->intr_modr_tx_usec = ec->tx_coalesce_usecs; | |
| apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames; | |
| if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled || | |
| !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled) | |
| dim_changed = true; | |
| saved.rx_dim_enabled = apc->rx_dim_enabled; | |
| saved.tx_dim_enabled = apc->tx_dim_enabled; | |
| saved.cqe_coalescing_enable = apc->cqe_coalescing_enable; | |
| saved.cqe8_coalescing_enable = apc->cqe8_coalescing_enable; | |
| apc->cqe_coalescing_enable = | |
| kernel_coal->rx_cqe_frames >= MANA_RXCOMP_OOB_NUM_PPI; | |
| apc->cqe8_coalescing_enable = | |
| kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8; | |
| if (!apc->port_is_up) { | |
| WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce); | |
| WRITE_ONCE(apc->tx_dim_enabled, !!ec->use_adaptive_tx_coalesce); | |
| return 0; | |
| } | |
| if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable || | |
| apc->cqe8_coalescing_enable != saved.cqe8_coalescing_enable) { | |
| /* CQE coalescing setting is applied via RSS configuration. */ | |
| err = mana_config_rss(apc, TRI_STATE_TRUE, false, false); | |
| if (err) { | |
| netdev_err(ndev, "Change CQE coalescing failed: %d\n", | |
| err); | |
| apc->cqe_coalescing_enable = | |
| saved.cqe_coalescing_enable; | |
| apc->cqe8_coalescing_enable = | |
| saved.cqe8_coalescing_enable; | |
| apc->intr_modr_rx_usec = saved.intr_modr_rx_usec; | |
| apc->intr_modr_rx_comp = saved.intr_modr_rx_comp; | |
| apc->intr_modr_tx_usec = saved.intr_modr_tx_usec; | |
| apc->intr_modr_tx_comp = saved.intr_modr_tx_comp; | |
| return err; | |
| } | |
| } | |
| if (modr_changed || dim_changed) { | |
| bool new_rx_dim = !!ec->use_adaptive_rx_coalesce; | |
| bool new_tx_dim = !!ec->use_adaptive_tx_coalesce; | |
| bool disable_rx_dim = saved.rx_dim_enabled && !new_rx_dim; | |
| bool disable_tx_dim = saved.tx_dim_enabled && !new_tx_dim; | |
| bool enable_rx_dim = !saved.rx_dim_enabled && new_rx_dim; | |
| bool enable_tx_dim = !saved.tx_dim_enabled && new_tx_dim; | |
| int q; | |
| /* On disable: clear the per-port flag first and | |
| * synchronize_net() so any in-flight NAPI poll observes | |
| * the new value and will not schedule further DIM work; | |
| * then drain pending work and restore the static | |
| * moderation values. | |
| */ | |
| if (disable_rx_dim) | |
| WRITE_ONCE(apc->rx_dim_enabled, false); | |
| if (disable_tx_dim) | |
| WRITE_ONCE(apc->tx_dim_enabled, false); | |
| if (disable_rx_dim || disable_tx_dim) | |
| synchronize_net(); | |
| for (q = 0; q < apc->num_queues; q++) { | |
| struct mana_cq *rx_cq = &apc->rxqs[q]->rx_cq; | |
| struct mana_cq *tx_cq = &apc->tx_qp[q]->tx_cq; | |
| if (disable_rx_dim) | |
| mana_dim_change(rx_cq, false); | |
| else if (enable_rx_dim) | |
| mana_dim_change(rx_cq, true); | |
| else if (!new_rx_dim && modr_changed) | |
| mana_gd_ring_dim(rx_cq->gdma_cq, | |
| apc->intr_modr_rx_usec, true, | |
| apc->intr_modr_rx_comp, true); | |
| if (disable_tx_dim) | |
| mana_dim_change(tx_cq, false); | |
| else if (enable_tx_dim) | |
| mana_dim_change(tx_cq, true); | |
| else if (!new_tx_dim && modr_changed) | |
| mana_gd_ring_dim(tx_cq->gdma_cq, | |
| apc->intr_modr_tx_usec, true, | |
| apc->intr_modr_tx_comp, true); | |
| } | |
| /* Publish the enable flag with release semantics so a | |
| * concurrent NAPI poll that observes it set also sees the DIM | |
| * (re)init done by mana_dim_change() above. | |
| */ | |
| if (enable_rx_dim) | |
| /* pairs with smp_load_acquire() in mana_update_rx_dim() */ | |
| smp_store_release(&apc->rx_dim_enabled, true); | |
| if (enable_tx_dim) | |
| /* pairs with smp_load_acquire() in mana_update_tx_dim() */ | |
| smp_store_release(&apc->tx_dim_enabled, true); | |
| } | |
| return 0; | |
| } | |
| /* mana_set_channels - change the number of queues on a port | |
| * | |
| * Returns -EBUSY if RDMA holds the vport with EQs sized to the | |
| * current num_queues. | |
| */ | |
| static int mana_set_channels(struct net_device *ndev, | |
| struct ethtool_channels *channels) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| unsigned int new_count = channels->combined_count; | |
| unsigned int old_count = apc->num_queues; | |
| int err; | |
| /* Set channel_changing to block RDMA from grabbing the vport | |
| * during the detach/attach window. mana_cfg_vport() checks | |
| * this flag under vport_mutex and returns -EBUSY if set. | |
| */ | |
| mutex_lock(&apc->vport_mutex); | |
| if (!apc->port_is_up && apc->vport_use_count) { | |
| mutex_unlock(&apc->vport_mutex); | |
| return -EBUSY; | |
| } | |
| apc->channel_changing = true; | |
| mutex_unlock(&apc->vport_mutex); | |
| err = mana_pre_alloc_rxbufs(apc, ndev->mtu, new_count); | |
| if (err) { | |
| netdev_err(ndev, "Insufficient memory for new allocations"); | |
| goto clear_flag; | |
| } | |
| err = mana_detach(ndev, false); | |
| if (err) { | |
| netdev_err(ndev, "mana_detach failed: %d\n", err); | |
| goto out; | |
| } | |
| apc->num_queues = new_count; | |
| err = mana_attach(ndev); | |
| if (err) { | |
| apc->num_queues = old_count; | |
| netdev_err(ndev, "mana_attach failed: %d\n", err); | |
| } | |
| out: | |
| mana_pre_dealloc_rxbufs(apc); | |
| clear_flag: | |
| mutex_lock(&apc->vport_mutex); | |
| apc->channel_changing = false; | |
| mutex_unlock(&apc->vport_mutex); | |
| return err; | |
| } | |
| static void mana_get_ringparam(struct net_device *ndev, | |
| struct ethtool_ringparam *ring, | |
| struct kernel_ethtool_ringparam *kernel_ring, | |
| struct netlink_ext_ack *extack) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| ring->rx_pending = apc->rx_queue_size; | |
| ring->tx_pending = apc->tx_queue_size; | |
| ring->rx_max_pending = MAX_RX_BUFFERS_PER_QUEUE; | |
| ring->tx_max_pending = MAX_TX_BUFFERS_PER_QUEUE; | |
| } | |
| static int mana_set_ringparam(struct net_device *ndev, | |
| struct ethtool_ringparam *ring, | |
| struct kernel_ethtool_ringparam *kernel_ring, | |
| struct netlink_ext_ack *extack) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| u32 new_tx, new_rx; | |
| u32 old_tx, old_rx; | |
| int err; | |
| old_tx = apc->tx_queue_size; | |
| old_rx = apc->rx_queue_size; | |
| if (ring->tx_pending < MIN_TX_BUFFERS_PER_QUEUE) { | |
| NL_SET_ERR_MSG_FMT(extack, "tx:%d less than the min:%d", ring->tx_pending, | |
| MIN_TX_BUFFERS_PER_QUEUE); | |
| return -EINVAL; | |
| } | |
| if (ring->rx_pending < MIN_RX_BUFFERS_PER_QUEUE) { | |
| NL_SET_ERR_MSG_FMT(extack, "rx:%d less than the min:%d", ring->rx_pending, | |
| MIN_RX_BUFFERS_PER_QUEUE); | |
| return -EINVAL; | |
| } | |
| new_rx = roundup_pow_of_two(ring->rx_pending); | |
| new_tx = roundup_pow_of_two(ring->tx_pending); | |
| netdev_info(ndev, "Using nearest power of 2 values for Txq:%d Rxq:%d\n", | |
| new_tx, new_rx); | |
| /* pre-allocating new buffers to prevent failures in mana_attach() later */ | |
| apc->rx_queue_size = new_rx; | |
| err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); | |
| apc->rx_queue_size = old_rx; | |
| if (err) { | |
| netdev_err(ndev, "Insufficient memory for new allocations\n"); | |
| return err; | |
| } | |
| err = mana_detach(ndev, false); | |
| if (err) { | |
| netdev_err(ndev, "mana_detach failed: %d\n", err); | |
| goto out; | |
| } | |
| apc->tx_queue_size = new_tx; | |
| apc->rx_queue_size = new_rx; | |
| err = mana_attach(ndev); | |
| if (err) { | |
| netdev_err(ndev, "mana_attach failed: %d\n", err); | |
| apc->tx_queue_size = old_tx; | |
| apc->rx_queue_size = old_rx; | |
| } | |
| out: | |
| mana_pre_dealloc_rxbufs(apc); | |
| return err; | |
| } | |
| static int mana_get_link_ksettings(struct net_device *ndev, | |
| struct ethtool_link_ksettings *cmd) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| int err; | |
| err = mana_query_link_cfg(apc); | |
| cmd->base.speed = (err) ? SPEED_UNKNOWN : apc->max_speed; | |
| cmd->base.duplex = DUPLEX_FULL; | |
| cmd->base.port = PORT_OTHER; | |
| return 0; | |
| } | |
| static u32 mana_get_priv_flags(struct net_device *ndev) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| return apc->priv_flags; | |
| } | |
| static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags) | |
| { | |
| struct mana_port_context *apc = netdev_priv(ndev); | |
| u32 changed = apc->priv_flags ^ priv_flags; | |
| u32 old_priv_flags = apc->priv_flags; | |
| int err = 0; | |
| if (!changed) | |
| return 0; | |
| /* Reject unknown bits */ | |
| if (priv_flags & ~GENMASK(MANA_PRIV_FLAG_MAX - 1, 0)) | |
| return -EINVAL; | |
| apc->priv_flags = priv_flags; | |
| if (changed & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) { | |
| if (!apc->port_is_up) | |
| return 0; | |
| /* If XDP is attached or MTU is jumbo, single-buffer-per-page | |
| * is already forced regardless of this flag. Skip the | |
| * expensive detach/attach cycle since nothing changes. | |
| */ | |
| if (ndev->mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || | |
| mana_xdp_get(apc)) | |
| return 0; | |
| /* Block RDMA from grabbing the vport during detach/attach */ | |
| mutex_lock(&apc->vport_mutex); | |
| apc->channel_changing = true; | |
| mutex_unlock(&apc->vport_mutex); | |
| err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); | |
| if (err) { | |
| netdev_err(ndev, | |
| "Insufficient memory for new allocations\n"); | |
| apc->priv_flags = old_priv_flags; | |
| goto clear_flag; | |
| } | |
| err = mana_detach(ndev, false); | |
| if (err) { | |
| netdev_err(ndev, "mana_detach failed: %d\n", err); | |
| apc->priv_flags = old_priv_flags; | |
| goto out; | |
| } | |
| err = mana_attach(ndev); | |
| if (err) { | |
| netdev_err(ndev, "mana_attach failed: %d\n", err); | |
| apc->priv_flags = old_priv_flags; | |
| } | |
| } | |
| out: | |
| mana_pre_dealloc_rxbufs(apc); | |
| clear_flag: | |
| mutex_lock(&apc->vport_mutex); | |
| apc->channel_changing = false; | |
| mutex_unlock(&apc->vport_mutex); | |
| return err; | |
| } | |
| const struct ethtool_ops mana_ethtool_ops = { | |
| .supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES | | |
| ETHTOOL_COALESCE_RX_USECS | | |
| ETHTOOL_COALESCE_RX_MAX_FRAMES | | |
| ETHTOOL_COALESCE_TX_USECS | | |
| ETHTOOL_COALESCE_TX_MAX_FRAMES | | |
| ETHTOOL_COALESCE_USE_ADAPTIVE_RX | | |
| ETHTOOL_COALESCE_USE_ADAPTIVE_TX, | |
| .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | | |
| ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | | |
| ETHTOOL_OP_NEEDS_RTNL_SPFLAGS | | |
| ETHTOOL_OP_NEEDS_RTNL_GLINK, | |
| .get_ethtool_stats = mana_get_ethtool_stats, | |
| .get_sset_count = mana_get_sset_count, | |
| .get_strings = mana_get_strings, | |
| .get_rx_ring_count = mana_get_rx_ring_count, | |
| .get_rxfh_key_size = mana_get_rxfh_key_size, | |
| .get_rxfh_indir_size = mana_rss_indir_size, | |
| .get_rxfh = mana_get_rxfh, | |
| .set_rxfh = mana_set_rxfh, | |
| .get_channels = mana_get_channels, | |
| .set_channels = mana_set_channels, | |
| .get_coalesce = mana_get_coalesce, | |
| .set_coalesce = mana_set_coalesce, | |
| .get_ringparam = mana_get_ringparam, | |
| .set_ringparam = mana_set_ringparam, | |
| .get_link_ksettings = mana_get_link_ksettings, | |
| .get_link = ethtool_op_get_link, | |
| .get_priv_flags = mana_get_priv_flags, | |
| .set_priv_flags = mana_set_priv_flags, | |
| }; |