| // SPDX-License-Identifier: GPL-2.0-or-later | |
| /* | |
| * Driver for Alibaba Elastic Ethernet Adapter. | |
| * | |
| * Copyright (C) 2025 Alibaba Inc. | |
| */ | |
| #include <net/netdev_queues.h> | |
| #include "eea_net.h" | |
| #include "eea_pci.h" | |
| #include "eea_ring.h" | |
| struct eea_sq_free_stats { | |
| u64 packets; | |
| u64 bytes; | |
| }; | |
| struct eea_tx_meta { | |
| struct eea_tx_meta *next; | |
| u32 id; | |
| union { | |
| struct sk_buff *skb; | |
| void *data; | |
| }; | |
| u32 num; | |
| dma_addr_t dma_addr; | |
| struct eea_tx_desc *desc; | |
| u32 dma_len; | |
| bool unmap; | |
| bool unmap_single; | |
| }; | |
| static struct eea_tx_meta *eea_tx_meta_get(struct eea_net_tx *tx) | |
| { | |
| struct eea_tx_meta *meta; | |
| if (!tx->free) | |
| return NULL; | |
| meta = tx->free; | |
| tx->free = meta->next; | |
| return meta; | |
| } | |
| static void eea_tx_meta_put_and_unmap(struct eea_net_tx *tx, | |
| struct eea_tx_meta *meta) | |
| { | |
| struct eea_tx_meta *head; | |
| head = meta; | |
| while (true) { | |
| if (meta->unmap) { | |
| if (meta->unmap_single) | |
| dma_unmap_single(tx->dma_dev, meta->dma_addr, | |
| meta->dma_len, DMA_TO_DEVICE); | |
| else | |
| dma_unmap_page(tx->dma_dev, meta->dma_addr, | |
| meta->dma_len, DMA_TO_DEVICE); | |
| } | |
| if (meta->next) { | |
| meta = meta->next; | |
| continue; | |
| } | |
| break; | |
| } | |
| meta->next = tx->free; | |
| tx->free = head; | |
| } | |
| static void eea_meta_free_xmit(struct eea_net_tx *tx, | |
| struct eea_tx_meta *meta, | |
| int budget, | |
| struct eea_tx_cdesc *desc, | |
| struct eea_sq_free_stats *stats) | |
| { | |
| struct sk_buff *skb = meta->skb; | |
| if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && desc)) { | |
| struct skb_shared_hwtstamps ts = {}; | |
| ts.hwtstamp = EEA_DESC_TS(desc) + tx->enet->hw_ts_offset; | |
| skb_tstamp_tx(skb, &ts); | |
| } | |
| ++stats->packets; | |
| stats->bytes += meta->skb->len; | |
| napi_consume_skb(meta->skb, budget); | |
| meta->data = NULL; | |
| } | |
| static int eea_clean_tx(struct eea_net_tx *tx, int budget) | |
| { | |
| struct eea_sq_free_stats stats = {0}; | |
| struct eea_tx_cdesc *desc; | |
| struct eea_tx_meta *meta; | |
| int desc_n; | |
| u16 id; | |
| while (stats.packets < budget) { | |
| desc = eea_ering_cq_get_desc(tx->ering); | |
| if (!desc) | |
| break; | |
| id = le16_to_cpu(desc->id); | |
| if (unlikely(id >= tx->ering->num)) { | |
| if (net_ratelimit()) | |
| netdev_err(tx->enet->netdev, "tx invalid id %d\n", | |
| id); | |
| eea_ering_cq_ack_desc(tx->ering, 1); | |
| continue; | |
| } | |
| meta = &tx->meta[id]; | |
| if (meta->data) { | |
| eea_tx_meta_put_and_unmap(tx, meta); | |
| eea_meta_free_xmit(tx, meta, budget, desc, &stats); | |
| desc_n = meta->num; | |
| } else { | |
| if (net_ratelimit()) | |
| netdev_err(tx->enet->netdev, | |
| "tx meta->data is null. id %d num: %d\n", | |
| meta->id, meta->num); | |
| desc_n = 1; | |
| } | |
| eea_ering_cq_ack_desc(tx->ering, desc_n); | |
| } | |
| if (stats.packets) { | |
| u64_stats_update_begin(&tx->stats.syncp); | |
| u64_stats_add(&tx->stats.bytes, stats.bytes); | |
| u64_stats_add(&tx->stats.packets, stats.packets); | |
| u64_stats_update_end(&tx->stats.syncp); | |
| } | |
| return stats.packets; | |
| } | |
| int eea_poll_tx(struct eea_net_tx *tx, int budget) | |
| { | |
| struct eea_net *enet = tx->enet; | |
| u32 index = tx - enet->tx; | |
| struct netdev_queue *txq; | |
| int num; | |
| txq = netdev_get_tx_queue(enet->netdev, index); | |
| __netif_tx_lock(txq, smp_processor_id()); | |
| num = eea_clean_tx(tx, budget); | |
| if (netif_tx_queue_stopped(txq) && | |
| tx->ering->num_free >= MAX_SKB_FRAGS + 2) | |
| netif_tx_wake_queue(txq); | |
| __netif_tx_unlock(txq); | |
| return num; | |
| } | |
| static int eea_fill_desc_from_skb(const struct sk_buff *skb, | |
| struct eea_tx_desc *desc) | |
| { | |
| if (skb_is_gso(skb)) { | |
| struct skb_shared_info *sinfo = skb_shinfo(skb); | |
| desc->gso_size = cpu_to_le16(sinfo->gso_size); | |
| if (sinfo->gso_type & SKB_GSO_TCPV4) | |
| desc->gso_type = EEA_TX_GSO_TCPV4; | |
| else if (sinfo->gso_type & SKB_GSO_TCPV6) | |
| desc->gso_type = EEA_TX_GSO_TCPV6; | |
| else if (sinfo->gso_type & SKB_GSO_UDP_L4) | |
| desc->gso_type = EEA_TX_GSO_UDP_L4; | |
| else | |
| return -EINVAL; | |
| if (sinfo->gso_type & SKB_GSO_TCP_ECN) | |
| desc->gso_type |= EEA_TX_GSO_ECN; | |
| } else { | |
| desc->gso_type = EEA_TX_GSO_NONE; | |
| } | |
| if (skb->ip_summed == CHECKSUM_PARTIAL) { | |
| desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); | |
| desc->csum_offset = cpu_to_le16(skb->csum_offset); | |
| } | |
| return 0; | |
| } | |
| static struct eea_tx_meta *__eea_tx_desc_fill(struct eea_net_tx *tx, | |
| struct eea_tx_meta *head_meta, | |
| dma_addr_t addr, u32 data_len, | |
| u32 dma_len, bool last, | |
| void *data, u16 flags, | |
| bool unmap) | |
| { | |
| struct eea_tx_meta *meta; | |
| struct eea_tx_desc *desc; | |
| meta = eea_tx_meta_get(tx); | |
| desc = eea_ering_sq_alloc_desc(tx->ering, meta->id, last, flags); | |
| desc->addr = cpu_to_le64(addr); | |
| desc->len = cpu_to_le16(data_len); | |
| meta->next = NULL; | |
| meta->dma_len = dma_len; | |
| meta->dma_addr = addr; | |
| meta->data = data; | |
| meta->num = 1; | |
| meta->desc = desc; | |
| meta->unmap = unmap; | |
| meta->unmap_single = false; | |
| if (head_meta) { | |
| meta->next = head_meta->next; | |
| head_meta->next = meta; | |
| ++head_meta->num; | |
| } | |
| return meta; | |
| } | |
| static struct eea_tx_meta *eea_tx_desc_fill(struct eea_net_tx *tx, | |
| struct eea_tx_meta *head_meta, | |
| dma_addr_t addr, u32 length, | |
| bool is_last, void *data, u16 flags) | |
| { | |
| struct eea_tx_meta *meta; | |
| u16 len, last; | |
| WARN_ON_ONCE(length >= 2 * USHRT_MAX); | |
| /* Since eea does not support BIG TCP, the maximum GSO size is capped at | |
| * 64KB. Consequently, a single skb buffer (head or fragment) will not | |
| * require more than two descriptors | |
| */ | |
| if (length > USHRT_MAX) { | |
| len = USHRT_MAX; | |
| last = false; | |
| } else { | |
| len = length; | |
| last = is_last; | |
| } | |
| meta = __eea_tx_desc_fill(tx, head_meta, addr, len, length, | |
| last, data, flags, true); | |
| if (length > USHRT_MAX) { | |
| if (!head_meta) | |
| head_meta = meta; | |
| addr += USHRT_MAX; | |
| len = length - USHRT_MAX; | |
| __eea_tx_desc_fill(tx, head_meta, addr, len, 0, is_last, | |
| NULL, 0, false); | |
| } | |
| return meta; | |
| } | |
| static int eea_tx_add_skb_frag(struct eea_net_tx *tx, | |
| struct eea_tx_meta *head_meta, | |
| const skb_frag_t *frag, bool is_last) | |
| { | |
| u32 len = skb_frag_size(frag); | |
| dma_addr_t addr; | |
| addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, DMA_TO_DEVICE); | |
| if (unlikely(dma_mapping_error(tx->dma_dev, addr))) | |
| return -ENOMEM; | |
| eea_tx_desc_fill(tx, head_meta, addr, len, is_last, NULL, 0); | |
| return 0; | |
| } | |
| static int eea_tx_post_skb(struct eea_net_tx *tx, struct sk_buff *skb) | |
| { | |
| const struct skb_shared_info *shinfo = skb_shinfo(skb); | |
| u32 hlen = skb_headlen(skb); | |
| struct eea_tx_meta *meta; | |
| const skb_frag_t *frag; | |
| dma_addr_t addr; | |
| u32 len = hlen; | |
| int i, err; | |
| u16 flags; | |
| bool last; | |
| if (len) { | |
| addr = dma_map_single(tx->dma_dev, skb->data, len, | |
| DMA_TO_DEVICE); | |
| if (unlikely(dma_mapping_error(tx->dma_dev, addr))) | |
| return -ENOMEM; | |
| last = !shinfo->nr_frags; | |
| i = 0; | |
| } else { | |
| /* The net stack will never submit an skb with an skb->len of | |
| * 0. If the head len is 0, the number of frags must be greater | |
| * than 0. | |
| */ | |
| frag = &shinfo->frags[0]; | |
| len = skb_frag_size(frag); | |
| addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, | |
| DMA_TO_DEVICE); | |
| if (unlikely(dma_mapping_error(tx->dma_dev, addr))) | |
| return -ENOMEM; | |
| last = shinfo->nr_frags == 1; | |
| i = 1; | |
| } | |
| flags = skb->ip_summed == CHECKSUM_PARTIAL ? EEA_DESC_F_DO_CSUM : 0; | |
| meta = eea_tx_desc_fill(tx, NULL, addr, len, last, skb, flags); | |
| meta->unmap_single = !!hlen; | |
| err = eea_fill_desc_from_skb(skb, meta->desc); | |
| if (err) | |
| goto err_cancel; | |
| for (; i < shinfo->nr_frags; i++) { | |
| frag = &shinfo->frags[i]; | |
| bool is_last = i == (shinfo->nr_frags - 1); | |
| err = eea_tx_add_skb_frag(tx, meta, frag, is_last); | |
| if (err) | |
| goto err_cancel; | |
| } | |
| eea_ering_sq_commit_desc(tx->ering); | |
| u64_stats_update_begin(&tx->stats.syncp); | |
| u64_stats_add(&tx->stats.descs, meta->num); | |
| u64_stats_update_end(&tx->stats.syncp); | |
| return 0; | |
| err_cancel: | |
| eea_ering_sq_cancel(tx->ering); | |
| eea_tx_meta_put_and_unmap(tx, meta); | |
| meta->data = NULL; | |
| return err; | |
| } | |
| static void eea_tx_kick(struct eea_net_tx *tx) | |
| { | |
| eea_ering_kick(tx->ering); | |
| u64_stats_update_begin(&tx->stats.syncp); | |
| u64_stats_inc(&tx->stats.kicks); | |
| u64_stats_update_end(&tx->stats.syncp); | |
| } | |
| static int eea_tx_check_free_num(struct eea_net_tx *tx, | |
| struct netdev_queue *txq) | |
| { | |
| int n; | |
| /* MAX_SKB_FRAGS + 1: Covers the skb linear head and all paged fragments | |
| * 1: Extra slot for a head or fragment that exceeds 64KB. | |
| */ | |
| n = MAX_SKB_FRAGS + 2; | |
| return netif_txq_maybe_stop(txq, tx->ering->num_free, n, n); | |
| } | |
| netdev_tx_t eea_tx_xmit(struct sk_buff *skb, struct net_device *netdev) | |
| { | |
| struct eea_net *enet = netdev_priv(netdev); | |
| int qnum = skb_get_queue_mapping(skb); | |
| struct eea_net_tx *tx = &enet->tx[qnum]; | |
| struct netdev_queue *txq; | |
| int err, enable; | |
| txq = netdev_get_tx_queue(netdev, qnum); | |
| enable = eea_tx_check_free_num(tx, txq); | |
| if (!enable) | |
| return NETDEV_TX_BUSY; | |
| err = eea_tx_post_skb(tx, skb); | |
| if (unlikely(err)) { | |
| u64_stats_update_begin(&tx->stats.syncp); | |
| u64_stats_inc(&tx->stats.drops); | |
| u64_stats_update_end(&tx->stats.syncp); | |
| dev_kfree_skb_any(skb); | |
| } else { | |
| if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) | |
| skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; | |
| skb_tx_timestamp(skb); | |
| } | |
| /* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */ | |
| eea_tx_check_free_num(tx, txq); | |
| if (!netdev_xmit_more() || netif_xmit_stopped(txq)) | |
| eea_tx_kick(tx); | |
| return NETDEV_TX_OK; | |
| } | |
| static void eea_free_meta(struct eea_net_tx *tx, struct eea_net_cfg *cfg) | |
| { | |
| struct eea_sq_free_stats stats = {0}; | |
| struct eea_tx_meta *meta; | |
| int i; | |
| while ((meta = eea_tx_meta_get(tx))) | |
| meta->skb = NULL; | |
| for (i = 0; i < cfg->tx_ring_depth; i++) { | |
| meta = &tx->meta[i]; | |
| if (!meta->skb) | |
| continue; | |
| eea_tx_meta_put_and_unmap(tx, meta); | |
| eea_meta_free_xmit(tx, meta, 0, NULL, &stats); | |
| } | |
| kvfree(tx->meta); | |
| tx->meta = NULL; | |
| } | |
| /* Maybe called before eea_bind_q_and_cfg. So the cfg must be passed. */ | |
| void eea_free_tx(struct eea_net_tx *tx, struct eea_net_cfg *cfg) | |
| { | |
| if (!tx) | |
| return; | |
| if (tx->ering) { | |
| eea_ering_free(tx->ering); | |
| tx->ering = NULL; | |
| } | |
| if (tx->meta) | |
| eea_free_meta(tx, cfg); | |
| } | |
| int eea_alloc_tx(struct eea_net_init_ctx *ctx, struct eea_net_tx *tx, u32 idx) | |
| { | |
| struct eea_tx_meta *meta; | |
| struct eea_ring *ering; | |
| u32 i; | |
| u64_stats_init(&tx->stats.syncp); | |
| snprintf(tx->name, sizeof(tx->name), "tx.%u", idx); | |
| ering = eea_ering_alloc(idx * 2 + 1, ctx->cfg.tx_ring_depth, ctx->edev, | |
| ctx->cfg.tx_sq_desc_size, | |
| ctx->cfg.tx_cq_desc_size, | |
| tx->name); | |
| if (!ering) | |
| goto err_free_tx; | |
| tx->ering = ering; | |
| tx->index = idx; | |
| tx->dma_dev = ctx->edev->dma_dev; | |
| /* meta */ | |
| tx->meta = kvzalloc_objs(*tx->meta, ctx->cfg.tx_ring_depth); | |
| if (!tx->meta) | |
| goto err_free_tx; | |
| for (i = 0; i < ctx->cfg.tx_ring_depth; ++i) { | |
| meta = &tx->meta[i]; | |
| meta->id = i; | |
| meta->next = tx->free; | |
| tx->free = meta; | |
| } | |
| return 0; | |
| err_free_tx: | |
| eea_free_tx(tx, &ctx->cfg); | |
| return -ENOMEM; | |
| } |