| /* SPDX-License-Identifier: GPL-2.0 */ |
| /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ |
| |
| #ifndef _IONIC_IBDEV_H_ |
| #define _IONIC_IBDEV_H_ |
| |
| #include <rdma/ib_umem.h> |
| #include <rdma/ib_verbs.h> |
| #include <rdma/ib_pack.h> |
| #include <rdma/uverbs_ioctl.h> |
| |
| #include <rdma/ionic-abi.h> |
| #include <ionic_api.h> |
| #include <ionic_regs.h> |
| |
| #include "ionic_fw.h" |
| #include "ionic_queue.h" |
| #include "ionic_res.h" |
| |
| #include "ionic_lif_cfg.h" |
| |
| /* Config knobs */ |
| #define IONIC_EQ_DEPTH 511 |
| #define IONIC_EQ_COUNT 32 |
| #define IONIC_AQ_DEPTH 63 |
| #define IONIC_AQ_COUNT 4 |
| #define IONIC_EQ_ISR_BUDGET 10 |
| #define IONIC_EQ_WORK_BUDGET 1000 |
| #define IONIC_MAX_RD_ATOM 16 |
| #define IONIC_PKEY_TBL_LEN 1 |
| #define IONIC_GID_TBL_LEN 256 |
| |
| #define IONIC_MAX_QPID 0xffffff |
| #define IONIC_SPEC_HIGH 8 |
| #define IONIC_MAX_PD 1024 |
| #define IONIC_SPEC_HIGH 8 |
| #define IONIC_SQCMB_ORDER 5 |
| #define IONIC_RQCMB_ORDER 0 |
| |
| #define IONIC_META_LAST ((void *)1ul) |
| #define IONIC_META_POSTED ((void *)2ul) |
| |
| #define IONIC_CQ_GRACE 100 |
| |
| #define IONIC_ROCE_UDP_SPORT 28272 |
| #define IONIC_DMA_LKEY 0 |
| #define IONIC_DMA_RKEY IONIC_DMA_LKEY |
| |
| #define IONIC_CMB_SUPPORTED \ |
| (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ |
| IONIC_CMB_WC | IONIC_CMB_UC) |
| |
| /* resource is not reserved on the device, indicated in tbl_order */ |
| #define IONIC_RES_INVALID -1 |
| |
| struct ionic_aq; |
| struct ionic_cq; |
| struct ionic_eq; |
| struct ionic_vcq; |
| |
| enum ionic_admin_state { |
| IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ |
| IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ |
| IONIC_ADMIN_KILLED, /* not submitting, locally completed */ |
| }; |
| |
| enum ionic_admin_flags { |
| IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ |
| IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ |
| IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ |
| }; |
| |
| enum ionic_mmap_flag { |
| IONIC_MMAP_WC = BIT(0), |
| }; |
| |
| struct ionic_mmap_entry { |
| struct rdma_user_mmap_entry rdma_entry; |
| unsigned long size; |
| unsigned long pfn; |
| u8 mmap_flags; |
| }; |
| |
| struct ionic_ibdev { |
| struct ib_device ibdev; |
| |
| struct ionic_lif_cfg lif_cfg; |
| |
| struct xarray qp_tbl; |
| struct xarray cq_tbl; |
| |
| struct ionic_resid_bits inuse_dbid; |
| struct ionic_resid_bits inuse_pdid; |
| struct ionic_resid_bits inuse_ahid; |
| struct ionic_resid_bits inuse_mrid; |
| struct ionic_resid_bits inuse_qpid; |
| struct ionic_resid_bits inuse_cqid; |
| |
| u8 half_cqid_udma_shift; |
| u8 half_qpid_udma_shift; |
| u8 next_qpid_udma_idx; |
| u8 next_mrkey; |
| |
| struct work_struct reset_work; |
| bool reset_posted; |
| u32 reset_cnt; |
| |
| struct delayed_work admin_dwork; |
| struct ionic_aq **aq_vec; |
| atomic_t admin_state; |
| |
| struct ionic_eq **eq_vec; |
| |
| struct ionic_v1_stat *hw_stats; |
| void *hw_stats_buf; |
| struct rdma_stat_desc *hw_stats_hdrs; |
| struct ionic_counter_stats *counter_stats; |
| int hw_stats_count; |
| }; |
| |
| struct ionic_eq { |
| struct ionic_ibdev *dev; |
| |
| u32 eqid; |
| u32 intr; |
| |
| struct ionic_queue q; |
| |
| int armed; |
| bool enable; |
| |
| struct work_struct work; |
| |
| int irq; |
| char name[32]; |
| }; |
| |
| struct ionic_admin_wr { |
| struct completion work; |
| struct list_head aq_ent; |
| struct ionic_v1_admin_wqe wqe; |
| struct ionic_v1_cqe cqe; |
| struct ionic_aq *aq; |
| int status; |
| }; |
| |
| struct ionic_admin_wr_q { |
| struct ionic_admin_wr *wr; |
| int wqe_strides; |
| }; |
| |
| struct ionic_aq { |
| struct ionic_ibdev *dev; |
| struct ionic_vcq *vcq; |
| |
| struct work_struct work; |
| |
| atomic_t admin_state; |
| unsigned long stamp; |
| bool armed; |
| |
| u32 aqid; |
| u32 cqid; |
| |
| spinlock_t lock; /* for posting */ |
| struct ionic_queue q; |
| struct ionic_admin_wr_q *q_wr; |
| struct list_head wr_prod; |
| struct list_head wr_post; |
| }; |
| |
| struct ionic_ctx { |
| struct ib_ucontext ibctx; |
| u32 dbid; |
| struct rdma_user_mmap_entry *mmap_dbell; |
| }; |
| |
| struct ionic_tbl_buf { |
| u32 tbl_limit; |
| u32 tbl_pages; |
| size_t tbl_size; |
| __le64 *tbl_buf; |
| dma_addr_t tbl_dma; |
| u8 page_size_log2; |
| }; |
| |
| struct ionic_pd { |
| struct ib_pd ibpd; |
| |
| u32 pdid; |
| u32 flags; |
| }; |
| |
| struct ionic_cq { |
| struct ionic_vcq *vcq; |
| |
| u32 cqid; |
| u32 eqid; |
| |
| spinlock_t lock; /* for polling */ |
| struct list_head poll_sq; |
| bool flush; |
| struct list_head flush_sq; |
| struct list_head flush_rq; |
| struct list_head ibkill_flush_ent; |
| |
| struct ionic_queue q; |
| bool color; |
| int credit; |
| u16 arm_any_prod; |
| u16 arm_sol_prod; |
| |
| struct kref cq_kref; |
| struct completion cq_rel_comp; |
| |
| /* infrequently accessed, keep at end */ |
| struct ib_umem *umem; |
| }; |
| |
| struct ionic_vcq { |
| struct ib_cq ibcq; |
| struct ionic_cq cq[2]; |
| u8 udma_mask; |
| u8 poll_idx; |
| }; |
| |
| struct ionic_sq_meta { |
| u64 wrid; |
| u32 len; |
| u16 seq; |
| u8 ibop; |
| u8 ibsts; |
| u8 remote:1; |
| u8 signal:1; |
| u8 local_comp:1; |
| }; |
| |
| struct ionic_rq_meta { |
| struct ionic_rq_meta *next; |
| u64 wrid; |
| }; |
| |
| struct ionic_qp { |
| struct ib_qp ibqp; |
| enum ib_qp_state state; |
| |
| u32 qpid; |
| u32 ahid; |
| u32 sq_cqid; |
| u32 rq_cqid; |
| u8 udma_idx; |
| u8 has_ah:1; |
| u8 has_sq:1; |
| u8 has_rq:1; |
| u8 sig_all:1; |
| |
| struct list_head qp_list_counter; |
| |
| struct list_head cq_poll_sq; |
| struct list_head cq_flush_sq; |
| struct list_head cq_flush_rq; |
| struct list_head ibkill_flush_ent; |
| |
| spinlock_t sq_lock; /* for posting and polling */ |
| struct ionic_queue sq; |
| struct ionic_sq_meta *sq_meta; |
| u16 *sq_msn_idx; |
| int sq_spec; |
| u16 sq_old_prod; |
| u16 sq_msn_prod; |
| u16 sq_msn_cons; |
| u8 sq_cmb; |
| bool sq_flush; |
| bool sq_flush_rcvd; |
| |
| spinlock_t rq_lock; /* for posting and polling */ |
| struct ionic_queue rq; |
| struct ionic_rq_meta *rq_meta; |
| struct ionic_rq_meta *rq_meta_head; |
| int rq_spec; |
| u16 rq_old_prod; |
| u8 rq_cmb; |
| bool rq_flush; |
| |
| struct kref qp_kref; |
| struct completion qp_rel_comp; |
| |
| /* infrequently accessed, keep at end */ |
| int sgid_index; |
| int sq_cmb_order; |
| u32 sq_cmb_pgid; |
| phys_addr_t sq_cmb_addr; |
| struct rdma_user_mmap_entry *mmap_sq_cmb; |
| |
| struct ib_umem *sq_umem; |
| |
| int rq_cmb_order; |
| u32 rq_cmb_pgid; |
| phys_addr_t rq_cmb_addr; |
| struct rdma_user_mmap_entry *mmap_rq_cmb; |
| |
| struct ib_umem *rq_umem; |
| |
| int dcqcn_profile; |
| |
| struct ib_ud_header *hdr; |
| }; |
| |
| struct ionic_ah { |
| struct ib_ah ibah; |
| u32 ahid; |
| int sgid_index; |
| struct ib_ud_header hdr; |
| }; |
| |
| struct ionic_mr { |
| union { |
| struct ib_mr ibmr; |
| struct ib_mw ibmw; |
| }; |
| |
| u32 mrid; |
| int flags; |
| |
| struct ib_umem *umem; |
| struct ionic_tbl_buf buf; |
| bool created; |
| }; |
| |
| struct ionic_counter_stats { |
| int queue_stats_count; |
| struct ionic_v1_stat *hdr; |
| struct rdma_stat_desc *stats_hdrs; |
| struct xarray xa_counters; |
| }; |
| |
| struct ionic_counter { |
| void *vals; |
| struct list_head qp_list; |
| }; |
| |
| static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) |
| { |
| return container_of(ibdev, struct ionic_ibdev, ibdev); |
| } |
| |
| static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) |
| { |
| return container_of(ibctx, struct ionic_ctx, ibctx); |
| } |
| |
| static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) |
| { |
| if (!uobj) |
| return NULL; |
| |
| if (!uobj->context) |
| return NULL; |
| |
| return to_ionic_ctx(uobj->context); |
| } |
| |
| static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) |
| { |
| return container_of(ibpd, struct ionic_pd, ibpd); |
| } |
| |
| static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) |
| { |
| return container_of(ibmr, struct ionic_mr, ibmr); |
| } |
| |
| static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) |
| { |
| return container_of(ibmw, struct ionic_mr, ibmw); |
| } |
| |
| static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) |
| { |
| return container_of(ibcq, struct ionic_vcq, ibcq); |
| } |
| |
| static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, |
| uint8_t udma_idx) |
| { |
| return &to_ionic_vcq(ibcq)->cq[udma_idx]; |
| } |
| |
| static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) |
| { |
| return container_of(ibqp, struct ionic_qp, ibqp); |
| } |
| |
| static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) |
| { |
| return container_of(ibah, struct ionic_ah, ibah); |
| } |
| |
| static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, |
| struct ionic_ctx *ctx) |
| { |
| if (!ctx) |
| return dev->lif_cfg.dbid; |
| |
| return ctx->dbid; |
| } |
| |
| static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, |
| struct ib_uobject *uobj) |
| { |
| return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); |
| } |
| |
| static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) |
| { |
| return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; |
| } |
| |
| static inline void ionic_qp_complete(struct kref *kref) |
| { |
| struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); |
| |
| complete(&qp->qp_rel_comp); |
| } |
| |
| static inline void ionic_cq_complete(struct kref *kref) |
| { |
| struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); |
| |
| complete(&cq->cq_rel_comp); |
| } |
| |
| /* ionic_admin.c */ |
| extern struct workqueue_struct *ionic_evt_workq; |
| void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); |
| int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, |
| enum ionic_admin_flags); |
| |
| int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); |
| |
| int ionic_create_rdma_admin(struct ionic_ibdev *dev); |
| void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); |
| void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); |
| |
| /* ionic_controlpath.c */ |
| int ionic_create_cq_common(struct ionic_vcq *vcq, |
| struct ionic_tbl_buf *buf, |
| const struct ib_cq_init_attr *attr, |
| struct ionic_ctx *ctx, |
| struct ib_udata *udata, |
| struct ionic_qdesc *req_cq, |
| __u32 *resp_cqid, |
| int udma_idx); |
| void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); |
| void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); |
| void ionic_notify_flush_cq(struct ionic_cq *cq); |
| |
| int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); |
| void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); |
| int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); |
| void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); |
| int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); |
| int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); |
| int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, |
| struct ib_udata *udata); |
| int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); |
| int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); |
| struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); |
| struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, |
| u64 addr, int access, struct ib_dmah *dmah, |
| struct ib_udata *udata); |
| struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, |
| u64 length, u64 addr, int fd, int access, |
| struct ib_dmah *dmah, |
| struct uverbs_attr_bundle *attrs); |
| int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); |
| struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, |
| u32 max_sg); |
| int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, |
| unsigned int *sg_offset); |
| int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); |
| int ionic_dealloc_mw(struct ib_mw *ibmw); |
| int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, |
| struct uverbs_attr_bundle *attrs); |
| int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); |
| int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, |
| struct ib_udata *udata); |
| int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, |
| struct ib_udata *udata); |
| int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, |
| struct ib_qp_init_attr *init_attr); |
| int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); |
| |
| /* ionic_datapath.c */ |
| int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, |
| const struct ib_send_wr **bad); |
| int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, |
| const struct ib_recv_wr **bad); |
| int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc); |
| int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); |
| |
| /* ionic_hw_stats.c */ |
| void ionic_stats_init(struct ionic_ibdev *dev); |
| void ionic_stats_cleanup(struct ionic_ibdev *dev); |
| |
| /* ionic_pgtbl.c */ |
| __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va); |
| __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va); |
| int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma); |
| int ionic_pgtbl_init(struct ionic_ibdev *dev, |
| struct ionic_tbl_buf *buf, |
| struct ib_umem *umem, |
| dma_addr_t dma, |
| int limit, |
| u64 page_size); |
| void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf); |
| #endif /* _IONIC_IBDEV_H_ */ |
