| // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB | |
| /* | |
| * mlx5 VFIO selftest driver | |
| * | |
| * Programs mlx5 ConnectX VFs and PFs through the bare-metal command interface | |
| * and RDMA WRITE self-loopback to perform DMA. Implements vfio_pci_driver_ops | |
| * (probe/init/remove) and plugs into the VFIO selftest framework. | |
| */ | |
| #include <sched.h> | |
| #include <stdbool.h> | |
| #include <stdint.h> | |
| #include <stdlib.h> | |
| #include <string.h> | |
| #include <time.h> | |
| #include <unistd.h> | |
| #include <linux/errno.h> | |
| #include <linux/io.h> | |
| #include <linux/log2.h> | |
| #include <linux/pci_regs.h> | |
| #include <linux/pci_ids.h> | |
| #include <libvfio.h> | |
| #include "mlx5_hw.h" | |
| /* | |
| * Driver state — overlaid on device->driver.region.vaddr. | |
| * | |
| * Contains both software-only state and HW-visible DMA buffers. HW buffers need | |
| * strict IOVA alignment. | |
| */ | |
| struct mlx5st_device { | |
| /* Back pointer */ | |
| struct vfio_pci_device *device; | |
| /* BAR0 */ | |
| struct mlx5st_initial_seg __iomem *init_seg; | |
| void __iomem *bar0; | |
| /* Command interface */ | |
| struct mlx5st_cmd_queue_entry *cmd_lay; | |
| struct mlx5st_cmd_queue_entry *pages_cmd_lay; | |
| u8 cmd_log_stride; | |
| unsigned int pages_slot; | |
| u8 cmd_token; | |
| bool cmd_sig_enabled; | |
| /* PD */ | |
| u32 pdn; | |
| /* Global PA-mode MKEY */ | |
| u32 global_lkey; | |
| u32 global_rkey; | |
| u32 mkey_index; | |
| /* CQ */ | |
| u32 cqn; | |
| u32 cq_ci; | |
| u32 cq_arm_sn; | |
| /* UAR */ | |
| u32 uar_page; | |
| void __iomem *uar_base; | |
| unsigned int uar_bf_offset; | |
| /* EQ (cmd/pages events — polled, not interrupt-driven) */ | |
| u32 eqn; | |
| u32 eq_cons_index; | |
| bool have_eq; | |
| /* MSI EQ (CQ completion events — fires MSI-X) */ | |
| u32 msi_eqn; | |
| u32 msi_eq_cons_index; | |
| bool have_msi_eq; | |
| /* Async pages slot state */ | |
| bool pages_slot_in_use; | |
| bool pages_slot_is_reclaim; | |
| unsigned int pages_reclaim_npages; | |
| unsigned int pages_pending_give; | |
| unsigned int pages_pending_reclaim; | |
| u16 pages_pending_func_id; | |
| bool pages_func_id_seen; | |
| /* QP */ | |
| u32 qpn; | |
| u32 sq_pi; | |
| u32 sq_ci; | |
| /* FW pages bitmap */ | |
| u64 fw_pages_bitmap[MAX_FW_PAGES / 64]; | |
| u32 fw_pages_given; | |
| u16 fw_func_id; | |
| /* Capabilities */ | |
| bool fl_supported; | |
| u8 log_max_msg; | |
| /* Buffers used by send_msi() to trigger an interrupt */ | |
| u64 send_msi_src; | |
| u64 send_msi_dst; | |
| /* | |
| * HW-visible DMA buffers below — device reads/writes via DMA. | |
| */ | |
| struct mlx5st_cmd_queue_entry cmd_queue | |
| [MLX5_HW_PAGE_SIZE / sizeof(struct mlx5st_cmd_queue_entry)] | |
| __aligned(MLX5_HW_PAGE_SIZE); | |
| struct mlx5st_send_wqe sq_buf[SQ_WQE_CNT]; | |
| struct mlx5st_dbrec cq_dbrec; | |
| struct mlx5st_dbrec qp_dbrec; | |
| struct mlx5st_cqe64 cq_buf[CQ_CQE_CNT]; | |
| /* Slot 0 mailboxes (regular commands) */ | |
| struct mlx5st_mbox_entry cmd_in_mbox[CMD_MBOX_NENT]; | |
| struct mlx5st_mbox_entry cmd_out_mbox[CMD_MBOX_NENT]; | |
| /* Pages slot mailboxes (async MANAGE_PAGES) */ | |
| struct mlx5st_mbox_entry pages_in_mbox[CMD_MBOX_NENT]; | |
| struct mlx5st_mbox_entry pages_out_mbox[CMD_MBOX_NENT]; | |
| /* EQ does not support page_offset */ | |
| struct mlx5st_eqe eq_buf[EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); | |
| /* MSI EQ buffer — CQ completions generate EQEs here -> MSI-X */ | |
| struct mlx5st_eqe msi_eq_buf[MSI_EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE); | |
| u8 fw_pages[MAX_FW_PAGES][MLX5_HW_PAGE_SIZE] | |
| __aligned(MLX5_HW_PAGE_SIZE); | |
| }; | |
| /* Check against HW limits on IOVA alignment */ | |
| static_assert(offsetof(struct mlx5st_device, cmd_in_mbox) % | |
| CMD_MBOX_STRIDE == 0, | |
| "cmd_in_mbox must be stride-aligned"); | |
| static_assert(offsetof(struct mlx5st_device, pages_in_mbox) % | |
| CMD_MBOX_STRIDE == 0, | |
| "pages_in_mbox must be stride-aligned"); | |
| static_assert(offsetof(struct mlx5st_device, cq_buf) % 64 == 0, | |
| "cq_buf must be 64-byte aligned"); | |
| static_assert(offsetof(struct mlx5st_device, sq_buf) % 64 == 0, | |
| "sq_buf must be 64-byte aligned"); | |
| static_assert(offsetof(struct mlx5st_device, cq_dbrec) % 64 == 0, | |
| "cq_dbrec must be 64-byte aligned"); | |
| static_assert(offsetof(struct mlx5st_device, qp_dbrec) % 64 == 0, | |
| "qp_dbrec must be 64-byte aligned"); | |
| static_assert(offsetof(struct mlx5st_device, eq_buf) % | |
| MLX5_HW_PAGE_SIZE == 0, | |
| "eq_buf must be page-aligned"); | |
| static_assert(offsetof(struct mlx5st_device, msi_eq_buf) % | |
| MLX5_HW_PAGE_SIZE == 0, | |
| "msi_eq_buf must be page-aligned"); | |
| static_assert(offsetof(struct mlx5st_device, fw_pages) % | |
| MLX5_HW_PAGE_SIZE == 0, | |
| "fw_pages must be page-aligned"); | |
| static struct mlx5st_device *to_mlx5st(struct vfio_pci_device *device) | |
| { | |
| return device->driver.region.vaddr; | |
| } | |
| /* | |
| * Fill a PAS (Physical Address Segment) for a buffer in the driver region. | |
| * Sets pas[0] to the page-aligned IOVA and returns the page_offset (the | |
| * buffer's byte offset within that page, in units of 64 bytes). | |
| */ | |
| static unsigned int mlx5st_fill_pas(struct vfio_pci_device *device, void *buf, | |
| __be64 *pas) | |
| { | |
| u64 iova = to_iova(device, buf); | |
| pas[0] = cpu_to_be64(iova & ~(u64)(MLX5_HW_PAGE_SIZE - 1)); | |
| return (iova & (MLX5_HW_PAGE_SIZE - 1)) / 64; | |
| } | |
| /* | |
| * Probe — match mlx5 devices by PCI vendor/device ID. | |
| */ | |
| static int mlx5st_probe(struct vfio_pci_device *device) | |
| { | |
| static const u16 mlx5st_pci_ids[] = { | |
| 0x1011, /* Connect-IB */ | |
| 0x1012, /* Connect-IB VF */ | |
| 0x1013, /* ConnectX-4 */ | |
| 0x1014, /* ConnectX-4 VF */ | |
| 0x1015, /* ConnectX-4LX */ | |
| 0x1016, /* ConnectX-4LX VF */ | |
| 0x1017, /* ConnectX-5 */ | |
| 0x1018, /* ConnectX-5 VF */ | |
| 0x1019, /* ConnectX-5 Ex */ | |
| 0x101a, /* ConnectX-5 Ex VF */ | |
| 0x101b, /* ConnectX-6 */ | |
| 0x101c, /* ConnectX-6 VF */ | |
| 0x101d, /* ConnectX-6 Dx */ | |
| 0x101e, /* ConnectX-6 Dx VF */ | |
| 0x101f, /* ConnectX-6 LX */ | |
| 0x1021, /* ConnectX-7 */ | |
| 0x1023, /* ConnectX-8 */ | |
| 0x1025, /* ConnectX-9 */ | |
| 0x1027, /* ConnectX-10 */ | |
| 0x2101, /* ConnectX-10 NVLink-C2C */ | |
| 0xa2d2, /* BlueField integrated ConnectX-5 */ | |
| 0xa2d3, /* BlueField integrated ConnectX-5 VF */ | |
| 0xa2d6, /* BlueField-2 integrated ConnectX-6 Dx */ | |
| 0xa2dc, /* BlueField-3 integrated ConnectX-7 */ | |
| 0xa2df, /* BlueField-4 integrated ConnectX-8 */ | |
| }; | |
| unsigned int i; | |
| u16 did; | |
| if (vfio_pci_config_readw(device, PCI_VENDOR_ID) != | |
| PCI_VENDOR_ID_MELLANOX) | |
| return -ENODEV; | |
| did = vfio_pci_config_readw(device, PCI_DEVICE_ID); | |
| for (i = 0; i < ARRAY_SIZE(mlx5st_pci_ids); i++) { | |
| if (mlx5st_pci_ids[i] == did) | |
| return 0; | |
| } | |
| return -ENODEV; | |
| } | |
| /* | |
| * Command interface | |
| */ | |
| static u8 xor8_buf(const void *buf, size_t offset, size_t len) | |
| { | |
| const u8 *p = buf; | |
| u8 sum = 0; | |
| size_t i; | |
| for (i = offset; i < offset + len; i++) | |
| sum ^= p[i]; | |
| return sum; | |
| } | |
| #define CMD_IF_BOX_CTRL_OFF MLX5_BYTE_OFF(cmd_if_box, reserved_at_1000) | |
| #define CMD_IF_BOX_CTRL_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, ctrl_signature) | |
| #define CMD_IF_BOX_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, signature) | |
| static void mlx5st_cmd_calc_block_sig(struct mlx5st_cmd_if_box *blk) | |
| { | |
| MLX5_SET(cmd_if_box, blk, ctrl_signature, | |
| ~xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, | |
| CMD_IF_BOX_CTRL_SIG_OFF - CMD_IF_BOX_CTRL_OFF)); | |
| MLX5_SET(cmd_if_box, blk, signature, | |
| ~xor8_buf(blk, 0, CMD_IF_BOX_SIG_OFF)); | |
| } | |
| static int mlx5st_cmd_verify_block_sig(struct mlx5st_cmd_if_box *blk) | |
| { | |
| if (xor8_buf(blk, CMD_IF_BOX_CTRL_OFF, | |
| CMD_IF_BOX_SIG_OFF - CMD_IF_BOX_CTRL_OFF) != 0xff) | |
| return -1; | |
| if (xor8_buf(blk, 0, sizeof(struct mlx5st_cmd_if_box)) != 0xff) | |
| return -1; | |
| return 0; | |
| } | |
| static unsigned int mlx5st_cmd_setup_mbox_chain(struct vfio_pci_device *device, | |
| struct mlx5st_mbox_entry *mbox, | |
| unsigned int nblocks, u8 token) | |
| { | |
| unsigned int i; | |
| for (i = 0; i < nblocks; i++) { | |
| struct mlx5st_cmd_if_box *blk = &mbox[i].block; | |
| u64 next_iova; | |
| memset(blk, 0, sizeof(struct mlx5st_cmd_if_box)); | |
| MLX5_SET(cmd_if_box, blk, block_number, i); | |
| MLX5_SET(cmd_if_box, blk, token, token); | |
| if (i < nblocks - 1) { | |
| next_iova = to_iova(device, &mbox[i + 1]); | |
| MLX5_SET(cmd_if_box, blk, next_pointer_63_32, | |
| next_iova >> 32); | |
| MLX5_SET(cmd_if_box, blk, next_pointer_31_10, | |
| (u32)next_iova >> 10); | |
| } | |
| } | |
| return nblocks; | |
| } | |
| static void mlx5st_cmd_copy_to_mbox(struct mlx5st_mbox_entry *mbox, | |
| const void *data, unsigned int len) | |
| { | |
| const u8 *src = data; | |
| unsigned int i = 0; | |
| while (len > 0) { | |
| unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? | |
| len : | |
| MLX5_CMD_DATA_BLOCK_SIZE; | |
| memcpy(MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), | |
| src, chunk); | |
| src += chunk; | |
| len -= chunk; | |
| i++; | |
| } | |
| } | |
| static void mlx5st_cmd_copy_from_mbox(void *data, | |
| const struct mlx5st_mbox_entry *mbox, | |
| unsigned int len) | |
| { | |
| unsigned int i = 0; | |
| u8 *dst = data; | |
| while (len > 0) { | |
| unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ? | |
| len : | |
| MLX5_CMD_DATA_BLOCK_SIZE; | |
| memcpy(dst, | |
| MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data), | |
| chunk); | |
| dst += chunk; | |
| len -= chunk; | |
| i++; | |
| } | |
| } | |
| /* Forward declaration — cmd_exec polls events during command wait */ | |
| static void mlx5st_process_events(struct mlx5st_device *dev); | |
| static const char *mlx5st_cmd_name(u16 opcode) | |
| { | |
| switch (opcode) { | |
| case MLX5_CMD_OP_QUERY_HCA_CAP: return "QUERY_HCA_CAP"; | |
| case MLX5_CMD_OP_INIT_HCA: return "INIT_HCA"; | |
| case MLX5_CMD_OP_TEARDOWN_HCA: return "TEARDOWN_HCA"; | |
| case MLX5_CMD_OP_ENABLE_HCA: return "ENABLE_HCA"; | |
| case MLX5_CMD_OP_DISABLE_HCA: return "DISABLE_HCA"; | |
| case MLX5_CMD_OP_QUERY_PAGES: return "QUERY_PAGES"; | |
| case MLX5_CMD_OP_MANAGE_PAGES: return "MANAGE_PAGES"; | |
| case MLX5_CMD_OP_SET_HCA_CAP: return "SET_HCA_CAP"; | |
| case MLX5_CMD_OP_SET_ISSI: return "SET_ISSI"; | |
| case MLX5_CMD_OP_CREATE_MKEY: return "CREATE_MKEY"; | |
| case MLX5_CMD_OP_DESTROY_MKEY: return "DESTROY_MKEY"; | |
| case MLX5_CMD_OP_CREATE_EQ: return "CREATE_EQ"; | |
| case MLX5_CMD_OP_DESTROY_EQ: return "DESTROY_EQ"; | |
| case MLX5_CMD_OP_CREATE_CQ: return "CREATE_CQ"; | |
| case MLX5_CMD_OP_DESTROY_CQ: return "DESTROY_CQ"; | |
| case MLX5_CMD_OP_CREATE_QP: return "CREATE_QP"; | |
| case MLX5_CMD_OP_DESTROY_QP: return "DESTROY_QP"; | |
| case MLX5_CMD_OP_RST2INIT_QP: return "RST2INIT_QP"; | |
| case MLX5_CMD_OP_INIT2RTR_QP: return "INIT2RTR_QP"; | |
| case MLX5_CMD_OP_RTR2RTS_QP: return "RTR2RTS_QP"; | |
| case MLX5_CMD_OP_ALLOC_PD: return "ALLOC_PD"; | |
| case MLX5_CMD_OP_DEALLOC_PD: return "DEALLOC_PD"; | |
| case MLX5_CMD_OP_ALLOC_UAR: return "ALLOC_UAR"; | |
| case MLX5_CMD_OP_DEALLOC_UAR: return "DEALLOC_UAR"; | |
| default: return "UNKNOWN"; | |
| } | |
| } | |
| /* | |
| * Post a command on a given slot: fill the cmd_queue_entry, set up mailbox | |
| * chains, compute signatures, hand ownership to FW, and ring the doorbell. | |
| */ | |
| static void mlx5st_cmd_post(struct mlx5st_device *dev, | |
| struct mlx5st_cmd_queue_entry *cmd, | |
| struct mlx5st_mbox_entry *in_mbox, | |
| struct mlx5st_mbox_entry *out_mbox, | |
| void *in, unsigned int ilen, unsigned int olen, | |
| u32 doorbell) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| unsigned int in_remain, out_remain, in_nblk, out_nblk; | |
| unsigned int i; | |
| void *cin, *cout; | |
| u8 token; | |
| /* Rotating non-zero token ties cmd entry to its mailbox blocks */ | |
| token = ++dev->cmd_token; | |
| if (!token) | |
| token = ++dev->cmd_token; | |
| in_remain = ilen > MLX5_CMD_INLINE_SZ ? ilen - MLX5_CMD_INLINE_SZ : 0; | |
| out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; | |
| in_nblk = (in_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / | |
| MLX5_CMD_DATA_BLOCK_SIZE; | |
| out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / | |
| MLX5_CMD_DATA_BLOCK_SIZE; | |
| /* Set up mailbox chains */ | |
| if (in_nblk > 0) { | |
| mlx5st_cmd_setup_mbox_chain(device, in_mbox, in_nblk, token); | |
| mlx5st_cmd_copy_to_mbox(in_mbox, | |
| (u8 *)in + MLX5_CMD_INLINE_SZ, | |
| in_remain); | |
| } | |
| if (out_nblk > 0) | |
| mlx5st_cmd_setup_mbox_chain(device, out_mbox, out_nblk, token); | |
| /* Copy inline input */ | |
| cin = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_input_inline_data); | |
| memset(cin, 0, MLX5_CMD_INLINE_SZ); | |
| memcpy(cin, in, ilen < MLX5_CMD_INLINE_SZ ? ilen : MLX5_CMD_INLINE_SZ); | |
| MLX5_SET(cmd_queue_entry, cmd, input_length, ilen); | |
| MLX5_SET(cmd_queue_entry, cmd, token, token); | |
| /* Zero inline output */ | |
| cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); | |
| memset(cout, 0, MLX5_CMD_INLINE_SZ); | |
| MLX5_SET(cmd_queue_entry, cmd, output_length, olen); | |
| /* | |
| * Compute signatures: mailbox blocks first, then cmd_queue_entry. | |
| * The sig must cover the final state including ownership=0x1, but | |
| * we must not set ownership until after the sig is in place - | |
| * XOR in the 0x1 without storing it to memory. | |
| */ | |
| for (i = 0; i < in_nblk; i++) | |
| mlx5st_cmd_calc_block_sig(&in_mbox[i].block); | |
| for (i = 0; i < out_nblk; i++) | |
| mlx5st_cmd_calc_block_sig(&out_mbox[i].block); | |
| MLX5_SET(cmd_queue_entry, cmd, signature, 0); | |
| MLX5_SET(cmd_queue_entry, cmd, signature, | |
| ~(xor8_buf(cmd, 0, sizeof(struct mlx5st_cmd_queue_entry)) ^ | |
| 0x1)); | |
| /* Ensure all cmd data (including sig) is visible, then hand to FW */ | |
| dma_wmb(); | |
| MLX5_SET_ONCE(cmd_queue_entry, cmd, ownership, 1); | |
| /* Ring doorbell */ | |
| MLX5_SET_MMIO(initial_seg, dev->init_seg, command_doorbell_vector, | |
| doorbell); | |
| } | |
| static void mlx5st_cmd_exec(struct mlx5st_device *dev, void *in, | |
| unsigned int ilen, void *out, unsigned int olen) | |
| { | |
| struct mlx5st_cmd_queue_entry *cmd = dev->cmd_lay; | |
| unsigned int out_remain, out_nblk; | |
| struct timespec start, now; | |
| unsigned int elapsed; | |
| unsigned int i; | |
| void *cout; | |
| mlx5st_cmd_post(dev, cmd, dev->cmd_in_mbox, dev->cmd_out_mbox, in, | |
| ilen, olen, 1); | |
| out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0; | |
| out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) / | |
| MLX5_CMD_DATA_BLOCK_SIZE; | |
| /* Poll for completion — also process EQ events for PF page requests */ | |
| clock_gettime(CLOCK_MONOTONIC, &start); | |
| for (;;) { | |
| if (!MLX5_GET_ONCE(cmd_queue_entry, cmd, ownership)) | |
| break; | |
| if (dev->have_eq) | |
| mlx5st_process_events(dev); | |
| sched_yield(); | |
| clock_gettime(CLOCK_MONOTONIC, &now); | |
| elapsed = (now.tv_sec - start.tv_sec) * 1000 + | |
| (now.tv_nsec - start.tv_nsec) / 1000000; | |
| if (elapsed > MLX5_CMD_TIMEOUT_MS) | |
| VFIO_FAIL("cmd timeout after %d ms", elapsed); | |
| } | |
| /* Ensure output data reads happen after ownership is seen clear */ | |
| dma_rmb(); | |
| /* Verify output signatures when FW has checksums enabled */ | |
| if (dev->cmd_sig_enabled) { | |
| if (xor8_buf(cmd, 0, | |
| sizeof(struct mlx5st_cmd_queue_entry)) != 0xff) | |
| VFIO_FAIL("cmd output signature mismatch"); | |
| for (i = 0; i < out_nblk; i++) { | |
| if (mlx5st_cmd_verify_block_sig( | |
| &dev->cmd_out_mbox[i].block)) | |
| VFIO_FAIL("cmd output mailbox block %d signature mismatch", | |
| i); | |
| } | |
| } | |
| /* Copy output: inline first */ | |
| cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); | |
| memcpy(out, cout, olen < MLX5_CMD_INLINE_SZ ? olen : MLX5_CMD_INLINE_SZ); | |
| /* Copy remaining from output mailbox chain */ | |
| if (out_remain > 0) | |
| mlx5st_cmd_copy_from_mbox((u8 *)out + MLX5_CMD_INLINE_SZ, | |
| dev->cmd_out_mbox, out_remain); | |
| /* Check command status */ | |
| if (MLX5_GET(enable_hca_out, out, status) != MLX5_CMD_STAT_OK) | |
| VFIO_FAIL("%s: status=0x%x syndrome=0x%x", | |
| mlx5st_cmd_name(MLX5_GET(enable_hca_in, in, opcode)), | |
| MLX5_GET(enable_hca_out, out, status), | |
| MLX5_GET(enable_hca_out, out, syndrome)); | |
| } | |
| static struct mlx5st_cmd_queue_entry * | |
| mlx5st_cmd_slot_init(struct mlx5st_device *dev, unsigned int slot, | |
| struct mlx5st_mbox_entry *in_mbox, | |
| struct mlx5st_mbox_entry *out_mbox) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| struct mlx5st_cmd_queue_entry *cmd = | |
| &dev->cmd_queue[(slot << dev->cmd_log_stride) / | |
| sizeof(struct mlx5st_cmd_queue_entry)]; | |
| u64 iova; | |
| MLX5_SET(cmd_queue_entry, cmd, type, | |
| MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT); | |
| iova = to_iova(device, in_mbox); | |
| MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_63_32, | |
| iova >> 32); | |
| MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_31_9, iova >> 9); | |
| iova = to_iova(device, out_mbox); | |
| MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_63_32, | |
| iova >> 32); | |
| MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_31_9, | |
| iova >> 9); | |
| return cmd; | |
| } | |
| static void mlx5st_cmd_init(struct mlx5st_device *dev) | |
| { | |
| struct mlx5st_initial_seg __iomem *seg = dev->init_seg; | |
| struct vfio_pci_device *device = dev->device; | |
| u16 cmdif_rev; | |
| u8 log_sz; | |
| u64 iova; | |
| cmdif_rev = MLX5_GET_MMIO(initial_seg, seg, cmd_interface_rev); | |
| VFIO_ASSERT_EQ(cmdif_rev, 5); | |
| /* Read command queue geometry from BAR */ | |
| log_sz = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_size); | |
| dev->cmd_log_stride = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_stride); | |
| dev->pages_slot = (1 << log_sz) - 1; | |
| VFIO_ASSERT_LE((unsigned int)(1 << log_sz), 32u); | |
| VFIO_ASSERT_GE((unsigned int)(1 << dev->cmd_log_stride), | |
| (unsigned int)sizeof(struct mlx5st_cmd_queue_entry)); | |
| VFIO_ASSERT_LE((unsigned int)((dev->pages_slot + 1) << | |
| dev->cmd_log_stride), | |
| (unsigned int)sizeof(dev->cmd_queue)); | |
| /* Set up slot 0 — regular commands */ | |
| dev->cmd_lay = mlx5st_cmd_slot_init(dev, 0, dev->cmd_in_mbox, | |
| dev->cmd_out_mbox); | |
| /* Set up pages slot — async MANAGE_PAGES */ | |
| dev->pages_cmd_lay = mlx5st_cmd_slot_init(dev, dev->pages_slot, | |
| dev->pages_in_mbox, | |
| dev->pages_out_mbox); | |
| /* Write command queue page address to BAR0 */ | |
| iova = to_iova(device, dev->cmd_queue); | |
| MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_63_32, iova >> 32); | |
| MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_31_12, iova >> 12); | |
| dev_dbg(device, | |
| "Command interface initialized (cmdif_rev=5, log_sz=%u, log_stride=%u, pages_slot=%u)\n", | |
| log_sz, dev->cmd_log_stride, dev->pages_slot); | |
| } | |
| /* | |
| * FW pages: bitmap allocator + MANAGE_PAGES | |
| */ | |
| static void mlx5st_fw_pages_alloc(struct mlx5st_device *dev, | |
| unsigned int npages, u64 *iovas) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| unsigned int found = 0; | |
| unsigned int w, b; | |
| u64 word; | |
| for (w = 0; w < MAX_FW_PAGES / 64 && found < npages; w++) { | |
| word = dev->fw_pages_bitmap[w]; | |
| for (b = 0; b < 64 && found < npages; b++) { | |
| if (!(word & (1ULL << b))) { | |
| unsigned int idx = w * 64 + b; | |
| dev->fw_pages_bitmap[w] |= (1ULL << b); | |
| iovas[found++] = to_iova(device, | |
| dev->fw_pages[idx]); | |
| } | |
| } | |
| } | |
| VFIO_ASSERT_EQ(found, npages); | |
| dev->fw_pages_given += npages; | |
| } | |
| static void mlx5st_fw_pages_free(struct mlx5st_device *dev, | |
| unsigned int npages, const u64 *iovas) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| unsigned int i, idx; | |
| u64 off; | |
| for (i = 0; i < npages; i++) { | |
| off = iovas[i] - to_iova(device, dev->fw_pages); | |
| idx = off / MLX5_HW_PAGE_SIZE; | |
| VFIO_ASSERT_TRUE(idx < MAX_FW_PAGES); | |
| dev->fw_pages_bitmap[idx / 64] &= ~(1ULL << (idx % 64)); | |
| } | |
| dev->fw_pages_given -= npages; | |
| } | |
| static void *mlx5st_build_manage_pages_give(u16 func_id, unsigned int npages, | |
| const u64 *iovas, | |
| unsigned int *out_inlen) | |
| { | |
| unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in) + npages * 8; | |
| unsigned int i; | |
| void *in; | |
| in = calloc(1, inlen); | |
| VFIO_ASSERT_NOT_NULL(in); | |
| MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); | |
| MLX5_SET(manage_pages_in, in, op_mod, | |
| MLX5_MANAGE_PAGES_IN_OP_MOD_ALLOCATION_SUCCESS); | |
| MLX5_SET(manage_pages_in, in, function_id, func_id); | |
| MLX5_SET(manage_pages_in, in, input_num_entries, npages); | |
| for (i = 0; i < npages; i++) | |
| MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, iovas[i]); | |
| *out_inlen = inlen; | |
| return in; | |
| } | |
| static void mlx5st_fw_pages_give_one(struct mlx5st_device *dev, u16 func_id, | |
| unsigned int npages, u64 *iovas) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {}; | |
| unsigned int inlen; | |
| void *in; | |
| in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); | |
| mlx5st_cmd_exec(dev, in, inlen, out, sizeof(out)); | |
| free(in); | |
| } | |
| static void mlx5st_fw_pages_give(struct mlx5st_device *dev, u16 func_id, | |
| unsigned int npages) | |
| { | |
| unsigned int remaining = npages; | |
| u64 *iovas; | |
| if (!npages) | |
| return; | |
| iovas = calloc(npages, sizeof(u64)); | |
| VFIO_ASSERT_NOT_NULL(iovas); | |
| mlx5st_fw_pages_alloc(dev, npages, iovas); | |
| /* Batch into chunks that fit in one mailbox */ | |
| for (unsigned int off = 0; remaining > 0;) { | |
| unsigned int batch = remaining < MAX_FW_PAGES_PER_CMD ? | |
| remaining : | |
| MAX_FW_PAGES_PER_CMD; | |
| mlx5st_fw_pages_give_one(dev, func_id, batch, iovas + off); | |
| off += batch; | |
| remaining -= batch; | |
| } | |
| dev_dbg(dev->device, "MANAGE_PAGES GIVE: %d pages to func_id=%u\n", | |
| npages, func_id); | |
| free(iovas); | |
| } | |
| static void mlx5st_fw_pages_satisfy(struct mlx5st_device *dev, int boot) | |
| { | |
| u32 qo[MLX5_ST_SZ_DW(query_pages_out)] = {}; | |
| u32 qi[MLX5_ST_SZ_DW(query_pages_in)] = {}; | |
| u16 func_id; | |
| int npages; | |
| MLX5_SET(query_pages_in, qi, opcode, MLX5_CMD_OP_QUERY_PAGES); | |
| MLX5_SET(query_pages_in, qi, op_mod, boot ? 0x01 : 0x02); | |
| mlx5st_cmd_exec(dev, qi, sizeof(qi), qo, sizeof(qo)); | |
| npages = MLX5_GET(query_pages_out, qo, num_pages); | |
| func_id = MLX5_GET(query_pages_out, qo, function_id); | |
| dev_dbg(dev->device, "QUERY_PAGES (%s): %d pages (func_id=%u)\n", | |
| boot ? "boot" : "init", npages, func_id); | |
| if (npages > 0) { | |
| dev->fw_func_id = func_id; | |
| mlx5st_fw_pages_give(dev, func_id, npages); | |
| } | |
| } | |
| /* | |
| * Async MANAGE_PAGES on the pages command slot. | |
| * | |
| * On PFs, firmware sends PAGE_REQUEST events via the EQ during command | |
| * execution. We must respond with MANAGE_PAGES on a second command slot | |
| * before the first (regular) command can complete. | |
| */ | |
| static void mlx5st_pages_slot_post(struct mlx5st_device *dev, void *in, | |
| unsigned int ilen, unsigned int olen) | |
| { | |
| mlx5st_cmd_post(dev, dev->pages_cmd_lay, dev->pages_in_mbox, | |
| dev->pages_out_mbox, in, ilen, olen, | |
| 1 << dev->pages_slot); | |
| } | |
| static void mlx5st_pages_slot_give(struct mlx5st_device *dev, u16 func_id, | |
| unsigned int npages) | |
| { | |
| unsigned int inlen; | |
| u64 *iovas; | |
| void *in; | |
| iovas = calloc(npages, sizeof(u64)); | |
| VFIO_ASSERT_NOT_NULL(iovas); | |
| mlx5st_fw_pages_alloc(dev, npages, iovas); | |
| in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen); | |
| free(iovas); | |
| mlx5st_pages_slot_post(dev, in, inlen, | |
| MLX5_ST_SZ_BYTES(manage_pages_out)); | |
| dev->pages_slot_in_use = true; | |
| dev->pages_slot_is_reclaim = false; | |
| free(in); | |
| dev_dbg(dev->device, | |
| "PAGE_REQUEST: %d pages given async to func_id=%u\n", | |
| npages, func_id); | |
| } | |
| static void mlx5st_pages_slot_reclaim(struct mlx5st_device *dev, u16 func_id, | |
| unsigned int npages) | |
| { | |
| unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); | |
| unsigned int outlen = | |
| MLX5_ST_SZ_BYTES(manage_pages_out) + npages * 8; | |
| void *in; | |
| in = calloc(1, inlen); | |
| VFIO_ASSERT_NOT_NULL(in); | |
| MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); | |
| MLX5_SET(manage_pages_in, in, op_mod, | |
| MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); | |
| MLX5_SET(manage_pages_in, in, function_id, func_id); | |
| MLX5_SET(manage_pages_in, in, input_num_entries, npages); | |
| mlx5st_pages_slot_post(dev, in, inlen, outlen); | |
| dev->pages_slot_in_use = true; | |
| dev->pages_slot_is_reclaim = true; | |
| dev->pages_reclaim_npages = npages; | |
| free(in); | |
| dev_dbg(dev->device, | |
| "PAGE_REQUEST: reclaim %d pages async from func_id=%u\n", | |
| npages, func_id); | |
| } | |
| static void mlx5st_pages_slot_kick(struct mlx5st_device *dev) | |
| { | |
| unsigned int batch; | |
| if (dev->pages_slot_in_use) | |
| return; | |
| if (dev->pages_pending_give) { | |
| batch = dev->pages_pending_give < MAX_FW_PAGES_PER_CMD ? | |
| dev->pages_pending_give : | |
| MAX_FW_PAGES_PER_CMD; | |
| dev->pages_pending_give -= batch; | |
| mlx5st_pages_slot_give(dev, dev->pages_pending_func_id, batch); | |
| } else if (dev->pages_pending_reclaim) { | |
| batch = dev->pages_pending_reclaim < MAX_FW_PAGES_PER_CMD ? | |
| dev->pages_pending_reclaim : | |
| MAX_FW_PAGES_PER_CMD; | |
| dev->pages_pending_reclaim -= batch; | |
| mlx5st_pages_slot_reclaim(dev, dev->pages_pending_func_id, | |
| batch); | |
| } | |
| } | |
| static void mlx5st_fw_pages_give_async(struct mlx5st_device *dev, | |
| u16 func_id, unsigned int npages) | |
| { | |
| if (!npages) | |
| return; | |
| dev->pages_pending_give += npages; | |
| dev->pages_pending_func_id = func_id; | |
| mlx5st_pages_slot_kick(dev); | |
| } | |
| static void mlx5st_fw_pages_reclaim_async(struct mlx5st_device *dev, | |
| u16 func_id, unsigned int npages) | |
| { | |
| dev->pages_pending_reclaim += npages; | |
| dev->pages_pending_func_id = func_id; | |
| mlx5st_pages_slot_kick(dev); | |
| } | |
| static void mlx5st_pages_slot_complete(struct mlx5st_device *dev) | |
| { | |
| struct mlx5st_cmd_queue_entry *cmd = dev->pages_cmd_lay; | |
| void *cout; | |
| dma_rmb(); | |
| cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data); | |
| if (MLX5_GET(enable_hca_out, cout, status) != MLX5_CMD_STAT_OK) | |
| VFIO_FAIL("async MANAGE_PAGES failed: status=0x%x syndrome=0x%x", | |
| MLX5_GET(enable_hca_out, cout, status), | |
| MLX5_GET(enable_hca_out, cout, syndrome)); | |
| if (dev->pages_slot_is_reclaim) { | |
| unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) + | |
| dev->pages_reclaim_npages * 8; | |
| unsigned int num_claimed; | |
| unsigned int i; | |
| void *out; | |
| u64 *iovas; | |
| out = calloc(1, outlen); | |
| iovas = calloc(dev->pages_reclaim_npages, sizeof(u64)); | |
| VFIO_ASSERT_NOT_NULL(out); | |
| VFIO_ASSERT_NOT_NULL(iovas); | |
| /* Copy inline output */ | |
| memcpy(out, cout, MLX5_CMD_INLINE_SZ); | |
| if (outlen > MLX5_CMD_INLINE_SZ) | |
| mlx5st_cmd_copy_from_mbox( | |
| (u8 *)out + MLX5_CMD_INLINE_SZ, | |
| dev->pages_out_mbox, | |
| outlen - MLX5_CMD_INLINE_SZ); | |
| num_claimed = | |
| MLX5_GET(manage_pages_out, out, output_num_entries); | |
| for (i = 0; i < num_claimed; i++) | |
| iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, | |
| i); | |
| mlx5st_fw_pages_free(dev, num_claimed, iovas); | |
| dev_dbg(dev->device, "PAGE_REQUEST: reclaimed %d pages\n", | |
| num_claimed); | |
| free(iovas); | |
| free(out); | |
| } | |
| dev->pages_slot_in_use = false; | |
| mlx5st_pages_slot_kick(dev); | |
| } | |
| /* | |
| * UAR alloc/dealloc | |
| */ | |
| static void mlx5st_alloc_uar(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {}; | |
| MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->uar_page = MLX5_GET(alloc_uar_out, out, uar); | |
| dev->uar_base = (u8 __iomem*)dev->bar0 + dev->uar_page * MLX5_HW_PAGE_SIZE; | |
| dev->uar_bf_offset = MLX5_BF_OFFSET; | |
| dev_dbg(dev->device, | |
| "Allocated UAR page_id=%u, doorbell offset=0x%x\n", | |
| dev->uar_page, | |
| dev->uar_page * MLX5_HW_PAGE_SIZE + MLX5_BF_OFFSET); | |
| } | |
| static void mlx5st_dealloc_uar(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {}; | |
| MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR); | |
| MLX5_SET(dealloc_uar_in, in, uar, dev->uar_page); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * EQ infrastructure | |
| */ | |
| static struct mlx5st_eqe *mlx5st_eq_get_eqe(struct mlx5st_device *dev, u32 cc) | |
| { | |
| u32 ci = dev->eq_cons_index + cc; | |
| struct mlx5st_eqe *eqe = &dev->eq_buf[ci % EQ_NENT]; | |
| u8 owner = MLX5_GET_ONCE(eqe, eqe, owner); | |
| u8 expected = !!(ci & EQ_NENT); | |
| if (owner != expected) | |
| return NULL; | |
| dma_rmb(); | |
| return eqe; | |
| } | |
| static void mlx5st_eq_update_ci(struct mlx5st_device *dev, u32 cc, bool arm) | |
| { | |
| u32 val; | |
| dev->eq_cons_index += cc; | |
| val = (dev->eq_cons_index & 0xffffff) | (dev->eqn << 24); | |
| iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET + | |
| (arm ? 0 : 8)); | |
| } | |
| static void mlx5st_create_eq(struct mlx5st_device *dev) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; | |
| u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; | |
| struct mlx5_ifc_eqc_bits *eqc; | |
| unsigned int i; | |
| __be64 *pas; | |
| /* Initialize EQE owner bits */ | |
| for (i = 0; i < EQ_NENT; i++) { | |
| struct mlx5st_eqe *eqe = &dev->eq_buf[i]; | |
| MLX5_SET_ONCE(eqe, eqe, owner, 1); | |
| } | |
| MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); | |
| /* Subscribe to CMD completions and PAGE_REQUEST events */ | |
| MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, 0, | |
| (1ULL << MLX5_EVENT_TYPE_CMD) | | |
| (1ULL << MLX5_EVENT_TYPE_PAGE_REQUEST)); | |
| eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); | |
| MLX5_SET(eqc, eqc, log_eq_size, LOG_EQ_SIZE); | |
| MLX5_SET(eqc, eqc, uar_page, dev->uar_page); | |
| pas = MLX5_ADDR_OF(create_eq_in, in, pas); | |
| VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->eq_buf, pas), 0u); | |
| MLX5_SET(eqc, eqc, log_page_size, 0); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->eqn = MLX5_GET(create_eq_out, out, eq_number); | |
| dev->eq_cons_index = 0; | |
| mlx5st_eq_update_ci(dev, 0, 0); | |
| dev->have_eq = true; | |
| dev_dbg(device, "Created EQ: eqn=%u, %d entries (CMD+PAGE_REQUEST)\n", | |
| dev->eqn, EQ_NENT); | |
| } | |
| static void mlx5st_destroy_eq(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; | |
| MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); | |
| MLX5_SET(destroy_eq_in, in, eq_number, dev->eqn); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * Drain all pending EQ events. Dispatches PAGE_REQUEST to the async pages | |
| * slot and CMD completions to the pages slot completion handler. | |
| */ | |
| static void mlx5st_process_events(struct mlx5st_device *dev) | |
| { | |
| struct mlx5st_eqe *eqe; | |
| u32 cc = 0; | |
| while ((eqe = mlx5st_eq_get_eqe(dev, cc))) { | |
| u8 type = MLX5_GET(eqe, eqe, event_type); | |
| switch (type) { | |
| case MLX5_EVENT_TYPE_PAGE_REQUEST: { | |
| void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); | |
| u16 func_id = MLX5_GET(pages_req_event, evdata, | |
| function_id); | |
| s32 npages = (s32)MLX5_GET(pages_req_event, evdata, | |
| num_pages); | |
| /* | |
| * The selftest doesn't use more than one func_id so a | |
| * simple counter approach is possible. | |
| */ | |
| if (dev->pages_func_id_seen) | |
| VFIO_ASSERT_EQ(func_id, | |
| dev->pages_pending_func_id); | |
| dev->pages_func_id_seen = true; | |
| if (npages > 0) | |
| mlx5st_fw_pages_give_async(dev, func_id, | |
| npages); | |
| else if (npages < 0) | |
| mlx5st_fw_pages_reclaim_async(dev, func_id, | |
| -npages); | |
| break; | |
| } | |
| case MLX5_EVENT_TYPE_CMD: { | |
| void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data); | |
| u32 vector = MLX5_GET(cmd_inter_comp_event, evdata, | |
| command_completion_vector); | |
| if (vector & (1U << dev->pages_slot)) | |
| mlx5st_pages_slot_complete(dev); | |
| break; | |
| } | |
| default: | |
| break; | |
| } | |
| cc++; | |
| } | |
| if (cc) | |
| mlx5st_eq_update_ci(dev, cc, 0); | |
| } | |
| /* | |
| * MSI EQ — dedicated EQ for CQ completion events that fires MSI-X. | |
| * Separate from the cmd/pages EQ so that only CQ completions (from | |
| * send_msi or memcpy) trigger the interrupt vector. | |
| */ | |
| static void mlx5st_msi_eq_drain(struct mlx5st_device *dev) | |
| { | |
| u32 cc = 0; | |
| u32 val; | |
| while (cc < MSI_EQ_NENT) { | |
| u32 ci = dev->msi_eq_cons_index + cc; | |
| struct mlx5st_eqe *eqe = | |
| &dev->msi_eq_buf[ci % MSI_EQ_NENT]; | |
| if (MLX5_GET_ONCE(eqe, eqe, owner) != !!(ci & MSI_EQ_NENT)) | |
| break; | |
| cc++; | |
| } | |
| /* Update consumer index and re-arm for next interrupt */ | |
| dev->msi_eq_cons_index += cc; | |
| val = (dev->msi_eq_cons_index & 0xffffff) | (dev->msi_eqn << 24); | |
| iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET); | |
| } | |
| static void mlx5st_create_msi_eq(struct mlx5st_device *dev) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {}; | |
| u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {}; | |
| struct mlx5_ifc_eqc_bits *eqc; | |
| unsigned int i; | |
| __be64 *pas; | |
| /* Initialize EQE owner bits */ | |
| for (i = 0; i < MSI_EQ_NENT; i++) { | |
| struct mlx5st_eqe *eqe = &dev->msi_eq_buf[i]; | |
| MLX5_SET_ONCE(eqe, eqe, owner, 1); | |
| } | |
| MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); | |
| /* | |
| * No event_bitmask — completion events are routed to this EQ via | |
| * the CQ's c_eqn field, not through CREATE_EQ subscription. | |
| */ | |
| eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); | |
| MLX5_SET(eqc, eqc, log_eq_size, LOG_MSI_EQ_SIZE); | |
| MLX5_SET(eqc, eqc, uar_page, dev->uar_page); | |
| MLX5_SET(eqc, eqc, intr, MSI_VECTOR); | |
| pas = MLX5_ADDR_OF(create_eq_in, in, pas); | |
| VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->msi_eq_buf, pas), 0u); | |
| MLX5_SET(eqc, eqc, log_page_size, 0); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->msi_eqn = MLX5_GET(create_eq_out, out, eq_number); | |
| dev->msi_eq_cons_index = 0; | |
| dev->have_msi_eq = true; | |
| mlx5st_msi_eq_drain(dev); | |
| dev_dbg(device, | |
| "Created MSI EQ: eqn=%u, %d entries (COMP), vector=%d\n", | |
| dev->msi_eqn, MSI_EQ_NENT, MSI_VECTOR); | |
| } | |
| static void mlx5st_destroy_msi_eq(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; | |
| MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); | |
| MLX5_SET(destroy_eq_in, in, eq_number, dev->msi_eqn); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * HCA init / teardown | |
| */ | |
| #define FW_INIT_TIMEOUT_MS 120000 | |
| #define FW_INIT_WAIT_MS 200 | |
| static void mlx5st_wait_fw_init(struct mlx5st_device *dev) | |
| { | |
| struct timespec start, now; | |
| unsigned int elapsed; | |
| clock_gettime(CLOCK_MONOTONIC, &start); | |
| while (MLX5_GET_MMIO(initial_seg, dev->init_seg, initializing)) { | |
| usleep(FW_INIT_WAIT_MS * 1000); | |
| clock_gettime(CLOCK_MONOTONIC, &now); | |
| elapsed = (now.tv_sec - start.tv_sec) * 1000 + | |
| (now.tv_nsec - start.tv_nsec) / 1000000; | |
| if (elapsed > FW_INIT_TIMEOUT_MS) | |
| VFIO_FAIL("FW init timeout after %d ms", elapsed); | |
| } | |
| } | |
| static void mlx5st_set_issi(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(set_issi_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(set_issi_in)] = {}; | |
| MLX5_SET(set_issi_in, in, opcode, MLX5_CMD_OP_SET_ISSI); | |
| MLX5_SET(set_issi_in, in, current_issi, 1); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "SET_ISSI: OK (issi=1)\n"); | |
| } | |
| static void mlx5st_set_hca_caps(struct mlx5st_device *dev) | |
| { | |
| u32 qout[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; | |
| u32 qin[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; | |
| u32 sout[MLX5_ST_SZ_DW(set_hca_cap_out)] = {}; | |
| u32 sin[MLX5_ST_SZ_DW(set_hca_cap_in)] = {}; | |
| struct mlx5_ifc_cmd_hca_cap_bits *set_hca_cap; | |
| u32 max_checksum; | |
| /* Query max caps to learn cmdif_checksum support */ | |
| MLX5_SET(query_hca_cap_in, qin, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); | |
| MLX5_SET(query_hca_cap_in, qin, op_mod, | |
| (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1)); | |
| mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); | |
| max_checksum = MLX5_GET( | |
| cmd_hca_cap, | |
| MLX5_ADDR_OF(query_hca_cap_out, qout, capability), | |
| cmdif_checksum); | |
| /* Query current caps as base for SET */ | |
| memset(qout, 0, sizeof(qout)); | |
| MLX5_SET(query_hca_cap_in, qin, op_mod, | |
| (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | | |
| HCA_CAP_OPMOD_GET_CUR); | |
| mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout)); | |
| set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, sin, capability); | |
| memcpy(set_hca_cap, | |
| MLX5_ADDR_OF(query_hca_cap_out, qout, capability), | |
| MLX5_ST_SZ_BYTES(cmd_hca_cap)); | |
| MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, max_checksum); | |
| MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, 0); | |
| MLX5_SET(set_hca_cap_in, sin, opcode, MLX5_CMD_OP_SET_HCA_CAP); | |
| MLX5_SET(set_hca_cap_in, sin, op_mod, | |
| MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1); | |
| mlx5st_cmd_exec(dev, sin, sizeof(sin), sout, sizeof(sout)); | |
| dev->cmd_sig_enabled = max_checksum == 0x3; | |
| dev_dbg(dev->device, "SET_HCA_CAP: OK (cmdif_checksum=%u)\n", | |
| max_checksum); | |
| } | |
| static void mlx5st_hca_init(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {}; | |
| mlx5st_wait_fw_init(dev); | |
| dev_dbg(dev->device, "Firmware ready\n"); | |
| MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "ENABLE_HCA: OK\n"); | |
| mlx5st_set_issi(dev); | |
| mlx5st_fw_pages_satisfy(dev, 1); | |
| mlx5st_set_hca_caps(dev); | |
| mlx5st_fw_pages_satisfy(dev, 0); | |
| memset(in, 0, sizeof(in)); | |
| memset(out, 0, sizeof(out)); | |
| MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "INIT_HCA: OK\n"); | |
| /* | |
| * Create EQ immediately after INIT_HCA so PAGE_REQUEST events | |
| * are captured during all subsequent commands. | |
| */ | |
| mlx5st_alloc_uar(dev); | |
| mlx5st_create_eq(dev); | |
| } | |
| static void mlx5st_disable_hca(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {}; | |
| MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| static void mlx5st_fw_pages_reclaim(struct mlx5st_device *dev, u16 func_id) | |
| { | |
| unsigned int npages = dev->fw_pages_given; | |
| unsigned int total_claimed = 0; | |
| while (npages > 0) { | |
| unsigned int batch = npages < MAX_FW_PAGES_PER_CMD ? | |
| npages : | |
| MAX_FW_PAGES_PER_CMD; | |
| unsigned int outlen = | |
| MLX5_ST_SZ_BYTES(manage_pages_out) + batch * 8; | |
| unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); | |
| unsigned int num_claimed; | |
| unsigned int i; | |
| void *in, *out; | |
| u64 *iovas; | |
| in = calloc(1, inlen); | |
| out = calloc(1, outlen); | |
| iovas = calloc(batch, sizeof(u64)); | |
| VFIO_ASSERT_NOT_NULL(in); | |
| VFIO_ASSERT_NOT_NULL(out); | |
| VFIO_ASSERT_NOT_NULL(iovas); | |
| MLX5_SET(manage_pages_in, in, opcode, | |
| MLX5_CMD_OP_MANAGE_PAGES); | |
| MLX5_SET(manage_pages_in, in, op_mod, | |
| MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES); | |
| MLX5_SET(manage_pages_in, in, function_id, func_id); | |
| MLX5_SET(manage_pages_in, in, input_num_entries, batch); | |
| mlx5st_cmd_exec(dev, in, inlen, out, outlen); | |
| num_claimed = | |
| MLX5_GET(manage_pages_out, out, output_num_entries); | |
| for (i = 0; i < num_claimed; i++) | |
| iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas, | |
| i); | |
| mlx5st_fw_pages_free(dev, num_claimed, iovas); | |
| total_claimed += num_claimed; | |
| npages -= num_claimed; | |
| free(iovas); | |
| free(in); | |
| free(out); | |
| if (!num_claimed && !dev->fw_pages_given) | |
| break; | |
| if (!num_claimed) | |
| VFIO_FAIL("MANAGE_PAGES RECLAIM: FW returned 0 but %d pages still given", | |
| dev->fw_pages_given); | |
| } | |
| dev_dbg(dev->device, | |
| "MANAGE_PAGES RECLAIM: %d pages (%d still given)\n", | |
| total_claimed, dev->fw_pages_given); | |
| } | |
| static void mlx5st_hca_teardown(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {}; | |
| /* Drain async pages slot, then stop EQ processing */ | |
| while (dev->pages_slot_in_use) { | |
| if (!MLX5_GET_ONCE(cmd_queue_entry, dev->pages_cmd_lay, | |
| ownership)) | |
| mlx5st_pages_slot_complete(dev); | |
| else | |
| sched_yield(); | |
| } | |
| dev->have_eq = false; | |
| if (dev->eqn) { | |
| mlx5st_destroy_eq(dev); | |
| dev->eqn = 0; | |
| } | |
| if (dev->uar_page) { | |
| mlx5st_dealloc_uar(dev); | |
| dev->uar_page = 0; | |
| } | |
| dev_dbg(dev->device, " hca_teardown: TEARDOWN_HCA\n"); | |
| MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA); | |
| MLX5_SET(teardown_hca_in, in, profile, | |
| MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| if (dev->fw_pages_given > 0) { | |
| dev_dbg(dev->device, " hca_teardown: reclaim %d pages\n", | |
| dev->fw_pages_given); | |
| mlx5st_fw_pages_reclaim(dev, dev->fw_func_id); | |
| } | |
| dev_dbg(dev->device, " hca_teardown: DISABLE_HCA\n"); | |
| mlx5st_disable_hca(dev); | |
| } | |
| /* | |
| * Query capabilities | |
| */ | |
| static void mlx5st_query_fl_caps(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {}; | |
| bool fl_roce_en, fl_roce_dis; | |
| /* Query RoCE capabilities */ | |
| MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); | |
| MLX5_SET(query_hca_cap_in, in, op_mod, | |
| (MLX5_SET_HCA_CAP_OP_MOD_ROCE << 1) | HCA_CAP_OPMOD_GET_CUR); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| fl_roce_en = MLX5_GET(query_hca_cap_out, out, | |
| capability.roce_cap.fl_rc_qp_when_roce_enabled); | |
| fl_roce_dis = MLX5_GET(query_hca_cap_out, out, | |
| capability.roce_cap.fl_rc_qp_when_roce_disabled); | |
| /* Also check general caps */ | |
| memset(in, 0, sizeof(in)); | |
| memset(out, 0, sizeof(out)); | |
| MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); | |
| MLX5_SET(query_hca_cap_in, in, op_mod, | |
| (MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) | | |
| HCA_CAP_OPMOD_GET_CUR); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| fl_roce_dis |= | |
| MLX5_GET(query_hca_cap_out, out, | |
| capability.cmd_hca_cap.fl_rc_qp_when_roce_disabled); | |
| dev->fl_supported = fl_roce_en || fl_roce_dis; | |
| dev->log_max_msg = MLX5_GET(query_hca_cap_out, out, | |
| capability.cmd_hca_cap.log_max_msg); | |
| dev_dbg(dev->device, | |
| "HCA capabilities: fl_roce_enabled=%d fl_roce_disabled=%d log_max_msg=%u\n", | |
| fl_roce_en, fl_roce_dis, dev->log_max_msg); | |
| VFIO_ASSERT_TRUE(dev->fl_supported, | |
| "Force-loopback not supported on this device"); | |
| } | |
| /* | |
| * Resource allocation | |
| */ | |
| static void mlx5st_alloc_pd(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; | |
| MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->pdn = MLX5_GET(alloc_pd_out, out, pd); | |
| dev_dbg(dev->device, "Allocated PD pdn=%u\n", dev->pdn); | |
| } | |
| static void mlx5st_dealloc_pd(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; | |
| MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); | |
| MLX5_SET(dealloc_pd_in, in, pd, dev->pdn); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| static void mlx5st_create_mkey(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(create_mkey_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; | |
| struct mlx5_ifc_mkc_bits *mkc; | |
| MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); | |
| mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); | |
| MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); | |
| MLX5_SET(mkc, mkc, length64, 1); | |
| MLX5_SET(mkc, mkc, pd, dev->pdn); | |
| MLX5_SET(mkc, mkc, qpn, 0xffffff); | |
| MLX5_SET(mkc, mkc, lr, 1); | |
| MLX5_SET(mkc, mkc, lw, 1); | |
| MLX5_SET(mkc, mkc, rw, 1); | |
| MLX5_SET(mkc, mkc, rr, 1); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->mkey_index = MLX5_GET(create_mkey_out, out, mkey_index); | |
| dev->global_lkey = mlx5st_idx_to_mkey(dev->mkey_index); | |
| dev->global_rkey = dev->global_lkey; | |
| dev_dbg(dev->device, "Created global PA-mode MKEY: lkey=0x%x\n", | |
| dev->global_lkey); | |
| } | |
| static void mlx5st_destroy_mkey(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; | |
| MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); | |
| MLX5_SET(destroy_mkey_in, in, mkey_index, dev->mkey_index); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * CQ create/destroy | |
| */ | |
| static void mlx5st_create_cq(struct mlx5st_device *dev) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| u64 in[MLX5_ST_SZ_QW(create_cq_in) + 1] = {}; | |
| u32 out[MLX5_ST_SZ_DW(create_cq_out)] = {}; | |
| struct mlx5_ifc_cqc_bits *cqc; | |
| unsigned int i; | |
| __be64 *pas; | |
| /* Initialize CQEs before CREATE_CQ: opcode=0xF, owner=1 */ | |
| for (i = 0; i < CQ_CQE_CNT; i++) { | |
| struct mlx5st_cqe64 *cqe = &dev->cq_buf[i]; | |
| MLX5_SET(cqe64, cqe, opcode, 0xF); | |
| MLX5_SET_ONCE(cqe64, cqe, owner, 1); | |
| } | |
| MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ); | |
| cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); | |
| MLX5_SET(cqc, cqc, log_cq_size, LOG_CQ_SIZE); | |
| MLX5_SET(cqc, cqc, uar_page, dev->uar_page); | |
| MLX5_SET(cqc, cqc, c_eqn_or_apu_element, dev->msi_eqn); | |
| MLX5_SET(cqc, cqc, cqe_sz, 0); | |
| pas = MLX5_ADDR_OF(create_cq_in, in, pas); | |
| MLX5_SET(cqc, cqc, page_offset, mlx5st_fill_pas(device, dev->cq_buf, pas)); | |
| MLX5_SET(cqc, cqc, log_page_size, 0); | |
| MLX5_SET64(cqc, cqc, dbr_addr, to_iova(device, &dev->cq_dbrec)); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->cqn = MLX5_GET(create_cq_out, out, cqn); | |
| dev->cq_ci = 0; | |
| dev_dbg(device, "Created CQ: cqn=%u, %d entries\n", dev->cqn, | |
| CQ_CQE_CNT); | |
| } | |
| static void mlx5st_destroy_cq(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {}; | |
| MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ); | |
| MLX5_SET(destroy_cq_in, in, cqn, dev->cqn); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * Arm CQ for event generation. The CQ event delivery state machine is | |
| * single-shot: after generating one EQE the CQ enters "Fired" state and | |
| * won't generate another until re-armed via ARM_NEXT. Both the CQ doorbell | |
| * record and the UAR CQ doorbell register must be written. | |
| */ | |
| static void mlx5st_arm_cq(struct mlx5st_device *dev) | |
| { | |
| u32 sn = dev->cq_arm_sn & 3; | |
| u32 ci = dev->cq_ci & 0xffffff; | |
| u64 doorbell; | |
| /* Update CQ doorbell record arm word */ | |
| WRITE_ONCE(dev->cq_dbrec.send_counter, | |
| cpu_to_be32(sn << 28 | ci)); | |
| /* Ring CQ doorbell register, iowrite has an internal dma_wmb() */ | |
| doorbell = ((u64)(sn << 28 | ci) << 32) | dev->cqn; | |
| iowrite64be(doorbell, | |
| (u8 __iomem *)dev->uar_base + MLX5_CQ_DOORBELL_OFFSET); | |
| dev->cq_arm_sn++; | |
| } | |
| /* | |
| * QP create/destroy | |
| */ | |
| static void mlx5st_create_qp(struct mlx5st_device *dev) | |
| { | |
| struct vfio_pci_device *device = dev->device; | |
| u64 in[MLX5_ST_SZ_QW(create_qp_in) + 1] = {}; | |
| u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; | |
| struct mlx5_ifc_qpc_bits *qpc; | |
| __be64 *pas; | |
| MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); | |
| qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); | |
| MLX5_SET(qpc, qpc, st, MLX5_QPC_ST_RC); | |
| MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); | |
| MLX5_SET(qpc, qpc, pd, dev->pdn); | |
| MLX5_SET(qpc, qpc, uar_page, dev->uar_page); | |
| MLX5_SET(qpc, qpc, cqn_snd, dev->cqn); | |
| MLX5_SET(qpc, qpc, cqn_rcv, dev->cqn); | |
| MLX5_SET(qpc, qpc, log_sq_size, LOG_SQ_SIZE); | |
| MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); | |
| MLX5_SET(qpc, qpc, rq_type, 0x3); | |
| MLX5_SET(qpc, qpc, ts_format, 1); | |
| pas = MLX5_ADDR_OF(create_qp_in, in, pas); | |
| MLX5_SET(qpc, qpc, page_offset, | |
| mlx5st_fill_pas(device, dev->sq_buf, pas)); | |
| MLX5_SET(qpc, qpc, log_page_size, 0); | |
| MLX5_SET64(qpc, qpc, dbr_addr, to_iova(device, &dev->qp_dbrec)); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev->qpn = MLX5_GET(create_qp_out, out, qpn); | |
| dev->sq_pi = 0; | |
| dev_dbg(device, "Created QP: qpn=%u, RC, sq=%d wqes\n", dev->qpn, | |
| SQ_WQE_CNT); | |
| } | |
| static void mlx5st_destroy_qp(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(destroy_qp_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; | |
| MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); | |
| MLX5_SET(destroy_qp_in, in, qpn, dev->qpn); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| } | |
| /* | |
| * QP state transitions | |
| */ | |
| static void mlx5st_qp_rst2init(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(rst2init_qp_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; | |
| struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc); | |
| MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP); | |
| MLX5_SET(rst2init_qp_in, in, qpn, dev->qpn); | |
| MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); | |
| MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); | |
| MLX5_SET(qpc, qpc, rre, 1); | |
| MLX5_SET(qpc, qpc, rwe, 1); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "QP RST->INIT\n"); | |
| } | |
| static void mlx5st_qp_init2rtr(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(init2rtr_qp_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; | |
| struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc); | |
| MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP); | |
| MLX5_SET(init2rtr_qp_in, in, qpn, dev->qpn); | |
| MLX5_SET(qpc, qpc, mtu, 3); | |
| MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg); | |
| MLX5_SET(qpc, qpc, remote_qpn, dev->qpn); | |
| MLX5_SET(qpc, qpc, min_rnr_nak, 12); | |
| MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); | |
| MLX5_SET(qpc, qpc, primary_address_path.fl, 1); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "QP INIT->RTR (fl=1)\n"); | |
| } | |
| static void mlx5st_qp_rtr2rts(struct mlx5st_device *dev) | |
| { | |
| u32 out[MLX5_ST_SZ_DW(rtr2rts_qp_out)] = {}; | |
| u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; | |
| struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc); | |
| MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP); | |
| MLX5_SET(rtr2rts_qp_in, in, qpn, dev->qpn); | |
| MLX5_SET(qpc, qpc, log_ack_req_freq, 0); | |
| MLX5_SET(qpc, qpc, retry_count, 7); | |
| MLX5_SET(qpc, qpc, rnr_retry, 7); | |
| MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 14); | |
| mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
| dev_dbg(dev->device, "QP RTR->RTS\n"); | |
| } | |
| /* | |
| * Post RDMA Write WQE | |
| */ | |
| static void mlx5st_post_rdma_write(struct mlx5st_device *dev, u64 src_addr, | |
| u32 src_lkey, u64 dst_addr, u32 dst_rkey, | |
| u32 length, bool signaled) | |
| { | |
| struct mlx5st_send_wqe *wqe; | |
| unsigned int idx; | |
| idx = dev->sq_pi % SQ_WQE_CNT; | |
| wqe = &dev->sq_buf[idx]; | |
| memset(wqe, 0, sizeof(*wqe)); | |
| MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, opcode, MLX5_OPCODE_RDMA_WRITE); | |
| MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, wqe_index, dev->sq_pi); | |
| MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, qp_or_sq, dev->qpn); | |
| MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ds, MLX5_RDMA_WRITE_DS); | |
| if (signaled) | |
| MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ce, MLX5_WQE_CE_CQE_ALWAYS); | |
| MLX5_SET64(wqe_raddr_seg, &wqe->raddr, raddr, dst_addr); | |
| MLX5_SET(wqe_raddr_seg, &wqe->raddr, rkey, dst_rkey); | |
| MLX5_SET(wqe_data_seg, &wqe->data, byte_count, length); | |
| MLX5_SET(wqe_data_seg, &wqe->data, lkey, src_lkey); | |
| MLX5_SET64(wqe_data_seg, &wqe->data, addr, src_addr); | |
| dev->sq_pi++; | |
| /* Ensure WQE is visible to device before doorbell record */ | |
| dma_wmb(); | |
| WRITE_ONCE(dev->qp_dbrec.send_counter, | |
| cpu_to_be32(dev->sq_pi & 0xffff)); | |
| /* | |
| * Ring doorbell: write first 8 bytes of ctrl to UAR BF register, | |
| * iowrite has an internal dma_wmb() so the doorbell record will be | |
| * visible. | |
| */ | |
| iowrite64be(be64_to_cpu(*(__be64 *)wqe), | |
| (u8 __iomem *)dev->uar_base + dev->uar_bf_offset); | |
| dev->uar_bf_offset ^= MLX5_BF_SIZE; | |
| } | |
| /* | |
| * Poll CQ | |
| */ | |
| static int mlx5st_poll_cq_batch(struct mlx5st_device *dev, | |
| unsigned int max_cqe) | |
| { | |
| unsigned int polled = 0; | |
| while (polled < max_cqe) { | |
| unsigned int idx = dev->cq_ci % CQ_CQE_CNT; | |
| struct mlx5st_cqe64 *cqe = &dev->cq_buf[idx]; | |
| u8 owner, opcode; | |
| owner = MLX5_GET_ONCE(cqe64, cqe, owner); | |
| if (owner != ((dev->cq_ci >> LOG_CQ_SIZE) & 1)) | |
| break; | |
| dma_rmb(); | |
| opcode = MLX5_GET(cqe64, cqe, opcode); | |
| dev->cq_ci++; | |
| WRITE_ONCE(dev->cq_dbrec.recv_counter, | |
| cpu_to_be32(dev->cq_ci & 0xffffff)); | |
| if (opcode == MLX5_CQE_REQ) { | |
| dev->sq_ci = | |
| (u16)(MLX5_GET(cqe64, cqe, wqe_counter) + 1); | |
| polled++; | |
| continue; | |
| } | |
| if (opcode == MLX5_CQE_REQ_ERR || | |
| opcode == MLX5_CQE_RESP_ERR) { | |
| dev_dbg(dev->device, | |
| "CQE error: opcode=0x%x syndrome=0x%x vendor=0x%x\n", | |
| opcode, | |
| MLX5_GET(cqe64, cqe, error_syndrome.syndrome), | |
| MLX5_GET(cqe64, cqe, | |
| error_syndrome.vendor_error_syndrome)); | |
| return -1; | |
| } | |
| dev_err(dev->device, "CQE unexpected opcode=0x%x\n", opcode); | |
| return -1; | |
| } | |
| return polled; | |
| } | |
| static int mlx5st_poll_cq(struct mlx5st_device *dev, unsigned int timeout_ms) | |
| { | |
| struct timespec start, now; | |
| unsigned int elapsed; | |
| int ret; | |
| clock_gettime(CLOCK_MONOTONIC, &start); | |
| for (;;) { | |
| ret = mlx5st_poll_cq_batch(dev, 1); | |
| if (ret < 0) | |
| return -1; | |
| if (ret > 0) | |
| return 0; | |
| if (dev->have_eq) | |
| mlx5st_process_events(dev); | |
| clock_gettime(CLOCK_MONOTONIC, &now); | |
| elapsed = (now.tv_sec - start.tv_sec) * 1000 + | |
| (now.tv_nsec - start.tv_nsec) / 1000000; | |
| if (elapsed > timeout_ms) { | |
| dev_err(dev->device, "CQ poll timeout after %u ms\n", | |
| timeout_ms); | |
| return -1; | |
| } | |
| } | |
| } | |
| /* | |
| * Data path setup/teardown helpers | |
| */ | |
| static void mlx5st_setup_datapath(struct mlx5st_device *dev) | |
| { | |
| mlx5st_create_cq(dev); | |
| mlx5st_create_qp(dev); | |
| mlx5st_qp_rst2init(dev); | |
| mlx5st_qp_init2rtr(dev); | |
| mlx5st_qp_rtr2rts(dev); | |
| } | |
| static void mlx5st_teardown_datapath(struct mlx5st_device *dev) | |
| { | |
| if (dev->qpn) { | |
| mlx5st_destroy_qp(dev); | |
| dev->qpn = 0; | |
| } | |
| if (dev->cqn) { | |
| mlx5st_destroy_cq(dev); | |
| dev->cqn = 0; | |
| } | |
| dev->sq_pi = 0; | |
| dev->sq_ci = 0; | |
| dev->cq_arm_sn = 0; | |
| memset(&dev->qp_dbrec, 0, sizeof(dev->qp_dbrec)); | |
| memset(&dev->cq_dbrec, 0, sizeof(dev->cq_dbrec)); | |
| } | |
| /* | |
| * memcpy callbacks | |
| */ | |
| #define MLX5ST_MEMCPY_TIMEOUT_MS 60000 | |
| static void mlx5st_memcpy_start(struct vfio_pci_device *device, | |
| iova_t src, iova_t dst, u64 size, u64 count) | |
| { | |
| struct mlx5st_device *dev = to_mlx5st(device); | |
| u64 i; | |
| for (i = 0; i < count; i++) { | |
| bool signaled = (i == count - 1); | |
| mlx5st_post_rdma_write(dev, src, dev->global_lkey, dst, | |
| dev->global_rkey, size, signaled); | |
| } | |
| } | |
| static int mlx5st_memcpy_wait(struct vfio_pci_device *device) | |
| { | |
| struct mlx5st_device *dev = to_mlx5st(device); | |
| int ret; | |
| ret = mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS); | |
| if (ret) { | |
| /* | |
| * CQE error puts the QP in error state. Rebuild the data path | |
| * so subsequent operations can succeed. | |
| */ | |
| mlx5st_teardown_datapath(dev); | |
| mlx5st_setup_datapath(dev); | |
| } | |
| return ret; | |
| } | |
| /* | |
| * send_msi callback — trigger CQE -> EQE -> MSI-X via a small RDMA Write. | |
| * | |
| * Both the CQ and MSI EQ use single-shot arming: the CQ must be armed so the | |
| * CQE generates an EQE, and the MSI EQ must be armed so the EQE fires MSI-X. | |
| */ | |
| static void mlx5st_send_msi(struct vfio_pci_device *device) | |
| { | |
| struct mlx5st_device *dev = to_mlx5st(device); | |
| /* Drain accumulated MSI EQ events and re-arm for next interrupt */ | |
| mlx5st_msi_eq_drain(dev); | |
| /* Arm CQ so the next CQE generates an EQE on the MSI EQ */ | |
| mlx5st_arm_cq(dev); | |
| /* Post a signaled RDMA Write to trigger CQE -> EQE -> MSI-X */ | |
| mlx5st_post_rdma_write(dev, | |
| to_iova(device, &dev->send_msi_src), | |
| dev->global_lkey, | |
| to_iova(device, &dev->send_msi_dst), | |
| dev->global_rkey, | |
| sizeof(dev->send_msi_src), true); | |
| /* Consume the CQE to avoid stale completions */ | |
| VFIO_ASSERT_EQ(mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS), 0); | |
| } | |
| /* | |
| * Driver ops callbacks | |
| */ | |
| static void mlx5st_init(struct vfio_pci_device *device) | |
| { | |
| struct mlx5st_device *dev = to_mlx5st(device); | |
| iova_t iova_align = | |
| device->driver.region.iova % __alignof__(struct mlx5st_device); | |
| VFIO_ASSERT_GE(device->driver.region.size, sizeof(*dev)); | |
| VFIO_ASSERT_EQ(iova_align, 0); | |
| memset(dev, 0, sizeof(*dev)); | |
| dev->device = device; | |
| dev->bar0 = device->bars[0].vaddr; | |
| dev->init_seg = dev->bar0; | |
| vfio_pci_config_writew(device, PCI_COMMAND, | |
| vfio_pci_config_readw(device, PCI_COMMAND) | | |
| PCI_COMMAND_MASTER); | |
| mlx5st_wait_fw_init(dev); | |
| mlx5st_cmd_init(dev); | |
| mlx5st_hca_init(dev); | |
| mlx5st_query_fl_caps(dev); | |
| mlx5st_alloc_pd(dev); | |
| mlx5st_create_mkey(dev); | |
| /* MSI EQ must be created before CQ so CQ can reference its eqn */ | |
| mlx5st_create_msi_eq(dev); | |
| mlx5st_setup_datapath(dev); | |
| vfio_pci_msix_enable(device, MSI_VECTOR, 1); | |
| device->driver.msi = MSI_VECTOR; | |
| device->driver.max_memcpy_size = 1ULL << dev->log_max_msg; | |
| device->driver.max_memcpy_count = SQ_WQE_CNT - 1; | |
| dev_dbg(device, "mlx5 driver initialized\n"); | |
| } | |
| static void mlx5st_remove(struct vfio_pci_device *device) | |
| { | |
| struct mlx5st_device *dev = to_mlx5st(device); | |
| vfio_pci_msix_disable(device); | |
| mlx5st_teardown_datapath(dev); | |
| if (dev->have_msi_eq) { | |
| mlx5st_destroy_msi_eq(dev); | |
| dev->have_msi_eq = false; | |
| } | |
| dev_dbg(device, "teardown: destroy_mkey\n"); | |
| if (dev->mkey_index) { | |
| mlx5st_destroy_mkey(dev); | |
| dev->mkey_index = 0; | |
| } | |
| dev_dbg(device, "teardown: dealloc_pd\n"); | |
| if (dev->pdn) { | |
| mlx5st_dealloc_pd(dev); | |
| dev->pdn = 0; | |
| } | |
| dev_dbg(device, "teardown: hca_teardown\n"); | |
| mlx5st_hca_teardown(dev); | |
| vfio_pci_config_writew(device, PCI_COMMAND, | |
| vfio_pci_config_readw(device, PCI_COMMAND) & | |
| ~(u16)PCI_COMMAND_MASTER); | |
| dev_dbg(device, "Teardown complete\n"); | |
| } | |
| struct vfio_pci_driver_ops mlx5st_ops = { | |
| .name = "mlx5", | |
| .region_size = sizeof(struct mlx5st_device), | |
| .probe = mlx5st_probe, | |
| .init = mlx5st_init, | |
| .remove = mlx5st_remove, | |
| .memcpy_start = mlx5st_memcpy_start, | |
| .memcpy_wait = mlx5st_memcpy_wait, | |
| .send_msi = mlx5st_send_msi, | |
| }; |