blob: a65b52e183bc6cced93a8723da697d60ad062796 [file] [edit]
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* mlx5 VFIO selftest driver
*
* Programs mlx5 ConnectX VFs and PFs through the bare-metal command interface
* and RDMA WRITE self-loopback to perform DMA. Implements vfio_pci_driver_ops
* (probe/init/remove) and plugs into the VFIO selftest framework.
*/
#include <sched.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <linux/errno.h>
#include <linux/io.h>
#include <linux/log2.h>
#include <linux/pci_regs.h>
#include <linux/pci_ids.h>
#include <libvfio.h>
#include "mlx5_hw.h"
/*
* Driver state — overlaid on device->driver.region.vaddr.
*
* Contains both software-only state and HW-visible DMA buffers. HW buffers need
* strict IOVA alignment.
*/
struct mlx5st_device {
/* Back pointer */
struct vfio_pci_device *device;
/* BAR0 */
struct mlx5st_initial_seg __iomem *init_seg;
void __iomem *bar0;
/* Command interface */
struct mlx5st_cmd_queue_entry *cmd_lay;
struct mlx5st_cmd_queue_entry *pages_cmd_lay;
u8 cmd_log_stride;
unsigned int pages_slot;
u8 cmd_token;
bool cmd_sig_enabled;
/* PD */
u32 pdn;
/* Global PA-mode MKEY */
u32 global_lkey;
u32 global_rkey;
u32 mkey_index;
/* CQ */
u32 cqn;
u32 cq_ci;
u32 cq_arm_sn;
/* UAR */
u32 uar_page;
void __iomem *uar_base;
unsigned int uar_bf_offset;
/* EQ (cmd/pages events — polled, not interrupt-driven) */
u32 eqn;
u32 eq_cons_index;
bool have_eq;
/* MSI EQ (CQ completion events — fires MSI-X) */
u32 msi_eqn;
u32 msi_eq_cons_index;
bool have_msi_eq;
/* Async pages slot state */
bool pages_slot_in_use;
bool pages_slot_is_reclaim;
unsigned int pages_reclaim_npages;
unsigned int pages_pending_give;
unsigned int pages_pending_reclaim;
u16 pages_pending_func_id;
bool pages_func_id_seen;
/* QP */
u32 qpn;
u32 sq_pi;
u32 sq_ci;
/* FW pages bitmap */
u64 fw_pages_bitmap[MAX_FW_PAGES / 64];
u32 fw_pages_given;
u16 fw_func_id;
/* Capabilities */
bool fl_supported;
u8 log_max_msg;
/* Buffers used by send_msi() to trigger an interrupt */
u64 send_msi_src;
u64 send_msi_dst;
/*
* HW-visible DMA buffers below — device reads/writes via DMA.
*/
struct mlx5st_cmd_queue_entry cmd_queue
[MLX5_HW_PAGE_SIZE / sizeof(struct mlx5st_cmd_queue_entry)]
__aligned(MLX5_HW_PAGE_SIZE);
struct mlx5st_send_wqe sq_buf[SQ_WQE_CNT];
struct mlx5st_dbrec cq_dbrec;
struct mlx5st_dbrec qp_dbrec;
struct mlx5st_cqe64 cq_buf[CQ_CQE_CNT];
/* Slot 0 mailboxes (regular commands) */
struct mlx5st_mbox_entry cmd_in_mbox[CMD_MBOX_NENT];
struct mlx5st_mbox_entry cmd_out_mbox[CMD_MBOX_NENT];
/* Pages slot mailboxes (async MANAGE_PAGES) */
struct mlx5st_mbox_entry pages_in_mbox[CMD_MBOX_NENT];
struct mlx5st_mbox_entry pages_out_mbox[CMD_MBOX_NENT];
/* EQ does not support page_offset */
struct mlx5st_eqe eq_buf[EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE);
/* MSI EQ buffer — CQ completions generate EQEs here -> MSI-X */
struct mlx5st_eqe msi_eq_buf[MSI_EQ_NENT] __aligned(MLX5_HW_PAGE_SIZE);
u8 fw_pages[MAX_FW_PAGES][MLX5_HW_PAGE_SIZE]
__aligned(MLX5_HW_PAGE_SIZE);
};
/* Check against HW limits on IOVA alignment */
static_assert(offsetof(struct mlx5st_device, cmd_in_mbox) %
CMD_MBOX_STRIDE == 0,
"cmd_in_mbox must be stride-aligned");
static_assert(offsetof(struct mlx5st_device, pages_in_mbox) %
CMD_MBOX_STRIDE == 0,
"pages_in_mbox must be stride-aligned");
static_assert(offsetof(struct mlx5st_device, cq_buf) % 64 == 0,
"cq_buf must be 64-byte aligned");
static_assert(offsetof(struct mlx5st_device, sq_buf) % 64 == 0,
"sq_buf must be 64-byte aligned");
static_assert(offsetof(struct mlx5st_device, cq_dbrec) % 64 == 0,
"cq_dbrec must be 64-byte aligned");
static_assert(offsetof(struct mlx5st_device, qp_dbrec) % 64 == 0,
"qp_dbrec must be 64-byte aligned");
static_assert(offsetof(struct mlx5st_device, eq_buf) %
MLX5_HW_PAGE_SIZE == 0,
"eq_buf must be page-aligned");
static_assert(offsetof(struct mlx5st_device, msi_eq_buf) %
MLX5_HW_PAGE_SIZE == 0,
"msi_eq_buf must be page-aligned");
static_assert(offsetof(struct mlx5st_device, fw_pages) %
MLX5_HW_PAGE_SIZE == 0,
"fw_pages must be page-aligned");
static struct mlx5st_device *to_mlx5st(struct vfio_pci_device *device)
{
return device->driver.region.vaddr;
}
/*
* Fill a PAS (Physical Address Segment) for a buffer in the driver region.
* Sets pas[0] to the page-aligned IOVA and returns the page_offset (the
* buffer's byte offset within that page, in units of 64 bytes).
*/
static unsigned int mlx5st_fill_pas(struct vfio_pci_device *device, void *buf,
__be64 *pas)
{
u64 iova = to_iova(device, buf);
pas[0] = cpu_to_be64(iova & ~(u64)(MLX5_HW_PAGE_SIZE - 1));
return (iova & (MLX5_HW_PAGE_SIZE - 1)) / 64;
}
/*
* Probe — match mlx5 devices by PCI vendor/device ID.
*/
static int mlx5st_probe(struct vfio_pci_device *device)
{
static const u16 mlx5st_pci_ids[] = {
0x1011, /* Connect-IB */
0x1012, /* Connect-IB VF */
0x1013, /* ConnectX-4 */
0x1014, /* ConnectX-4 VF */
0x1015, /* ConnectX-4LX */
0x1016, /* ConnectX-4LX VF */
0x1017, /* ConnectX-5 */
0x1018, /* ConnectX-5 VF */
0x1019, /* ConnectX-5 Ex */
0x101a, /* ConnectX-5 Ex VF */
0x101b, /* ConnectX-6 */
0x101c, /* ConnectX-6 VF */
0x101d, /* ConnectX-6 Dx */
0x101e, /* ConnectX-6 Dx VF */
0x101f, /* ConnectX-6 LX */
0x1021, /* ConnectX-7 */
0x1023, /* ConnectX-8 */
0x1025, /* ConnectX-9 */
0x1027, /* ConnectX-10 */
0x2101, /* ConnectX-10 NVLink-C2C */
0xa2d2, /* BlueField integrated ConnectX-5 */
0xa2d3, /* BlueField integrated ConnectX-5 VF */
0xa2d6, /* BlueField-2 integrated ConnectX-6 Dx */
0xa2dc, /* BlueField-3 integrated ConnectX-7 */
0xa2df, /* BlueField-4 integrated ConnectX-8 */
};
unsigned int i;
u16 did;
if (vfio_pci_config_readw(device, PCI_VENDOR_ID) !=
PCI_VENDOR_ID_MELLANOX)
return -ENODEV;
did = vfio_pci_config_readw(device, PCI_DEVICE_ID);
for (i = 0; i < ARRAY_SIZE(mlx5st_pci_ids); i++) {
if (mlx5st_pci_ids[i] == did)
return 0;
}
return -ENODEV;
}
/*
* Command interface
*/
static u8 xor8_buf(const void *buf, size_t offset, size_t len)
{
const u8 *p = buf;
u8 sum = 0;
size_t i;
for (i = offset; i < offset + len; i++)
sum ^= p[i];
return sum;
}
#define CMD_IF_BOX_CTRL_OFF MLX5_BYTE_OFF(cmd_if_box, reserved_at_1000)
#define CMD_IF_BOX_CTRL_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, ctrl_signature)
#define CMD_IF_BOX_SIG_OFF MLX5_BYTE_OFF(cmd_if_box, signature)
static void mlx5st_cmd_calc_block_sig(struct mlx5st_cmd_if_box *blk)
{
MLX5_SET(cmd_if_box, blk, ctrl_signature,
~xor8_buf(blk, CMD_IF_BOX_CTRL_OFF,
CMD_IF_BOX_CTRL_SIG_OFF - CMD_IF_BOX_CTRL_OFF));
MLX5_SET(cmd_if_box, blk, signature,
~xor8_buf(blk, 0, CMD_IF_BOX_SIG_OFF));
}
static int mlx5st_cmd_verify_block_sig(struct mlx5st_cmd_if_box *blk)
{
if (xor8_buf(blk, CMD_IF_BOX_CTRL_OFF,
CMD_IF_BOX_SIG_OFF - CMD_IF_BOX_CTRL_OFF) != 0xff)
return -1;
if (xor8_buf(blk, 0, sizeof(struct mlx5st_cmd_if_box)) != 0xff)
return -1;
return 0;
}
static unsigned int mlx5st_cmd_setup_mbox_chain(struct vfio_pci_device *device,
struct mlx5st_mbox_entry *mbox,
unsigned int nblocks, u8 token)
{
unsigned int i;
for (i = 0; i < nblocks; i++) {
struct mlx5st_cmd_if_box *blk = &mbox[i].block;
u64 next_iova;
memset(blk, 0, sizeof(struct mlx5st_cmd_if_box));
MLX5_SET(cmd_if_box, blk, block_number, i);
MLX5_SET(cmd_if_box, blk, token, token);
if (i < nblocks - 1) {
next_iova = to_iova(device, &mbox[i + 1]);
MLX5_SET(cmd_if_box, blk, next_pointer_63_32,
next_iova >> 32);
MLX5_SET(cmd_if_box, blk, next_pointer_31_10,
(u32)next_iova >> 10);
}
}
return nblocks;
}
static void mlx5st_cmd_copy_to_mbox(struct mlx5st_mbox_entry *mbox,
const void *data, unsigned int len)
{
const u8 *src = data;
unsigned int i = 0;
while (len > 0) {
unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ?
len :
MLX5_CMD_DATA_BLOCK_SIZE;
memcpy(MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data),
src, chunk);
src += chunk;
len -= chunk;
i++;
}
}
static void mlx5st_cmd_copy_from_mbox(void *data,
const struct mlx5st_mbox_entry *mbox,
unsigned int len)
{
unsigned int i = 0;
u8 *dst = data;
while (len > 0) {
unsigned int chunk = len < MLX5_CMD_DATA_BLOCK_SIZE ?
len :
MLX5_CMD_DATA_BLOCK_SIZE;
memcpy(dst,
MLX5_ADDR_OF(cmd_if_box, &mbox[i].block, mailbox_data),
chunk);
dst += chunk;
len -= chunk;
i++;
}
}
/* Forward declaration — cmd_exec polls events during command wait */
static void mlx5st_process_events(struct mlx5st_device *dev);
static const char *mlx5st_cmd_name(u16 opcode)
{
switch (opcode) {
case MLX5_CMD_OP_QUERY_HCA_CAP: return "QUERY_HCA_CAP";
case MLX5_CMD_OP_INIT_HCA: return "INIT_HCA";
case MLX5_CMD_OP_TEARDOWN_HCA: return "TEARDOWN_HCA";
case MLX5_CMD_OP_ENABLE_HCA: return "ENABLE_HCA";
case MLX5_CMD_OP_DISABLE_HCA: return "DISABLE_HCA";
case MLX5_CMD_OP_QUERY_PAGES: return "QUERY_PAGES";
case MLX5_CMD_OP_MANAGE_PAGES: return "MANAGE_PAGES";
case MLX5_CMD_OP_SET_HCA_CAP: return "SET_HCA_CAP";
case MLX5_CMD_OP_SET_ISSI: return "SET_ISSI";
case MLX5_CMD_OP_CREATE_MKEY: return "CREATE_MKEY";
case MLX5_CMD_OP_DESTROY_MKEY: return "DESTROY_MKEY";
case MLX5_CMD_OP_CREATE_EQ: return "CREATE_EQ";
case MLX5_CMD_OP_DESTROY_EQ: return "DESTROY_EQ";
case MLX5_CMD_OP_CREATE_CQ: return "CREATE_CQ";
case MLX5_CMD_OP_DESTROY_CQ: return "DESTROY_CQ";
case MLX5_CMD_OP_CREATE_QP: return "CREATE_QP";
case MLX5_CMD_OP_DESTROY_QP: return "DESTROY_QP";
case MLX5_CMD_OP_RST2INIT_QP: return "RST2INIT_QP";
case MLX5_CMD_OP_INIT2RTR_QP: return "INIT2RTR_QP";
case MLX5_CMD_OP_RTR2RTS_QP: return "RTR2RTS_QP";
case MLX5_CMD_OP_ALLOC_PD: return "ALLOC_PD";
case MLX5_CMD_OP_DEALLOC_PD: return "DEALLOC_PD";
case MLX5_CMD_OP_ALLOC_UAR: return "ALLOC_UAR";
case MLX5_CMD_OP_DEALLOC_UAR: return "DEALLOC_UAR";
default: return "UNKNOWN";
}
}
/*
* Post a command on a given slot: fill the cmd_queue_entry, set up mailbox
* chains, compute signatures, hand ownership to FW, and ring the doorbell.
*/
static void mlx5st_cmd_post(struct mlx5st_device *dev,
struct mlx5st_cmd_queue_entry *cmd,
struct mlx5st_mbox_entry *in_mbox,
struct mlx5st_mbox_entry *out_mbox,
void *in, unsigned int ilen, unsigned int olen,
u32 doorbell)
{
struct vfio_pci_device *device = dev->device;
unsigned int in_remain, out_remain, in_nblk, out_nblk;
unsigned int i;
void *cin, *cout;
u8 token;
/* Rotating non-zero token ties cmd entry to its mailbox blocks */
token = ++dev->cmd_token;
if (!token)
token = ++dev->cmd_token;
in_remain = ilen > MLX5_CMD_INLINE_SZ ? ilen - MLX5_CMD_INLINE_SZ : 0;
out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0;
in_nblk = (in_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) /
MLX5_CMD_DATA_BLOCK_SIZE;
out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) /
MLX5_CMD_DATA_BLOCK_SIZE;
/* Set up mailbox chains */
if (in_nblk > 0) {
mlx5st_cmd_setup_mbox_chain(device, in_mbox, in_nblk, token);
mlx5st_cmd_copy_to_mbox(in_mbox,
(u8 *)in + MLX5_CMD_INLINE_SZ,
in_remain);
}
if (out_nblk > 0)
mlx5st_cmd_setup_mbox_chain(device, out_mbox, out_nblk, token);
/* Copy inline input */
cin = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_input_inline_data);
memset(cin, 0, MLX5_CMD_INLINE_SZ);
memcpy(cin, in, ilen < MLX5_CMD_INLINE_SZ ? ilen : MLX5_CMD_INLINE_SZ);
MLX5_SET(cmd_queue_entry, cmd, input_length, ilen);
MLX5_SET(cmd_queue_entry, cmd, token, token);
/* Zero inline output */
cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data);
memset(cout, 0, MLX5_CMD_INLINE_SZ);
MLX5_SET(cmd_queue_entry, cmd, output_length, olen);
/*
* Compute signatures: mailbox blocks first, then cmd_queue_entry.
* The sig must cover the final state including ownership=0x1, but
* we must not set ownership until after the sig is in place -
* XOR in the 0x1 without storing it to memory.
*/
for (i = 0; i < in_nblk; i++)
mlx5st_cmd_calc_block_sig(&in_mbox[i].block);
for (i = 0; i < out_nblk; i++)
mlx5st_cmd_calc_block_sig(&out_mbox[i].block);
MLX5_SET(cmd_queue_entry, cmd, signature, 0);
MLX5_SET(cmd_queue_entry, cmd, signature,
~(xor8_buf(cmd, 0, sizeof(struct mlx5st_cmd_queue_entry)) ^
0x1));
/* Ensure all cmd data (including sig) is visible, then hand to FW */
dma_wmb();
MLX5_SET_ONCE(cmd_queue_entry, cmd, ownership, 1);
/* Ring doorbell */
MLX5_SET_MMIO(initial_seg, dev->init_seg, command_doorbell_vector,
doorbell);
}
static void mlx5st_cmd_exec(struct mlx5st_device *dev, void *in,
unsigned int ilen, void *out, unsigned int olen)
{
struct mlx5st_cmd_queue_entry *cmd = dev->cmd_lay;
unsigned int out_remain, out_nblk;
struct timespec start, now;
unsigned int elapsed;
unsigned int i;
void *cout;
mlx5st_cmd_post(dev, cmd, dev->cmd_in_mbox, dev->cmd_out_mbox, in,
ilen, olen, 1);
out_remain = olen > MLX5_CMD_INLINE_SZ ? olen - MLX5_CMD_INLINE_SZ : 0;
out_nblk = (out_remain + MLX5_CMD_DATA_BLOCK_SIZE - 1) /
MLX5_CMD_DATA_BLOCK_SIZE;
/* Poll for completion — also process EQ events for PF page requests */
clock_gettime(CLOCK_MONOTONIC, &start);
for (;;) {
if (!MLX5_GET_ONCE(cmd_queue_entry, cmd, ownership))
break;
if (dev->have_eq)
mlx5st_process_events(dev);
sched_yield();
clock_gettime(CLOCK_MONOTONIC, &now);
elapsed = (now.tv_sec - start.tv_sec) * 1000 +
(now.tv_nsec - start.tv_nsec) / 1000000;
if (elapsed > MLX5_CMD_TIMEOUT_MS)
VFIO_FAIL("cmd timeout after %d ms", elapsed);
}
/* Ensure output data reads happen after ownership is seen clear */
dma_rmb();
/* Verify output signatures when FW has checksums enabled */
if (dev->cmd_sig_enabled) {
if (xor8_buf(cmd, 0,
sizeof(struct mlx5st_cmd_queue_entry)) != 0xff)
VFIO_FAIL("cmd output signature mismatch");
for (i = 0; i < out_nblk; i++) {
if (mlx5st_cmd_verify_block_sig(
&dev->cmd_out_mbox[i].block))
VFIO_FAIL("cmd output mailbox block %d signature mismatch",
i);
}
}
/* Copy output: inline first */
cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data);
memcpy(out, cout, olen < MLX5_CMD_INLINE_SZ ? olen : MLX5_CMD_INLINE_SZ);
/* Copy remaining from output mailbox chain */
if (out_remain > 0)
mlx5st_cmd_copy_from_mbox((u8 *)out + MLX5_CMD_INLINE_SZ,
dev->cmd_out_mbox, out_remain);
/* Check command status */
if (MLX5_GET(enable_hca_out, out, status) != MLX5_CMD_STAT_OK)
VFIO_FAIL("%s: status=0x%x syndrome=0x%x",
mlx5st_cmd_name(MLX5_GET(enable_hca_in, in, opcode)),
MLX5_GET(enable_hca_out, out, status),
MLX5_GET(enable_hca_out, out, syndrome));
}
static struct mlx5st_cmd_queue_entry *
mlx5st_cmd_slot_init(struct mlx5st_device *dev, unsigned int slot,
struct mlx5st_mbox_entry *in_mbox,
struct mlx5st_mbox_entry *out_mbox)
{
struct vfio_pci_device *device = dev->device;
struct mlx5st_cmd_queue_entry *cmd =
&dev->cmd_queue[(slot << dev->cmd_log_stride) /
sizeof(struct mlx5st_cmd_queue_entry)];
u64 iova;
MLX5_SET(cmd_queue_entry, cmd, type,
MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT);
iova = to_iova(device, in_mbox);
MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_63_32,
iova >> 32);
MLX5_SET(cmd_queue_entry, cmd, input_mailbox_pointer_31_9, iova >> 9);
iova = to_iova(device, out_mbox);
MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_63_32,
iova >> 32);
MLX5_SET(cmd_queue_entry, cmd, output_mailbox_pointer_31_9,
iova >> 9);
return cmd;
}
static void mlx5st_cmd_init(struct mlx5st_device *dev)
{
struct mlx5st_initial_seg __iomem *seg = dev->init_seg;
struct vfio_pci_device *device = dev->device;
u16 cmdif_rev;
u8 log_sz;
u64 iova;
cmdif_rev = MLX5_GET_MMIO(initial_seg, seg, cmd_interface_rev);
VFIO_ASSERT_EQ(cmdif_rev, 5);
/* Read command queue geometry from BAR */
log_sz = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_size);
dev->cmd_log_stride = MLX5_GET_MMIO(initial_seg, seg, log_cmdq_stride);
dev->pages_slot = (1 << log_sz) - 1;
VFIO_ASSERT_LE((unsigned int)(1 << log_sz), 32u);
VFIO_ASSERT_GE((unsigned int)(1 << dev->cmd_log_stride),
(unsigned int)sizeof(struct mlx5st_cmd_queue_entry));
VFIO_ASSERT_LE((unsigned int)((dev->pages_slot + 1) <<
dev->cmd_log_stride),
(unsigned int)sizeof(dev->cmd_queue));
/* Set up slot 0 — regular commands */
dev->cmd_lay = mlx5st_cmd_slot_init(dev, 0, dev->cmd_in_mbox,
dev->cmd_out_mbox);
/* Set up pages slot — async MANAGE_PAGES */
dev->pages_cmd_lay = mlx5st_cmd_slot_init(dev, dev->pages_slot,
dev->pages_in_mbox,
dev->pages_out_mbox);
/* Write command queue page address to BAR0 */
iova = to_iova(device, dev->cmd_queue);
MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_63_32, iova >> 32);
MLX5_SET_MMIO(initial_seg, seg, cmdq_phy_addr_31_12, iova >> 12);
dev_dbg(device,
"Command interface initialized (cmdif_rev=5, log_sz=%u, log_stride=%u, pages_slot=%u)\n",
log_sz, dev->cmd_log_stride, dev->pages_slot);
}
/*
* FW pages: bitmap allocator + MANAGE_PAGES
*/
static void mlx5st_fw_pages_alloc(struct mlx5st_device *dev,
unsigned int npages, u64 *iovas)
{
struct vfio_pci_device *device = dev->device;
unsigned int found = 0;
unsigned int w, b;
u64 word;
for (w = 0; w < MAX_FW_PAGES / 64 && found < npages; w++) {
word = dev->fw_pages_bitmap[w];
for (b = 0; b < 64 && found < npages; b++) {
if (!(word & (1ULL << b))) {
unsigned int idx = w * 64 + b;
dev->fw_pages_bitmap[w] |= (1ULL << b);
iovas[found++] = to_iova(device,
dev->fw_pages[idx]);
}
}
}
VFIO_ASSERT_EQ(found, npages);
dev->fw_pages_given += npages;
}
static void mlx5st_fw_pages_free(struct mlx5st_device *dev,
unsigned int npages, const u64 *iovas)
{
struct vfio_pci_device *device = dev->device;
unsigned int i, idx;
u64 off;
for (i = 0; i < npages; i++) {
off = iovas[i] - to_iova(device, dev->fw_pages);
idx = off / MLX5_HW_PAGE_SIZE;
VFIO_ASSERT_TRUE(idx < MAX_FW_PAGES);
dev->fw_pages_bitmap[idx / 64] &= ~(1ULL << (idx % 64));
}
dev->fw_pages_given -= npages;
}
static void *mlx5st_build_manage_pages_give(u16 func_id, unsigned int npages,
const u64 *iovas,
unsigned int *out_inlen)
{
unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in) + npages * 8;
unsigned int i;
void *in;
in = calloc(1, inlen);
VFIO_ASSERT_NOT_NULL(in);
MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
MLX5_SET(manage_pages_in, in, op_mod,
MLX5_MANAGE_PAGES_IN_OP_MOD_ALLOCATION_SUCCESS);
MLX5_SET(manage_pages_in, in, function_id, func_id);
MLX5_SET(manage_pages_in, in, input_num_entries, npages);
for (i = 0; i < npages; i++)
MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, iovas[i]);
*out_inlen = inlen;
return in;
}
static void mlx5st_fw_pages_give_one(struct mlx5st_device *dev, u16 func_id,
unsigned int npages, u64 *iovas)
{
u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {};
unsigned int inlen;
void *in;
in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen);
mlx5st_cmd_exec(dev, in, inlen, out, sizeof(out));
free(in);
}
static void mlx5st_fw_pages_give(struct mlx5st_device *dev, u16 func_id,
unsigned int npages)
{
unsigned int remaining = npages;
u64 *iovas;
if (!npages)
return;
iovas = calloc(npages, sizeof(u64));
VFIO_ASSERT_NOT_NULL(iovas);
mlx5st_fw_pages_alloc(dev, npages, iovas);
/* Batch into chunks that fit in one mailbox */
for (unsigned int off = 0; remaining > 0;) {
unsigned int batch = remaining < MAX_FW_PAGES_PER_CMD ?
remaining :
MAX_FW_PAGES_PER_CMD;
mlx5st_fw_pages_give_one(dev, func_id, batch, iovas + off);
off += batch;
remaining -= batch;
}
dev_dbg(dev->device, "MANAGE_PAGES GIVE: %d pages to func_id=%u\n",
npages, func_id);
free(iovas);
}
static void mlx5st_fw_pages_satisfy(struct mlx5st_device *dev, int boot)
{
u32 qo[MLX5_ST_SZ_DW(query_pages_out)] = {};
u32 qi[MLX5_ST_SZ_DW(query_pages_in)] = {};
u16 func_id;
int npages;
MLX5_SET(query_pages_in, qi, opcode, MLX5_CMD_OP_QUERY_PAGES);
MLX5_SET(query_pages_in, qi, op_mod, boot ? 0x01 : 0x02);
mlx5st_cmd_exec(dev, qi, sizeof(qi), qo, sizeof(qo));
npages = MLX5_GET(query_pages_out, qo, num_pages);
func_id = MLX5_GET(query_pages_out, qo, function_id);
dev_dbg(dev->device, "QUERY_PAGES (%s): %d pages (func_id=%u)\n",
boot ? "boot" : "init", npages, func_id);
if (npages > 0) {
dev->fw_func_id = func_id;
mlx5st_fw_pages_give(dev, func_id, npages);
}
}
/*
* Async MANAGE_PAGES on the pages command slot.
*
* On PFs, firmware sends PAGE_REQUEST events via the EQ during command
* execution. We must respond with MANAGE_PAGES on a second command slot
* before the first (regular) command can complete.
*/
static void mlx5st_pages_slot_post(struct mlx5st_device *dev, void *in,
unsigned int ilen, unsigned int olen)
{
mlx5st_cmd_post(dev, dev->pages_cmd_lay, dev->pages_in_mbox,
dev->pages_out_mbox, in, ilen, olen,
1 << dev->pages_slot);
}
static void mlx5st_pages_slot_give(struct mlx5st_device *dev, u16 func_id,
unsigned int npages)
{
unsigned int inlen;
u64 *iovas;
void *in;
iovas = calloc(npages, sizeof(u64));
VFIO_ASSERT_NOT_NULL(iovas);
mlx5st_fw_pages_alloc(dev, npages, iovas);
in = mlx5st_build_manage_pages_give(func_id, npages, iovas, &inlen);
free(iovas);
mlx5st_pages_slot_post(dev, in, inlen,
MLX5_ST_SZ_BYTES(manage_pages_out));
dev->pages_slot_in_use = true;
dev->pages_slot_is_reclaim = false;
free(in);
dev_dbg(dev->device,
"PAGE_REQUEST: %d pages given async to func_id=%u\n",
npages, func_id);
}
static void mlx5st_pages_slot_reclaim(struct mlx5st_device *dev, u16 func_id,
unsigned int npages)
{
unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in);
unsigned int outlen =
MLX5_ST_SZ_BYTES(manage_pages_out) + npages * 8;
void *in;
in = calloc(1, inlen);
VFIO_ASSERT_NOT_NULL(in);
MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
MLX5_SET(manage_pages_in, in, op_mod,
MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES);
MLX5_SET(manage_pages_in, in, function_id, func_id);
MLX5_SET(manage_pages_in, in, input_num_entries, npages);
mlx5st_pages_slot_post(dev, in, inlen, outlen);
dev->pages_slot_in_use = true;
dev->pages_slot_is_reclaim = true;
dev->pages_reclaim_npages = npages;
free(in);
dev_dbg(dev->device,
"PAGE_REQUEST: reclaim %d pages async from func_id=%u\n",
npages, func_id);
}
static void mlx5st_pages_slot_kick(struct mlx5st_device *dev)
{
unsigned int batch;
if (dev->pages_slot_in_use)
return;
if (dev->pages_pending_give) {
batch = dev->pages_pending_give < MAX_FW_PAGES_PER_CMD ?
dev->pages_pending_give :
MAX_FW_PAGES_PER_CMD;
dev->pages_pending_give -= batch;
mlx5st_pages_slot_give(dev, dev->pages_pending_func_id, batch);
} else if (dev->pages_pending_reclaim) {
batch = dev->pages_pending_reclaim < MAX_FW_PAGES_PER_CMD ?
dev->pages_pending_reclaim :
MAX_FW_PAGES_PER_CMD;
dev->pages_pending_reclaim -= batch;
mlx5st_pages_slot_reclaim(dev, dev->pages_pending_func_id,
batch);
}
}
static void mlx5st_fw_pages_give_async(struct mlx5st_device *dev,
u16 func_id, unsigned int npages)
{
if (!npages)
return;
dev->pages_pending_give += npages;
dev->pages_pending_func_id = func_id;
mlx5st_pages_slot_kick(dev);
}
static void mlx5st_fw_pages_reclaim_async(struct mlx5st_device *dev,
u16 func_id, unsigned int npages)
{
dev->pages_pending_reclaim += npages;
dev->pages_pending_func_id = func_id;
mlx5st_pages_slot_kick(dev);
}
static void mlx5st_pages_slot_complete(struct mlx5st_device *dev)
{
struct mlx5st_cmd_queue_entry *cmd = dev->pages_cmd_lay;
void *cout;
dma_rmb();
cout = MLX5_ADDR_OF(cmd_queue_entry, cmd, command_output_inline_data);
if (MLX5_GET(enable_hca_out, cout, status) != MLX5_CMD_STAT_OK)
VFIO_FAIL("async MANAGE_PAGES failed: status=0x%x syndrome=0x%x",
MLX5_GET(enable_hca_out, cout, status),
MLX5_GET(enable_hca_out, cout, syndrome));
if (dev->pages_slot_is_reclaim) {
unsigned int outlen = MLX5_ST_SZ_BYTES(manage_pages_out) +
dev->pages_reclaim_npages * 8;
unsigned int num_claimed;
unsigned int i;
void *out;
u64 *iovas;
out = calloc(1, outlen);
iovas = calloc(dev->pages_reclaim_npages, sizeof(u64));
VFIO_ASSERT_NOT_NULL(out);
VFIO_ASSERT_NOT_NULL(iovas);
/* Copy inline output */
memcpy(out, cout, MLX5_CMD_INLINE_SZ);
if (outlen > MLX5_CMD_INLINE_SZ)
mlx5st_cmd_copy_from_mbox(
(u8 *)out + MLX5_CMD_INLINE_SZ,
dev->pages_out_mbox,
outlen - MLX5_CMD_INLINE_SZ);
num_claimed =
MLX5_GET(manage_pages_out, out, output_num_entries);
for (i = 0; i < num_claimed; i++)
iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas,
i);
mlx5st_fw_pages_free(dev, num_claimed, iovas);
dev_dbg(dev->device, "PAGE_REQUEST: reclaimed %d pages\n",
num_claimed);
free(iovas);
free(out);
}
dev->pages_slot_in_use = false;
mlx5st_pages_slot_kick(dev);
}
/*
* UAR alloc/dealloc
*/
static void mlx5st_alloc_uar(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {};
u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {};
MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->uar_page = MLX5_GET(alloc_uar_out, out, uar);
dev->uar_base = (u8 __iomem*)dev->bar0 + dev->uar_page * MLX5_HW_PAGE_SIZE;
dev->uar_bf_offset = MLX5_BF_OFFSET;
dev_dbg(dev->device,
"Allocated UAR page_id=%u, doorbell offset=0x%x\n",
dev->uar_page,
dev->uar_page * MLX5_HW_PAGE_SIZE + MLX5_BF_OFFSET);
}
static void mlx5st_dealloc_uar(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {};
u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {};
MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);
MLX5_SET(dealloc_uar_in, in, uar, dev->uar_page);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* EQ infrastructure
*/
static struct mlx5st_eqe *mlx5st_eq_get_eqe(struct mlx5st_device *dev, u32 cc)
{
u32 ci = dev->eq_cons_index + cc;
struct mlx5st_eqe *eqe = &dev->eq_buf[ci % EQ_NENT];
u8 owner = MLX5_GET_ONCE(eqe, eqe, owner);
u8 expected = !!(ci & EQ_NENT);
if (owner != expected)
return NULL;
dma_rmb();
return eqe;
}
static void mlx5st_eq_update_ci(struct mlx5st_device *dev, u32 cc, bool arm)
{
u32 val;
dev->eq_cons_index += cc;
val = (dev->eq_cons_index & 0xffffff) | (dev->eqn << 24);
iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET +
(arm ? 0 : 8));
}
static void mlx5st_create_eq(struct mlx5st_device *dev)
{
struct vfio_pci_device *device = dev->device;
u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {};
u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {};
struct mlx5_ifc_eqc_bits *eqc;
unsigned int i;
__be64 *pas;
/* Initialize EQE owner bits */
for (i = 0; i < EQ_NENT; i++) {
struct mlx5st_eqe *eqe = &dev->eq_buf[i];
MLX5_SET_ONCE(eqe, eqe, owner, 1);
}
MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
/* Subscribe to CMD completions and PAGE_REQUEST events */
MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, 0,
(1ULL << MLX5_EVENT_TYPE_CMD) |
(1ULL << MLX5_EVENT_TYPE_PAGE_REQUEST));
eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
MLX5_SET(eqc, eqc, log_eq_size, LOG_EQ_SIZE);
MLX5_SET(eqc, eqc, uar_page, dev->uar_page);
pas = MLX5_ADDR_OF(create_eq_in, in, pas);
VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->eq_buf, pas), 0u);
MLX5_SET(eqc, eqc, log_page_size, 0);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->eqn = MLX5_GET(create_eq_out, out, eq_number);
dev->eq_cons_index = 0;
mlx5st_eq_update_ci(dev, 0, 0);
dev->have_eq = true;
dev_dbg(device, "Created EQ: eqn=%u, %d entries (CMD+PAGE_REQUEST)\n",
dev->eqn, EQ_NENT);
}
static void mlx5st_destroy_eq(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {};
u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
MLX5_SET(destroy_eq_in, in, eq_number, dev->eqn);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* Drain all pending EQ events. Dispatches PAGE_REQUEST to the async pages
* slot and CMD completions to the pages slot completion handler.
*/
static void mlx5st_process_events(struct mlx5st_device *dev)
{
struct mlx5st_eqe *eqe;
u32 cc = 0;
while ((eqe = mlx5st_eq_get_eqe(dev, cc))) {
u8 type = MLX5_GET(eqe, eqe, event_type);
switch (type) {
case MLX5_EVENT_TYPE_PAGE_REQUEST: {
void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data);
u16 func_id = MLX5_GET(pages_req_event, evdata,
function_id);
s32 npages = (s32)MLX5_GET(pages_req_event, evdata,
num_pages);
/*
* The selftest doesn't use more than one func_id so a
* simple counter approach is possible.
*/
if (dev->pages_func_id_seen)
VFIO_ASSERT_EQ(func_id,
dev->pages_pending_func_id);
dev->pages_func_id_seen = true;
if (npages > 0)
mlx5st_fw_pages_give_async(dev, func_id,
npages);
else if (npages < 0)
mlx5st_fw_pages_reclaim_async(dev, func_id,
-npages);
break;
}
case MLX5_EVENT_TYPE_CMD: {
void *evdata = MLX5_ADDR_OF(eqe, eqe, event_data);
u32 vector = MLX5_GET(cmd_inter_comp_event, evdata,
command_completion_vector);
if (vector & (1U << dev->pages_slot))
mlx5st_pages_slot_complete(dev);
break;
}
default:
break;
}
cc++;
}
if (cc)
mlx5st_eq_update_ci(dev, cc, 0);
}
/*
* MSI EQ — dedicated EQ for CQ completion events that fires MSI-X.
* Separate from the cmd/pages EQ so that only CQ completions (from
* send_msi or memcpy) trigger the interrupt vector.
*/
static void mlx5st_msi_eq_drain(struct mlx5st_device *dev)
{
u32 cc = 0;
u32 val;
while (cc < MSI_EQ_NENT) {
u32 ci = dev->msi_eq_cons_index + cc;
struct mlx5st_eqe *eqe =
&dev->msi_eq_buf[ci % MSI_EQ_NENT];
if (MLX5_GET_ONCE(eqe, eqe, owner) != !!(ci & MSI_EQ_NENT))
break;
cc++;
}
/* Update consumer index and re-arm for next interrupt */
dev->msi_eq_cons_index += cc;
val = (dev->msi_eq_cons_index & 0xffffff) | (dev->msi_eqn << 24);
iowrite32be(val, (u8 __iomem *)dev->uar_base + MLX5_EQ_DOORBELL_OFFSET);
}
static void mlx5st_create_msi_eq(struct mlx5st_device *dev)
{
struct vfio_pci_device *device = dev->device;
u64 in[MLX5_ST_SZ_QW(create_eq_in) + 1] = {};
u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {};
struct mlx5_ifc_eqc_bits *eqc;
unsigned int i;
__be64 *pas;
/* Initialize EQE owner bits */
for (i = 0; i < MSI_EQ_NENT; i++) {
struct mlx5st_eqe *eqe = &dev->msi_eq_buf[i];
MLX5_SET_ONCE(eqe, eqe, owner, 1);
}
MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
/*
* No event_bitmask — completion events are routed to this EQ via
* the CQ's c_eqn field, not through CREATE_EQ subscription.
*/
eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
MLX5_SET(eqc, eqc, log_eq_size, LOG_MSI_EQ_SIZE);
MLX5_SET(eqc, eqc, uar_page, dev->uar_page);
MLX5_SET(eqc, eqc, intr, MSI_VECTOR);
pas = MLX5_ADDR_OF(create_eq_in, in, pas);
VFIO_ASSERT_EQ(mlx5st_fill_pas(device, dev->msi_eq_buf, pas), 0u);
MLX5_SET(eqc, eqc, log_page_size, 0);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->msi_eqn = MLX5_GET(create_eq_out, out, eq_number);
dev->msi_eq_cons_index = 0;
dev->have_msi_eq = true;
mlx5st_msi_eq_drain(dev);
dev_dbg(device,
"Created MSI EQ: eqn=%u, %d entries (COMP), vector=%d\n",
dev->msi_eqn, MSI_EQ_NENT, MSI_VECTOR);
}
static void mlx5st_destroy_msi_eq(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {};
u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
MLX5_SET(destroy_eq_in, in, eq_number, dev->msi_eqn);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* HCA init / teardown
*/
#define FW_INIT_TIMEOUT_MS 120000
#define FW_INIT_WAIT_MS 200
static void mlx5st_wait_fw_init(struct mlx5st_device *dev)
{
struct timespec start, now;
unsigned int elapsed;
clock_gettime(CLOCK_MONOTONIC, &start);
while (MLX5_GET_MMIO(initial_seg, dev->init_seg, initializing)) {
usleep(FW_INIT_WAIT_MS * 1000);
clock_gettime(CLOCK_MONOTONIC, &now);
elapsed = (now.tv_sec - start.tv_sec) * 1000 +
(now.tv_nsec - start.tv_nsec) / 1000000;
if (elapsed > FW_INIT_TIMEOUT_MS)
VFIO_FAIL("FW init timeout after %d ms", elapsed);
}
}
static void mlx5st_set_issi(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(set_issi_out)] = {};
u32 in[MLX5_ST_SZ_DW(set_issi_in)] = {};
MLX5_SET(set_issi_in, in, opcode, MLX5_CMD_OP_SET_ISSI);
MLX5_SET(set_issi_in, in, current_issi, 1);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "SET_ISSI: OK (issi=1)\n");
}
static void mlx5st_set_hca_caps(struct mlx5st_device *dev)
{
u32 qout[MLX5_ST_SZ_DW(query_hca_cap_out)] = {};
u32 qin[MLX5_ST_SZ_DW(query_hca_cap_in)] = {};
u32 sout[MLX5_ST_SZ_DW(set_hca_cap_out)] = {};
u32 sin[MLX5_ST_SZ_DW(set_hca_cap_in)] = {};
struct mlx5_ifc_cmd_hca_cap_bits *set_hca_cap;
u32 max_checksum;
/* Query max caps to learn cmdif_checksum support */
MLX5_SET(query_hca_cap_in, qin, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
MLX5_SET(query_hca_cap_in, qin, op_mod,
(MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1));
mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout));
max_checksum = MLX5_GET(
cmd_hca_cap,
MLX5_ADDR_OF(query_hca_cap_out, qout, capability),
cmdif_checksum);
/* Query current caps as base for SET */
memset(qout, 0, sizeof(qout));
MLX5_SET(query_hca_cap_in, qin, op_mod,
(MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) |
HCA_CAP_OPMOD_GET_CUR);
mlx5st_cmd_exec(dev, qin, sizeof(qin), qout, sizeof(qout));
set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, sin, capability);
memcpy(set_hca_cap,
MLX5_ADDR_OF(query_hca_cap_out, qout, capability),
MLX5_ST_SZ_BYTES(cmd_hca_cap));
MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, max_checksum);
MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, 0);
MLX5_SET(set_hca_cap_in, sin, opcode, MLX5_CMD_OP_SET_HCA_CAP);
MLX5_SET(set_hca_cap_in, sin, op_mod,
MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1);
mlx5st_cmd_exec(dev, sin, sizeof(sin), sout, sizeof(sout));
dev->cmd_sig_enabled = max_checksum == 0x3;
dev_dbg(dev->device, "SET_HCA_CAP: OK (cmdif_checksum=%u)\n",
max_checksum);
}
static void mlx5st_hca_init(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {};
u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {};
mlx5st_wait_fw_init(dev);
dev_dbg(dev->device, "Firmware ready\n");
MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "ENABLE_HCA: OK\n");
mlx5st_set_issi(dev);
mlx5st_fw_pages_satisfy(dev, 1);
mlx5st_set_hca_caps(dev);
mlx5st_fw_pages_satisfy(dev, 0);
memset(in, 0, sizeof(in));
memset(out, 0, sizeof(out));
MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "INIT_HCA: OK\n");
/*
* Create EQ immediately after INIT_HCA so PAGE_REQUEST events
* are captured during all subsequent commands.
*/
mlx5st_alloc_uar(dev);
mlx5st_create_eq(dev);
}
static void mlx5st_disable_hca(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {};
u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {};
MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static void mlx5st_fw_pages_reclaim(struct mlx5st_device *dev, u16 func_id)
{
unsigned int npages = dev->fw_pages_given;
unsigned int total_claimed = 0;
while (npages > 0) {
unsigned int batch = npages < MAX_FW_PAGES_PER_CMD ?
npages :
MAX_FW_PAGES_PER_CMD;
unsigned int outlen =
MLX5_ST_SZ_BYTES(manage_pages_out) + batch * 8;
unsigned int inlen = MLX5_ST_SZ_BYTES(manage_pages_in);
unsigned int num_claimed;
unsigned int i;
void *in, *out;
u64 *iovas;
in = calloc(1, inlen);
out = calloc(1, outlen);
iovas = calloc(batch, sizeof(u64));
VFIO_ASSERT_NOT_NULL(in);
VFIO_ASSERT_NOT_NULL(out);
VFIO_ASSERT_NOT_NULL(iovas);
MLX5_SET(manage_pages_in, in, opcode,
MLX5_CMD_OP_MANAGE_PAGES);
MLX5_SET(manage_pages_in, in, op_mod,
MLX5_MANAGE_PAGES_IN_OP_MOD_HCA_RETURN_PAGES);
MLX5_SET(manage_pages_in, in, function_id, func_id);
MLX5_SET(manage_pages_in, in, input_num_entries, batch);
mlx5st_cmd_exec(dev, in, inlen, out, outlen);
num_claimed =
MLX5_GET(manage_pages_out, out, output_num_entries);
for (i = 0; i < num_claimed; i++)
iovas[i] = MLX5_ARRAY_GET64(manage_pages_out, out, pas,
i);
mlx5st_fw_pages_free(dev, num_claimed, iovas);
total_claimed += num_claimed;
npages -= num_claimed;
free(iovas);
free(in);
free(out);
if (!num_claimed && !dev->fw_pages_given)
break;
if (!num_claimed)
VFIO_FAIL("MANAGE_PAGES RECLAIM: FW returned 0 but %d pages still given",
dev->fw_pages_given);
}
dev_dbg(dev->device,
"MANAGE_PAGES RECLAIM: %d pages (%d still given)\n",
total_claimed, dev->fw_pages_given);
}
static void mlx5st_hca_teardown(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {};
u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {};
/* Drain async pages slot, then stop EQ processing */
while (dev->pages_slot_in_use) {
if (!MLX5_GET_ONCE(cmd_queue_entry, dev->pages_cmd_lay,
ownership))
mlx5st_pages_slot_complete(dev);
else
sched_yield();
}
dev->have_eq = false;
if (dev->eqn) {
mlx5st_destroy_eq(dev);
dev->eqn = 0;
}
if (dev->uar_page) {
mlx5st_dealloc_uar(dev);
dev->uar_page = 0;
}
dev_dbg(dev->device, " hca_teardown: TEARDOWN_HCA\n");
MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
MLX5_SET(teardown_hca_in, in, profile,
MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
if (dev->fw_pages_given > 0) {
dev_dbg(dev->device, " hca_teardown: reclaim %d pages\n",
dev->fw_pages_given);
mlx5st_fw_pages_reclaim(dev, dev->fw_func_id);
}
dev_dbg(dev->device, " hca_teardown: DISABLE_HCA\n");
mlx5st_disable_hca(dev);
}
/*
* Query capabilities
*/
static void mlx5st_query_fl_caps(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {};
u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {};
bool fl_roce_en, fl_roce_dis;
/* Query RoCE capabilities */
MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
MLX5_SET(query_hca_cap_in, in, op_mod,
(MLX5_SET_HCA_CAP_OP_MOD_ROCE << 1) | HCA_CAP_OPMOD_GET_CUR);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
fl_roce_en = MLX5_GET(query_hca_cap_out, out,
capability.roce_cap.fl_rc_qp_when_roce_enabled);
fl_roce_dis = MLX5_GET(query_hca_cap_out, out,
capability.roce_cap.fl_rc_qp_when_roce_disabled);
/* Also check general caps */
memset(in, 0, sizeof(in));
memset(out, 0, sizeof(out));
MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
MLX5_SET(query_hca_cap_in, in, op_mod,
(MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1) |
HCA_CAP_OPMOD_GET_CUR);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
fl_roce_dis |=
MLX5_GET(query_hca_cap_out, out,
capability.cmd_hca_cap.fl_rc_qp_when_roce_disabled);
dev->fl_supported = fl_roce_en || fl_roce_dis;
dev->log_max_msg = MLX5_GET(query_hca_cap_out, out,
capability.cmd_hca_cap.log_max_msg);
dev_dbg(dev->device,
"HCA capabilities: fl_roce_enabled=%d fl_roce_disabled=%d log_max_msg=%u\n",
fl_roce_en, fl_roce_dis, dev->log_max_msg);
VFIO_ASSERT_TRUE(dev->fl_supported,
"Force-loopback not supported on this device");
}
/*
* Resource allocation
*/
static void mlx5st_alloc_pd(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->pdn = MLX5_GET(alloc_pd_out, out, pd);
dev_dbg(dev->device, "Allocated PD pdn=%u\n", dev->pdn);
}
static void mlx5st_dealloc_pd(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {};
u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};
MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
MLX5_SET(dealloc_pd_in, in, pd, dev->pdn);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
static void mlx5st_create_mkey(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(create_mkey_out)] = {};
u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
struct mlx5_ifc_mkc_bits *mkc;
MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, length64, 1);
MLX5_SET(mkc, mkc, pd, dev->pdn);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, lr, 1);
MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, rw, 1);
MLX5_SET(mkc, mkc, rr, 1);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->mkey_index = MLX5_GET(create_mkey_out, out, mkey_index);
dev->global_lkey = mlx5st_idx_to_mkey(dev->mkey_index);
dev->global_rkey = dev->global_lkey;
dev_dbg(dev->device, "Created global PA-mode MKEY: lkey=0x%x\n",
dev->global_lkey);
}
static void mlx5st_destroy_mkey(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {};
u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
MLX5_SET(destroy_mkey_in, in, mkey_index, dev->mkey_index);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* CQ create/destroy
*/
static void mlx5st_create_cq(struct mlx5st_device *dev)
{
struct vfio_pci_device *device = dev->device;
u64 in[MLX5_ST_SZ_QW(create_cq_in) + 1] = {};
u32 out[MLX5_ST_SZ_DW(create_cq_out)] = {};
struct mlx5_ifc_cqc_bits *cqc;
unsigned int i;
__be64 *pas;
/* Initialize CQEs before CREATE_CQ: opcode=0xF, owner=1 */
for (i = 0; i < CQ_CQE_CNT; i++) {
struct mlx5st_cqe64 *cqe = &dev->cq_buf[i];
MLX5_SET(cqe64, cqe, opcode, 0xF);
MLX5_SET_ONCE(cqe64, cqe, owner, 1);
}
MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ);
cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
MLX5_SET(cqc, cqc, log_cq_size, LOG_CQ_SIZE);
MLX5_SET(cqc, cqc, uar_page, dev->uar_page);
MLX5_SET(cqc, cqc, c_eqn_or_apu_element, dev->msi_eqn);
MLX5_SET(cqc, cqc, cqe_sz, 0);
pas = MLX5_ADDR_OF(create_cq_in, in, pas);
MLX5_SET(cqc, cqc, page_offset, mlx5st_fill_pas(device, dev->cq_buf, pas));
MLX5_SET(cqc, cqc, log_page_size, 0);
MLX5_SET64(cqc, cqc, dbr_addr, to_iova(device, &dev->cq_dbrec));
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->cqn = MLX5_GET(create_cq_out, out, cqn);
dev->cq_ci = 0;
dev_dbg(device, "Created CQ: cqn=%u, %d entries\n", dev->cqn,
CQ_CQE_CNT);
}
static void mlx5st_destroy_cq(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {};
u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {};
MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ);
MLX5_SET(destroy_cq_in, in, cqn, dev->cqn);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* Arm CQ for event generation. The CQ event delivery state machine is
* single-shot: after generating one EQE the CQ enters "Fired" state and
* won't generate another until re-armed via ARM_NEXT. Both the CQ doorbell
* record and the UAR CQ doorbell register must be written.
*/
static void mlx5st_arm_cq(struct mlx5st_device *dev)
{
u32 sn = dev->cq_arm_sn & 3;
u32 ci = dev->cq_ci & 0xffffff;
u64 doorbell;
/* Update CQ doorbell record arm word */
WRITE_ONCE(dev->cq_dbrec.send_counter,
cpu_to_be32(sn << 28 | ci));
/* Ring CQ doorbell register, iowrite has an internal dma_wmb() */
doorbell = ((u64)(sn << 28 | ci) << 32) | dev->cqn;
iowrite64be(doorbell,
(u8 __iomem *)dev->uar_base + MLX5_CQ_DOORBELL_OFFSET);
dev->cq_arm_sn++;
}
/*
* QP create/destroy
*/
static void mlx5st_create_qp(struct mlx5st_device *dev)
{
struct vfio_pci_device *device = dev->device;
u64 in[MLX5_ST_SZ_QW(create_qp_in) + 1] = {};
u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
struct mlx5_ifc_qpc_bits *qpc;
__be64 *pas;
MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
MLX5_SET(qpc, qpc, st, MLX5_QPC_ST_RC);
MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED);
MLX5_SET(qpc, qpc, pd, dev->pdn);
MLX5_SET(qpc, qpc, uar_page, dev->uar_page);
MLX5_SET(qpc, qpc, cqn_snd, dev->cqn);
MLX5_SET(qpc, qpc, cqn_rcv, dev->cqn);
MLX5_SET(qpc, qpc, log_sq_size, LOG_SQ_SIZE);
MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg);
MLX5_SET(qpc, qpc, rq_type, 0x3);
MLX5_SET(qpc, qpc, ts_format, 1);
pas = MLX5_ADDR_OF(create_qp_in, in, pas);
MLX5_SET(qpc, qpc, page_offset,
mlx5st_fill_pas(device, dev->sq_buf, pas));
MLX5_SET(qpc, qpc, log_page_size, 0);
MLX5_SET64(qpc, qpc, dbr_addr, to_iova(device, &dev->qp_dbrec));
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev->qpn = MLX5_GET(create_qp_out, out, qpn);
dev->sq_pi = 0;
dev_dbg(device, "Created QP: qpn=%u, RC, sq=%d wqes\n", dev->qpn,
SQ_WQE_CNT);
}
static void mlx5st_destroy_qp(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(destroy_qp_out)] = {};
u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
MLX5_SET(destroy_qp_in, in, qpn, dev->qpn);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
/*
* QP state transitions
*/
static void mlx5st_qp_rst2init(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(rst2init_qp_out)] = {};
u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {};
struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc);
MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP);
MLX5_SET(rst2init_qp_in, in, qpn, dev->qpn);
MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED);
MLX5_SET(qpc, qpc, rre, 1);
MLX5_SET(qpc, qpc, rwe, 1);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "QP RST->INIT\n");
}
static void mlx5st_qp_init2rtr(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(init2rtr_qp_out)] = {};
u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {};
struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc);
MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP);
MLX5_SET(init2rtr_qp_in, in, qpn, dev->qpn);
MLX5_SET(qpc, qpc, mtu, 3);
MLX5_SET(qpc, qpc, log_msg_max, dev->log_max_msg);
MLX5_SET(qpc, qpc, remote_qpn, dev->qpn);
MLX5_SET(qpc, qpc, min_rnr_nak, 12);
MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
MLX5_SET(qpc, qpc, primary_address_path.fl, 1);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "QP INIT->RTR (fl=1)\n");
}
static void mlx5st_qp_rtr2rts(struct mlx5st_device *dev)
{
u32 out[MLX5_ST_SZ_DW(rtr2rts_qp_out)] = {};
u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {};
struct mlx5_ifc_qpc_bits *qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc);
MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP);
MLX5_SET(rtr2rts_qp_in, in, qpn, dev->qpn);
MLX5_SET(qpc, qpc, log_ack_req_freq, 0);
MLX5_SET(qpc, qpc, retry_count, 7);
MLX5_SET(qpc, qpc, rnr_retry, 7);
MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 14);
mlx5st_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
dev_dbg(dev->device, "QP RTR->RTS\n");
}
/*
* Post RDMA Write WQE
*/
static void mlx5st_post_rdma_write(struct mlx5st_device *dev, u64 src_addr,
u32 src_lkey, u64 dst_addr, u32 dst_rkey,
u32 length, bool signaled)
{
struct mlx5st_send_wqe *wqe;
unsigned int idx;
idx = dev->sq_pi % SQ_WQE_CNT;
wqe = &dev->sq_buf[idx];
memset(wqe, 0, sizeof(*wqe));
MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, opcode, MLX5_OPCODE_RDMA_WRITE);
MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, wqe_index, dev->sq_pi);
MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, qp_or_sq, dev->qpn);
MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ds, MLX5_RDMA_WRITE_DS);
if (signaled)
MLX5_SET(wqe_ctrl_seg, &wqe->ctrl, ce, MLX5_WQE_CE_CQE_ALWAYS);
MLX5_SET64(wqe_raddr_seg, &wqe->raddr, raddr, dst_addr);
MLX5_SET(wqe_raddr_seg, &wqe->raddr, rkey, dst_rkey);
MLX5_SET(wqe_data_seg, &wqe->data, byte_count, length);
MLX5_SET(wqe_data_seg, &wqe->data, lkey, src_lkey);
MLX5_SET64(wqe_data_seg, &wqe->data, addr, src_addr);
dev->sq_pi++;
/* Ensure WQE is visible to device before doorbell record */
dma_wmb();
WRITE_ONCE(dev->qp_dbrec.send_counter,
cpu_to_be32(dev->sq_pi & 0xffff));
/*
* Ring doorbell: write first 8 bytes of ctrl to UAR BF register,
* iowrite has an internal dma_wmb() so the doorbell record will be
* visible.
*/
iowrite64be(be64_to_cpu(*(__be64 *)wqe),
(u8 __iomem *)dev->uar_base + dev->uar_bf_offset);
dev->uar_bf_offset ^= MLX5_BF_SIZE;
}
/*
* Poll CQ
*/
static int mlx5st_poll_cq_batch(struct mlx5st_device *dev,
unsigned int max_cqe)
{
unsigned int polled = 0;
while (polled < max_cqe) {
unsigned int idx = dev->cq_ci % CQ_CQE_CNT;
struct mlx5st_cqe64 *cqe = &dev->cq_buf[idx];
u8 owner, opcode;
owner = MLX5_GET_ONCE(cqe64, cqe, owner);
if (owner != ((dev->cq_ci >> LOG_CQ_SIZE) & 1))
break;
dma_rmb();
opcode = MLX5_GET(cqe64, cqe, opcode);
dev->cq_ci++;
WRITE_ONCE(dev->cq_dbrec.recv_counter,
cpu_to_be32(dev->cq_ci & 0xffffff));
if (opcode == MLX5_CQE_REQ) {
dev->sq_ci =
(u16)(MLX5_GET(cqe64, cqe, wqe_counter) + 1);
polled++;
continue;
}
if (opcode == MLX5_CQE_REQ_ERR ||
opcode == MLX5_CQE_RESP_ERR) {
dev_dbg(dev->device,
"CQE error: opcode=0x%x syndrome=0x%x vendor=0x%x\n",
opcode,
MLX5_GET(cqe64, cqe, error_syndrome.syndrome),
MLX5_GET(cqe64, cqe,
error_syndrome.vendor_error_syndrome));
return -1;
}
dev_err(dev->device, "CQE unexpected opcode=0x%x\n", opcode);
return -1;
}
return polled;
}
static int mlx5st_poll_cq(struct mlx5st_device *dev, unsigned int timeout_ms)
{
struct timespec start, now;
unsigned int elapsed;
int ret;
clock_gettime(CLOCK_MONOTONIC, &start);
for (;;) {
ret = mlx5st_poll_cq_batch(dev, 1);
if (ret < 0)
return -1;
if (ret > 0)
return 0;
if (dev->have_eq)
mlx5st_process_events(dev);
clock_gettime(CLOCK_MONOTONIC, &now);
elapsed = (now.tv_sec - start.tv_sec) * 1000 +
(now.tv_nsec - start.tv_nsec) / 1000000;
if (elapsed > timeout_ms) {
dev_err(dev->device, "CQ poll timeout after %u ms\n",
timeout_ms);
return -1;
}
}
}
/*
* Data path setup/teardown helpers
*/
static void mlx5st_setup_datapath(struct mlx5st_device *dev)
{
mlx5st_create_cq(dev);
mlx5st_create_qp(dev);
mlx5st_qp_rst2init(dev);
mlx5st_qp_init2rtr(dev);
mlx5st_qp_rtr2rts(dev);
}
static void mlx5st_teardown_datapath(struct mlx5st_device *dev)
{
if (dev->qpn) {
mlx5st_destroy_qp(dev);
dev->qpn = 0;
}
if (dev->cqn) {
mlx5st_destroy_cq(dev);
dev->cqn = 0;
}
dev->sq_pi = 0;
dev->sq_ci = 0;
dev->cq_arm_sn = 0;
memset(&dev->qp_dbrec, 0, sizeof(dev->qp_dbrec));
memset(&dev->cq_dbrec, 0, sizeof(dev->cq_dbrec));
}
/*
* memcpy callbacks
*/
#define MLX5ST_MEMCPY_TIMEOUT_MS 60000
static void mlx5st_memcpy_start(struct vfio_pci_device *device,
iova_t src, iova_t dst, u64 size, u64 count)
{
struct mlx5st_device *dev = to_mlx5st(device);
u64 i;
for (i = 0; i < count; i++) {
bool signaled = (i == count - 1);
mlx5st_post_rdma_write(dev, src, dev->global_lkey, dst,
dev->global_rkey, size, signaled);
}
}
static int mlx5st_memcpy_wait(struct vfio_pci_device *device)
{
struct mlx5st_device *dev = to_mlx5st(device);
int ret;
ret = mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS);
if (ret) {
/*
* CQE error puts the QP in error state. Rebuild the data path
* so subsequent operations can succeed.
*/
mlx5st_teardown_datapath(dev);
mlx5st_setup_datapath(dev);
}
return ret;
}
/*
* send_msi callback — trigger CQE -> EQE -> MSI-X via a small RDMA Write.
*
* Both the CQ and MSI EQ use single-shot arming: the CQ must be armed so the
* CQE generates an EQE, and the MSI EQ must be armed so the EQE fires MSI-X.
*/
static void mlx5st_send_msi(struct vfio_pci_device *device)
{
struct mlx5st_device *dev = to_mlx5st(device);
/* Drain accumulated MSI EQ events and re-arm for next interrupt */
mlx5st_msi_eq_drain(dev);
/* Arm CQ so the next CQE generates an EQE on the MSI EQ */
mlx5st_arm_cq(dev);
/* Post a signaled RDMA Write to trigger CQE -> EQE -> MSI-X */
mlx5st_post_rdma_write(dev,
to_iova(device, &dev->send_msi_src),
dev->global_lkey,
to_iova(device, &dev->send_msi_dst),
dev->global_rkey,
sizeof(dev->send_msi_src), true);
/* Consume the CQE to avoid stale completions */
VFIO_ASSERT_EQ(mlx5st_poll_cq(dev, MLX5ST_MEMCPY_TIMEOUT_MS), 0);
}
/*
* Driver ops callbacks
*/
static void mlx5st_init(struct vfio_pci_device *device)
{
struct mlx5st_device *dev = to_mlx5st(device);
iova_t iova_align =
device->driver.region.iova % __alignof__(struct mlx5st_device);
VFIO_ASSERT_GE(device->driver.region.size, sizeof(*dev));
VFIO_ASSERT_EQ(iova_align, 0);
memset(dev, 0, sizeof(*dev));
dev->device = device;
dev->bar0 = device->bars[0].vaddr;
dev->init_seg = dev->bar0;
vfio_pci_config_writew(device, PCI_COMMAND,
vfio_pci_config_readw(device, PCI_COMMAND) |
PCI_COMMAND_MASTER);
mlx5st_wait_fw_init(dev);
mlx5st_cmd_init(dev);
mlx5st_hca_init(dev);
mlx5st_query_fl_caps(dev);
mlx5st_alloc_pd(dev);
mlx5st_create_mkey(dev);
/* MSI EQ must be created before CQ so CQ can reference its eqn */
mlx5st_create_msi_eq(dev);
mlx5st_setup_datapath(dev);
vfio_pci_msix_enable(device, MSI_VECTOR, 1);
device->driver.msi = MSI_VECTOR;
device->driver.max_memcpy_size = 1ULL << dev->log_max_msg;
device->driver.max_memcpy_count = SQ_WQE_CNT - 1;
dev_dbg(device, "mlx5 driver initialized\n");
}
static void mlx5st_remove(struct vfio_pci_device *device)
{
struct mlx5st_device *dev = to_mlx5st(device);
vfio_pci_msix_disable(device);
mlx5st_teardown_datapath(dev);
if (dev->have_msi_eq) {
mlx5st_destroy_msi_eq(dev);
dev->have_msi_eq = false;
}
dev_dbg(device, "teardown: destroy_mkey\n");
if (dev->mkey_index) {
mlx5st_destroy_mkey(dev);
dev->mkey_index = 0;
}
dev_dbg(device, "teardown: dealloc_pd\n");
if (dev->pdn) {
mlx5st_dealloc_pd(dev);
dev->pdn = 0;
}
dev_dbg(device, "teardown: hca_teardown\n");
mlx5st_hca_teardown(dev);
vfio_pci_config_writew(device, PCI_COMMAND,
vfio_pci_config_readw(device, PCI_COMMAND) &
~(u16)PCI_COMMAND_MASTER);
dev_dbg(device, "Teardown complete\n");
}
struct vfio_pci_driver_ops mlx5st_ops = {
.name = "mlx5",
.region_size = sizeof(struct mlx5st_device),
.probe = mlx5st_probe,
.init = mlx5st_init,
.remove = mlx5st_remove,
.memcpy_start = mlx5st_memcpy_start,
.memcpy_wait = mlx5st_memcpy_wait,
.send_msi = mlx5st_send_msi,
};