drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c

// SPDX-License-Identifier: GPL-2.0
/* Marvell IPSEC offload driver
 *
 * Copyright (C) 2024 Marvell.
 */

#include <net/xfrm.h>
#include <linux/netdevice.h>
#include <linux/bitfield.h>
#include <crypto/aead.h>
#include <crypto/gcm.h>

#include "otx2_common.h"
#include "otx2_struct.h"
#include "cn10k_ipsec.h"

static bool is_dev_support_ipsec_offload(struct pci_dev *pdev)
{
	return is_dev_cn10ka_b0(pdev) || is_dev_cn10kb(pdev);
}

static bool cn10k_cpt_device_set_inuse(struct otx2_nic *pf)
{
	enum cn10k_cpt_hw_state_e state;

	while (true) {
		state = atomic_cmpxchg(&pf->ipsec.cpt_state,
				       CN10K_CPT_HW_AVAILABLE,
				       CN10K_CPT_HW_IN_USE);
		if (state == CN10K_CPT_HW_AVAILABLE)
			return true;
		if (state == CN10K_CPT_HW_UNAVAILABLE)
			return false;

		mdelay(1);
	}
}

static void cn10k_cpt_device_set_available(struct otx2_nic *pf)
{
	atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_AVAILABLE);
}

static void  cn10k_cpt_device_set_unavailable(struct otx2_nic *pf)
{
	atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_UNAVAILABLE);
}

static int cn10k_outb_cptlf_attach(struct otx2_nic *pf)
{
	struct rsrc_attach *attach;
	int ret = -ENOMEM;

	mutex_lock(&pf->mbox.lock);
	/* Get memory to put this msg */
	attach = otx2_mbox_alloc_msg_attach_resources(&pf->mbox);
	if (!attach)
		goto unlock;

	attach->cptlfs = true;
	attach->modify = true;

	/* Send attach request to AF */
	ret = otx2_sync_mbox_msg(&pf->mbox);

unlock:
	mutex_unlock(&pf->mbox.lock);
	return ret;
}

static int cn10k_outb_cptlf_detach(struct otx2_nic *pf)
{
	struct rsrc_detach *detach;
	int ret = -ENOMEM;

	mutex_lock(&pf->mbox.lock);
	detach = otx2_mbox_alloc_msg_detach_resources(&pf->mbox);
	if (!detach)
		goto unlock;

	detach->partial = true;
	detach->cptlfs = true;

	/* Send detach request to AF */
	ret = otx2_sync_mbox_msg(&pf->mbox);

unlock:
	mutex_unlock(&pf->mbox.lock);
	return ret;
}

static int cn10k_outb_cptlf_alloc(struct otx2_nic *pf)
{
	struct cpt_lf_alloc_req_msg *req;
	int ret = -ENOMEM;

	mutex_lock(&pf->mbox.lock);
	req = otx2_mbox_alloc_msg_cpt_lf_alloc(&pf->mbox);
	if (!req)
		goto unlock;

	/* PF function */
	req->nix_pf_func = pf->pcifunc;
	/* Enable SE-IE Engine Group */
	req->eng_grpmsk = 1 << CN10K_DEF_CPT_IPSEC_EGRP;

	ret = otx2_sync_mbox_msg(&pf->mbox);

unlock:
	mutex_unlock(&pf->mbox.lock);
	return ret;
}

static void cn10k_outb_cptlf_free(struct otx2_nic *pf)
{
	mutex_lock(&pf->mbox.lock);
	otx2_mbox_alloc_msg_cpt_lf_free(&pf->mbox);
	otx2_sync_mbox_msg(&pf->mbox);
	mutex_unlock(&pf->mbox.lock);
}

static int cn10k_outb_cptlf_config(struct otx2_nic *pf)
{
	struct cpt_inline_ipsec_cfg_msg *req;
	int ret = -ENOMEM;

	mutex_lock(&pf->mbox.lock);
	req = otx2_mbox_alloc_msg_cpt_inline_ipsec_cfg(&pf->mbox);
	if (!req)
		goto unlock;

	req->dir = CPT_INLINE_OUTBOUND;
	req->enable = 1;
	req->nix_pf_func = pf->pcifunc;
	ret = otx2_sync_mbox_msg(&pf->mbox);
unlock:
	mutex_unlock(&pf->mbox.lock);
	return ret;
}

static void cn10k_outb_cptlf_iq_enable(struct otx2_nic *pf)
{
	u64 reg_val;

	/* Set Execution Enable of instruction queue */
	reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
	reg_val |= BIT_ULL(16);
	otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);

	/* Set iqueue's enqueuing */
	reg_val = otx2_read64(pf, CN10K_CPT_LF_CTL);
	reg_val |= BIT_ULL(0);
	otx2_write64(pf, CN10K_CPT_LF_CTL, reg_val);
}

static void cn10k_outb_cptlf_iq_disable(struct otx2_nic *pf)
{
	u32 inflight, grb_cnt, gwb_cnt;
	u32 nq_ptr, dq_ptr;
	int timeout = 20;
	u64 reg_val;
	int cnt;

	/* Disable instructions enqueuing */
	otx2_write64(pf, CN10K_CPT_LF_CTL, 0ull);

	/* Wait for instruction queue to become empty.
	 * CPT_LF_INPROG.INFLIGHT count is zero
	 */
	do {
		reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
		inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
		if (!inflight)
			break;

		usleep_range(10000, 20000);
		if (timeout-- < 0) {
			netdev_err(pf->netdev, "Timeout to cleanup CPT IQ\n");
			break;
		}
	} while (1);

	/* Disable executions in the LF's queue,
	 * the queue should be empty at this point
	 */
	reg_val &= ~BIT_ULL(16);
	otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);

	/* Wait for instruction queue to become empty */
	cnt = 0;
	do {
		reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
		if (reg_val & BIT_ULL(31))
			cnt = 0;
		else
			cnt++;
		reg_val = otx2_read64(pf, CN10K_CPT_LF_Q_GRP_PTR);
		nq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
		dq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
	} while ((cnt < 10) && (nq_ptr != dq_ptr));

	cnt = 0;
	do {
		reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
		inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
		grb_cnt = FIELD_GET(CPT_LF_INPROG_GRB_CNT, reg_val);
		gwb_cnt = FIELD_GET(CPT_LF_INPROG_GWB_CNT, reg_val);
		if (inflight == 0 && gwb_cnt < 40 &&
		    (grb_cnt == 0 || grb_cnt == 40))
			cnt++;
		else
			cnt = 0;
	} while (cnt < 10);
}

/* Allocate memory for CPT outbound Instruction queue.
 * Instruction queue memory format is:
 *      -----------------------------
 *     | Instruction Group memory    |
 *     |  (CPT_LF_Q_SIZE[SIZE_DIV40] |
 *     |   x 16 Bytes)               |
 *     |                             |
 *      ----------------------------- <-- CPT_LF_Q_BASE[ADDR]
 *     | Flow Control (128 Bytes)    |
 *     |                             |
 *      -----------------------------
 *     |  Instruction Memory         |
 *     |  (CPT_LF_Q_SIZE[SIZE_DIV40] |
 *     |   × 40 × 64 bytes)          |
 *     |                             |
 *      -----------------------------
 */
static int cn10k_outb_cptlf_iq_alloc(struct otx2_nic *pf)
{
	struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;

	iq->size = CN10K_CPT_INST_QLEN_BYTES + CN10K_CPT_Q_FC_LEN +
		    CN10K_CPT_INST_GRP_QLEN_BYTES + OTX2_ALIGN;

	iq->real_vaddr = dma_alloc_coherent(pf->dev, iq->size,
					    &iq->real_dma_addr, GFP_KERNEL);
	if (!iq->real_vaddr)
		return -ENOMEM;

	/* iq->vaddr/dma_addr points to Flow Control location */
	iq->vaddr = iq->real_vaddr + CN10K_CPT_INST_GRP_QLEN_BYTES;
	iq->dma_addr = iq->real_dma_addr + CN10K_CPT_INST_GRP_QLEN_BYTES;

	/* Align pointers */
	iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_ALIGN);
	iq->dma_addr = PTR_ALIGN(iq->dma_addr, OTX2_ALIGN);
	return 0;
}

static void cn10k_outb_cptlf_iq_free(struct otx2_nic *pf)
{
	struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;

	if (iq->real_vaddr)
		dma_free_coherent(pf->dev, iq->size, iq->real_vaddr,
				  iq->real_dma_addr);

	iq->real_vaddr = NULL;
	iq->vaddr = NULL;
}

static int cn10k_outb_cptlf_iq_init(struct otx2_nic *pf)
{
	u64 reg_val;
	int ret;

	/* Allocate Memory for CPT IQ */
	ret = cn10k_outb_cptlf_iq_alloc(pf);
	if (ret)
		return ret;

	/* Disable IQ */
	cn10k_outb_cptlf_iq_disable(pf);

	/* Set IQ base address */
	otx2_write64(pf, CN10K_CPT_LF_Q_BASE, pf->ipsec.iq.dma_addr);

	/* Set IQ size */
	reg_val = FIELD_PREP(CPT_LF_Q_SIZE_DIV40, CN10K_CPT_SIZE_DIV40 +
			     CN10K_CPT_EXTRA_SIZE_DIV40);
	otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, reg_val);

	return 0;
}

static int cn10k_outb_cptlf_init(struct otx2_nic *pf)
{
	int ret;

	/* Initialize CPTLF Instruction Queue (IQ) */
	ret = cn10k_outb_cptlf_iq_init(pf);
	if (ret)
		return ret;

	/* Configure CPTLF for outbound ipsec offload */
	ret = cn10k_outb_cptlf_config(pf);
	if (ret)
		goto iq_clean;

	/* Enable CPTLF IQ */
	cn10k_outb_cptlf_iq_enable(pf);
	return 0;
iq_clean:
	cn10k_outb_cptlf_iq_free(pf);
	return ret;
}

static int cn10k_outb_cpt_init(struct net_device *netdev)
{
	struct otx2_nic *pf = netdev_priv(netdev);
	int ret;

	/* Attach a CPT LF for outbound ipsec offload */
	ret = cn10k_outb_cptlf_attach(pf);
	if (ret)
		return ret;

	/* Allocate a CPT LF for outbound ipsec offload */
	ret = cn10k_outb_cptlf_alloc(pf);
	if (ret)
		goto detach;

	/* Initialize the CPTLF for outbound ipsec offload */
	ret = cn10k_outb_cptlf_init(pf);
	if (ret)
		goto lf_free;

	pf->ipsec.io_addr = (__force u64)otx2_get_regaddr(pf,
						CN10K_CPT_LF_NQX(0));

	/* Set ipsec offload enabled for this device */
	pf->flags |= OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;

	cn10k_cpt_device_set_available(pf);
	return 0;

lf_free:
	cn10k_outb_cptlf_free(pf);
detach:
	cn10k_outb_cptlf_detach(pf);
	return ret;
}

static int cn10k_outb_cpt_clean(struct otx2_nic *pf)
{
	int ret;

	if (!cn10k_cpt_device_set_inuse(pf)) {
		netdev_err(pf->netdev, "CPT LF device unavailable\n");
		return -ENODEV;
	}

	/* Set ipsec offload disabled for this device */
	pf->flags &= ~OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;

	/* Disable CPTLF Instruction Queue (IQ) */
	cn10k_outb_cptlf_iq_disable(pf);

	/* Set IQ base address and size to 0 */
	otx2_write64(pf, CN10K_CPT_LF_Q_BASE, 0);
	otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, 0);

	/* Free CPTLF IQ */
	cn10k_outb_cptlf_iq_free(pf);

	/* Free and detach CPT LF */
	cn10k_outb_cptlf_free(pf);
	ret = cn10k_outb_cptlf_detach(pf);
	if (ret)
		netdev_err(pf->netdev, "Failed to detach CPT LF\n");

	cn10k_cpt_device_set_unavailable(pf);
	return ret;
}

static void cn10k_cpt_inst_flush(struct otx2_nic *pf, struct cpt_inst_s *inst,
				 u64 size)
{
	struct otx2_lmt_info *lmt_info;
	u64 val = 0, tar_addr = 0;

	lmt_info = per_cpu_ptr(pf->hw.lmt_info, smp_processor_id());
	/* FIXME: val[0:10] LMT_ID.
	 * [12:15] no of LMTST - 1 in the burst.
	 * [19:63] data size of each LMTST in the burst except first.
	 */
	val = (lmt_info->lmt_id & 0x7FF);
	/* Target address for LMTST flush tells HW how many 128bit
	 * words are present.
	 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
	 */
	tar_addr |= pf->ipsec.io_addr | (((size / 16) - 1) & 0x7) << 4;
	dma_wmb();
	memcpy((u64 *)lmt_info->lmt_addr, inst, size);
	cn10k_lmt_flush(val, tar_addr);
}

static int cn10k_wait_for_cpt_respose(struct otx2_nic *pf,
				      struct cpt_res_s *res)
{
	unsigned long timeout = jiffies + msecs_to_jiffies(100);
	u64 *completion_ptr = (u64 *)res;

	do {
		if (time_after(jiffies, timeout)) {
			netdev_err(pf->netdev, "CPT response timeout\n");
			return -EBUSY;
		}
	} while ((READ_ONCE(*completion_ptr) & CN10K_CPT_COMP_E_MASK) ==
		 CN10K_CPT_COMP_E_NOTDONE);

	if (!(res->compcode == CN10K_CPT_COMP_E_GOOD ||
	      res->compcode == CN10K_CPT_COMP_E_WARN) || res->uc_compcode) {
		netdev_err(pf->netdev, "compcode=%x doneint=%x\n",
			   res->compcode, res->doneint);
		netdev_err(pf->netdev, "uc_compcode=%x uc_info=%llx esn=%llx\n",
			   res->uc_compcode, (u64)res->uc_info, res->esn);
	}
	return 0;
}

static int cn10k_outb_write_sa(struct otx2_nic *pf, struct qmem *sa_info)
{
	dma_addr_t res_iova, dptr_iova, sa_iova;
	struct cn10k_tx_sa_s *sa_dptr;
	struct cpt_inst_s inst = {};
	struct cpt_res_s *res;
	u32 sa_size, off;
	u64 *sptr, *dptr;
	u64 reg_val;
	int ret;

	sa_iova = sa_info->iova;
	if (!sa_iova)
		return -EINVAL;

	res = dma_alloc_coherent(pf->dev, sizeof(struct cpt_res_s),
				 &res_iova, GFP_ATOMIC);
	if (!res)
		return -ENOMEM;

	sa_size = sizeof(struct cn10k_tx_sa_s);
	sa_dptr = dma_alloc_coherent(pf->dev, sa_size, &dptr_iova, GFP_ATOMIC);
	if (!sa_dptr) {
		dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res,
				  res_iova);
		return -ENOMEM;
	}

	sptr = (__force u64 *)sa_info->base;
	dptr =  (__force u64 *)sa_dptr;
	for (off = 0; off < (sa_size / 8); off++)
		*(dptr + off) = (__force u64)cpu_to_be64(*(sptr + off));

	res->compcode = CN10K_CPT_COMP_E_NOTDONE;
	inst.res_addr = res_iova;
	inst.dptr = (u64)dptr_iova;
	inst.param2 = sa_size >> 3;
	inst.dlen = sa_size;
	inst.opcode_major = CN10K_IPSEC_MAJOR_OP_WRITE_SA;
	inst.opcode_minor = CN10K_IPSEC_MINOR_OP_WRITE_SA;
	inst.cptr = sa_iova;
	inst.ctx_val = 1;
	inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;

	/* Check if CPT-LF available */
	if (!cn10k_cpt_device_set_inuse(pf)) {
		ret = -ENODEV;
		goto free_mem;
	}

	cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
	dma_wmb();
	ret = cn10k_wait_for_cpt_respose(pf, res);
	if (ret)
		goto set_available;

	/* Trigger CTX flush to write dirty data back to DRAM */
	reg_val = FIELD_PREP(CPT_LF_CTX_FLUSH_CPTR, sa_iova >> 7);
	otx2_write64(pf, CN10K_CPT_LF_CTX_FLUSH, reg_val);

set_available:
	cn10k_cpt_device_set_available(pf);
free_mem:
	dma_free_coherent(pf->dev, sa_size, sa_dptr, dptr_iova);
	dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res, res_iova);
	return ret;
}

static int cn10k_ipsec_get_hw_ctx_offset(void)
{
	/* Offset on Hardware-context offset in word */
	return (offsetof(struct cn10k_tx_sa_s, hw_ctx) / sizeof(u64)) & 0x7F;
}

static int cn10k_ipsec_get_ctx_push_size(void)
{
	/* Context push size is round up and in multiple of 8 Byte */
	return (roundup(offsetof(struct cn10k_tx_sa_s, hw_ctx), 8) / 8) & 0x7F;
}

static int cn10k_ipsec_get_aes_key_len(int key_len)
{
	/* key_len is aes key length in bytes */
	switch (key_len) {
	case 16:
		return CN10K_IPSEC_SA_AES_KEY_LEN_128;
	case 24:
		return CN10K_IPSEC_SA_AES_KEY_LEN_192;
	default:
		return CN10K_IPSEC_SA_AES_KEY_LEN_256;
	}
}

static void cn10k_outb_prepare_sa(struct xfrm_state *x,
				  struct cn10k_tx_sa_s *sa_entry)
{
	int key_len = (x->aead->alg_key_len + 7) / 8;
	struct net_device *netdev = x->xso.dev;
	u8 *key = x->aead->alg_key;
	struct otx2_nic *pf;
	u32 *tmp_salt;
	u64 *tmp_key;
	int idx;

	memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));

	/* context size, 128 Byte aligned up */
	pf = netdev_priv(netdev);
	sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN)  & 0xF;
	sa_entry->hw_ctx_off = cn10k_ipsec_get_hw_ctx_offset();
	sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();

	/* Ucode to skip two words of CPT_CTX_HW_S */
	sa_entry->ctx_hdr_size = 1;

	/* Allow Atomic operation (AOP) */
	sa_entry->aop_valid = 1;

	/* Outbound, ESP TRANSPORT/TUNNEL Mode, AES-GCM with */
	sa_entry->sa_dir = CN10K_IPSEC_SA_DIR_OUTB;
	sa_entry->ipsec_protocol = CN10K_IPSEC_SA_IPSEC_PROTO_ESP;
	sa_entry->enc_type = CN10K_IPSEC_SA_ENCAP_TYPE_AES_GCM;
	sa_entry->iv_src = CN10K_IPSEC_SA_IV_SRC_PACKET;
	if (x->props.mode == XFRM_MODE_TUNNEL)
		sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TUNNEL;
	else
		sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TRANSPORT;

	/* Last 4 bytes are salt */
	key_len -= 4;
	sa_entry->aes_key_len = cn10k_ipsec_get_aes_key_len(key_len);
	memcpy(sa_entry->cipher_key, key, key_len);
	tmp_key = (u64 *)sa_entry->cipher_key;

	for (idx = 0; idx < key_len / 8; idx++)
		tmp_key[idx] = (__force u64)cpu_to_be64(tmp_key[idx]);

	memcpy(&sa_entry->iv_gcm_salt, key + key_len, 4);
	tmp_salt = (u32 *)&sa_entry->iv_gcm_salt;
	*tmp_salt = (__force u32)cpu_to_be32(*tmp_salt);

	/* Write SA context data to memory before enabling */
	wmb();

	/* Enable SA */
	sa_entry->sa_valid = 1;
}

static int cn10k_ipsec_validate_state(struct xfrm_state *x,
				      struct netlink_ext_ack *extack)
{
	if (x->props.aalgo != SADB_AALG_NONE) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload authenticated xfrm states");
		return -EINVAL;
	}
	if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Only AES-GCM-ICV16 xfrm state may be offloaded");
		return -EINVAL;
	}
	if (x->props.calgo != SADB_X_CALG_NONE) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload compressed xfrm states");
		return -EINVAL;
	}
	if (x->props.flags & XFRM_STATE_ESN) {
		NL_SET_ERR_MSG_MOD(extack, "Cannot offload ESN xfrm states");
		return -EINVAL;
	}
	if (x->props.family != AF_INET && x->props.family != AF_INET6) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Only IPv4/v6 xfrm states may be offloaded");
		return -EINVAL;
	}
	if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload other than crypto-mode");
		return -EINVAL;
	}
	if (x->props.mode != XFRM_MODE_TRANSPORT &&
	    x->props.mode != XFRM_MODE_TUNNEL) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Only tunnel/transport xfrm states may be offloaded");
		return -EINVAL;
	}
	if (x->id.proto != IPPROTO_ESP) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Only ESP xfrm state may be offloaded");
		return -EINVAL;
	}
	if (x->encap) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Encapsulated xfrm state may not be offloaded");
		return -EINVAL;
	}
	if (!x->aead) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states without aead");
		return -EINVAL;
	}

	if (x->aead->alg_icv_len != 128) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states with AEAD ICV length other than 128bit");
		return -EINVAL;
	}
	if (x->aead->alg_key_len != 128 + 32 &&
	    x->aead->alg_key_len != 192 + 32 &&
	    x->aead->alg_key_len != 256 + 32) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states with AEAD key length other than 128/192/256bit");
		return -EINVAL;
	}
	if (x->tfcpad) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states with tfc padding");
		return -EINVAL;
	}
	if (!x->geniv) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states without geniv");
		return -EINVAL;
	}
	if (strcmp(x->geniv, "seqiv")) {
		NL_SET_ERR_MSG_MOD(extack,
				   "Cannot offload xfrm states with geniv other than seqiv");
		return -EINVAL;
	}
	return 0;
}

static int cn10k_ipsec_inb_add_state(struct xfrm_state *x,
				     struct netlink_ext_ack *extack)
{
	NL_SET_ERR_MSG_MOD(extack, "xfrm inbound offload not supported");
	return -EOPNOTSUPP;
}

static int cn10k_ipsec_outb_add_state(struct net_device *dev,
				      struct xfrm_state *x,
				      struct netlink_ext_ack *extack)
{
	struct cn10k_tx_sa_s *sa_entry;
	struct qmem *sa_info;
	struct otx2_nic *pf;
	int err;

	err = cn10k_ipsec_validate_state(x, extack);
	if (err)
		return err;

	pf = netdev_priv(dev);

	err = qmem_alloc(pf->dev, &sa_info, pf->ipsec.sa_size, OTX2_ALIGN);
	if (err)
		return err;

	sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
	cn10k_outb_prepare_sa(x, sa_entry);

	err = cn10k_outb_write_sa(pf, sa_info);
	if (err) {
		NL_SET_ERR_MSG_MOD(extack, "Error writing outbound SA");
		qmem_free(pf->dev, sa_info);
		return err;
	}

	x->xso.offload_handle = (unsigned long)sa_info;
	/* Enable static branch when first SA setup */
	if (!pf->ipsec.outb_sa_count)
		static_branch_enable(&cn10k_ipsec_sa_enabled);
	pf->ipsec.outb_sa_count++;
	return 0;
}

static int cn10k_ipsec_add_state(struct net_device *dev,
				 struct xfrm_state *x,
				 struct netlink_ext_ack *extack)
{
	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
		return cn10k_ipsec_inb_add_state(x, extack);
	else
		return cn10k_ipsec_outb_add_state(dev, x, extack);
}

static void cn10k_ipsec_del_state(struct net_device *dev, struct xfrm_state *x)
{
	struct cn10k_tx_sa_s *sa_entry;
	struct qmem *sa_info;
	struct otx2_nic *pf;
	int err;

	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
		return;

	pf = netdev_priv(dev);

	sa_info = (struct qmem *)x->xso.offload_handle;
	sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
	memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
	/* Disable SA in CPT h/w */
	sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
	sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN)  & 0xF;
	sa_entry->aop_valid = 1;

	err = cn10k_outb_write_sa(pf, sa_info);
	if (err)
		netdev_err(dev, "Error (%d) deleting SA\n", err);

	x->xso.offload_handle = 0;
	qmem_free(pf->dev, sa_info);

	/* If no more SA's then update netdev feature for potential change
	 * in NETIF_F_HW_ESP.
	 */
	if (!--pf->ipsec.outb_sa_count)
		queue_work(pf->ipsec.sa_workq, &pf->ipsec.sa_work);
}

static const struct xfrmdev_ops cn10k_ipsec_xfrmdev_ops = {
	.xdo_dev_state_add	= cn10k_ipsec_add_state,
	.xdo_dev_state_delete	= cn10k_ipsec_del_state,
};

static void cn10k_ipsec_sa_wq_handler(struct work_struct *work)
{
	struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
						 sa_work);
	struct otx2_nic *pf = container_of(ipsec, struct otx2_nic, ipsec);

	/* Disable static branch when no more SA enabled */
	static_branch_disable(&cn10k_ipsec_sa_enabled);
	rtnl_lock();
	netdev_update_features(pf->netdev);
	rtnl_unlock();
}

int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable)
{
	struct otx2_nic *pf = netdev_priv(netdev);

	/* IPsec offload supported on cn10k */
	if (!is_dev_support_ipsec_offload(pf->pdev))
		return -EOPNOTSUPP;

	/* Initialize CPT for outbound ipsec offload */
	if (enable)
		return cn10k_outb_cpt_init(netdev);

	/* Don't do CPT cleanup if SA installed */
	if (pf->ipsec.outb_sa_count) {
		netdev_err(pf->netdev, "SA installed on this device\n");
		return -EBUSY;
	}

	return cn10k_outb_cpt_clean(pf);
}

int cn10k_ipsec_init(struct net_device *netdev)
{
	struct otx2_nic *pf = netdev_priv(netdev);
	u32 sa_size;

	if (!is_dev_support_ipsec_offload(pf->pdev))
		return 0;

	/* Each SA entry size is 128 Byte round up in size */
	sa_size = sizeof(struct cn10k_tx_sa_s) % OTX2_ALIGN ?
			 (sizeof(struct cn10k_tx_sa_s) / OTX2_ALIGN + 1) *
			 OTX2_ALIGN : sizeof(struct cn10k_tx_sa_s);
	pf->ipsec.sa_size = sa_size;

	INIT_WORK(&pf->ipsec.sa_work, cn10k_ipsec_sa_wq_handler);
	pf->ipsec.sa_workq = alloc_workqueue("cn10k_ipsec_sa_workq", 0, 0);
	if (!pf->ipsec.sa_workq) {
		netdev_err(pf->netdev, "SA alloc workqueue failed\n");
		return -ENOMEM;
	}

	/* Set xfrm device ops */
	netdev->xfrmdev_ops = &cn10k_ipsec_xfrmdev_ops;
	netdev->hw_features |= NETIF_F_HW_ESP;
	netdev->hw_enc_features |= NETIF_F_HW_ESP;

	cn10k_cpt_device_set_unavailable(pf);
	return 0;
}
EXPORT_SYMBOL(cn10k_ipsec_init);

void cn10k_ipsec_clean(struct otx2_nic *pf)
{
	if (!is_dev_support_ipsec_offload(pf->pdev))
		return;

	if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
		return;

	if (pf->ipsec.sa_workq) {
		destroy_workqueue(pf->ipsec.sa_workq);
		pf->ipsec.sa_workq = NULL;
	}

	cn10k_outb_cpt_clean(pf);
}
EXPORT_SYMBOL(cn10k_ipsec_clean);

static u16 cn10k_ipsec_get_ip_data_len(struct xfrm_state *x,
				       struct sk_buff *skb)
{
	struct ipv6hdr *ipv6h;
	struct iphdr *iph;
	u8 *src;

	src = (u8 *)skb->data + ETH_HLEN;

	if (x->props.family == AF_INET) {
		iph = (struct iphdr *)src;
		return ntohs(iph->tot_len);
	}

	ipv6h = (struct ipv6hdr *)src;
	return ntohs(ipv6h->payload_len) + sizeof(struct ipv6hdr);
}

/* Prepare CPT and NIX SQE scatter/gather subdescriptor structure.
 * SG of NIX and CPT are same in size.
 * Layout of a NIX SQE and CPT SG entry:
 *      -----------------------------
 *     |     CPT Scatter Gather      |
 *     |       (SQE SIZE)            |
 *     |                             |
 *      -----------------------------
 *     |       NIX SQE               |
 *     |       (SQE SIZE)            |
 *     |                             |
 *      -----------------------------
 */
bool otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
			   struct sk_buff *skb, int num_segs, int *offset)
{
	struct cpt_sg_s *cpt_sg = NULL;
	struct nix_sqe_sg_s *sg = NULL;
	u64 dma_addr, *iova = NULL;
	u64 *cpt_iova = NULL;
	u16 *sg_lens = NULL;
	int seg, len;

	sq->sg[sq->head].num_segs = 0;
	cpt_sg = (struct cpt_sg_s *)(sq->sqe_base - sq->sqe_size);

	for (seg = 0; seg < num_segs; seg++) {
		if ((seg % MAX_SEGS_PER_SG) == 0) {
			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
			sg->ld_type = NIX_SEND_LDTYPE_LDD;
			sg->subdc = NIX_SUBDC_SG;
			sg->segs = 0;
			sg_lens = (void *)sg;
			iova = (void *)sg + sizeof(*sg);
			/* Next subdc always starts at a 16byte boundary.
			 * So if sg->segs is whether 2 or 3, offset += 16bytes.
			 */
			if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
				*offset += sizeof(*sg) + (3 * sizeof(u64));
			else
				*offset += sizeof(*sg) + sizeof(u64);

			cpt_sg += (seg / MAX_SEGS_PER_SG) * 4;
			cpt_iova = (void *)cpt_sg + sizeof(*cpt_sg);
		}
		dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
		if (dma_mapping_error(pfvf->dev, dma_addr))
			return false;

		sg_lens[seg % MAX_SEGS_PER_SG] = len;
		sg->segs++;
		*iova++ = dma_addr;
		*cpt_iova++ = dma_addr;

		/* Save DMA mapping info for later unmapping */
		sq->sg[sq->head].dma_addr[seg] = dma_addr;
		sq->sg[sq->head].size[seg] = len;
		sq->sg[sq->head].num_segs++;

		*cpt_sg = *(struct cpt_sg_s *)sg;
		cpt_sg->rsvd_63_50 = 0;
	}

	sq->sg[sq->head].skb = (u64)skb;
	return true;
}

static u16 cn10k_ipsec_get_param1(u8 iv_offset)
{
	u16 param1_val;

	/* Set Crypto mode, disable L3/L4 checksum */
	param1_val = CN10K_IPSEC_INST_PARAM1_DIS_L4_CSUM |
		      CN10K_IPSEC_INST_PARAM1_DIS_L3_CSUM;
	param1_val |= (u16)iv_offset << CN10K_IPSEC_INST_PARAM1_IV_OFFSET_SHIFT;
	return param1_val;
}

bool cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
			  struct otx2_snd_queue *sq, struct sk_buff *skb,
			  int num_segs, int size)
{
	struct cpt_inst_s inst;
	struct cpt_res_s *res;
	struct xfrm_state *x;
	struct qmem *sa_info;
	dma_addr_t dptr_iova;
	struct sec_path *sp;
	u8 encap_offset;
	u8 auth_offset;
	u8 gthr_size;
	u8 iv_offset;
	u16 dlen;

	/* Check for IPSEC offload enabled */
	if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
		goto drop;

	sp = skb_sec_path(skb);
	if (unlikely(!sp->len))
		goto drop;

	x = xfrm_input_state(skb);
	if (unlikely(!x))
		goto drop;

	if (x->props.mode != XFRM_MODE_TRANSPORT &&
	    x->props.mode != XFRM_MODE_TUNNEL)
		goto drop;

	dlen = cn10k_ipsec_get_ip_data_len(x, skb);
	if (dlen == 0 && netif_msg_tx_err(pf)) {
		netdev_err(pf->netdev, "Invalid IP header, ip-length zero\n");
		goto drop;
	}

	/* Check for valid SA context */
	sa_info = (struct qmem *)x->xso.offload_handle;
	if (!sa_info)
		goto drop;

	memset(&inst, 0, sizeof(struct cpt_inst_s));

	/* Get authentication offset */
	if (x->props.family == AF_INET)
		auth_offset = sizeof(struct iphdr);
	else
		auth_offset = sizeof(struct ipv6hdr);

	/* IV offset is after ESP header */
	iv_offset = auth_offset + sizeof(struct ip_esp_hdr);
	/* Encap will start after IV */
	encap_offset = iv_offset + GCM_RFC4106_IV_SIZE;

	/* CPT Instruction word-1 */
	res = (struct cpt_res_s *)(sq->cpt_resp->base + (64 * sq->head));
	res->compcode = 0;
	inst.res_addr = sq->cpt_resp->iova + (64 * sq->head);

	/* CPT Instruction word-2 */
	inst.rvu_pf_func = pf->pcifunc;

	/* CPT Instruction word-3:
	 * Set QORD to force CPT_RES_S write completion
	 */
	inst.qord = 1;

	/* CPT Instruction word-4 */
	/* inst.dlen should not include ICV length */
	inst.dlen = dlen + ETH_HLEN - (x->aead->alg_icv_len / 8);
	inst.opcode_major = CN10K_IPSEC_MAJOR_OP_OUTB_IPSEC;
	inst.param1 = cn10k_ipsec_get_param1(iv_offset);

	inst.param2 = encap_offset <<
		       CN10K_IPSEC_INST_PARAM2_ENC_DATA_OFFSET_SHIFT;
	inst.param2 |= (u16)auth_offset <<
			CN10K_IPSEC_INST_PARAM2_AUTH_DATA_OFFSET_SHIFT;

	/* CPT Instruction word-5 */
	gthr_size = num_segs / MAX_SEGS_PER_SG;
	gthr_size = (num_segs % MAX_SEGS_PER_SG) ? gthr_size + 1 : gthr_size;

	gthr_size &= 0xF;
	dptr_iova = (sq->sqe_ring->iova + (sq->head * (sq->sqe_size * 2)));
	inst.dptr = dptr_iova | ((u64)gthr_size << 60);

	/* CPT Instruction word-6 */
	inst.rptr = inst.dptr;

	/* CPT Instruction word-7 */
	inst.cptr = sa_info->iova;
	inst.ctx_val = 1;
	inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;

	/* CPT Instruction word-0 */
	inst.nixtxl = (size / 16) - 1;
	inst.dat_offset = ETH_HLEN;
	inst.nixtx_offset = sq->sqe_size;

	netdev_tx_sent_queue(txq, skb->len);

	/* Finally Flush the CPT instruction */
	sq->head++;
	sq->head &= (sq->sqe_cnt - 1);
	cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
	return true;
drop:
	dev_kfree_skb_any(skb);
	return false;
}