| // SPDX-License-Identifier: GPL-2.0 | |
| /* Copyright(c) 2023 Advanced Micro Devices, Inc */ | |
| #include <linux/dynamic_debug.h> | |
| #include "core.h" | |
| static int pdsc_process_notifyq(struct pdsc_qcq *qcq) | |
| { | |
| union pds_core_notifyq_comp *comp; | |
| struct pdsc *pdsc = qcq->pdsc; | |
| struct pdsc_cq *cq = &qcq->cq; | |
| struct pdsc_cq_info *cq_info; | |
| int nq_work = 0; | |
| u64 eid; | |
| cq_info = &cq->info[cq->tail_idx]; | |
| comp = cq_info->comp; | |
| eid = le64_to_cpu(comp->event.eid); | |
| while (eid > pdsc->last_eid) { | |
| u16 ecode; | |
| /* Order the payload read after the event id, the field the | |
| * driver uses to detect a new completion. | |
| */ | |
| dma_rmb(); | |
| ecode = le16_to_cpu(comp->event.ecode); | |
| switch (ecode) { | |
| case PDS_EVENT_LINK_CHANGE: | |
| dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n", | |
| ecode, eid); | |
| pdsc_notify(PDS_EVENT_LINK_CHANGE, comp); | |
| break; | |
| case PDS_EVENT_RESET: | |
| dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n", | |
| ecode, eid); | |
| pdsc_notify(PDS_EVENT_RESET, comp); | |
| break; | |
| case PDS_EVENT_XCVR: | |
| dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n", | |
| ecode, eid); | |
| break; | |
| default: | |
| dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n", | |
| ecode, eid); | |
| break; | |
| } | |
| pdsc->last_eid = eid; | |
| cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); | |
| cq_info = &cq->info[cq->tail_idx]; | |
| comp = cq_info->comp; | |
| eid = le64_to_cpu(comp->event.eid); | |
| nq_work++; | |
| } | |
| qcq->accum_work += nq_work; | |
| return nq_work; | |
| } | |
| static bool pdsc_adminq_inc_if_up(struct pdsc *pdsc) | |
| { | |
| if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER) || | |
| pdsc->state & BIT_ULL(PDSC_S_FW_DEAD)) | |
| return false; | |
| return refcount_inc_not_zero(&pdsc->adminq_refcnt); | |
| } | |
| void pdsc_process_adminq(struct pdsc_qcq *qcq) | |
| { | |
| union pds_core_adminq_comp *comp; | |
| struct pdsc_queue *q = &qcq->q; | |
| struct pdsc *pdsc = qcq->pdsc; | |
| struct pdsc_cq *cq = &qcq->cq; | |
| struct pdsc_q_info *q_info; | |
| unsigned long irqflags; | |
| int nq_work = 0; | |
| int aq_work = 0; | |
| /* Don't process AdminQ when it's not up */ | |
| if (!pdsc_adminq_inc_if_up(pdsc)) { | |
| dev_err(pdsc->dev, "%s: called while adminq is unavailable\n", | |
| __func__); | |
| return; | |
| } | |
| /* Check for NotifyQ event */ | |
| nq_work = pdsc_process_notifyq(&pdsc->notifyqcq); | |
| /* Check for empty queue, which can happen if the interrupt was | |
| * for a NotifyQ event and there are no new AdminQ completions. | |
| */ | |
| if (q->tail_idx == q->head_idx) | |
| goto credits; | |
| /* Find the first completion to clean, | |
| * run the callback in the related q_info, | |
| * and continue while we still match done color | |
| */ | |
| spin_lock_irqsave(&pdsc->adminq_lock, irqflags); | |
| comp = cq->info[cq->tail_idx].comp; | |
| while (pdsc_color_match(comp->color, cq->done_color)) { | |
| /* Order the payload reads after the color bit, the field the | |
| * driver uses to detect a new completion. | |
| */ | |
| dma_rmb(); | |
| q_info = &q->info[q->tail_idx]; | |
| q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); | |
| if (!completion_done(&q_info->completion)) { | |
| memcpy(q_info->dest, comp, sizeof(*comp)); | |
| complete(&q_info->completion); | |
| } | |
| if (cq->tail_idx == cq->num_descs - 1) | |
| cq->done_color = !cq->done_color; | |
| cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); | |
| comp = cq->info[cq->tail_idx].comp; | |
| aq_work++; | |
| } | |
| spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); | |
| qcq->accum_work += aq_work; | |
| credits: | |
| /* Return the interrupt credits, one for each completion */ | |
| pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx], | |
| nq_work + aq_work, | |
| PDS_CORE_INTR_CRED_REARM); | |
| refcount_dec(&pdsc->adminq_refcnt); | |
| } | |
| void pdsc_work_thread(struct work_struct *work) | |
| { | |
| struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work); | |
| pdsc_process_adminq(qcq); | |
| } | |
| irqreturn_t pdsc_adminq_isr(int irq, void *data) | |
| { | |
| struct pdsc *pdsc = data; | |
| struct pdsc_qcq *qcq; | |
| /* Don't process AdminQ when it's not up */ | |
| if (!pdsc_adminq_inc_if_up(pdsc)) { | |
| dev_err(pdsc->dev, "%s: called while adminq is unavailable\n", | |
| __func__); | |
| return IRQ_HANDLED; | |
| } | |
| qcq = &pdsc->adminqcq; | |
| queue_work(pdsc->wq, &qcq->work); | |
| refcount_dec(&pdsc->adminq_refcnt); | |
| return IRQ_HANDLED; | |
| } | |
| static int __pdsc_adminq_post(struct pdsc *pdsc, | |
| struct pdsc_qcq *qcq, | |
| union pds_core_adminq_cmd *cmd, | |
| union pds_core_adminq_comp *comp) | |
| { | |
| struct pdsc_queue *q = &qcq->q; | |
| struct pdsc_q_info *q_info; | |
| unsigned long irqflags; | |
| unsigned int avail; | |
| int index; | |
| int ret; | |
| spin_lock_irqsave(&pdsc->adminq_lock, irqflags); | |
| /* Check for space in the queue */ | |
| avail = q->tail_idx; | |
| if (q->head_idx >= avail) | |
| avail += q->num_descs - q->head_idx - 1; | |
| else | |
| avail -= q->head_idx + 1; | |
| if (!avail) { | |
| ret = -ENOSPC; | |
| goto err_out_unlock; | |
| } | |
| /* Check that the FW is running */ | |
| if (!pdsc_is_fw_running(pdsc)) { | |
| if (pdsc->info_regs) { | |
| u8 fw_status = | |
| ioread8(&pdsc->info_regs->fw_status); | |
| dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n", | |
| __func__, fw_status); | |
| } else { | |
| dev_info(pdsc->dev, "%s: post failed - BARs not setup\n", | |
| __func__); | |
| } | |
| ret = -ENXIO; | |
| goto err_out_unlock; | |
| } | |
| /* Post the request */ | |
| index = q->head_idx; | |
| q_info = &q->info[index]; | |
| q_info->dest = comp; | |
| memcpy(q_info->desc, cmd, sizeof(*cmd)); | |
| reinit_completion(&q_info->completion); | |
| dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n", | |
| q->head_idx, q->tail_idx); | |
| dev_dbg(pdsc->dev, "post admin queue command:\n"); | |
| dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1, | |
| cmd, sizeof(*cmd), true); | |
| q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); | |
| pds_core_dbell_ring(pdsc->kern_dbpage, | |
| q->hw_type, q->dbval | q->head_idx); | |
| ret = index; | |
| err_out_unlock: | |
| spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); | |
| return ret; | |
| } | |
| int pdsc_adminq_post(struct pdsc *pdsc, | |
| union pds_core_adminq_cmd *cmd, | |
| union pds_core_adminq_comp *comp, | |
| bool fast_poll) | |
| { | |
| unsigned long poll_interval = 200; | |
| unsigned long poll_jiffies; | |
| unsigned long time_limit; | |
| unsigned long time_start; | |
| unsigned long time_done; | |
| unsigned long remaining; | |
| struct completion *wc; | |
| int err = 0; | |
| int index; | |
| if (!pdsc_adminq_inc_if_up(pdsc)) { | |
| dev_dbg(pdsc->dev, "%s: preventing adminq cmd %u\n", | |
| __func__, cmd->opcode); | |
| return -ENXIO; | |
| } | |
| index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp); | |
| if (index < 0) { | |
| err = index; | |
| goto err_out; | |
| } | |
| wc = &pdsc->adminqcq.q.info[index].completion; | |
| time_start = jiffies; | |
| time_limit = time_start + HZ * pdsc->devcmd_timeout; | |
| do { | |
| /* Timeslice the actual wait to catch IO errors etc early */ | |
| poll_jiffies = usecs_to_jiffies(poll_interval); | |
| remaining = wait_for_completion_timeout(wc, poll_jiffies); | |
| if (remaining) | |
| break; | |
| if (!pdsc_is_fw_running(pdsc)) { | |
| if (pdsc->info_regs) { | |
| u8 fw_status = | |
| ioread8(&pdsc->info_regs->fw_status); | |
| dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n", | |
| __func__, fw_status); | |
| } else { | |
| dev_dbg(pdsc->dev, "%s: post wait failed - BARs not setup\n", | |
| __func__); | |
| } | |
| err = -ENXIO; | |
| break; | |
| } | |
| /* When fast_poll is not requested, prevent aggressive polling | |
| * on failures due to timeouts by doing exponential back off. | |
| */ | |
| if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL) | |
| poll_interval <<= 1; | |
| } while (time_before(jiffies, time_limit)); | |
| time_done = jiffies; | |
| dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n", | |
| __func__, jiffies_to_msecs(time_done - time_start)); | |
| /* Check the results and clear an un-completed timeout */ | |
| if (time_after_eq(time_done, time_limit) && !completion_done(wc)) { | |
| err = -ETIMEDOUT; | |
| complete(wc); | |
| } | |
| dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index); | |
| dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1, | |
| comp, sizeof(*comp), true); | |
| if (remaining && comp->status) | |
| err = pdsc_err_to_errno(comp->status); | |
| err_out: | |
| if (err) { | |
| dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n", | |
| __func__, cmd->opcode, comp->status, ERR_PTR(err)); | |
| if (err == -ENXIO || err == -ETIMEDOUT) | |
| queue_work(pdsc->wq, &pdsc->health_work); | |
| } | |
| refcount_dec(&pdsc->adminq_refcnt); | |
| return err; | |
| } | |
| EXPORT_SYMBOL_GPL(pdsc_adminq_post); |