| // SPDX-License-Identifier: GPL-2.0 | |
| /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ | |
| #include <linux/kernel.h> | |
| #include <linux/types.h> | |
| #include <linux/errno.h> | |
| #include <linux/io.h> | |
| #include <linux/slab.h> | |
| #include <linux/etherdevice.h> | |
| #include "ionic.h" | |
| #include "ionic_dev.h" | |
| #include "ionic_lif.h" | |
| static void ionic_watchdog_cb(struct timer_list *t) | |
| { | |
| struct ionic *ionic = timer_container_of(ionic, t, watchdog_timer); | |
| struct ionic_lif *lif = ionic->lif; | |
| struct ionic_deferred_work *work; | |
| int hb; | |
| mod_timer(&ionic->watchdog_timer, | |
| round_jiffies(jiffies + ionic->watchdog_period)); | |
| if (!lif) | |
| return; | |
| hb = ionic_heartbeat_check(ionic); | |
| dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n", | |
| __func__, hb, netif_running(lif->netdev), | |
| test_bit(IONIC_LIF_F_UP, lif->state)); | |
| if (hb >= 0 && | |
| !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) | |
| ionic_link_status_check_request(lif, CAN_NOT_SLEEP); | |
| if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) && | |
| !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { | |
| work = kzalloc_obj(*work, GFP_ATOMIC); | |
| if (!work) { | |
| netdev_err(lif->netdev, "rxmode change dropped\n"); | |
| return; | |
| } | |
| work->type = IONIC_DW_TYPE_RX_MODE; | |
| netdev_dbg(lif->netdev, "deferred: rx_mode\n"); | |
| ionic_lif_deferred_enqueue(lif, work); | |
| } | |
| } | |
| static void ionic_napi_schedule_do_softirq(struct napi_struct *napi) | |
| { | |
| local_bh_disable(); | |
| napi_schedule(napi); | |
| local_bh_enable(); | |
| } | |
| void ionic_doorbell_napi_work(struct work_struct *work) | |
| { | |
| struct ionic_qcq *qcq = container_of(work, struct ionic_qcq, | |
| doorbell_napi_work); | |
| unsigned long now, then, dif; | |
| now = READ_ONCE(jiffies); | |
| then = qcq->q.dbell_jiffies; | |
| dif = now - then; | |
| if (dif > qcq->q.dbell_deadline) | |
| ionic_napi_schedule_do_softirq(&qcq->napi); | |
| } | |
| static int ionic_get_preferred_cpu(struct ionic *ionic, | |
| struct ionic_intr_info *intr) | |
| { | |
| int cpu; | |
| cpu = cpumask_first_and(*intr->affinity_mask, cpu_online_mask); | |
| if (cpu >= nr_cpu_ids) | |
| cpu = cpumask_local_spread(0, dev_to_node(ionic->dev)); | |
| return cpu; | |
| } | |
| static void ionic_queue_dbell_napi_work(struct ionic *ionic, | |
| struct ionic_qcq *qcq) | |
| { | |
| int cpu; | |
| if (!(qcq->flags & IONIC_QCQ_F_INTR)) | |
| return; | |
| cpu = ionic_get_preferred_cpu(ionic, &qcq->intr); | |
| queue_work_on(cpu, ionic->wq, &qcq->doorbell_napi_work); | |
| } | |
| static void ionic_doorbell_check_dwork(struct work_struct *work) | |
| { | |
| struct ionic *ionic = container_of(work, struct ionic, | |
| doorbell_check_dwork.work); | |
| struct ionic_lif *lif = ionic->lif; | |
| mutex_lock(&lif->queue_lock); | |
| if (test_bit(IONIC_LIF_F_FW_STOPPING, lif->state) || | |
| test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { | |
| mutex_unlock(&lif->queue_lock); | |
| return; | |
| } | |
| ionic_napi_schedule_do_softirq(&lif->adminqcq->napi); | |
| if (test_bit(IONIC_LIF_F_UP, lif->state)) { | |
| int i; | |
| for (i = 0; i < lif->nxqs; i++) { | |
| ionic_queue_dbell_napi_work(ionic, lif->txqcqs[i]); | |
| ionic_queue_dbell_napi_work(ionic, lif->rxqcqs[i]); | |
| } | |
| if (lif->hwstamp_txq && | |
| lif->hwstamp_txq->flags & IONIC_QCQ_F_INTR) | |
| ionic_napi_schedule_do_softirq(&lif->hwstamp_txq->napi); | |
| if (lif->hwstamp_rxq && | |
| lif->hwstamp_rxq->flags & IONIC_QCQ_F_INTR) | |
| ionic_napi_schedule_do_softirq(&lif->hwstamp_rxq->napi); | |
| } | |
| mutex_unlock(&lif->queue_lock); | |
| ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE); | |
| } | |
| bool ionic_doorbell_wa(struct ionic *ionic) | |
| { | |
| u8 asic_type = ionic->idev.dev_info.asic_type; | |
| return !asic_type || asic_type == IONIC_ASIC_TYPE_ELBA; | |
| } | |
| static int ionic_watchdog_init(struct ionic *ionic) | |
| { | |
| struct ionic_dev *idev = &ionic->idev; | |
| timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0); | |
| ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ; | |
| /* set times to ensure the first check will proceed */ | |
| atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ); | |
| idev->last_hb_time = jiffies - 2 * ionic->watchdog_period; | |
| /* init as ready, so no transition if the first check succeeds */ | |
| idev->last_fw_hb = 0; | |
| idev->fw_hb_ready = true; | |
| idev->fw_status_ready = true; | |
| idev->fw_generation = IONIC_FW_STS_F_GENERATION & | |
| ioread8(&idev->dev_info_regs->fw_status); | |
| ionic->wq = alloc_workqueue("%s-wq", WQ_UNBOUND, 0, | |
| dev_name(ionic->dev)); | |
| if (!ionic->wq) { | |
| dev_err(ionic->dev, "alloc_workqueue failed"); | |
| return -ENOMEM; | |
| } | |
| if (ionic_doorbell_wa(ionic)) | |
| INIT_DELAYED_WORK(&ionic->doorbell_check_dwork, | |
| ionic_doorbell_check_dwork); | |
| return 0; | |
| } | |
| void ionic_queue_doorbell_check(struct ionic *ionic, int delay) | |
| { | |
| int cpu; | |
| if (!ionic->lif->doorbell_wa) | |
| return; | |
| cpu = ionic_get_preferred_cpu(ionic, &ionic->lif->adminqcq->intr); | |
| queue_delayed_work_on(cpu, ionic->wq, &ionic->doorbell_check_dwork, | |
| delay); | |
| } | |
| void ionic_init_devinfo(struct ionic *ionic) | |
| { | |
| struct ionic_dev *idev = &ionic->idev; | |
| idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type); | |
| idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev); | |
| memcpy_fromio(idev->dev_info.fw_version, | |
| idev->dev_info_regs->fw_version, | |
| IONIC_DEVINFO_FWVERS_BUFLEN); | |
| memcpy_fromio(idev->dev_info.serial_num, | |
| idev->dev_info_regs->serial_num, | |
| IONIC_DEVINFO_SERIAL_BUFLEN); | |
| idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0; | |
| idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0; | |
| dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version); | |
| } | |
| static void ionic_map_disc_cmb(struct ionic *ionic) | |
| { | |
| struct ionic_identity *ident = &ionic->ident; | |
| u32 length_reg0, length, offset, num_regions; | |
| struct ionic_dev_bar *bar = ionic->bars; | |
| struct ionic_dev *idev = &ionic->idev; | |
| struct device *dev = ionic->dev; | |
| int err, sz, i; | |
| u64 end; | |
| mutex_lock(&ionic->dev_cmd_lock); | |
| ionic_dev_cmd_discover_cmb(idev); | |
| err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); | |
| if (!err) { | |
| sz = min(sizeof(ident->cmb_layout), | |
| sizeof(idev->dev_cmd_regs->data)); | |
| memcpy_fromio(&ident->cmb_layout, | |
| &idev->dev_cmd_regs->data, sz); | |
| } | |
| mutex_unlock(&ionic->dev_cmd_lock); | |
| if (err) { | |
| dev_warn(dev, "Cannot discover CMB layout, disabling CMB\n"); | |
| return; | |
| } | |
| bar += 2; | |
| num_regions = le32_to_cpu(ident->cmb_layout.num_regions); | |
| if (!num_regions || num_regions > IONIC_MAX_CMB_REGIONS) { | |
| dev_warn(dev, "Invalid number of CMB entries (%d)\n", | |
| num_regions); | |
| return; | |
| } | |
| dev_dbg(dev, "ionic_cmb_layout_identity num_regions %d flags %x:\n", | |
| num_regions, ident->cmb_layout.flags); | |
| for (i = 0; i < num_regions; i++) { | |
| offset = le32_to_cpu(ident->cmb_layout.region[i].offset); | |
| length = le32_to_cpu(ident->cmb_layout.region[i].length); | |
| end = offset + length; | |
| dev_dbg(dev, "CMB entry %d: bar_num %u cmb_type %u offset %x length %u\n", | |
| i, ident->cmb_layout.region[i].bar_num, | |
| ident->cmb_layout.region[i].cmb_type, | |
| offset, length); | |
| if (end > (bar->len >> IONIC_CMB_SHIFT_64K)) { | |
| dev_warn(dev, "Out of bounds CMB region %d offset %x length %u\n", | |
| i, offset, length); | |
| return; | |
| } | |
| } | |
| /* if first entry matches PCI config, expdb is not supported */ | |
| if (ident->cmb_layout.region[0].bar_num == bar->res_index && | |
| le32_to_cpu(ident->cmb_layout.region[0].length) == bar->len && | |
| !ident->cmb_layout.region[0].offset) { | |
| dev_warn(dev, "No CMB mapping discovered\n"); | |
| return; | |
| } | |
| /* process first entry for regular mapping */ | |
| length_reg0 = le32_to_cpu(ident->cmb_layout.region[0].length); | |
| if (!length_reg0) { | |
| dev_warn(dev, "region len = 0. No CMB mapping discovered\n"); | |
| return; | |
| } | |
| /* Verify first entry size matches expected 8MB size (in 64KB pages) */ | |
| if (length_reg0 != IONIC_BAR2_CMB_ENTRY_SIZE >> IONIC_CMB_SHIFT_64K) { | |
| dev_warn(dev, "Unexpected CMB size in entry 0: %u pages\n", | |
| length_reg0); | |
| return; | |
| } | |
| sz = BITS_TO_LONGS((length_reg0 << IONIC_CMB_SHIFT_64K) / | |
| PAGE_SIZE) * sizeof(long); | |
| idev->cmb_inuse = kzalloc(sz, GFP_KERNEL); | |
| if (!idev->cmb_inuse) { | |
| dev_warn(dev, "No memory for CMB, disabling\n"); | |
| idev->phy_cmb_pages = 0; | |
| idev->phy_cmb_expdb64_pages = 0; | |
| idev->phy_cmb_expdb128_pages = 0; | |
| idev->phy_cmb_expdb256_pages = 0; | |
| idev->phy_cmb_expdb512_pages = 0; | |
| idev->cmb_npages = 0; | |
| return; | |
| } | |
| for (i = 0; i < num_regions; i++) { | |
| /* check this region matches first region length as to | |
| * ease implementation | |
| */ | |
| if (le32_to_cpu(ident->cmb_layout.region[i].length) != | |
| length_reg0) | |
| continue; | |
| offset = le32_to_cpu(ident->cmb_layout.region[i].offset); | |
| switch (ident->cmb_layout.region[i].cmb_type) { | |
| case IONIC_CMB_TYPE_DEVMEM: | |
| idev->phy_cmb_pages = bar->bus_addr + offset; | |
| idev->cmb_npages = | |
| (length_reg0 << IONIC_CMB_SHIFT_64K) / PAGE_SIZE; | |
| dev_dbg(dev, "regular cmb mapping: bar->bus_addr %pa region[%d].length %u\n", | |
| &bar->bus_addr, i, length); | |
| dev_dbg(dev, "idev->phy_cmb_pages %pad, idev->cmb_npages %u\n", | |
| &idev->phy_cmb_pages, idev->cmb_npages); | |
| break; | |
| case IONIC_CMB_TYPE_EXPDB64: | |
| idev->phy_cmb_expdb64_pages = | |
| bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K); | |
| dev_dbg(dev, "idev->phy_cmb_expdb64_pages %pad\n", | |
| &idev->phy_cmb_expdb64_pages); | |
| break; | |
| case IONIC_CMB_TYPE_EXPDB128: | |
| idev->phy_cmb_expdb128_pages = | |
| bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K); | |
| dev_dbg(dev, "idev->phy_cmb_expdb128_pages %pad\n", | |
| &idev->phy_cmb_expdb128_pages); | |
| break; | |
| case IONIC_CMB_TYPE_EXPDB256: | |
| idev->phy_cmb_expdb256_pages = | |
| bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K); | |
| dev_dbg(dev, "idev->phy_cmb_expdb256_pages %pad\n", | |
| &idev->phy_cmb_expdb256_pages); | |
| break; | |
| case IONIC_CMB_TYPE_EXPDB512: | |
| idev->phy_cmb_expdb512_pages = | |
| bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K); | |
| dev_dbg(dev, "idev->phy_cmb_expdb512_pages %pad\n", | |
| &idev->phy_cmb_expdb512_pages); | |
| break; | |
| default: | |
| dev_warn(dev, "[%d] Invalid cmb_type (%d)\n", | |
| i, ident->cmb_layout.region[i].cmb_type); | |
| break; | |
| } | |
| } | |
| } | |
| static void ionic_map_classic_cmb(struct ionic *ionic) | |
| { | |
| struct ionic_dev_bar *bar = ionic->bars; | |
| struct ionic_dev *idev = &ionic->idev; | |
| struct device *dev = ionic->dev; | |
| int sz; | |
| bar += 2; | |
| /* classic CMB mapping */ | |
| idev->phy_cmb_pages = bar->bus_addr; | |
| idev->cmb_npages = bar->len / PAGE_SIZE; | |
| dev_dbg(dev, "classic cmb mapping: bar->bus_addr %pa bar->len %lu\n", | |
| &bar->bus_addr, bar->len); | |
| dev_dbg(dev, "idev->phy_cmb_pages %pad, idev->cmb_npages %u\n", | |
| &idev->phy_cmb_pages, idev->cmb_npages); | |
| sz = BITS_TO_LONGS(idev->cmb_npages) * sizeof(long); | |
| idev->cmb_inuse = kzalloc(sz, GFP_KERNEL); | |
| if (!idev->cmb_inuse) { | |
| idev->phy_cmb_pages = 0; | |
| idev->cmb_npages = 0; | |
| } | |
| } | |
| void ionic_map_cmb(struct ionic *ionic) | |
| { | |
| struct pci_dev *pdev = ionic->pdev; | |
| struct device *dev = ionic->dev; | |
| if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM)) { | |
| dev_dbg(dev, "No CMB, disabling\n"); | |
| return; | |
| } | |
| if (ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_DISC_CMB)) | |
| ionic_map_disc_cmb(ionic); | |
| else | |
| ionic_map_classic_cmb(ionic); | |
| } | |
| int ionic_dev_setup(struct ionic *ionic) | |
| { | |
| struct ionic_dev_bar *bar = ionic->bars; | |
| unsigned int num_bars = ionic->num_bars; | |
| struct ionic_dev *idev = &ionic->idev; | |
| struct device *dev = ionic->dev; | |
| u32 sig; | |
| int err; | |
| /* BAR0: dev_cmd and interrupts */ | |
| if (num_bars < 1) { | |
| dev_err(dev, "No bars found, aborting\n"); | |
| return -EFAULT; | |
| } | |
| if (bar->len < IONIC_BAR0_SIZE) { | |
| dev_err(dev, "Resource bar size %lu too small, aborting\n", | |
| bar->len); | |
| return -EFAULT; | |
| } | |
| idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET; | |
| idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET; | |
| idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET; | |
| idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET; | |
| idev->hwstamp_regs = &idev->dev_info_regs->hwstamp; | |
| sig = ioread32(&idev->dev_info_regs->signature); | |
| if (sig != IONIC_DEV_INFO_SIGNATURE) { | |
| dev_err(dev, "Incompatible firmware signature %x", sig); | |
| return -EFAULT; | |
| } | |
| ionic_init_devinfo(ionic); | |
| /* BAR1: doorbells */ | |
| bar++; | |
| if (num_bars < 2) { | |
| dev_err(dev, "Doorbell bar missing, aborting\n"); | |
| return -EFAULT; | |
| } | |
| err = ionic_watchdog_init(ionic); | |
| if (err) | |
| return err; | |
| idev->db_pages = bar->vaddr; | |
| idev->phy_db_pages = bar->bus_addr; | |
| /* BAR2: optional controller memory mapping */ | |
| bar++; | |
| mutex_init(&idev->cmb_inuse_lock); | |
| if (num_bars < 3 || !ionic->bars[IONIC_PCI_BAR_CMB].len) { | |
| idev->cmb_inuse = NULL; | |
| idev->phy_cmb_pages = 0; | |
| idev->cmb_npages = 0; | |
| return 0; | |
| } | |
| return 0; | |
| } | |
| void ionic_dev_teardown(struct ionic *ionic) | |
| { | |
| struct ionic_dev *idev = &ionic->idev; | |
| kfree(idev->cmb_inuse); | |
| idev->cmb_inuse = NULL; | |
| idev->phy_cmb_pages = 0; | |
| idev->cmb_npages = 0; | |
| idev->phy_cmb_expdb64_pages = 0; | |
| idev->phy_cmb_expdb128_pages = 0; | |
| idev->phy_cmb_expdb256_pages = 0; | |
| idev->phy_cmb_expdb512_pages = 0; | |
| if (ionic->wq) { | |
| destroy_workqueue(ionic->wq); | |
| ionic->wq = NULL; | |
| } | |
| mutex_destroy(&idev->cmb_inuse_lock); | |
| } | |
| /* Devcmd Interface */ | |
| static bool __ionic_is_fw_running(struct ionic_dev *idev, u8 *status_ptr) | |
| { | |
| u8 fw_status; | |
| if (!idev->dev_info_regs) { | |
| if (status_ptr) | |
| *status_ptr = 0xff; | |
| return false; | |
| } | |
| fw_status = ioread8(&idev->dev_info_regs->fw_status); | |
| if (status_ptr) | |
| *status_ptr = fw_status; | |
| /* firmware is useful only if the running bit is set and | |
| * fw_status != 0xff (bad PCI read) | |
| */ | |
| return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING); | |
| } | |
| bool ionic_is_fw_running(struct ionic_dev *idev) | |
| { | |
| return __ionic_is_fw_running(idev, NULL); | |
| } | |
| int ionic_heartbeat_check(struct ionic *ionic) | |
| { | |
| unsigned long check_time, last_check_time; | |
| struct ionic_dev *idev = &ionic->idev; | |
| struct ionic_lif *lif = ionic->lif; | |
| bool fw_status_ready = true; | |
| bool fw_hb_ready; | |
| u8 fw_generation; | |
| u8 fw_status; | |
| u32 fw_hb; | |
| /* wait a least one second before testing again */ | |
| check_time = jiffies; | |
| last_check_time = atomic_long_read(&idev->last_check_time); | |
| do_check_time: | |
| if (time_before(check_time, last_check_time + HZ)) | |
| return 0; | |
| if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time, | |
| &last_check_time, check_time)) { | |
| /* if called concurrently, only the first should proceed. */ | |
| dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__); | |
| goto do_check_time; | |
| } | |
| /* If fw_status is not ready don't bother with the generation */ | |
| if (!__ionic_is_fw_running(idev, &fw_status)) { | |
| fw_status_ready = false; | |
| } else { | |
| fw_generation = fw_status & IONIC_FW_STS_F_GENERATION; | |
| if (idev->fw_generation != fw_generation) { | |
| dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n", | |
| idev->fw_generation, fw_generation); | |
| idev->fw_generation = fw_generation; | |
| /* If the generation changed, the fw status is not | |
| * ready so we need to trigger a fw-down cycle. After | |
| * the down, the next watchdog will see the fw is up | |
| * and the generation value stable, so will trigger | |
| * the fw-up activity. | |
| * | |
| * If we had already moved to FW_RESET from a RESET event, | |
| * it is possible that we never saw the fw_status go to 0, | |
| * so we fake the current idev->fw_status_ready here to | |
| * force the transition and get FW up again. | |
| */ | |
| if (test_bit(IONIC_LIF_F_FW_RESET, lif->state)) | |
| idev->fw_status_ready = false; /* go to running */ | |
| else | |
| fw_status_ready = false; /* go to down */ | |
| } | |
| } | |
| dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n", | |
| fw_status, fw_status_ready, idev->fw_status_ready, | |
| idev->last_fw_hb, lif->state[0]); | |
| /* is this a transition? */ | |
| if (fw_status_ready != idev->fw_status_ready && | |
| !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { | |
| bool trigger = false; | |
| idev->fw_status_ready = fw_status_ready; | |
| if (!fw_status_ready && | |
| !test_bit(IONIC_LIF_F_FW_RESET, lif->state) && | |
| !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { | |
| dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status); | |
| trigger = true; | |
| } else if (fw_status_ready && | |
| test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { | |
| dev_info(ionic->dev, "FW running 0x%02x\n", fw_status); | |
| trigger = true; | |
| } | |
| if (trigger) { | |
| struct ionic_deferred_work *work; | |
| work = kzalloc_obj(*work, GFP_ATOMIC); | |
| if (work) { | |
| work->type = IONIC_DW_TYPE_LIF_RESET; | |
| work->fw_status = fw_status_ready; | |
| ionic_lif_deferred_enqueue(lif, work); | |
| } | |
| } | |
| } | |
| if (!idev->fw_status_ready) | |
| return -ENXIO; | |
| /* Because of some variability in the actual FW heartbeat, we | |
| * wait longer than the DEVCMD_TIMEOUT before checking again. | |
| */ | |
| last_check_time = idev->last_hb_time; | |
| if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ)) | |
| return 0; | |
| fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat); | |
| fw_hb_ready = fw_hb != idev->last_fw_hb; | |
| /* early FW version had no heartbeat, so fake it */ | |
| if (!fw_hb_ready && !fw_hb) | |
| fw_hb_ready = true; | |
| dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n", | |
| __func__, fw_hb, idev->last_fw_hb, fw_hb_ready); | |
| idev->last_fw_hb = fw_hb; | |
| /* log a transition */ | |
| if (fw_hb_ready != idev->fw_hb_ready) { | |
| idev->fw_hb_ready = fw_hb_ready; | |
| if (!fw_hb_ready) | |
| dev_info(ionic->dev, "FW heartbeat stalled at %u\n", fw_hb); | |
| else | |
| dev_info(ionic->dev, "FW heartbeat restored at %u\n", fw_hb); | |
| } | |
| if (!fw_hb_ready) | |
| return -ENXIO; | |
| idev->last_hb_time = check_time; | |
| return 0; | |
| } | |
| u8 ionic_dev_cmd_status(struct ionic_dev *idev) | |
| { | |
| if (!idev->dev_cmd_regs) | |
| return (u8)PCI_ERROR_RESPONSE; | |
| return ioread8(&idev->dev_cmd_regs->comp.comp.status); | |
| } | |
| bool ionic_dev_cmd_done(struct ionic_dev *idev) | |
| { | |
| if (!idev->dev_cmd_regs) | |
| return false; | |
| return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE; | |
| } | |
| void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp) | |
| { | |
| if (!idev->dev_cmd_regs) | |
| return; | |
| memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp)); | |
| } | |
| void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd) | |
| { | |
| idev->opcode = cmd->cmd.opcode; | |
| if (!idev->dev_cmd_regs) | |
| return; | |
| memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd)); | |
| iowrite32(0, &idev->dev_cmd_regs->done); | |
| iowrite32(1, &idev->dev_cmd_regs->doorbell); | |
| } | |
| /* Device commands */ | |
| void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .identify.opcode = IONIC_CMD_IDENTIFY, | |
| .identify.ver = ver, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_init(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .init.opcode = IONIC_CMD_INIT, | |
| .init.type = 0, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_reset(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .reset.opcode = IONIC_CMD_RESET, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| /* Port commands */ | |
| void ionic_dev_cmd_port_identify(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_init.opcode = IONIC_CMD_PORT_IDENTIFY, | |
| .port_init.index = 0, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_init(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_init.opcode = IONIC_CMD_PORT_INIT, | |
| .port_init.index = 0, | |
| .port_init.info_pa = cpu_to_le64(idev->port_info_pa), | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_reset(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_reset.opcode = IONIC_CMD_PORT_RESET, | |
| .port_reset.index = 0, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, | |
| .port_setattr.index = 0, | |
| .port_setattr.attr = IONIC_PORT_ATTR_STATE, | |
| .port_setattr.state = state, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, | |
| .port_setattr.index = 0, | |
| .port_setattr.attr = IONIC_PORT_ATTR_SPEED, | |
| .port_setattr.speed = cpu_to_le32(speed), | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, | |
| .port_setattr.index = 0, | |
| .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG, | |
| .port_setattr.an_enable = an_enable, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, | |
| .port_setattr.index = 0, | |
| .port_setattr.attr = IONIC_PORT_ATTR_FEC, | |
| .port_setattr.fec_type = fec_type, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, | |
| .port_setattr.index = 0, | |
| .port_setattr.attr = IONIC_PORT_ATTR_PAUSE, | |
| .port_setattr.pause_type = pause_type, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| /* VF commands */ | |
| int ionic_set_vf_config(struct ionic *ionic, int vf, | |
| struct ionic_vf_setattr_cmd *vfc) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .vf_setattr.opcode = IONIC_CMD_VF_SETATTR, | |
| .vf_setattr.attr = vfc->attr, | |
| .vf_setattr.vf_index = cpu_to_le16(vf), | |
| }; | |
| int err; | |
| memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad)); | |
| mutex_lock(&ionic->dev_cmd_lock); | |
| ionic_dev_cmd_go(&ionic->idev, &cmd); | |
| err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); | |
| mutex_unlock(&ionic->dev_cmd_lock); | |
| return err; | |
| } | |
| void ionic_vf_start(struct ionic *ionic) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .vf_ctrl.opcode = IONIC_CMD_VF_CTRL, | |
| .vf_ctrl.ctrl_opcode = IONIC_VF_CTRL_START_ALL, | |
| }; | |
| if (!(ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_VF_CTRL))) | |
| return; | |
| ionic_dev_cmd_go(&ionic->idev, &cmd); | |
| ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); | |
| } | |
| /* LIF commands */ | |
| void ionic_dev_cmd_queue_identify(struct ionic_dev *idev, | |
| u16 lif_type, u8 qtype, u8 qver) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .q_identify.opcode = IONIC_CMD_Q_IDENTIFY, | |
| .q_identify.lif_type = cpu_to_le16(lif_type), | |
| .q_identify.type = qtype, | |
| .q_identify.ver = qver, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY, | |
| .lif_identify.type = type, | |
| .lif_identify.ver = ver, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index, | |
| dma_addr_t info_pa) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .lif_init.opcode = IONIC_CMD_LIF_INIT, | |
| .lif_init.index = cpu_to_le16(lif_index), | |
| .lif_init.info_pa = cpu_to_le64(info_pa), | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .lif_init.opcode = IONIC_CMD_LIF_RESET, | |
| .lif_init.index = cpu_to_le16(lif_index), | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq, | |
| u16 lif_index, u16 intr_index) | |
| { | |
| struct ionic_queue *q = &qcq->q; | |
| struct ionic_cq *cq = &qcq->cq; | |
| union ionic_dev_cmd cmd = { | |
| .q_init.opcode = IONIC_CMD_Q_INIT, | |
| .q_init.lif_index = cpu_to_le16(lif_index), | |
| .q_init.type = q->type, | |
| .q_init.ver = qcq->q.lif->qtype_info[q->type].version, | |
| .q_init.index = cpu_to_le32(q->index), | |
| .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ | | |
| IONIC_QINIT_F_ENA), | |
| .q_init.pid = cpu_to_le16(q->pid), | |
| .q_init.intr_index = cpu_to_le16(intr_index), | |
| .q_init.ring_size = ilog2(q->num_descs), | |
| .q_init.ring_base = cpu_to_le64(q->base_pa), | |
| .q_init.cq_ring_base = cpu_to_le64(cq->base_pa), | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| void ionic_dev_cmd_discover_cmb(struct ionic_dev *idev) | |
| { | |
| union ionic_dev_cmd cmd = { | |
| .discover_cmb.opcode = IONIC_CMD_DISCOVER_CMB, | |
| }; | |
| ionic_dev_cmd_go(idev, &cmd); | |
| } | |
| int ionic_db_page_num(struct ionic_lif *lif, int pid) | |
| { | |
| return (lif->hw_index * lif->dbid_count) + pid; | |
| } | |
| int ionic_get_cmb(struct ionic_lif *lif, u32 *pgid, phys_addr_t *pgaddr, | |
| int order, u8 stride_log2, bool *expdb) | |
| { | |
| struct ionic_dev *idev = &lif->ionic->idev; | |
| void __iomem *nonexpdb_pgptr; | |
| phys_addr_t nonexpdb_pgaddr; | |
| int i, idx; | |
| mutex_lock(&idev->cmb_inuse_lock); | |
| idx = bitmap_find_free_region(idev->cmb_inuse, idev->cmb_npages, order); | |
| mutex_unlock(&idev->cmb_inuse_lock); | |
| if (idx < 0) | |
| return idx; | |
| *pgid = (u32)idx; | |
| if (idev->phy_cmb_expdb64_pages && | |
| stride_log2 == IONIC_EXPDB_64B_WQE_LG2) { | |
| *pgaddr = idev->phy_cmb_expdb64_pages + idx * PAGE_SIZE; | |
| if (expdb) | |
| *expdb = true; | |
| } else if (idev->phy_cmb_expdb128_pages && | |
| stride_log2 == IONIC_EXPDB_128B_WQE_LG2) { | |
| *pgaddr = idev->phy_cmb_expdb128_pages + idx * PAGE_SIZE; | |
| if (expdb) | |
| *expdb = true; | |
| } else if (idev->phy_cmb_expdb256_pages && | |
| stride_log2 == IONIC_EXPDB_256B_WQE_LG2) { | |
| *pgaddr = idev->phy_cmb_expdb256_pages + idx * PAGE_SIZE; | |
| if (expdb) | |
| *expdb = true; | |
| } else if (idev->phy_cmb_expdb512_pages && | |
| stride_log2 == IONIC_EXPDB_512B_WQE_LG2) { | |
| *pgaddr = idev->phy_cmb_expdb512_pages + idx * PAGE_SIZE; | |
| if (expdb) | |
| *expdb = true; | |
| } else { | |
| *pgaddr = idev->phy_cmb_pages + idx * PAGE_SIZE; | |
| if (expdb) | |
| *expdb = false; | |
| } | |
| /* clear the requested CMB region, 1 PAGE_SIZE ioremap at a time */ | |
| nonexpdb_pgaddr = idev->phy_cmb_pages + idx * PAGE_SIZE; | |
| for (i = 0; i < (1 << order); i++) { | |
| nonexpdb_pgptr = | |
| ioremap_wc(nonexpdb_pgaddr + i * PAGE_SIZE, PAGE_SIZE); | |
| if (!nonexpdb_pgptr) { | |
| ionic_put_cmb(lif, *pgid, order); | |
| return -ENOMEM; | |
| } | |
| memset_io(nonexpdb_pgptr, 0, PAGE_SIZE); | |
| iounmap(nonexpdb_pgptr); | |
| } | |
| return 0; | |
| } | |
| EXPORT_SYMBOL_NS(ionic_get_cmb, "NET_IONIC"); | |
| void ionic_put_cmb(struct ionic_lif *lif, u32 pgid, int order) | |
| { | |
| struct ionic_dev *idev = &lif->ionic->idev; | |
| mutex_lock(&idev->cmb_inuse_lock); | |
| bitmap_release_region(idev->cmb_inuse, pgid, order); | |
| mutex_unlock(&idev->cmb_inuse_lock); | |
| } | |
| EXPORT_SYMBOL_NS(ionic_put_cmb, "NET_IONIC"); | |
| int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, | |
| struct ionic_intr_info *intr, | |
| unsigned int num_descs, size_t desc_size) | |
| { | |
| unsigned int ring_size; | |
| if (desc_size == 0 || !is_power_of_2(num_descs)) | |
| return -EINVAL; | |
| ring_size = ilog2(num_descs); | |
| if (ring_size < 2 || ring_size > 16) | |
| return -EINVAL; | |
| cq->lif = lif; | |
| cq->bound_intr = intr; | |
| cq->num_descs = num_descs; | |
| cq->desc_size = desc_size; | |
| cq->tail_idx = 0; | |
| cq->done_color = 1; | |
| cq->idev = &lif->ionic->idev; | |
| return 0; | |
| } | |
| unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do, | |
| ionic_cq_cb cb, ionic_cq_done_cb done_cb, | |
| void *done_arg) | |
| { | |
| unsigned int work_done = 0; | |
| if (work_to_do == 0) | |
| return 0; | |
| while (cb(cq)) { | |
| if (cq->tail_idx == cq->num_descs - 1) | |
| cq->done_color = !cq->done_color; | |
| cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); | |
| if (++work_done >= work_to_do) | |
| break; | |
| } | |
| if (work_done && done_cb) | |
| done_cb(done_arg); | |
| return work_done; | |
| } | |
| int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, | |
| struct ionic_queue *q, unsigned int index, const char *name, | |
| unsigned int num_descs, size_t desc_size, | |
| size_t sg_desc_size, unsigned int pid) | |
| { | |
| unsigned int ring_size; | |
| if (desc_size == 0 || !is_power_of_2(num_descs)) | |
| return -EINVAL; | |
| ring_size = ilog2(num_descs); | |
| if (ring_size < 2 || ring_size > 16) | |
| return -EINVAL; | |
| q->lif = lif; | |
| q->index = index; | |
| q->num_descs = num_descs; | |
| q->desc_size = desc_size; | |
| q->sg_desc_size = sg_desc_size; | |
| q->tail_idx = 0; | |
| q->head_idx = 0; | |
| q->pid = pid; | |
| snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index); | |
| return 0; | |
| } | |
| void ionic_q_post(struct ionic_queue *q, bool ring_doorbell) | |
| { | |
| struct ionic_lif *lif = q->lif; | |
| struct device *dev = q->dev; | |
| q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); | |
| dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n", | |
| q->lif->index, q->name, q->hw_type, q->hw_index, | |
| q->head_idx, ring_doorbell); | |
| if (ring_doorbell) { | |
| ionic_dbell_ring(lif->kern_dbpage, q->hw_type, | |
| q->dbval | q->head_idx); | |
| q->dbell_jiffies = jiffies; | |
| } | |
| } | |
| bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos) | |
| { | |
| unsigned int mask, tail, head; | |
| mask = q->num_descs - 1; | |
| tail = q->tail_idx; | |
| head = q->head_idx; | |
| return ((pos - tail) & mask) < ((head - tail) & mask); | |
| } | |
| void ionic_reset_link_down_count(struct ionic_dev *idev) | |
| { | |
| if (!READ_ONCE(idev->link_down_count_init)) { | |
| idev->link_down_count_total = 0; | |
| idev->link_down_count_last = | |
| le16_to_cpu(idev->port_info->status.link_down_count); | |
| WRITE_ONCE(idev->link_down_count_init, true); | |
| } | |
| } |