| // SPDX-License-Identifier: MIT | |
| /* | |
| * Copyright © 2022 Intel Corporation | |
| */ | |
| #include "xe_reg_sr.h" | |
| #include <kunit/visibility.h> | |
| #include <linux/align.h> | |
| #include <linux/string_helpers.h> | |
| #include <linux/xarray.h> | |
| #include <drm/drm_managed.h> | |
| #include <drm/drm_print.h> | |
| #include "xe_assert.h" | |
| #include "xe_device.h" | |
| #include "xe_device_types.h" | |
| #include "xe_force_wake.h" | |
| #include "xe_gt_mcr.h" | |
| #include "xe_gt_printk.h" | |
| #include "xe_gt_types.h" | |
| #include "xe_hw_engine_types.h" | |
| #include "xe_lrc.h" | |
| #include "xe_mmio.h" | |
| #include "xe_rtp_types.h" | |
| static void reg_sr_fini(struct drm_device *drm, void *arg) | |
| { | |
| struct xe_reg_sr *sr = arg; | |
| struct xe_reg_sr_entry *entry; | |
| unsigned long reg; | |
| xa_for_each(&sr->xa, reg, entry) | |
| kfree(entry); | |
| xa_destroy(&sr->xa); | |
| } | |
| int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) | |
| { | |
| xa_init(&sr->xa); | |
| sr->name = name; | |
| return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); | |
| } | |
| EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init); | |
| static bool compatible_entries(const struct xe_reg_sr_entry *e1, | |
| const struct xe_reg_sr_entry *e2) | |
| { | |
| /* | |
| * Don't allow overwriting values: clr_bits/set_bits should be disjoint | |
| * when operating in the same register | |
| */ | |
| if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || | |
| e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) | |
| return false; | |
| if (e1->reg.raw != e2->reg.raw) | |
| return false; | |
| return true; | |
| } | |
| static void reg_sr_inc_error(struct xe_reg_sr *sr) | |
| { | |
| #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) | |
| sr->errors++; | |
| #endif | |
| } | |
| static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr, | |
| const struct xe_reg_sr_entry *e, | |
| struct xe_gt *gt) | |
| { | |
| struct xe_reg reg = e->reg; | |
| bool is_mcr; | |
| /* | |
| * We need the gt structure to check MCR ranges. | |
| */ | |
| if (!gt) | |
| return reg; | |
| is_mcr = xe_gt_mcr_check_reg(gt, reg); | |
| if (is_mcr && !reg.mcr) { | |
| reg.mcr = 1; | |
| xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n", | |
| reg.addr); | |
| reg_sr_inc_error(sr); | |
| } | |
| if (!is_mcr && reg.mcr) { | |
| reg.mcr = 0; | |
| xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n", | |
| reg.addr); | |
| reg_sr_inc_error(sr); | |
| } | |
| return reg; | |
| } | |
| int xe_reg_sr_add(struct xe_reg_sr *sr, | |
| const struct xe_reg_sr_entry *e, | |
| struct xe_gt *gt) | |
| { | |
| unsigned long idx = e->reg.addr; | |
| struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); | |
| struct xe_reg reg; | |
| int ret; | |
| reg = sanitize_mcr(sr, e, gt); | |
| if (pentry) { | |
| if (!compatible_entries(pentry, e)) { | |
| ret = -EINVAL; | |
| goto fail; | |
| } | |
| pentry->clr_bits |= e->clr_bits; | |
| pentry->set_bits |= e->set_bits; | |
| pentry->read_mask |= e->read_mask; | |
| return 0; | |
| } | |
| pentry = kmalloc_obj(*pentry); | |
| if (!pentry) { | |
| ret = -ENOMEM; | |
| goto fail; | |
| } | |
| *pentry = *e; | |
| pentry->reg = reg; | |
| ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); | |
| if (ret) | |
| goto fail_free; | |
| return 0; | |
| fail_free: | |
| kfree(pentry); | |
| fail: | |
| xe_gt_err(gt, | |
| "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", | |
| idx, e->clr_bits, e->set_bits, | |
| str_yes_no(e->reg.masked), | |
| str_yes_no(e->reg.mcr), | |
| ret); | |
| reg_sr_inc_error(sr); | |
| return ret; | |
| } | |
| /* | |
| * Convert back from encoded value to type-safe, only to be used when reg.mcr | |
| * is true | |
| */ | |
| static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) | |
| { | |
| return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; | |
| } | |
| static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry) | |
| { | |
| struct xe_reg reg = entry->reg; | |
| struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); | |
| u32 val; | |
| /* | |
| * If this is a masked register, need to set the upper 16 bits. | |
| * Set them to clr_bits since that is always a superset of the bits | |
| * being modified. | |
| * | |
| * When it's not masked, we have to read it from hardware, unless we are | |
| * supposed to set all bits. | |
| */ | |
| if (reg.masked) | |
| val = entry->clr_bits << 16; | |
| else if (entry->clr_bits + 1) | |
| val = (reg.mcr ? | |
| xe_gt_mcr_unicast_read_any(gt, reg_mcr) : | |
| xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits); | |
| else | |
| val = 0; | |
| /* | |
| * TODO: add selftest to validate all tables, regardless of platform: | |
| * - Masked registers can't have set_bits with upper bits set | |
| * - set_bits must be contained in clr_bits | |
| */ | |
| val |= entry->set_bits; | |
| xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); | |
| if (entry->reg.mcr) | |
| xe_gt_mcr_multicast_write(gt, reg_mcr, val); | |
| else | |
| xe_mmio_write32(>->mmio, reg, val); | |
| } | |
| void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) | |
| { | |
| struct xe_reg_sr_entry *entry; | |
| unsigned long reg; | |
| if (xa_empty(&sr->xa)) | |
| return; | |
| /* | |
| * We don't process non-LRC reg_sr lists in VF, so they should have | |
| * been empty in the check above. | |
| */ | |
| xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); | |
| xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); | |
| CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
| if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { | |
| xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); | |
| return; | |
| } | |
| xa_for_each(&sr->xa, reg, entry) | |
| apply_one_mmio(gt, entry); | |
| } | |
| /** | |
| * xe_reg_sr_dump - print all save/restore entries | |
| * @sr: Save/restore entries | |
| * @p: DRM printer | |
| */ | |
| void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p) | |
| { | |
| struct xe_reg_sr_entry *entry; | |
| unsigned long reg; | |
| if (!sr->name || xa_empty(&sr->xa)) | |
| return; | |
| drm_printf(p, "%s\n", sr->name); | |
| xa_for_each(&sr->xa, reg, entry) | |
| drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n", | |
| reg, entry->clr_bits, entry->set_bits, | |
| str_yes_no(entry->reg.masked), | |
| str_yes_no(entry->reg.mcr)); | |
| } | |
| static u32 readback_reg(struct xe_gt *gt, struct xe_reg reg) | |
| { | |
| struct xe_reg_mcr mcr_reg = to_xe_reg_mcr(reg); | |
| if (reg.mcr) | |
| return xe_gt_mcr_unicast_read_any(gt, mcr_reg); | |
| else | |
| return xe_mmio_read32(>->mmio, reg); | |
| } | |
| /** | |
| * xe_reg_sr_readback_check() - Readback registers referenced in save/restore | |
| * entries and check whether the programming is in place. | |
| * @sr: Save/restore entries | |
| * @gt: GT to read register from | |
| * @p: DRM printer to report discrepancies on | |
| */ | |
| void xe_reg_sr_readback_check(struct xe_reg_sr *sr, | |
| struct xe_gt *gt, | |
| struct drm_printer *p) | |
| { | |
| struct xe_reg_sr_entry *entry; | |
| unsigned long offset; | |
| xa_for_each(&sr->xa, offset, entry) { | |
| u32 val = readback_reg(gt, entry->reg); | |
| u32 mask = entry->clr_bits | entry->set_bits; | |
| if ((val & mask) != entry->set_bits) | |
| drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", | |
| offset, mask, entry->set_bits, val & mask); | |
| } | |
| } | |
| /** | |
| * xe_reg_sr_lrc_check() - Check LRC for registers referenced in save/restore | |
| * entries and check whether the programming is in place. | |
| * @sr: Save/restore entries | |
| * @gt: GT to read register from | |
| * @hwe: Hardware engine type to check LRC for | |
| * @p: DRM printer to report discrepancies on | |
| */ | |
| void xe_reg_sr_lrc_check(struct xe_reg_sr *sr, | |
| struct xe_gt *gt, | |
| struct xe_hw_engine *hwe, | |
| struct drm_printer *p) | |
| { | |
| struct xe_reg_sr_entry *entry; | |
| unsigned long offset; | |
| xa_for_each(&sr->xa, offset, entry) { | |
| u32 val; | |
| int ret = xe_lrc_lookup_default_reg_value(gt, hwe->class, offset, &val); | |
| u32 mask = entry->clr_bits | entry->set_bits; | |
| if (ret == -ENOENT) | |
| drm_printf(p, "%#8lx :: not found in LRC for %s\n", offset, hwe->name); | |
| else if ((val & mask) != entry->set_bits) | |
| drm_printf(p, "%#8lx & %#10x :: expected %#10x got %#10x\n", | |
| offset, mask, entry->set_bits, val & mask); | |
| } | |
| } |