| /* | |
| * Copyright 2025 Advanced Micro Devices, Inc. | |
| * | |
| * Permission is hereby granted, free of charge, to any person obtaining a | |
| * copy of this software and associated documentation files (the "Software"), | |
| * to deal in the Software without restriction, including without limitation | |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| * and/or sell copies of the Software, and to permit persons to whom the | |
| * Software is furnished to do so, subject to the following conditions: | |
| * | |
| * The above copyright notice and this permission notice shall be included in | |
| * all copies or substantial portions of the Software. | |
| * | |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| * OTHER DEALINGS IN THE SOFTWARE. | |
| * | |
| */ | |
| #include "amdgpu.h" | |
| #include "gmc_v12_1.h" | |
| #include "soc15_common.h" | |
| #include "soc_v1_0.h" | |
| #include "soc_v1_0_enum.h" | |
| #include "oss/osssys_7_1_0_offset.h" | |
| #include "oss/osssys_7_1_0_sh_mask.h" | |
| #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" | |
| static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev, | |
| unsigned int *mtype_local, | |
| unsigned int *mtype_remote); | |
| static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev, | |
| struct amdgpu_irq_src *src, | |
| unsigned int type, | |
| enum amdgpu_interrupt_state state) | |
| { | |
| struct amdgpu_vmhub *hub; | |
| u32 tmp, reg, i, j; | |
| switch (state) { | |
| case AMDGPU_IRQ_STATE_DISABLE: | |
| for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { | |
| hub = &adev->vmhub[j]; | |
| for (i = 0; i < 16; i++) { | |
| reg = hub->vm_context0_cntl + i; | |
| /* This works because this interrupt is only | |
| * enabled at init/resume and disabled in | |
| * fini/suspend, so the overall state doesn't | |
| * change over the course of suspend/resume. | |
| */ | |
| if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) | |
| continue; | |
| if (j >= AMDGPU_MMHUB0(0)) | |
| tmp = RREG32_SOC15_IP(MMHUB, reg); | |
| else | |
| tmp = RREG32_XCC(reg, j); | |
| tmp &= ~hub->vm_cntx_cntl_vm_fault; | |
| if (j >= AMDGPU_MMHUB0(0)) | |
| WREG32_SOC15_IP(MMHUB, reg, tmp); | |
| else | |
| WREG32_XCC(reg, tmp, j); | |
| } | |
| } | |
| break; | |
| case AMDGPU_IRQ_STATE_ENABLE: | |
| for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { | |
| hub = &adev->vmhub[j]; | |
| for (i = 0; i < 16; i++) { | |
| reg = hub->vm_context0_cntl + i; | |
| /* This works because this interrupt is only | |
| * enabled at init/resume and disabled in | |
| * fini/suspend, so the overall state doesn't | |
| * change over the course of suspend/resume. | |
| */ | |
| if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) | |
| continue; | |
| if (j >= AMDGPU_MMHUB0(0)) | |
| tmp = RREG32_SOC15_IP(MMHUB, reg); | |
| else | |
| tmp = RREG32_XCC(reg, j); | |
| tmp |= hub->vm_cntx_cntl_vm_fault; | |
| if (j >= AMDGPU_MMHUB0(0)) | |
| WREG32_SOC15_IP(MMHUB, reg, tmp); | |
| else | |
| WREG32_XCC(reg, tmp, j); | |
| } | |
| } | |
| break; | |
| default: | |
| break; | |
| } | |
| return 0; | |
| } | |
| /* | |
| * NodeID definition in interrupt cookie for gmc v12.1.0 | |
| * | |
| * DIE IH Cookie NodeID | |
| * MID0 0x0 | |
| * AID0 0x1 | |
| * AID0.XCD0 0x2 | |
| * AID0.XCD1 0x3 | |
| * AID0.XCD2 0x4 | |
| * AID0.XCD3 0x5 | |
| * MID1 0x8 | |
| * AID1 0x9 | |
| * AID1.XCD0 0xA | |
| * AID1.XCD1 0xB | |
| * AID1.XCD2 0xC | |
| * AID1.XCD3 0xD | |
| */ | |
| static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, | |
| struct amdgpu_irq_src *source, | |
| struct amdgpu_iv_entry *entry) | |
| { | |
| struct amdgpu_task_info *task_info; | |
| bool retry_fault = false, write_fault = false; | |
| unsigned int vmhub, node_id; | |
| struct amdgpu_vmhub *hub; | |
| uint32_t cam_index = 0; | |
| const char *hub_name; | |
| int xcc_id = 0; | |
| uint32_t status = 0, status_hi = 0; | |
| const char *die_name; | |
| char die_name_buf[32]; | |
| u64 addr; | |
| node_id = entry->node_id; | |
| addr = (u64)entry->src_data[0] << 12; | |
| addr |= ((u64)entry->src_data[1] & 0x1fff) << 44; | |
| if (entry->src_id == UTCL2_1_0__SRCID__RETRY) { | |
| retry_fault = true; | |
| write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE); | |
| } | |
| if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) { | |
| hub_name = "mmhub0"; | |
| vmhub = AMDGPU_MMHUB0(node_id / 8); | |
| } else { | |
| hub_name = "gfxhub0"; | |
| if (adev->gfx.funcs->ih_node_to_logical_xcc) { | |
| xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, | |
| node_id); | |
| if (xcc_id < 0) | |
| xcc_id = 0; | |
| } | |
| vmhub = xcc_id; | |
| } | |
| hub = &adev->vmhub[vmhub]; | |
| if (retry_fault) { | |
| cam_index = entry->src_data[3] & 0x3ff; | |
| int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, node_id, | |
| write_fault); | |
| /* Returning 1 here also prevents sending the IV to the KFD */ | |
| if (ret == 1) | |
| return 1; | |
| } | |
| if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault)) | |
| return 1; | |
| if (!printk_ratelimit()) | |
| return 0; | |
| dev_err(adev->dev, | |
| "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name, | |
| retry_fault ? "retry" : "no-retry", | |
| entry->src_id, entry->ring_id, entry->vmid, entry->pasid); | |
| task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); | |
| if (task_info) { | |
| amdgpu_vm_print_task_info(adev, task_info); | |
| amdgpu_vm_put_task_info(task_info); | |
| } | |
| dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", | |
| addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); | |
| if (adev->irq.ih_funcs && | |
| adev->irq.ih_funcs->node_id_to_die_name) { | |
| die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id, | |
| die_name_buf, | |
| sizeof(die_name_buf)); | |
| if (die_name) | |
| dev_err(adev->dev, | |
| " cookie node_id %d fault from die %s\n", | |
| node_id, die_name); | |
| } | |
| if (amdgpu_sriov_vf(adev)) | |
| return 0; | |
| /* | |
| * Issue a dummy read to wait for the status register to | |
| * be updated to avoid reading an incorrect value due to | |
| * the new fast GRBM interface. | |
| */ | |
| if (entry->vmid_src == AMDGPU_GFXHUB(0)) | |
| RREG32(hub->vm_l2_pro_fault_status); | |
| status = RREG32(hub->vm_l2_pro_fault_status); | |
| if (hub->vmhub_funcs && | |
| hub->vmhub_funcs->print_l2_protection_fault_status_hi) | |
| status_hi = RREG32(hub->vm_l2_pro_fault_status_hi); | |
| /* Only print L2 fault status if the status register could be read and | |
| * contains useful information | |
| */ | |
| if (!status && !status_hi) | |
| return 0; | |
| WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); | |
| amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub); | |
| if (!hub->vmhub_funcs || !hub->vmhub_funcs->print_l2_protection_fault_status) { | |
| dev_warn_once(adev->dev, "vmhub %d print fault status func not defined\n", vmhub); | |
| return 0; | |
| } | |
| hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); | |
| if (hub->vmhub_funcs->print_l2_protection_fault_status_hi) | |
| hub->vmhub_funcs->print_l2_protection_fault_status_hi(adev, | |
| status_hi); | |
| return 0; | |
| } | |
| static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev, | |
| uint8_t vmid, uint8_t inst, | |
| uint16_t *p_pasid) | |
| { | |
| uint16_t index; | |
| if (inst/4) | |
| index = 0xA + inst%4; | |
| else | |
| index = 0x2 + inst%4; | |
| WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index); | |
| *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff; | |
| return !!(*p_pasid); | |
| } | |
| /* | |
| * GART | |
| * VMID 0 is the physical GPU addresses as used by the kernel. | |
| * VMIDs 1-15 are used for userspace clients and are handled | |
| * by the amdgpu vm/hsa code. | |
| */ | |
| /** | |
| * gmc_v12_1_use_invalidate_semaphore - judge whether to use semaphore | |
| * | |
| * @adev: amdgpu_device pointer | |
| * @vmhub: vmhub type | |
| * | |
| */ | |
| static bool gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device *adev, | |
| uint32_t vmhub) | |
| { | |
| return ((!AMDGPU_IS_GFXHUB(vmhub)) && | |
| (!amdgpu_sriov_vf(adev))); | |
| } | |
| static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, | |
| unsigned int vmhub, uint32_t flush_type) | |
| { | |
| bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(adev, vmhub); | |
| struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; | |
| u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); | |
| u32 tmp; | |
| /* Use register 17 for GART */ | |
| const unsigned eng = 17; | |
| unsigned int i; | |
| u32 inst; | |
| inst = AMDGPU_IS_GFXHUB(vmhub) ? vmhub - AMDGPU_GFXHUB_START : 0; | |
| spin_lock(&adev->gmc.invalidate_lock); | |
| if (use_semaphore) { | |
| for (i = 0; i < adev->usec_timeout; i++) { | |
| /* a read return value of 1 means semaphore acuqire */ | |
| if (AMDGPU_IS_GFXHUB(vmhub)) | |
| tmp = RREG32_SOC15_IP_NO_KIQ(GC, | |
| hub->vm_inv_eng0_sem + | |
| hub->eng_distance * eng, | |
| inst); | |
| else | |
| tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, | |
| hub->vm_inv_eng0_sem + | |
| hub->eng_distance * eng, | |
| inst); | |
| if (tmp & 0x1) | |
| break; | |
| udelay(1); | |
| } | |
| if (i >= adev->usec_timeout) | |
| DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); | |
| } | |
| if (AMDGPU_IS_GFXHUB(vmhub)) | |
| WREG32_SOC15_IP_NO_KIQ(GC, | |
| hub->vm_inv_eng0_req + hub->eng_distance * eng, | |
| inv_req, inst); | |
| else | |
| WREG32_SOC15_IP_NO_KIQ(MMHUB, | |
| hub->vm_inv_eng0_req + hub->eng_distance * eng, | |
| inv_req, inst); | |
| /* Wait for ACK with a delay.*/ | |
| for (i = 0; i < adev->usec_timeout; i++) { | |
| if (AMDGPU_IS_GFXHUB(vmhub)) | |
| tmp = RREG32_SOC15_IP_NO_KIQ(GC, | |
| hub->vm_inv_eng0_ack + | |
| hub->eng_distance * eng, | |
| inst); | |
| else | |
| tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, | |
| hub->vm_inv_eng0_ack + | |
| hub->eng_distance * eng, | |
| inst); | |
| tmp &= 1 << vmid; | |
| if (tmp) | |
| break; | |
| udelay(1); | |
| } | |
| if (use_semaphore) { | |
| if (AMDGPU_IS_GFXHUB(vmhub)) | |
| WREG32_SOC15_IP_NO_KIQ(GC, | |
| hub->vm_inv_eng0_sem + hub->eng_distance * eng, | |
| 0, inst); | |
| else | |
| WREG32_SOC15_IP_NO_KIQ(MMHUB, | |
| hub->vm_inv_eng0_sem + hub->eng_distance * eng, | |
| 0, inst); | |
| } | |
| /* Issue additional private vm invalidation to MMHUB */ | |
| if (!AMDGPU_IS_GFXHUB(vmhub) && | |
| (hub->vm_l2_bank_select_reserved_cid2) && | |
| !amdgpu_sriov_vf(adev)) { | |
| inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); | |
| /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ | |
| inv_req |= (1 << 25); | |
| /* Issue private invalidation */ | |
| WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req); | |
| /* Read back to ensure invalidation is done*/ | |
| RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); | |
| } | |
| spin_unlock(&adev->gmc.invalidate_lock); | |
| if (i < adev->usec_timeout) | |
| return; | |
| dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); | |
| } | |
| /** | |
| * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback | |
| * | |
| * @adev: amdgpu_device pointer | |
| * @vmid: vm instance to flush | |
| * @vmhub: which hub to flush | |
| * @flush_type: the flush type | |
| * | |
| * Flush the TLB for the requested page table. | |
| */ | |
| static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, | |
| uint32_t vmhub, uint32_t flush_type) | |
| { | |
| u32 inst; | |
| if (AMDGPU_IS_GFXHUB(vmhub) && | |
| !adev->gfx.is_poweron) | |
| return; | |
| if (vmhub >= AMDGPU_MMHUB0(0)) | |
| inst = 0; | |
| else | |
| inst = vmhub; | |
| /* This is necessary for SRIOV as well as for GFXOFF to function | |
| * properly under bare metal | |
| */ | |
| if (adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready && | |
| (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { | |
| struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; | |
| const unsigned eng = 17; | |
| u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); | |
| u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; | |
| u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; | |
| amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, | |
| 1 << vmid, inst); | |
| return; | |
| } | |
| gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, flush_type); | |
| return; | |
| } | |
| /** | |
| * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid | |
| * | |
| * @adev: amdgpu_device pointer | |
| * @pasid: pasid to be flush | |
| * @flush_type: the flush type | |
| * @all_hub: flush all hubs | |
| * @inst: XCC instance to use for the invalidation | |
| * | |
| * Flush the TLB for the requested pasid. | |
| */ | |
| static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, | |
| uint16_t pasid, uint32_t flush_type, | |
| bool all_hub, uint32_t inst) | |
| { | |
| uint16_t queried; | |
| int vmid, i; | |
| if (adev->mes.ring[0].sched.ready && | |
| (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { | |
| struct mes_inv_tlbs_pasid_input input = {0}; | |
| input.xcc_id = inst; | |
| input.pasid = pasid; | |
| input.flush_type = flush_type; | |
| /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ | |
| if (!amdgpu_gfx_is_master_xcc(adev, inst)) | |
| return; | |
| input.hub_id = AMDGPU_GFXHUB(0); | |
| adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); | |
| if (all_hub) { | |
| /* invalidate mm_hub */ | |
| if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) { | |
| input.hub_id = AMDGPU_MMHUB0(0); | |
| adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); | |
| } | |
| if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { | |
| input.hub_id = AMDGPU_MMHUB1(0); | |
| adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); | |
| } | |
| } | |
| return; | |
| } | |
| for (vmid = 1; vmid < 16; vmid++) { | |
| bool valid; | |
| valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst, | |
| &queried); | |
| if (!valid || queried != pasid) | |
| continue; | |
| if (all_hub) { | |
| for_each_set_bit(i, adev->vmhubs_mask, | |
| AMDGPU_MAX_VMHUBS) | |
| gmc_v12_1_flush_gpu_tlb(adev, vmid, i, | |
| flush_type); | |
| } else { | |
| gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst), | |
| flush_type); | |
| } | |
| } | |
| } | |
| static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring, | |
| unsigned vmid, uint64_t pd_addr) | |
| { | |
| bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(ring->adev, ring->vm_hub); | |
| struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; | |
| uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); | |
| unsigned eng = ring->vm_inv_eng; | |
| if (use_semaphore) | |
| /* a read return value of 1 means semaphore acuqire */ | |
| amdgpu_ring_emit_reg_wait(ring, | |
| hub->vm_inv_eng0_sem + | |
| hub->eng_distance * eng, 0x1, 0x1); | |
| amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + | |
| (hub->ctx_addr_distance * vmid), | |
| lower_32_bits(pd_addr)); | |
| amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + | |
| (hub->ctx_addr_distance * vmid), | |
| upper_32_bits(pd_addr)); | |
| amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + | |
| hub->eng_distance * eng, | |
| hub->vm_inv_eng0_ack + | |
| hub->eng_distance * eng, | |
| req, 1 << vmid); | |
| if (use_semaphore) | |
| /* | |
| * add semaphore release after invalidation, | |
| * write with 0 means semaphore release | |
| */ | |
| amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + | |
| hub->eng_distance * eng, 0); | |
| return pd_addr; | |
| } | |
| static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring, | |
| unsigned vmid, unsigned pasid) | |
| { | |
| struct amdgpu_device *adev = ring->adev; | |
| uint32_t reg; | |
| if (ring->vm_hub == AMDGPU_GFXHUB(0)) | |
| reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid; | |
| else | |
| reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid; | |
| amdgpu_ring_emit_wreg(ring, reg, pasid); | |
| } | |
| /* | |
| * PTE format: | |
| * 63 P | |
| * 62:59 reserved | |
| * 58 D | |
| * 57 G | |
| * 56 T | |
| * 55:54 M | |
| * 53:52 SW | |
| * 51:48 reserved for future | |
| * 47:12 4k physical page base address | |
| * 11:7 fragment | |
| * 6 write | |
| * 5 read | |
| * 4 exe | |
| * 3 Z | |
| * 2 snooped | |
| * 1 system | |
| * 0 valid | |
| * | |
| * PDE format: | |
| * 63 P | |
| * 62:58 block fragment size | |
| * 57 reserved | |
| * 56 A | |
| * 55:54 M | |
| * 53:52 reserved | |
| * 51:48 reserved for future | |
| * 47:6 physical base address of PD or PTE | |
| * 5:3 reserved | |
| * 2 C | |
| * 1 system | |
| * 0 valid | |
| */ | |
| static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level, | |
| uint64_t *addr, uint64_t *flags) | |
| { | |
| if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM)) | |
| *addr = adev->vm_manager.vram_base_offset + *addr - | |
| adev->gmc.vram_start; | |
| if (*addr & ~adev->gmc.pte_addr_mask) { | |
| dev_err(adev->dev, "Invalid PDE address: %llx\n", *addr); | |
| *addr = 0; | |
| *flags &= ~AMDGPU_PTE_VALID; | |
| } | |
| *flags |= AMDGPU_PTE_SNOOPED; | |
| if (!adev->gmc.translate_further) | |
| return; | |
| if (level == AMDGPU_VM_PDB1) { | |
| /* Set the block fragment size */ | |
| if (!(*flags & AMDGPU_PDE_PTE_GFX12)) | |
| *flags |= AMDGPU_PDE_BFS_GFX12(0x9); | |
| } | |
| } | |
| static void gmc_v12_1_get_npa_flags(struct amdgpu_device *adev, | |
| uint64_t *flags) | |
| { | |
| unsigned int mtype_local, mtype_remote; | |
| gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote); | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype_remote); | |
| /* VSCT = 0011 to identify NPA. Additionally PTE.B = 1 */ | |
| *flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | | |
| AMDGPU_PTE_BUS_ATOMICS; | |
| *flags &= ~AMDGPU_PTE_VALID; | |
| *flags &= ~AMDGPU_PTE_SYSTEM; | |
| *flags &= ~AMDGPU_PTE_EXECUTABLE; | |
| } | |
| /* | |
| * Resolve the MTYPEs used for local and remote memory accesses on GFX 12.1. | |
| * Both default to an ASIC-dependent value that can be overridden through the | |
| * amdgpu_mtype_local and amdgpu_mtype_remote module parameters. | |
| */ | |
| static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev, | |
| unsigned int *mtype_local, | |
| unsigned int *mtype_remote) | |
| { | |
| bool is_aid_a1 = SOC_V1_0_DIE_REV_AID(adev->rev_id) == 1; | |
| /* Local memory: ASIC default depends on the AID stepping. */ | |
| *mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC; | |
| if (amdgpu_mtype_local == 0) | |
| *mtype_local = MTYPE_RW; | |
| else if (amdgpu_mtype_local == 1) | |
| *mtype_local = MTYPE_NC; | |
| else if (amdgpu_mtype_local == 2) | |
| DRM_INFO_ONCE("MTYPE_CC not supported for local memory\n"); | |
| /* Remote memory defaults to MTYPE_UC on GFX 12.1. */ | |
| *mtype_remote = MTYPE_UC; | |
| if (amdgpu_mtype_remote == 0) | |
| *mtype_remote = MTYPE_NC; | |
| else if (amdgpu_mtype_remote == 1) | |
| *mtype_remote = MTYPE_UC; | |
| DRM_INFO_ONCE("Using %s for local memory and %s for remote memory\n", | |
| *mtype_local == MTYPE_RW ? "MTYPE_RW" : "MTYPE_NC", | |
| *mtype_remote == MTYPE_NC ? "MTYPE_NC" : "MTYPE_UC"); | |
| } | |
| /* | |
| * The compute MQD coherent_aql_mtype field (offset 509) must be programmed to | |
| * 0 whenever the driver maps local or remote memory as MTYPE_NC, and to 1 in | |
| * all other cases. | |
| */ | |
| u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev) | |
| { | |
| unsigned int mtype_local, mtype_remote; | |
| gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote); | |
| return (mtype_local == MTYPE_NC || mtype_remote == MTYPE_NC) ? 0 : 1; | |
| } | |
| static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev, | |
| struct amdgpu_bo *bo, | |
| uint64_t *flags) | |
| { | |
| struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
| bool is_vram = bo->tbo.resource && | |
| bo->tbo.resource->mem_type == TTM_PL_VRAM; | |
| bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | | |
| AMDGPU_GEM_CREATE_EXT_COHERENT); | |
| bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT; | |
| uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); | |
| bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; | |
| unsigned int mtype, mtype_local, mtype_remote; | |
| bool snoop = false; | |
| bool is_local = false; | |
| bool is_aid_a1, is_spx; | |
| switch (gc_ip_version) { | |
| case IP_VERSION(12, 1, 0): | |
| gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote); | |
| is_local = (is_vram && adev == bo_adev); | |
| snoop = true; | |
| if (uncached) { | |
| mtype = MTYPE_UC; | |
| } else if (ext_coherent) { | |
| is_aid_a1 = SOC_V1_0_DIE_REV_AID(adev->rev_id) == 1; | |
| is_spx = amdgpu_xcp_query_partition_mode(adev->xcp_mgr, | |
| AMDGPU_XCP_FL_NONE) == | |
| AMDGPU_SPX_PARTITION_MODE; | |
| /* AID A0 requires MTYPE_UC for extended-scope coherent | |
| * local memory in DPX/QPX/CPX modes. | |
| */ | |
| if (is_local && (is_aid_a1 || is_spx)) | |
| mtype = mtype_local; | |
| else | |
| mtype = MTYPE_UC; | |
| } else { | |
| mtype = is_local ? mtype_local : mtype_remote; | |
| } | |
| break; | |
| default: | |
| if (uncached || coherent) | |
| mtype = MTYPE_UC; | |
| else | |
| mtype = MTYPE_NC; | |
| } | |
| if (mtype != MTYPE_NC) | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype); | |
| if (is_local || adev->have_atomics_support) | |
| *flags |= AMDGPU_PTE_BUS_ATOMICS; | |
| *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; | |
| } | |
| static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev, | |
| struct amdgpu_vm *vm, | |
| struct amdgpu_bo *bo, | |
| uint32_t vm_flags, | |
| uint64_t *flags) | |
| { | |
| struct ttm_resource *mem; | |
| if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) | |
| *flags |= AMDGPU_PTE_EXECUTABLE; | |
| else | |
| *flags &= ~AMDGPU_PTE_EXECUTABLE; | |
| switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { | |
| case AMDGPU_VM_MTYPE_DEFAULT: | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); | |
| break; | |
| case AMDGPU_VM_MTYPE_NC: | |
| default: | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); | |
| break; | |
| case AMDGPU_VM_MTYPE_RW: | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW); | |
| break; | |
| case AMDGPU_VM_MTYPE_UC: | |
| *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC); | |
| break; | |
| } | |
| if (bo) { | |
| mem = bo->tbo.resource; | |
| if (mem && mem->mem_type == AMDGPU_PL_NPA) { | |
| dev_dbg(adev->dev, | |
| "Setting PTE for NPA BO, mem->type: %d, mem->start: %lx, mem->size: %u, cur_flags: %llx\n", | |
| mem->mem_type, mem->start, (u32)mem->size, *flags); | |
| gmc_v12_1_get_npa_flags(adev, flags); | |
| } else if (*flags & AMDGPU_PTE_VALID) { | |
| gmc_v12_1_get_coherence_flags(adev, bo, flags); | |
| } | |
| } | |
| } | |
| static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = { | |
| .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb, | |
| .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid, | |
| .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb, | |
| .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping, | |
| .get_vm_pde = gmc_v12_1_get_vm_pde, | |
| .get_vm_pte = gmc_v12_1_get_vm_pte, | |
| .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition, | |
| .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition, | |
| }; | |
| void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev) | |
| { | |
| adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs; | |
| } | |
| static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = { | |
| .set = gmc_v12_1_vm_fault_interrupt_state, | |
| .process = gmc_v12_1_process_interrupt, | |
| }; | |
| static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = { | |
| .process = amdgpu_umc_uniras_process_ecc_irq, | |
| }; | |
| void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev) | |
| { | |
| adev->gmc.vm_fault.num_types = 1; | |
| adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs; | |
| adev->gmc.ecc_irq.num_types = 1; | |
| adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs; | |
| } | |
| void gmc_v12_1_init_vram_info(struct amdgpu_device *adev) | |
| { | |
| /* TODO: query vram_info from ip discovery binary */ | |
| adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4; | |
| adev->gmc.vram_width = 384 * 64; | |
| } | |
| void gmc_v12_1_init_nps_details(struct amdgpu_device *adev) | |
| { | |
| enum amdgpu_memory_partition mode; | |
| uint32_t supp_modes; | |
| int i; | |
| adev->gmc.supported_nps_modes = 0; | |
| if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) | |
| return; | |
| mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes); | |
| /* Mode detected by hardware and supported modes available */ | |
| if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && supp_modes) { | |
| while ((i = ffs(supp_modes))) { | |
| if (AMDGPU_ALL_NPS_MASK & BIT(i)) | |
| adev->gmc.supported_nps_modes |= BIT(i); | |
| supp_modes &= supp_modes - 1; | |
| } | |
| } else { | |
| switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| case IP_VERSION(12, 1, 0): | |
| adev->gmc.supported_nps_modes = | |
| BIT(AMDGPU_NPS1_PARTITION_MODE) | | |
| BIT(AMDGPU_NPS2_PARTITION_MODE); | |
| break; | |
| default: | |
| break; | |
| } | |
| } | |
| } |