[magma] Import AMD's MIT-licensed Linux driver. This is a pristine copy, with the GPL-licensed build configs removed. A LICENSE and METADATA file have been added, per the third-party guidelines. This driver coordinates with the AMD-VLK ICD, which will be imported in a separate change. Both are permissively licensed, but need to be ported from Linux to Fuchsia. Source: "v5.3" tag from github.com/torvalds/linux Change-Id: I0b4c2fc58e556d6923d00c4c3933a55b4febeb19
diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f248d4 --- /dev/null +++ b/LICENSE
@@ -0,0 +1,19 @@ +Copyright 2015 Advanced Micro Devices, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.fuchsia b/README.fuchsia new file mode 100644 index 0000000..23256b7 --- /dev/null +++ b/README.fuchsia
@@ -0,0 +1,12 @@ +# AMD GPU driver + +The AMD GPU driver for Linux/DRM. Imported from tags/v5.3 of the kernel tree. + +Upstream Maintainer: http://www.amd.com +License: MIT +Source: https://github.com/torvalds/linux +Version: Tag "5.3", commit 4d856f72c10ecb060868ed10ff1b1453943fc6c8 + +Modifications: + - added LICENSE with a copy of the license from include/amd_shared.h + - added README.fuchsia
diff --git a/acp/Makefile b/acp/Makefile new file mode 100644 index 0000000..d4176a3 --- /dev/null +++ b/acp/Makefile
@@ -0,0 +1,27 @@ +# +# Copyright 2017 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +# Makefile for the ACP, which is a sub-component +# of AMDSOC/AMDGPU drm driver. +# It provides the HW control for ACP related functionalities. + +AMD_ACP_FILES := $(AMDACPPATH)/acp_hw.o
diff --git a/acp/acp_hw.c b/acp/acp_hw.c new file mode 100644 index 0000000..c7d7205 --- /dev/null +++ b/acp/acp_hw.c
@@ -0,0 +1,50 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/delay.h> +#include <linux/errno.h> + +#include "acp_gfx_if.h" + +#define ACP_MODE_I2S 0 +#define ACP_MODE_AZ 1 + +#define mmACP_AZALIA_I2S_SELECT 0x51d4 + +int amd_acp_hw_init(struct cgs_device *cgs_device, + unsigned acp_version_major, unsigned acp_version_minor) +{ + unsigned int acp_mode = ACP_MODE_I2S; + + if ((acp_version_major == 2) && (acp_version_minor == 2)) + acp_mode = cgs_read_register(cgs_device, + mmACP_AZALIA_I2S_SELECT); + + if (acp_mode != ACP_MODE_I2S) + return -ENODEV; + + return 0; +}
diff --git a/acp/include/acp_gfx_if.h b/acp/include/acp_gfx_if.h new file mode 100644 index 0000000..feab8eb --- /dev/null +++ b/acp/include/acp_gfx_if.h
@@ -0,0 +1,33 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * +*/ + +#ifndef _ACP_GFX_IF_H +#define _ACP_GFX_IF_H + +#include <linux/types.h> +#include "cgs_common.h" + +int amd_acp_hw_init(struct cgs_device *cgs_device, + unsigned acp_version_major, unsigned acp_version_minor); + +#endif /* _ACP_GFX_IF_H */
diff --git a/amdgpu/Makefile b/amdgpu/Makefile new file mode 100644 index 0000000..56e0843 --- /dev/null +++ b/amdgpu/Makefile
@@ -0,0 +1,207 @@ +# +# Copyright 2017 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# +# Makefile for the drm device driver. This driver provides support for the +# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. + +FULL_AMD_PATH=$(srctree)/$(src)/.. +DISPLAY_FOLDER_NAME=display +FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME) + +ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \ + -I$(FULL_AMD_PATH)/include \ + -I$(FULL_AMD_PATH)/amdgpu \ + -I$(FULL_AMD_PATH)/powerplay/inc \ + -I$(FULL_AMD_PATH)/acp/include \ + -I$(FULL_AMD_DISPLAY_PATH) \ + -I$(FULL_AMD_DISPLAY_PATH)/include \ + -I$(FULL_AMD_DISPLAY_PATH)/dc \ + -I$(FULL_AMD_DISPLAY_PATH)/amdgpu_dm \ + -I$(FULL_AMD_PATH)/amdkfd + +amdgpu-y := amdgpu_drv.o + +# add KMS driver +amdgpu-y += amdgpu_device.o amdgpu_kms.o \ + amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \ + atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \ + amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \ + amdgpu_fb.o amdgpu_gem.o amdgpu_ring.o \ + amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o amdgpu_test.o \ + amdgpu_pm.o atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \ + atombios_encoders.o amdgpu_sa.o atombios_i2c.o \ + amdgpu_dma_buf.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \ + amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \ + amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \ + amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \ + amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \ + amdgpu_vm_sdma.o amdgpu_discovery.o + +amdgpu-$(CONFIG_PERF_EVENTS) += amdgpu_pmu.o + +# add asic specific block +amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ + dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o + +amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o + +amdgpu-y += \ + vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ + vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o navi10_reg_init.o + +# add DF block +amdgpu-y += \ + df_v1_7.o \ + df_v3_6.o + +# add GMC block +amdgpu-y += \ + gmc_v7_0.o \ + gmc_v8_0.o \ + gfxhub_v1_0.o mmhub_v1_0.o gmc_v9_0.o gfxhub_v1_1.o \ + gfxhub_v2_0.o mmhub_v2_0.o gmc_v10_0.o + +# add IH block +amdgpu-y += \ + amdgpu_irq.o \ + amdgpu_ih.o \ + iceland_ih.o \ + tonga_ih.o \ + cz_ih.o \ + vega10_ih.o \ + navi10_ih.o + +# add PSP block +amdgpu-y += \ + amdgpu_psp.o \ + psp_v3_1.o \ + psp_v10_0.o \ + psp_v11_0.o + +# add SMC block +amdgpu-y += \ + amdgpu_dpm.o + +# add DCE block +amdgpu-y += \ + dce_v10_0.o \ + dce_v11_0.o \ + dce_virtual.o + +# add GFX block +amdgpu-y += \ + amdgpu_gfx.o \ + amdgpu_rlc.o \ + gfx_v8_0.o \ + gfx_v9_0.o \ + gfx_v10_0.o + +# add async DMA block +amdgpu-y += \ + amdgpu_sdma.o \ + sdma_v2_4.o \ + sdma_v3_0.o \ + sdma_v4_0.o \ + sdma_v5_0.o + +# add MES block +amdgpu-y += \ + mes_v10_1.o + +# add UVD block +amdgpu-y += \ + amdgpu_uvd.o \ + uvd_v5_0.o \ + uvd_v6_0.o \ + uvd_v7_0.o + +# add VCE block +amdgpu-y += \ + amdgpu_vce.o \ + vce_v3_0.o \ + vce_v4_0.o + +# add VCN block +amdgpu-y += \ + amdgpu_vcn.o \ + vcn_v1_0.o \ + vcn_v2_0.o + +# add ATHUB block +amdgpu-y += \ + athub_v2_0.o + +# add amdkfd interfaces +amdgpu-y += amdgpu_amdkfd.o + +ifneq ($(CONFIG_HSA_AMD),) +AMDKFD_PATH := ../amdkfd +include $(FULL_AMD_PATH)/amdkfd/Makefile +amdgpu-y += $(AMDKFD_FILES) +amdgpu-y += \ + amdgpu_amdkfd_fence.o \ + amdgpu_amdkfd_gpuvm.o \ + amdgpu_amdkfd_gfx_v8.o \ + amdgpu_amdkfd_gfx_v9.o \ + amdgpu_amdkfd_gfx_v10.o + +ifneq ($(CONFIG_DRM_AMDGPU_CIK),) +amdgpu-y += amdgpu_amdkfd_gfx_v7.o +endif + +endif + +# add cgs +amdgpu-y += amdgpu_cgs.o + +# GPU scheduler +amdgpu-y += amdgpu_job.o + +# ACP componet +ifneq ($(CONFIG_DRM_AMD_ACP),) +amdgpu-y += amdgpu_acp.o + +AMDACPPATH := ../acp +include $(FULL_AMD_PATH)/acp/Makefile + +amdgpu-y += $(AMD_ACP_FILES) +endif + +amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o +amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o +amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o +amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o + +include $(FULL_AMD_PATH)/powerplay/Makefile + +amdgpu-y += $(AMD_POWERPLAY_FILES) + +ifneq ($(CONFIG_DRM_AMD_DC),) + +RELATIVE_AMD_DISPLAY_PATH = ../$(DISPLAY_FOLDER_NAME) +include $(FULL_AMD_DISPLAY_PATH)/Makefile + +amdgpu-y += $(AMD_DISPLAY_FILES) + +endif + +obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
diff --git a/amdgpu/ObjectID.h b/amdgpu/ObjectID.h new file mode 100644 index 0000000..5b39362 --- /dev/null +++ b/amdgpu/ObjectID.h
@@ -0,0 +1,744 @@ +/* +* Copyright 2006-2007 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +* OTHER DEALINGS IN THE SOFTWARE. +*/ +/* based on stg/asic_reg/drivers/inc/asic_reg/ObjectID.h ver 23 */ + +#ifndef _OBJECTID_H +#define _OBJECTID_H + +#if defined(_X86_) +#pragma pack(1) +#endif + +/****************************************************/ +/* Graphics Object Type Definition */ +/****************************************************/ +#define GRAPH_OBJECT_TYPE_NONE 0x0 +#define GRAPH_OBJECT_TYPE_GPU 0x1 +#define GRAPH_OBJECT_TYPE_ENCODER 0x2 +#define GRAPH_OBJECT_TYPE_CONNECTOR 0x3 +#define GRAPH_OBJECT_TYPE_ROUTER 0x4 +/* deleted */ +#define GRAPH_OBJECT_TYPE_DISPLAY_PATH 0x6 +#define GRAPH_OBJECT_TYPE_GENERIC 0x7 + +/****************************************************/ +/* Encoder Object ID Definition */ +/****************************************************/ +#define ENCODER_OBJECT_ID_NONE 0x00 + +/* Radeon Class Display Hardware */ +#define ENCODER_OBJECT_ID_INTERNAL_LVDS 0x01 +#define ENCODER_OBJECT_ID_INTERNAL_TMDS1 0x02 +#define ENCODER_OBJECT_ID_INTERNAL_TMDS2 0x03 +#define ENCODER_OBJECT_ID_INTERNAL_DAC1 0x04 +#define ENCODER_OBJECT_ID_INTERNAL_DAC2 0x05 /* TV/CV DAC */ +#define ENCODER_OBJECT_ID_INTERNAL_SDVOA 0x06 +#define ENCODER_OBJECT_ID_INTERNAL_SDVOB 0x07 + +/* External Third Party Encoders */ +#define ENCODER_OBJECT_ID_SI170B 0x08 +#define ENCODER_OBJECT_ID_CH7303 0x09 +#define ENCODER_OBJECT_ID_CH7301 0x0A +#define ENCODER_OBJECT_ID_INTERNAL_DVO1 0x0B /* This belongs to Radeon Class Display Hardware */ +#define ENCODER_OBJECT_ID_EXTERNAL_SDVOA 0x0C +#define ENCODER_OBJECT_ID_EXTERNAL_SDVOB 0x0D +#define ENCODER_OBJECT_ID_TITFP513 0x0E +#define ENCODER_OBJECT_ID_INTERNAL_LVTM1 0x0F /* not used for Radeon */ +#define ENCODER_OBJECT_ID_VT1623 0x10 +#define ENCODER_OBJECT_ID_HDMI_SI1930 0x11 +#define ENCODER_OBJECT_ID_HDMI_INTERNAL 0x12 +#define ENCODER_OBJECT_ID_ALMOND 0x22 +#define ENCODER_OBJECT_ID_TRAVIS 0x23 +#define ENCODER_OBJECT_ID_NUTMEG 0x22 +#define ENCODER_OBJECT_ID_HDMI_ANX9805 0x26 + +/* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */ +#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 0x13 +#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1 0x14 +#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1 0x15 +#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2 0x16 /* Shared with CV/TV and CRT */ +#define ENCODER_OBJECT_ID_SI178 0X17 /* External TMDS (dual link, no HDCP.) */ +#define ENCODER_OBJECT_ID_MVPU_FPGA 0x18 /* MVPU FPGA chip */ +#define ENCODER_OBJECT_ID_INTERNAL_DDI 0x19 +#define ENCODER_OBJECT_ID_VT1625 0x1A +#define ENCODER_OBJECT_ID_HDMI_SI1932 0x1B +#define ENCODER_OBJECT_ID_DP_AN9801 0x1C +#define ENCODER_OBJECT_ID_DP_DP501 0x1D +#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY 0x1E +#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA 0x1F +#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY1 0x20 +#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 0x21 +#define ENCODER_OBJECT_ID_INTERNAL_VCE 0x24 +#define ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 0x25 +#define ENCODER_OBJECT_ID_INTERNAL_AMCLK 0x27 + +#define ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO 0xFF + +/****************************************************/ +/* Connector Object ID Definition */ +/****************************************************/ +#define CONNECTOR_OBJECT_ID_NONE 0x00 +#define CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I 0x01 +#define CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I 0x02 +#define CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D 0x03 +#define CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D 0x04 +#define CONNECTOR_OBJECT_ID_VGA 0x05 +#define CONNECTOR_OBJECT_ID_COMPOSITE 0x06 +#define CONNECTOR_OBJECT_ID_SVIDEO 0x07 +#define CONNECTOR_OBJECT_ID_YPbPr 0x08 +#define CONNECTOR_OBJECT_ID_D_CONNECTOR 0x09 +#define CONNECTOR_OBJECT_ID_9PIN_DIN 0x0A /* Supports both CV & TV */ +#define CONNECTOR_OBJECT_ID_SCART 0x0B +#define CONNECTOR_OBJECT_ID_HDMI_TYPE_A 0x0C +#define CONNECTOR_OBJECT_ID_HDMI_TYPE_B 0x0D +#define CONNECTOR_OBJECT_ID_LVDS 0x0E +#define CONNECTOR_OBJECT_ID_7PIN_DIN 0x0F +#define CONNECTOR_OBJECT_ID_PCIE_CONNECTOR 0x10 +#define CONNECTOR_OBJECT_ID_CROSSFIRE 0x11 +#define CONNECTOR_OBJECT_ID_HARDCODE_DVI 0x12 +#define CONNECTOR_OBJECT_ID_DISPLAYPORT 0x13 +#define CONNECTOR_OBJECT_ID_eDP 0x14 +#define CONNECTOR_OBJECT_ID_MXM 0x15 +#define CONNECTOR_OBJECT_ID_LVDS_eDP 0x16 + +/* deleted */ + +/****************************************************/ +/* Router Object ID Definition */ +/****************************************************/ +#define ROUTER_OBJECT_ID_NONE 0x00 +#define ROUTER_OBJECT_ID_I2C_EXTENDER_CNTL 0x01 + +/****************************************************/ +/* Generic Object ID Definition */ +/****************************************************/ +#define GENERIC_OBJECT_ID_NONE 0x00 +#define GENERIC_OBJECT_ID_GLSYNC 0x01 +#define GENERIC_OBJECT_ID_PX2_NON_DRIVABLE 0x02 +#define GENERIC_OBJECT_ID_MXM_OPM 0x03 +#define GENERIC_OBJECT_ID_STEREO_PIN 0x04 //This object could show up from Misc Object table, it follows ATOM_OBJECT format, and contains one ATOM_OBJECT_GPIO_CNTL_RECORD for the stereo pin +#define GENERIC_OBJECT_ID_BRACKET_LAYOUT 0x05 + +/****************************************************/ +/* Graphics Object ENUM ID Definition */ +/****************************************************/ +#define GRAPH_OBJECT_ENUM_ID1 0x01 +#define GRAPH_OBJECT_ENUM_ID2 0x02 +#define GRAPH_OBJECT_ENUM_ID3 0x03 +#define GRAPH_OBJECT_ENUM_ID4 0x04 +#define GRAPH_OBJECT_ENUM_ID5 0x05 +#define GRAPH_OBJECT_ENUM_ID6 0x06 +#define GRAPH_OBJECT_ENUM_ID7 0x07 + +/****************************************************/ +/* Graphics Object ID Bit definition */ +/****************************************************/ +#define OBJECT_ID_MASK 0x00FF +#define ENUM_ID_MASK 0x0700 +#define RESERVED1_ID_MASK 0x0800 +#define OBJECT_TYPE_MASK 0x7000 +#define RESERVED2_ID_MASK 0x8000 + +#define OBJECT_ID_SHIFT 0x00 +#define ENUM_ID_SHIFT 0x08 +#define OBJECT_TYPE_SHIFT 0x0C + + +/****************************************************/ +/* Graphics Object family definition */ +/****************************************************/ +#define CONSTRUCTOBJECTFAMILYID(GRAPHICS_OBJECT_TYPE, GRAPHICS_OBJECT_ID) (GRAPHICS_OBJECT_TYPE << OBJECT_TYPE_SHIFT | \ + GRAPHICS_OBJECT_ID << OBJECT_ID_SHIFT) +/****************************************************/ +/* GPU Object ID definition - Shared with BIOS */ +/****************************************************/ +#define GPU_ENUM_ID1 ( GRAPH_OBJECT_TYPE_GPU << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT) + +/****************************************************/ +/* Encoder Object ID definition - Shared with BIOS */ +/****************************************************/ +/* +#define ENCODER_INTERNAL_LVDS_ENUM_ID1 0x2101 +#define ENCODER_INTERNAL_TMDS1_ENUM_ID1 0x2102 +#define ENCODER_INTERNAL_TMDS2_ENUM_ID1 0x2103 +#define ENCODER_INTERNAL_DAC1_ENUM_ID1 0x2104 +#define ENCODER_INTERNAL_DAC2_ENUM_ID1 0x2105 +#define ENCODER_INTERNAL_SDVOA_ENUM_ID1 0x2106 +#define ENCODER_INTERNAL_SDVOB_ENUM_ID1 0x2107 +#define ENCODER_SIL170B_ENUM_ID1 0x2108 +#define ENCODER_CH7303_ENUM_ID1 0x2109 +#define ENCODER_CH7301_ENUM_ID1 0x210A +#define ENCODER_INTERNAL_DVO1_ENUM_ID1 0x210B +#define ENCODER_EXTERNAL_SDVOA_ENUM_ID1 0x210C +#define ENCODER_EXTERNAL_SDVOB_ENUM_ID1 0x210D +#define ENCODER_TITFP513_ENUM_ID1 0x210E +#define ENCODER_INTERNAL_LVTM1_ENUM_ID1 0x210F +#define ENCODER_VT1623_ENUM_ID1 0x2110 +#define ENCODER_HDMI_SI1930_ENUM_ID1 0x2111 +#define ENCODER_HDMI_INTERNAL_ENUM_ID1 0x2112 +#define ENCODER_INTERNAL_KLDSCP_TMDS1_ENUM_ID1 0x2113 +#define ENCODER_INTERNAL_KLDSCP_DVO1_ENUM_ID1 0x2114 +#define ENCODER_INTERNAL_KLDSCP_DAC1_ENUM_ID1 0x2115 +#define ENCODER_INTERNAL_KLDSCP_DAC2_ENUM_ID1 0x2116 +#define ENCODER_SI178_ENUM_ID1 0x2117 +#define ENCODER_MVPU_FPGA_ENUM_ID1 0x2118 +#define ENCODER_INTERNAL_DDI_ENUM_ID1 0x2119 +#define ENCODER_VT1625_ENUM_ID1 0x211A +#define ENCODER_HDMI_SI1932_ENUM_ID1 0x211B +#define ENCODER_ENCODER_DP_AN9801_ENUM_ID1 0x211C +#define ENCODER_DP_DP501_ENUM_ID1 0x211D +#define ENCODER_INTERNAL_UNIPHY_ENUM_ID1 0x211E +*/ +#define ENCODER_INTERNAL_LVDS_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_LVDS << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_TMDS1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_TMDS1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_TMDS2_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_TMDS2 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_DAC1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_DAC1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_DAC2_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_DAC2 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_SDVOA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_SDVOA << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_SDVOA_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_SDVOA << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_SDVOB_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_SDVOB << OBJECT_ID_SHIFT) + +#define ENCODER_SIL170B_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_SI170B << OBJECT_ID_SHIFT) + +#define ENCODER_CH7303_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_CH7303 << OBJECT_ID_SHIFT) + +#define ENCODER_CH7301_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_CH7301 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_DVO1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_DVO1 << OBJECT_ID_SHIFT) + +#define ENCODER_EXTERNAL_SDVOA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_EXTERNAL_SDVOA << OBJECT_ID_SHIFT) + +#define ENCODER_EXTERNAL_SDVOA_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_EXTERNAL_SDVOA << OBJECT_ID_SHIFT) + + +#define ENCODER_EXTERNAL_SDVOB_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_EXTERNAL_SDVOB << OBJECT_ID_SHIFT) + + +#define ENCODER_TITFP513_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_TITFP513 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_LVTM1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_LVTM1 << OBJECT_ID_SHIFT) + +#define ENCODER_VT1623_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_VT1623 << OBJECT_ID_SHIFT) + +#define ENCODER_HDMI_SI1930_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_HDMI_SI1930 << OBJECT_ID_SHIFT) + +#define ENCODER_HDMI_INTERNAL_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_HDMI_INTERNAL << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_KLDSCP_TMDS1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 << OBJECT_ID_SHIFT) + + +#define ENCODER_INTERNAL_KLDSCP_TMDS1_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 << OBJECT_ID_SHIFT) + + +#define ENCODER_INTERNAL_KLDSCP_DVO1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_KLDSCP_DAC1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_KLDSCP_DAC2_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2 << OBJECT_ID_SHIFT) // Shared with CV/TV and CRT + +#define ENCODER_SI178_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_SI178 << OBJECT_ID_SHIFT) + +#define ENCODER_MVPU_FPGA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_MVPU_FPGA << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_DDI_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_DDI << OBJECT_ID_SHIFT) + +#define ENCODER_VT1625_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_VT1625 << OBJECT_ID_SHIFT) + +#define ENCODER_HDMI_SI1932_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_HDMI_SI1932 << OBJECT_ID_SHIFT) + +#define ENCODER_DP_DP501_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_DP_DP501 << OBJECT_ID_SHIFT) + +#define ENCODER_DP_AN9801_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_DP_AN9801 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_KLDSCP_LVTMA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY1_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY1_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY1 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY2_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY2_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY2 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT) + +#define ENCODER_INTERNAL_UNIPHY3_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_UNIPHY3 << OBJECT_ID_SHIFT) + +#define ENCODER_GENERAL_EXTERNAL_DVO_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT) + +#define ENCODER_ALMOND_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT) + +#define ENCODER_ALMOND_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT) + +#define ENCODER_TRAVIS_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT) + +#define ENCODER_TRAVIS_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT) + +#define ENCODER_NUTMEG_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_NUTMEG << OBJECT_ID_SHIFT) + +#define ENCODER_VCE_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_INTERNAL_VCE << OBJECT_ID_SHIFT) + +#define ENCODER_HDMI_ANX9805_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ENCODER_OBJECT_ID_HDMI_ANX9805 << OBJECT_ID_SHIFT) + +/****************************************************/ +/* Connector Object ID definition - Shared with BIOS */ +/****************************************************/ +/* +#define CONNECTOR_SINGLE_LINK_DVI_I_ENUM_ID1 0x3101 +#define CONNECTOR_DUAL_LINK_DVI_I_ENUM_ID1 0x3102 +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID1 0x3103 +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1 0x3104 +#define CONNECTOR_VGA_ENUM_ID1 0x3105 +#define CONNECTOR_COMPOSITE_ENUM_ID1 0x3106 +#define CONNECTOR_SVIDEO_ENUM_ID1 0x3107 +#define CONNECTOR_YPbPr_ENUM_ID1 0x3108 +#define CONNECTOR_D_CONNECTORE_ENUM_ID1 0x3109 +#define CONNECTOR_9PIN_DIN_ENUM_ID1 0x310A +#define CONNECTOR_SCART_ENUM_ID1 0x310B +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID1 0x310C +#define CONNECTOR_HDMI_TYPE_B_ENUM_ID1 0x310D +#define CONNECTOR_LVDS_ENUM_ID1 0x310E +#define CONNECTOR_7PIN_DIN_ENUM_ID1 0x310F +#define CONNECTOR_PCIE_CONNECTOR_ENUM_ID1 0x3110 +*/ +#define CONNECTOR_LVDS_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_LVDS << OBJECT_ID_SHIFT) + +#define CONNECTOR_LVDS_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_LVDS << OBJECT_ID_SHIFT) + +#define CONNECTOR_eDP_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_eDP << OBJECT_ID_SHIFT) + +#define CONNECTOR_eDP_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_eDP << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_I_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_I_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_I << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_I_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_I_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) + +#define CONNECTOR_VGA_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_VGA << OBJECT_ID_SHIFT) + +#define CONNECTOR_VGA_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_VGA << OBJECT_ID_SHIFT) + +#define CONNECTOR_COMPOSITE_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_COMPOSITE << OBJECT_ID_SHIFT) + +#define CONNECTOR_COMPOSITE_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_COMPOSITE << OBJECT_ID_SHIFT) + +#define CONNECTOR_SVIDEO_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SVIDEO << OBJECT_ID_SHIFT) + +#define CONNECTOR_SVIDEO_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SVIDEO << OBJECT_ID_SHIFT) + +#define CONNECTOR_YPbPr_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_YPbPr << OBJECT_ID_SHIFT) + +#define CONNECTOR_YPbPr_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_YPbPr << OBJECT_ID_SHIFT) + +#define CONNECTOR_D_CONNECTOR_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_D_CONNECTOR << OBJECT_ID_SHIFT) + +#define CONNECTOR_D_CONNECTOR_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_D_CONNECTOR << OBJECT_ID_SHIFT) + +#define CONNECTOR_9PIN_DIN_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_9PIN_DIN << OBJECT_ID_SHIFT) + +#define CONNECTOR_9PIN_DIN_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_9PIN_DIN << OBJECT_ID_SHIFT) + +#define CONNECTOR_SCART_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SCART << OBJECT_ID_SHIFT) + +#define CONNECTOR_SCART_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_SCART << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_A_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_A << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_B_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_B << OBJECT_ID_SHIFT) + +#define CONNECTOR_HDMI_TYPE_B_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HDMI_TYPE_B << OBJECT_ID_SHIFT) + +#define CONNECTOR_7PIN_DIN_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT) + +#define CONNECTOR_7PIN_DIN_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT) + +#define CONNECTOR_PCIE_CONNECTOR_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_PCIE_CONNECTOR << OBJECT_ID_SHIFT) + +#define CONNECTOR_PCIE_CONNECTOR_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_PCIE_CONNECTOR << OBJECT_ID_SHIFT) + +#define CONNECTOR_CROSSFIRE_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_CROSSFIRE << OBJECT_ID_SHIFT) + +#define CONNECTOR_CROSSFIRE_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_CROSSFIRE << OBJECT_ID_SHIFT) + + +#define CONNECTOR_HARDCODE_DVI_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HARDCODE_DVI << OBJECT_ID_SHIFT) + +#define CONNECTOR_HARDCODE_DVI_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_HARDCODE_DVI << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_DISPLAYPORT_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_DISPLAYPORT << OBJECT_ID_SHIFT) + +#define CONNECTOR_MXM_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DP_A + +#define CONNECTOR_MXM_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DP_B + +#define CONNECTOR_MXM_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DP_C + +#define CONNECTOR_MXM_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DP_D + +#define CONNECTOR_MXM_ENUM_ID5 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID5 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_LVDS_TXxx + +#define CONNECTOR_MXM_ENUM_ID6 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID6 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_LVDS_UXxx + +#define CONNECTOR_MXM_ENUM_ID7 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID7 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DAC + +#define CONNECTOR_LVDS_eDP_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT) + +#define CONNECTOR_LVDS_eDP_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT) + +/****************************************************/ +/* Router Object ID definition - Shared with BIOS */ +/****************************************************/ +#define ROUTER_I2C_EXTENDER_CNTL_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ROUTER << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + ROUTER_OBJECT_ID_I2C_EXTENDER_CNTL << OBJECT_ID_SHIFT) + +/* deleted */ + +/****************************************************/ +/* Generic Object ID definition - Shared with BIOS */ +/****************************************************/ +#define GENERICOBJECT_GLSYNC_ENUM_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_GLSYNC << OBJECT_ID_SHIFT) + +#define GENERICOBJECT_PX2_NON_DRIVABLE_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_PX2_NON_DRIVABLE<< OBJECT_ID_SHIFT) + +#define GENERICOBJECT_PX2_NON_DRIVABLE_ID2 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_PX2_NON_DRIVABLE<< OBJECT_ID_SHIFT) + +#define GENERICOBJECT_MXM_OPM_ENUM_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_MXM_OPM << OBJECT_ID_SHIFT) + +#define GENERICOBJECT_STEREO_PIN_ENUM_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_STEREO_PIN << OBJECT_ID_SHIFT) + +#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT) + +#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\ + GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ + GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT) +/****************************************************/ +/* Object Cap definition - Shared with BIOS */ +/****************************************************/ +#define GRAPHICS_OBJECT_CAP_I2C 0x00000001L +#define GRAPHICS_OBJECT_CAP_TABLE_ID 0x00000002L + + +#define GRAPHICS_OBJECT_I2CCOMMAND_TABLE_ID 0x01 +#define GRAPHICS_OBJECT_HOTPLUGDETECTIONINTERUPT_TABLE_ID 0x02 +#define GRAPHICS_OBJECT_ENCODER_OUTPUT_PROTECTION_TABLE_ID 0x03 + +#if defined(_X86_) +#pragma pack() +#endif + +#endif /*GRAPHICTYPE */ + + + +
diff --git a/amdgpu/amdgpu.h b/amdgpu/amdgpu.h new file mode 100644 index 0000000..8199d20 --- /dev/null +++ b/amdgpu/amdgpu.h
@@ -0,0 +1,1241 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#ifndef __AMDGPU_H__ +#define __AMDGPU_H__ + +#include "amdgpu_ctx.h" + +#include <linux/atomic.h> +#include <linux/wait.h> +#include <linux/list.h> +#include <linux/kref.h> +#include <linux/rbtree.h> +#include <linux/hashtable.h> +#include <linux/dma-fence.h> + +#include <drm/ttm/ttm_bo_api.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_execbuf_util.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_gem.h> +#include <drm/drm_ioctl.h> +#include <drm/gpu_scheduler.h> + +#include <kgd_kfd_interface.h> +#include "dm_pp_interface.h" +#include "kgd_pp_interface.h" + +#include "amd_shared.h" +#include "amdgpu_mode.h" +#include "amdgpu_ih.h" +#include "amdgpu_irq.h" +#include "amdgpu_ucode.h" +#include "amdgpu_ttm.h" +#include "amdgpu_psp.h" +#include "amdgpu_gds.h" +#include "amdgpu_sync.h" +#include "amdgpu_ring.h" +#include "amdgpu_vm.h" +#include "amdgpu_dpm.h" +#include "amdgpu_acp.h" +#include "amdgpu_uvd.h" +#include "amdgpu_vce.h" +#include "amdgpu_vcn.h" +#include "amdgpu_mn.h" +#include "amdgpu_gmc.h" +#include "amdgpu_gfx.h" +#include "amdgpu_sdma.h" +#include "amdgpu_dm.h" +#include "amdgpu_virt.h" +#include "amdgpu_csa.h" +#include "amdgpu_gart.h" +#include "amdgpu_debugfs.h" +#include "amdgpu_job.h" +#include "amdgpu_bo_list.h" +#include "amdgpu_gem.h" +#include "amdgpu_doorbell.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_smu.h" +#include "amdgpu_discovery.h" +#include "amdgpu_mes.h" + +#define MAX_GPU_INSTANCE 16 + +struct amdgpu_gpu_instance +{ + struct amdgpu_device *adev; + int mgpu_fan_enabled; +}; + +struct amdgpu_mgpu_info +{ + struct amdgpu_gpu_instance gpu_ins[MAX_GPU_INSTANCE]; + struct mutex mutex; + uint32_t num_gpu; + uint32_t num_dgpu; + uint32_t num_apu; +}; + +/* + * Modules parameters. + */ +extern int amdgpu_modeset; +extern int amdgpu_vram_limit; +extern int amdgpu_vis_vram_limit; +extern int amdgpu_gart_size; +extern int amdgpu_gtt_size; +extern int amdgpu_moverate; +extern int amdgpu_benchmarking; +extern int amdgpu_testing; +extern int amdgpu_audio; +extern int amdgpu_disp_priority; +extern int amdgpu_hw_i2c; +extern int amdgpu_pcie_gen2; +extern int amdgpu_msi; +extern int amdgpu_dpm; +extern int amdgpu_fw_load_type; +extern int amdgpu_aspm; +extern int amdgpu_runtime_pm; +extern uint amdgpu_ip_block_mask; +extern int amdgpu_bapm; +extern int amdgpu_deep_color; +extern int amdgpu_vm_size; +extern int amdgpu_vm_block_size; +extern int amdgpu_vm_fragment_size; +extern int amdgpu_vm_fault_stop; +extern int amdgpu_vm_debug; +extern int amdgpu_vm_update_mode; +extern int amdgpu_dc; +extern int amdgpu_sched_jobs; +extern int amdgpu_sched_hw_submission; +extern uint amdgpu_pcie_gen_cap; +extern uint amdgpu_pcie_lane_cap; +extern uint amdgpu_cg_mask; +extern uint amdgpu_pg_mask; +extern uint amdgpu_sdma_phase_quantum; +extern char *amdgpu_disable_cu; +extern char *amdgpu_virtual_display; +extern uint amdgpu_pp_feature_mask; +extern int amdgpu_ngg; +extern int amdgpu_prim_buf_per_se; +extern int amdgpu_pos_buf_per_se; +extern int amdgpu_cntl_sb_buf_per_se; +extern int amdgpu_param_buf_per_se; +extern int amdgpu_job_hang_limit; +extern int amdgpu_lbpw; +extern int amdgpu_compute_multipipe; +extern int amdgpu_gpu_recovery; +extern int amdgpu_emu_mode; +extern uint amdgpu_smu_memory_pool_size; +extern uint amdgpu_dc_feature_mask; +extern uint amdgpu_dm_abm_level; +extern struct amdgpu_mgpu_info mgpu_info; +extern int amdgpu_ras_enable; +extern uint amdgpu_ras_mask; +extern int amdgpu_async_gfx_ring; +extern int amdgpu_mcbp; +extern int amdgpu_discovery; +extern int amdgpu_mes; +extern int amdgpu_noretry; + +#ifdef CONFIG_DRM_AMDGPU_SI +extern int amdgpu_si_support; +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK +extern int amdgpu_cik_support; +#endif + +#define AMDGPU_VM_MAX_NUM_CTX 4096 +#define AMDGPU_SG_THRESHOLD (256*1024*1024) +#define AMDGPU_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */ +#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 +#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ +#define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2) +/* AMDGPU_IB_POOL_SIZE must be a power of 2 */ +#define AMDGPU_IB_POOL_SIZE 16 +#define AMDGPU_DEBUGFS_MAX_COMPONENTS 32 +#define AMDGPUFB_CONN_LIMIT 4 +#define AMDGPU_BIOS_NUM_SCRATCH 16 + +/* hard reset data */ +#define AMDGPU_ASIC_RESET_DATA 0x39d5e86b + +/* reset flags */ +#define AMDGPU_RESET_GFX (1 << 0) +#define AMDGPU_RESET_COMPUTE (1 << 1) +#define AMDGPU_RESET_DMA (1 << 2) +#define AMDGPU_RESET_CP (1 << 3) +#define AMDGPU_RESET_GRBM (1 << 4) +#define AMDGPU_RESET_DMA1 (1 << 5) +#define AMDGPU_RESET_RLC (1 << 6) +#define AMDGPU_RESET_SEM (1 << 7) +#define AMDGPU_RESET_IH (1 << 8) +#define AMDGPU_RESET_VMC (1 << 9) +#define AMDGPU_RESET_MC (1 << 10) +#define AMDGPU_RESET_DISPLAY (1 << 11) +#define AMDGPU_RESET_UVD (1 << 12) +#define AMDGPU_RESET_VCE (1 << 13) +#define AMDGPU_RESET_VCE1 (1 << 14) + +/* max cursor sizes (in pixels) */ +#define CIK_CURSOR_WIDTH 128 +#define CIK_CURSOR_HEIGHT 128 + +struct amdgpu_device; +struct amdgpu_ib; +struct amdgpu_cs_parser; +struct amdgpu_job; +struct amdgpu_irq_src; +struct amdgpu_fpriv; +struct amdgpu_bo_va_mapping; +struct amdgpu_atif; +struct kfd_vm_fault_info; + +enum amdgpu_cp_irq { + AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0, + AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP, + AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP, + + AMDGPU_CP_IRQ_LAST +}; + +enum amdgpu_thermal_irq { + AMDGPU_THERMAL_IRQ_LOW_TO_HIGH = 0, + AMDGPU_THERMAL_IRQ_HIGH_TO_LOW, + + AMDGPU_THERMAL_IRQ_LAST +}; + +enum amdgpu_kiq_irq { + AMDGPU_CP_KIQ_IRQ_DRIVER0 = 0, + AMDGPU_CP_KIQ_IRQ_LAST +}; + +#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */ +#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ +#define MAX_KIQ_REG_TRY 80 /* 20 -> 80 */ + +int amdgpu_device_ip_set_clockgating_state(void *dev, + enum amd_ip_block_type block_type, + enum amd_clockgating_state state); +int amdgpu_device_ip_set_powergating_state(void *dev, + enum amd_ip_block_type block_type, + enum amd_powergating_state state); +void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags); +int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type); +bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type); + +#define AMDGPU_MAX_IP_NUM 16 + +struct amdgpu_ip_block_status { + bool valid; + bool sw; + bool hw; + bool late_initialized; + bool hang; +}; + +struct amdgpu_ip_block_version { + const enum amd_ip_block_type type; + const u32 major; + const u32 minor; + const u32 rev; + const struct amd_ip_funcs *funcs; +}; + +struct amdgpu_ip_block { + struct amdgpu_ip_block_status status; + const struct amdgpu_ip_block_version *version; +}; + +int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, + enum amd_ip_block_type type, + u32 major, u32 minor); + +struct amdgpu_ip_block * +amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, + enum amd_ip_block_type type); + +int amdgpu_device_ip_block_add(struct amdgpu_device *adev, + const struct amdgpu_ip_block_version *ip_block_version); + +/* + * BIOS. + */ +bool amdgpu_get_bios(struct amdgpu_device *adev); +bool amdgpu_read_bios(struct amdgpu_device *adev); + +/* + * Clocks + */ + +#define AMDGPU_MAX_PPLL 3 + +struct amdgpu_clock { + struct amdgpu_pll ppll[AMDGPU_MAX_PPLL]; + struct amdgpu_pll spll; + struct amdgpu_pll mpll; + /* 10 Khz units */ + uint32_t default_mclk; + uint32_t default_sclk; + uint32_t default_dispclk; + uint32_t current_dispclk; + uint32_t dp_extclk; + uint32_t max_pixel_clock; +}; + +/* sub-allocation manager, it has to be protected by another lock. + * By conception this is an helper for other part of the driver + * like the indirect buffer or semaphore, which both have their + * locking. + * + * Principe is simple, we keep a list of sub allocation in offset + * order (first entry has offset == 0, last entry has the highest + * offset). + * + * When allocating new object we first check if there is room at + * the end total_size - (last_object_offset + last_object_size) >= + * alloc_size. If so we allocate new object there. + * + * When there is not enough room at the end, we start waiting for + * each sub object until we reach object_offset+object_size >= + * alloc_size, this object then become the sub object we return. + * + * Alignment can't be bigger than page size. + * + * Hole are not considered for allocation to keep things simple. + * Assumption is that there won't be hole (all object on same + * alignment). + */ + +#define AMDGPU_SA_NUM_FENCE_LISTS 32 + +struct amdgpu_sa_manager { + wait_queue_head_t wq; + struct amdgpu_bo *bo; + struct list_head *hole; + struct list_head flist[AMDGPU_SA_NUM_FENCE_LISTS]; + struct list_head olist; + unsigned size; + uint64_t gpu_addr; + void *cpu_ptr; + uint32_t domain; + uint32_t align; +}; + +/* sub-allocation buffer */ +struct amdgpu_sa_bo { + struct list_head olist; + struct list_head flist; + struct amdgpu_sa_manager *manager; + unsigned soffset; + unsigned eoffset; + struct dma_fence *fence; +}; + +int amdgpu_fence_slab_init(void); +void amdgpu_fence_slab_fini(void); + +/* + * IRQS. + */ + +struct amdgpu_flip_work { + struct delayed_work flip_work; + struct work_struct unpin_work; + struct amdgpu_device *adev; + int crtc_id; + u32 target_vblank; + uint64_t base; + struct drm_pending_vblank_event *event; + struct amdgpu_bo *old_abo; + struct dma_fence *excl; + unsigned shared_count; + struct dma_fence **shared; + struct dma_fence_cb cb; + bool async; +}; + + +/* + * CP & rings. + */ + +struct amdgpu_ib { + struct amdgpu_sa_bo *sa_bo; + uint32_t length_dw; + uint64_t gpu_addr; + uint32_t *ptr; + uint32_t flags; +}; + +extern const struct drm_sched_backend_ops amdgpu_sched_ops; + +/* + * file private structure + */ + +struct amdgpu_fpriv { + struct amdgpu_vm vm; + struct amdgpu_bo_va *prt_va; + struct amdgpu_bo_va *csa_va; + struct mutex bo_list_lock; + struct idr bo_list_handles; + struct amdgpu_ctx_mgr ctx_mgr; +}; + +int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); +int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev); + +int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, + unsigned size, struct amdgpu_ib *ib); +void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib, + struct dma_fence *f); +int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, + struct amdgpu_ib *ibs, struct amdgpu_job *job, + struct dma_fence **f); +int amdgpu_ib_pool_init(struct amdgpu_device *adev); +void amdgpu_ib_pool_fini(struct amdgpu_device *adev); +int amdgpu_ib_ring_tests(struct amdgpu_device *adev); + +/* + * CS. + */ +struct amdgpu_cs_chunk { + uint32_t chunk_id; + uint32_t length_dw; + void *kdata; +}; + +struct amdgpu_cs_post_dep { + struct drm_syncobj *syncobj; + struct dma_fence_chain *chain; + u64 point; +}; + +struct amdgpu_cs_parser { + struct amdgpu_device *adev; + struct drm_file *filp; + struct amdgpu_ctx *ctx; + + /* chunks */ + unsigned nchunks; + struct amdgpu_cs_chunk *chunks; + + /* scheduler job object */ + struct amdgpu_job *job; + struct drm_sched_entity *entity; + + /* buffer objects */ + struct ww_acquire_ctx ticket; + struct amdgpu_bo_list *bo_list; + struct amdgpu_mn *mn; + struct amdgpu_bo_list_entry vm_pd; + struct list_head validated; + struct dma_fence *fence; + uint64_t bytes_moved_threshold; + uint64_t bytes_moved_vis_threshold; + uint64_t bytes_moved; + uint64_t bytes_moved_vis; + struct amdgpu_bo_list_entry *evictable; + + /* user fence */ + struct amdgpu_bo_list_entry uf_entry; + + unsigned num_post_deps; + struct amdgpu_cs_post_dep *post_deps; +}; + +static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, + uint32_t ib_idx, int idx) +{ + return p->job->ibs[ib_idx].ptr[idx]; +} + +static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p, + uint32_t ib_idx, int idx, + uint32_t value) +{ + p->job->ibs[ib_idx].ptr[idx] = value; +} + +/* + * Writeback + */ +#define AMDGPU_MAX_WB 128 /* Reserve at most 128 WB slots for amdgpu-owned rings. */ + +struct amdgpu_wb { + struct amdgpu_bo *wb_obj; + volatile uint32_t *wb; + uint64_t gpu_addr; + u32 num_wb; /* Number of wb slots actually reserved for amdgpu. */ + unsigned long used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)]; +}; + +int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb); +void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb); + +/* + * Benchmarking + */ +void amdgpu_benchmark(struct amdgpu_device *adev, int test_number); + + +/* + * Testing + */ +void amdgpu_test_moves(struct amdgpu_device *adev); + +/* + * ASIC specific register table accessible by UMD + */ +struct amdgpu_allowed_register_entry { + uint32_t reg_offset; + bool grbm_indexed; +}; + +/* + * ASIC specific functions. + */ +struct amdgpu_asic_funcs { + bool (*read_disabled_bios)(struct amdgpu_device *adev); + bool (*read_bios_from_rom)(struct amdgpu_device *adev, + u8 *bios, u32 length_bytes); + int (*read_register)(struct amdgpu_device *adev, u32 se_num, + u32 sh_num, u32 reg_offset, u32 *value); + void (*set_vga_state)(struct amdgpu_device *adev, bool state); + int (*reset)(struct amdgpu_device *adev); + /* get the reference clock */ + u32 (*get_xclk)(struct amdgpu_device *adev); + /* MM block clocks */ + int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk); + int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk); + /* static power management */ + int (*get_pcie_lanes)(struct amdgpu_device *adev); + void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes); + /* get config memsize register */ + u32 (*get_config_memsize)(struct amdgpu_device *adev); + /* flush hdp write queue */ + void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring); + /* invalidate hdp read cache */ + void (*invalidate_hdp)(struct amdgpu_device *adev, + struct amdgpu_ring *ring); + /* check if the asic needs a full reset of if soft reset will work */ + bool (*need_full_reset)(struct amdgpu_device *adev); + /* initialize doorbell layout for specific asic*/ + void (*init_doorbell_index)(struct amdgpu_device *adev); + /* PCIe bandwidth usage */ + void (*get_pcie_usage)(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1); + /* do we need to reset the asic at init time (e.g., kexec) */ + bool (*need_reset_on_init)(struct amdgpu_device *adev); + /* PCIe replay counter */ + uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev); +}; + +/* + * IOCTL. + */ +int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +/* VRAM scratch page for HDP bug, default vram page */ +struct amdgpu_vram_scratch { + struct amdgpu_bo *robj; + volatile uint32_t *ptr; + u64 gpu_addr; +}; + +/* + * ACPI + */ +struct amdgpu_atcs_functions { + bool get_ext_state; + bool pcie_perf_req; + bool pcie_dev_rdy; + bool pcie_bus_width; +}; + +struct amdgpu_atcs { + struct amdgpu_atcs_functions functions; +}; + +/* + * Firmware VRAM reservation + */ +struct amdgpu_fw_vram_usage { + u64 start_offset; + u64 size; + struct amdgpu_bo *reserved_bo; + void *va; +}; + +/* + * CGS + */ +struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev); +void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device); + +/* + * Core structure, functions and helpers. + */ +typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t); +typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t); + +typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t); +typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t); + + +/* + * amdgpu nbio functions + * + */ +struct nbio_hdp_flush_reg { + u32 ref_and_mask_cp0; + u32 ref_and_mask_cp1; + u32 ref_and_mask_cp2; + u32 ref_and_mask_cp3; + u32 ref_and_mask_cp4; + u32 ref_and_mask_cp5; + u32 ref_and_mask_cp6; + u32 ref_and_mask_cp7; + u32 ref_and_mask_cp8; + u32 ref_and_mask_cp9; + u32 ref_and_mask_sdma0; + u32 ref_and_mask_sdma1; +}; + +struct amdgpu_mmio_remap { + u32 reg_offset; + resource_size_t bus_addr; +}; + +struct amdgpu_nbio_funcs { + const struct nbio_hdp_flush_reg *hdp_flush_reg; + u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev); + u32 (*get_hdp_flush_done_offset)(struct amdgpu_device *adev); + u32 (*get_pcie_index_offset)(struct amdgpu_device *adev); + u32 (*get_pcie_data_offset)(struct amdgpu_device *adev); + u32 (*get_rev_id)(struct amdgpu_device *adev); + void (*mc_access_enable)(struct amdgpu_device *adev, bool enable); + void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring); + u32 (*get_memsize)(struct amdgpu_device *adev); + void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance, + bool use_doorbell, int doorbell_index, int doorbell_size); + void (*vcn_doorbell_range)(struct amdgpu_device *adev, bool use_doorbell, + int doorbell_index); + void (*enable_doorbell_aperture)(struct amdgpu_device *adev, + bool enable); + void (*enable_doorbell_selfring_aperture)(struct amdgpu_device *adev, + bool enable); + void (*ih_doorbell_range)(struct amdgpu_device *adev, + bool use_doorbell, int doorbell_index); + void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev, + bool enable); + void (*update_medium_grain_light_sleep)(struct amdgpu_device *adev, + bool enable); + void (*get_clockgating_state)(struct amdgpu_device *adev, + u32 *flags); + void (*ih_control)(struct amdgpu_device *adev); + void (*init_registers)(struct amdgpu_device *adev); + void (*detect_hw_virt)(struct amdgpu_device *adev); + void (*remap_hdp_registers)(struct amdgpu_device *adev); +}; + +struct amdgpu_df_funcs { + void (*sw_init)(struct amdgpu_device *adev); + void (*enable_broadcast_mode)(struct amdgpu_device *adev, + bool enable); + u32 (*get_fb_channel_number)(struct amdgpu_device *adev); + u32 (*get_hbm_channel_number)(struct amdgpu_device *adev); + void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev, + bool enable); + void (*get_clockgating_state)(struct amdgpu_device *adev, + u32 *flags); + void (*enable_ecc_force_par_wr_rmw)(struct amdgpu_device *adev, + bool enable); + int (*pmc_start)(struct amdgpu_device *adev, uint64_t config, + int is_enable); + int (*pmc_stop)(struct amdgpu_device *adev, uint64_t config, + int is_disable); + void (*pmc_get_count)(struct amdgpu_device *adev, uint64_t config, + uint64_t *count); +}; +/* Define the HW IP blocks will be used in driver , add more if necessary */ +enum amd_hw_ip_block_type { + GC_HWIP = 1, + HDP_HWIP, + SDMA0_HWIP, + SDMA1_HWIP, + MMHUB_HWIP, + ATHUB_HWIP, + NBIO_HWIP, + MP0_HWIP, + MP1_HWIP, + UVD_HWIP, + VCN_HWIP = UVD_HWIP, + VCE_HWIP, + DF_HWIP, + DCE_HWIP, + OSSSYS_HWIP, + SMUIO_HWIP, + PWR_HWIP, + NBIF_HWIP, + THM_HWIP, + CLK_HWIP, + MAX_HWIP +}; + +#define HWIP_MAX_INSTANCE 6 + +struct amd_powerplay { + void *pp_handle; + const struct amd_pm_funcs *pp_funcs; +}; + +#define AMDGPU_RESET_MAGIC_NUM 64 +#define AMDGPU_MAX_DF_PERFMONS 4 +struct amdgpu_device { + struct device *dev; + struct drm_device *ddev; + struct pci_dev *pdev; + +#ifdef CONFIG_DRM_AMD_ACP + struct amdgpu_acp acp; +#endif + + /* ASIC */ + enum amd_asic_type asic_type; + uint32_t family; + uint32_t rev_id; + uint32_t external_rev_id; + unsigned long flags; + int usec_timeout; + const struct amdgpu_asic_funcs *asic_funcs; + bool shutdown; + bool need_dma32; + bool need_swiotlb; + bool accel_working; + struct notifier_block acpi_nb; + struct amdgpu_i2c_chan *i2c_bus[AMDGPU_MAX_I2C_BUS]; + struct amdgpu_debugfs debugfs[AMDGPU_DEBUGFS_MAX_COMPONENTS]; + unsigned debugfs_count; +#if defined(CONFIG_DEBUG_FS) + struct dentry *debugfs_preempt; + struct dentry *debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS]; +#endif + struct amdgpu_atif *atif; + struct amdgpu_atcs atcs; + struct mutex srbm_mutex; + /* GRBM index mutex. Protects concurrent access to GRBM index */ + struct mutex grbm_idx_mutex; + struct dev_pm_domain vga_pm_domain; + bool have_disp_power_ref; + bool have_atomics_support; + + /* BIOS */ + bool is_atom_fw; + uint8_t *bios; + uint32_t bios_size; + struct amdgpu_bo *stolen_vga_memory; + uint32_t bios_scratch_reg_offset; + uint32_t bios_scratch[AMDGPU_BIOS_NUM_SCRATCH]; + + /* Register/doorbell mmio */ + resource_size_t rmmio_base; + resource_size_t rmmio_size; + void __iomem *rmmio; + /* protects concurrent MM_INDEX/DATA based register access */ + spinlock_t mmio_idx_lock; + struct amdgpu_mmio_remap rmmio_remap; + /* protects concurrent SMC based register access */ + spinlock_t smc_idx_lock; + amdgpu_rreg_t smc_rreg; + amdgpu_wreg_t smc_wreg; + /* protects concurrent PCIE register access */ + spinlock_t pcie_idx_lock; + amdgpu_rreg_t pcie_rreg; + amdgpu_wreg_t pcie_wreg; + amdgpu_rreg_t pciep_rreg; + amdgpu_wreg_t pciep_wreg; + /* protects concurrent UVD register access */ + spinlock_t uvd_ctx_idx_lock; + amdgpu_rreg_t uvd_ctx_rreg; + amdgpu_wreg_t uvd_ctx_wreg; + /* protects concurrent DIDT register access */ + spinlock_t didt_idx_lock; + amdgpu_rreg_t didt_rreg; + amdgpu_wreg_t didt_wreg; + /* protects concurrent gc_cac register access */ + spinlock_t gc_cac_idx_lock; + amdgpu_rreg_t gc_cac_rreg; + amdgpu_wreg_t gc_cac_wreg; + /* protects concurrent se_cac register access */ + spinlock_t se_cac_idx_lock; + amdgpu_rreg_t se_cac_rreg; + amdgpu_wreg_t se_cac_wreg; + /* protects concurrent ENDPOINT (audio) register access */ + spinlock_t audio_endpt_idx_lock; + amdgpu_block_rreg_t audio_endpt_rreg; + amdgpu_block_wreg_t audio_endpt_wreg; + void __iomem *rio_mem; + resource_size_t rio_mem_size; + struct amdgpu_doorbell doorbell; + + /* clock/pll info */ + struct amdgpu_clock clock; + + /* MC */ + struct amdgpu_gmc gmc; + struct amdgpu_gart gart; + dma_addr_t dummy_page_addr; + struct amdgpu_vm_manager vm_manager; + struct amdgpu_vmhub vmhub[AMDGPU_MAX_VMHUBS]; + + /* memory management */ + struct amdgpu_mman mman; + struct amdgpu_vram_scratch vram_scratch; + struct amdgpu_wb wb; + atomic64_t num_bytes_moved; + atomic64_t num_evictions; + atomic64_t num_vram_cpu_page_faults; + atomic_t gpu_reset_counter; + atomic_t vram_lost_counter; + + /* data for buffer migration throttling */ + struct { + spinlock_t lock; + s64 last_update_us; + s64 accum_us; /* accumulated microseconds */ + s64 accum_us_vis; /* for visible VRAM */ + u32 log2_max_MBps; + } mm_stats; + + /* display */ + bool enable_virtual_display; + struct amdgpu_mode_info mode_info; + /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */ + struct work_struct hotplug_work; + struct amdgpu_irq_src crtc_irq; + struct amdgpu_irq_src vupdate_irq; + struct amdgpu_irq_src pageflip_irq; + struct amdgpu_irq_src hpd_irq; + + /* rings */ + u64 fence_context; + unsigned num_rings; + struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; + bool ib_pool_ready; + struct amdgpu_sa_manager ring_tmp_bo; + + /* interrupts */ + struct amdgpu_irq irq; + + /* powerplay */ + struct amd_powerplay powerplay; + bool pp_force_state_enabled; + + /* smu */ + struct smu_context smu; + + /* dpm */ + struct amdgpu_pm pm; + u32 cg_flags; + u32 pg_flags; + + /* gfx */ + struct amdgpu_gfx gfx; + + /* sdma */ + struct amdgpu_sdma sdma; + + /* uvd */ + struct amdgpu_uvd uvd; + + /* vce */ + struct amdgpu_vce vce; + + /* vcn */ + struct amdgpu_vcn vcn; + + /* firmwares */ + struct amdgpu_firmware firmware; + + /* PSP */ + struct psp_context psp; + + /* GDS */ + struct amdgpu_gds gds; + + /* KFD */ + struct amdgpu_kfd_dev kfd; + + /* display related functionality */ + struct amdgpu_display_manager dm; + + /* discovery */ + uint8_t *discovery; + + /* mes */ + bool enable_mes; + struct amdgpu_mes mes; + + struct amdgpu_ip_block ip_blocks[AMDGPU_MAX_IP_NUM]; + int num_ip_blocks; + struct mutex mn_lock; + DECLARE_HASHTABLE(mn_hash, 7); + + /* tracking pinned memory */ + atomic64_t vram_pin_size; + atomic64_t visible_pin_size; + atomic64_t gart_pin_size; + + /* soc15 register offset based on ip, instance and segment */ + uint32_t *reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE]; + + const struct amdgpu_nbio_funcs *nbio_funcs; + const struct amdgpu_df_funcs *df_funcs; + + /* delayed work_func for deferring clockgating during resume */ + struct delayed_work delayed_init_work; + + struct amdgpu_virt virt; + /* firmware VRAM reservation */ + struct amdgpu_fw_vram_usage fw_vram_usage; + + /* link all shadow bo */ + struct list_head shadow_list; + struct mutex shadow_list_lock; + /* keep an lru list of rings by HW IP */ + struct list_head ring_lru_list; + spinlock_t ring_lru_list_lock; + + /* record hw reset is performed */ + bool has_hw_reset; + u8 reset_magic[AMDGPU_RESET_MAGIC_NUM]; + + /* s3/s4 mask */ + bool in_suspend; + + /* record last mm index being written through WREG32*/ + unsigned long last_mm_index; + bool in_gpu_reset; + struct mutex lock_reset; + struct amdgpu_doorbell_index doorbell_index; + + int asic_reset_res; + struct work_struct xgmi_reset_work; + + bool in_baco_reset; + + long gfx_timeout; + long sdma_timeout; + long video_timeout; + long compute_timeout; + + uint64_t unique_id; + uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS]; +}; + +static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) +{ + return container_of(bdev, struct amdgpu_device, mman.bdev); +} + +int amdgpu_device_init(struct amdgpu_device *adev, + struct drm_device *ddev, + struct pci_dev *pdev, + uint32_t flags); +void amdgpu_device_fini(struct amdgpu_device *adev); +int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev); + +uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, + uint32_t acc_flags); +void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, + uint32_t acc_flags); +void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value); +uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset); + +u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg); +void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v); + +bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type); +bool amdgpu_device_has_dc_support(struct amdgpu_device *adev); + +int emu_soc_asic_init(struct amdgpu_device *adev); + +/* + * Registers read & write functions. + */ + +#define AMDGPU_REGS_IDX (1<<0) +#define AMDGPU_REGS_NO_KIQ (1<<1) + +#define RREG32_NO_KIQ(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_NO_KIQ) +#define WREG32_NO_KIQ(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_NO_KIQ) + +#define RREG8(reg) amdgpu_mm_rreg8(adev, (reg)) +#define WREG8(reg, v) amdgpu_mm_wreg8(adev, (reg), (v)) + +#define RREG32(reg) amdgpu_mm_rreg(adev, (reg), 0) +#define RREG32_IDX(reg) amdgpu_mm_rreg(adev, (reg), AMDGPU_REGS_IDX) +#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", amdgpu_mm_rreg(adev, (reg), 0)) +#define WREG32(reg, v) amdgpu_mm_wreg(adev, (reg), (v), 0) +#define WREG32_IDX(reg, v) amdgpu_mm_wreg(adev, (reg), (v), AMDGPU_REGS_IDX) +#define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK) +#define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK) +#define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg)) +#define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v)) +#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg)) +#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v)) +#define RREG32_SMC(reg) adev->smc_rreg(adev, (reg)) +#define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v)) +#define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg)) +#define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v)) +#define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg)) +#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v)) +#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg)) +#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v)) +#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg)) +#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v)) +#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg)) +#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v)) +#define WREG32_P(reg, val, mask) \ + do { \ + uint32_t tmp_ = RREG32(reg); \ + tmp_ &= (mask); \ + tmp_ |= ((val) & ~(mask)); \ + WREG32(reg, tmp_); \ + } while (0) +#define WREG32_AND(reg, and) WREG32_P(reg, 0, and) +#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or)) +#define WREG32_PLL_P(reg, val, mask) \ + do { \ + uint32_t tmp_ = RREG32_PLL(reg); \ + tmp_ &= (mask); \ + tmp_ |= ((val) & ~(mask)); \ + WREG32_PLL(reg, tmp_); \ + } while (0) +#define DREG32_SYS(sqf, adev, reg) seq_printf((sqf), #reg " : 0x%08X\n", amdgpu_mm_rreg((adev), (reg), false)) +#define RREG32_IO(reg) amdgpu_io_rreg(adev, (reg)) +#define WREG32_IO(reg, v) amdgpu_io_wreg(adev, (reg), (v)) + +#define REG_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT +#define REG_FIELD_MASK(reg, field) reg##__##field##_MASK + +#define REG_SET_FIELD(orig_val, reg, field, field_val) \ + (((orig_val) & ~REG_FIELD_MASK(reg, field)) | \ + (REG_FIELD_MASK(reg, field) & ((field_val) << REG_FIELD_SHIFT(reg, field)))) + +#define REG_GET_FIELD(value, reg, field) \ + (((value) & REG_FIELD_MASK(reg, field)) >> REG_FIELD_SHIFT(reg, field)) + +#define WREG32_FIELD(reg, field, val) \ + WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field)) + +#define WREG32_FIELD_OFFSET(reg, offset, field, val) \ + WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field)) + +/* + * BIOS helpers. + */ +#define RBIOS8(i) (adev->bios[i]) +#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8)) +#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16)) + +/* + * ASICs macro. + */ +#define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state)) +#define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev)) +#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev)) +#define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d)) +#define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec)) +#define amdgpu_get_pcie_lanes(adev) (adev)->asic_funcs->get_pcie_lanes((adev)) +#define amdgpu_set_pcie_lanes(adev, l) (adev)->asic_funcs->set_pcie_lanes((adev), (l)) +#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev)) +#define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev)) +#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)) +#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v))) +#define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev)) +#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r)) +#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r)) +#define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev)) +#define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev)) +#define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1))) +#define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev)) +#define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev))) + +/* Common functions */ +bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); +int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + struct amdgpu_job* job); +void amdgpu_device_pci_config_reset(struct amdgpu_device *adev); +bool amdgpu_device_need_post(struct amdgpu_device *adev); + +void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, + u64 num_vis_bytes); +int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev); +void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, + const u32 *registers, + const u32 array_size); + +bool amdgpu_device_is_px(struct drm_device *dev); +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, + struct amdgpu_device *peer_adev); + +/* atpx handler */ +#if defined(CONFIG_VGA_SWITCHEROO) +void amdgpu_register_atpx_handler(void); +void amdgpu_unregister_atpx_handler(void); +bool amdgpu_has_atpx_dgpu_power_cntl(void); +bool amdgpu_is_atpx_hybrid(void); +bool amdgpu_atpx_dgpu_req_power_for_displays(void); +bool amdgpu_has_atpx(void); +#else +static inline void amdgpu_register_atpx_handler(void) {} +static inline void amdgpu_unregister_atpx_handler(void) {} +static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; } +static inline bool amdgpu_is_atpx_hybrid(void) { return false; } +static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; } +static inline bool amdgpu_has_atpx(void) { return false; } +#endif + +#if defined(CONFIG_VGA_SWITCHEROO) && defined(CONFIG_ACPI) +void *amdgpu_atpx_get_dhandle(void); +#else +static inline void *amdgpu_atpx_get_dhandle(void) { return NULL; } +#endif + +/* + * KMS + */ +extern const struct drm_ioctl_desc amdgpu_ioctls_kms[]; +extern const int amdgpu_max_kms_ioctl; + +int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags); +void amdgpu_driver_unload_kms(struct drm_device *dev); +void amdgpu_driver_lastclose_kms(struct drm_device *dev); +int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv); +void amdgpu_driver_postclose_kms(struct drm_device *dev, + struct drm_file *file_priv); +int amdgpu_device_ip_suspend(struct amdgpu_device *adev); +int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon); +int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon); +u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe); +int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe); +void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe); +long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); + +/* + * functions used by amdgpu_encoder.c + */ +struct amdgpu_afmt_acr { + u32 clock; + + int n_32khz; + int cts_32khz; + + int n_44_1khz; + int cts_44_1khz; + + int n_48khz; + int cts_48khz; + +}; + +struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock); + +/* amdgpu_acpi.c */ +#if defined(CONFIG_ACPI) +int amdgpu_acpi_init(struct amdgpu_device *adev); +void amdgpu_acpi_fini(struct amdgpu_device *adev); +bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev); +int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev, + u8 perf_req, bool advertise); +int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev); + +void amdgpu_acpi_get_backlight_caps(struct amdgpu_device *adev, + struct amdgpu_dm_backlight_caps *caps); +#else +static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; } +static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { } +#endif + +int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, + uint64_t addr, struct amdgpu_bo **bo, + struct amdgpu_bo_va_mapping **mapping); + +#if defined(CONFIG_DRM_AMD_DC) +int amdgpu_dm_display_resume(struct amdgpu_device *adev ); +#else +static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; } +#endif + + +void amdgpu_register_gpu_instance(struct amdgpu_device *adev); +void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev); + +#include "amdgpu_object.h" + +/* used by df_v3_6.c and amdgpu_pmu.c */ +#define AMDGPU_PMU_ATTR(_name, _object) \ +static ssize_t \ +_name##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *page) \ +{ \ + BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1); \ + return sprintf(page, _object "\n"); \ +} \ + \ +static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name) + +#endif +
diff --git a/amdgpu/amdgpu_acp.c b/amdgpu/amdgpu_acp.c new file mode 100644 index 0000000..eba42c7 --- /dev/null +++ b/amdgpu/amdgpu_acp.c
@@ -0,0 +1,553 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include <linux/irqdomain.h> +#include <linux/pci.h> +#include <linux/pm_domain.h> +#include <linux/platform_device.h> +#include <sound/designware_i2s.h> +#include <sound/pcm.h> + +#include "amdgpu.h" +#include "atom.h" +#include "amdgpu_acp.h" + +#include "acp_gfx_if.h" + +#define ACP_TILE_ON_MASK 0x03 +#define ACP_TILE_OFF_MASK 0x02 +#define ACP_TILE_ON_RETAIN_REG_MASK 0x1f +#define ACP_TILE_OFF_RETAIN_REG_MASK 0x20 + +#define ACP_TILE_P1_MASK 0x3e +#define ACP_TILE_P2_MASK 0x3d +#define ACP_TILE_DSP0_MASK 0x3b +#define ACP_TILE_DSP1_MASK 0x37 + +#define ACP_TILE_DSP2_MASK 0x2f + +#define ACP_DMA_REGS_END 0x146c0 +#define ACP_I2S_PLAY_REGS_START 0x14840 +#define ACP_I2S_PLAY_REGS_END 0x148b4 +#define ACP_I2S_CAP_REGS_START 0x148b8 +#define ACP_I2S_CAP_REGS_END 0x1496c + +#define ACP_I2S_COMP1_CAP_REG_OFFSET 0xac +#define ACP_I2S_COMP2_CAP_REG_OFFSET 0xa8 +#define ACP_I2S_COMP1_PLAY_REG_OFFSET 0x6c +#define ACP_I2S_COMP2_PLAY_REG_OFFSET 0x68 +#define ACP_BT_PLAY_REGS_START 0x14970 +#define ACP_BT_PLAY_REGS_END 0x14a24 +#define ACP_BT_COMP1_REG_OFFSET 0xac +#define ACP_BT_COMP2_REG_OFFSET 0xa8 + +#define mmACP_PGFSM_RETAIN_REG 0x51c9 +#define mmACP_PGFSM_CONFIG_REG 0x51ca +#define mmACP_PGFSM_READ_REG_0 0x51cc + +#define mmACP_MEM_SHUT_DOWN_REQ_LO 0x51f8 +#define mmACP_MEM_SHUT_DOWN_REQ_HI 0x51f9 +#define mmACP_MEM_SHUT_DOWN_STS_LO 0x51fa +#define mmACP_MEM_SHUT_DOWN_STS_HI 0x51fb + +#define mmACP_CONTROL 0x5131 +#define mmACP_STATUS 0x5133 +#define mmACP_SOFT_RESET 0x5134 +#define ACP_CONTROL__ClkEn_MASK 0x1 +#define ACP_SOFT_RESET__SoftResetAud_MASK 0x100 +#define ACP_SOFT_RESET__SoftResetAudDone_MASK 0x1000000 +#define ACP_CLOCK_EN_TIME_OUT_VALUE 0x000000FF +#define ACP_SOFT_RESET_DONE_TIME_OUT_VALUE 0x000000FF + +#define ACP_TIMEOUT_LOOP 0x000000FF +#define ACP_DEVS 4 +#define ACP_SRC_ID 162 + +enum { + ACP_TILE_P1 = 0, + ACP_TILE_P2, + ACP_TILE_DSP0, + ACP_TILE_DSP1, + ACP_TILE_DSP2, +}; + +static int acp_sw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + adev->acp.parent = adev->dev; + + adev->acp.cgs_device = + amdgpu_cgs_create_device(adev); + if (!adev->acp.cgs_device) + return -EINVAL; + + return 0; +} + +static int acp_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + if (adev->acp.cgs_device) + amdgpu_cgs_destroy_device(adev->acp.cgs_device); + + return 0; +} + +struct acp_pm_domain { + void *adev; + struct generic_pm_domain gpd; +}; + +static int acp_poweroff(struct generic_pm_domain *genpd) +{ + struct acp_pm_domain *apd; + struct amdgpu_device *adev; + + apd = container_of(genpd, struct acp_pm_domain, gpd); + if (apd != NULL) { + adev = apd->adev; + /* call smu to POWER GATE ACP block + * smu will + * 1. turn off the acp clock + * 2. power off the acp tiles + * 3. check and enter ulv state + */ + if (adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->set_powergating_by_smu) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, true); + } + return 0; +} + +static int acp_poweron(struct generic_pm_domain *genpd) +{ + struct acp_pm_domain *apd; + struct amdgpu_device *adev; + + apd = container_of(genpd, struct acp_pm_domain, gpd); + if (apd != NULL) { + adev = apd->adev; + /* call smu to UNGATE ACP block + * smu will + * 1. exit ulv + * 2. turn on acp clock + * 3. power on acp tiles + */ + if (adev->powerplay.pp_funcs->set_powergating_by_smu) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, false); + } + return 0; +} + +static struct device *get_mfd_cell_dev(const char *device_name, int r) +{ + char auto_dev_name[25]; + struct device *dev; + + snprintf(auto_dev_name, sizeof(auto_dev_name), + "%s.%d.auto", device_name, r); + dev = bus_find_device_by_name(&platform_bus_type, NULL, auto_dev_name); + dev_info(dev, "device %s added to pm domain\n", auto_dev_name); + + return dev; +} + +/** + * acp_hw_init - start and test ACP block + * + * @adev: amdgpu_device pointer + * + */ +static int acp_hw_init(void *handle) +{ + int r, i; + uint64_t acp_base; + u32 val = 0; + u32 count = 0; + struct device *dev; + struct i2s_platform_data *i2s_pdata; + + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + const struct amdgpu_ip_block *ip_block = + amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_ACP); + + if (!ip_block) + return -EINVAL; + + r = amd_acp_hw_init(adev->acp.cgs_device, + ip_block->version->major, ip_block->version->minor); + /* -ENODEV means board uses AZ rather than ACP */ + if (r == -ENODEV) { + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, true); + return 0; + } else if (r) { + return r; + } + + if (adev->rmmio_size == 0 || adev->rmmio_size < 0x5289) + return -EINVAL; + + acp_base = adev->rmmio_base; + + + adev->acp.acp_genpd = kzalloc(sizeof(struct acp_pm_domain), GFP_KERNEL); + if (adev->acp.acp_genpd == NULL) + return -ENOMEM; + + adev->acp.acp_genpd->gpd.name = "ACP_AUDIO"; + adev->acp.acp_genpd->gpd.power_off = acp_poweroff; + adev->acp.acp_genpd->gpd.power_on = acp_poweron; + + + adev->acp.acp_genpd->adev = adev; + + pm_genpd_init(&adev->acp.acp_genpd->gpd, NULL, false); + + adev->acp.acp_cell = kcalloc(ACP_DEVS, sizeof(struct mfd_cell), + GFP_KERNEL); + + if (adev->acp.acp_cell == NULL) + return -ENOMEM; + + adev->acp.acp_res = kcalloc(5, sizeof(struct resource), GFP_KERNEL); + if (adev->acp.acp_res == NULL) { + kfree(adev->acp.acp_cell); + return -ENOMEM; + } + + i2s_pdata = kcalloc(3, sizeof(struct i2s_platform_data), GFP_KERNEL); + if (i2s_pdata == NULL) { + kfree(adev->acp.acp_res); + kfree(adev->acp.acp_cell); + return -ENOMEM; + } + + switch (adev->asic_type) { + case CHIP_STONEY: + i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + break; + default: + i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + } + i2s_pdata[0].cap = DWC_I2S_PLAY; + i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; + i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET; + i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET; + switch (adev->asic_type) { + case CHIP_STONEY: + i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1 | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + break; + default: + i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1; + } + + i2s_pdata[1].cap = DWC_I2S_RECORD; + i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000; + i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; + i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; + + i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + switch (adev->asic_type) { + case CHIP_STONEY: + i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + break; + default: + break; + } + + i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; + i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000; + i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET; + i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET; + + adev->acp.acp_res[0].name = "acp2x_dma"; + adev->acp.acp_res[0].flags = IORESOURCE_MEM; + adev->acp.acp_res[0].start = acp_base; + adev->acp.acp_res[0].end = acp_base + ACP_DMA_REGS_END; + + adev->acp.acp_res[1].name = "acp2x_dw_i2s_play"; + adev->acp.acp_res[1].flags = IORESOURCE_MEM; + adev->acp.acp_res[1].start = acp_base + ACP_I2S_PLAY_REGS_START; + adev->acp.acp_res[1].end = acp_base + ACP_I2S_PLAY_REGS_END; + + adev->acp.acp_res[2].name = "acp2x_dw_i2s_cap"; + adev->acp.acp_res[2].flags = IORESOURCE_MEM; + adev->acp.acp_res[2].start = acp_base + ACP_I2S_CAP_REGS_START; + adev->acp.acp_res[2].end = acp_base + ACP_I2S_CAP_REGS_END; + + adev->acp.acp_res[3].name = "acp2x_dw_bt_i2s_play_cap"; + adev->acp.acp_res[3].flags = IORESOURCE_MEM; + adev->acp.acp_res[3].start = acp_base + ACP_BT_PLAY_REGS_START; + adev->acp.acp_res[3].end = acp_base + ACP_BT_PLAY_REGS_END; + + adev->acp.acp_res[4].name = "acp2x_dma_irq"; + adev->acp.acp_res[4].flags = IORESOURCE_IRQ; + adev->acp.acp_res[4].start = amdgpu_irq_create_mapping(adev, 162); + adev->acp.acp_res[4].end = adev->acp.acp_res[4].start; + + adev->acp.acp_cell[0].name = "acp_audio_dma"; + adev->acp.acp_cell[0].num_resources = 5; + adev->acp.acp_cell[0].resources = &adev->acp.acp_res[0]; + adev->acp.acp_cell[0].platform_data = &adev->asic_type; + adev->acp.acp_cell[0].pdata_size = sizeof(adev->asic_type); + + adev->acp.acp_cell[1].name = "designware-i2s"; + adev->acp.acp_cell[1].num_resources = 1; + adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1]; + adev->acp.acp_cell[1].platform_data = &i2s_pdata[0]; + adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data); + + adev->acp.acp_cell[2].name = "designware-i2s"; + adev->acp.acp_cell[2].num_resources = 1; + adev->acp.acp_cell[2].resources = &adev->acp.acp_res[2]; + adev->acp.acp_cell[2].platform_data = &i2s_pdata[1]; + adev->acp.acp_cell[2].pdata_size = sizeof(struct i2s_platform_data); + + adev->acp.acp_cell[3].name = "designware-i2s"; + adev->acp.acp_cell[3].num_resources = 1; + adev->acp.acp_cell[3].resources = &adev->acp.acp_res[3]; + adev->acp.acp_cell[3].platform_data = &i2s_pdata[2]; + adev->acp.acp_cell[3].pdata_size = sizeof(struct i2s_platform_data); + + r = mfd_add_hotplug_devices(adev->acp.parent, adev->acp.acp_cell, + ACP_DEVS); + if (r) + return r; + + for (i = 0; i < ACP_DEVS ; i++) { + dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i); + r = pm_genpd_add_device(&adev->acp.acp_genpd->gpd, dev); + if (r) { + dev_err(dev, "Failed to add dev to genpd\n"); + return r; + } + } + + + /* Assert Soft reset of ACP */ + val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET); + + val |= ACP_SOFT_RESET__SoftResetAud_MASK; + cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val); + + count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE; + while (true) { + val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET); + if (ACP_SOFT_RESET__SoftResetAudDone_MASK == + (val & ACP_SOFT_RESET__SoftResetAudDone_MASK)) + break; + if (--count == 0) { + dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); + return -ETIMEDOUT; + } + udelay(100); + } + /* Enable clock to ACP and wait until the clock is enabled */ + val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL); + val = val | ACP_CONTROL__ClkEn_MASK; + cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val); + + count = ACP_CLOCK_EN_TIME_OUT_VALUE; + + while (true) { + val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS); + if (val & (u32) 0x1) + break; + if (--count == 0) { + dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); + return -ETIMEDOUT; + } + udelay(100); + } + /* Deassert the SOFT RESET flags */ + val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET); + val &= ~ACP_SOFT_RESET__SoftResetAud_MASK; + cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val); + return 0; +} + +/** + * acp_hw_fini - stop the hardware block + * + * @adev: amdgpu_device pointer + * + */ +static int acp_hw_fini(void *handle) +{ + int i, ret; + u32 val = 0; + u32 count = 0; + struct device *dev; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + /* return early if no ACP */ + if (!adev->acp.acp_genpd) { + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, false); + return 0; + } + + /* Assert Soft reset of ACP */ + val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET); + + val |= ACP_SOFT_RESET__SoftResetAud_MASK; + cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val); + + count = ACP_SOFT_RESET_DONE_TIME_OUT_VALUE; + while (true) { + val = cgs_read_register(adev->acp.cgs_device, mmACP_SOFT_RESET); + if (ACP_SOFT_RESET__SoftResetAudDone_MASK == + (val & ACP_SOFT_RESET__SoftResetAudDone_MASK)) + break; + if (--count == 0) { + dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); + return -ETIMEDOUT; + } + udelay(100); + } + /* Disable ACP clock */ + val = cgs_read_register(adev->acp.cgs_device, mmACP_CONTROL); + val &= ~ACP_CONTROL__ClkEn_MASK; + cgs_write_register(adev->acp.cgs_device, mmACP_CONTROL, val); + + count = ACP_CLOCK_EN_TIME_OUT_VALUE; + + while (true) { + val = cgs_read_register(adev->acp.cgs_device, mmACP_STATUS); + if (val & (u32) 0x1) + break; + if (--count == 0) { + dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); + return -ETIMEDOUT; + } + udelay(100); + } + + for (i = 0; i < ACP_DEVS ; i++) { + dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i); + ret = pm_genpd_remove_device(dev); + /* If removal fails, dont giveup and try rest */ + if (ret) + dev_err(dev, "remove dev from genpd failed\n"); + } + + mfd_remove_devices(adev->acp.parent); + kfree(adev->acp.acp_res); + kfree(adev->acp.acp_genpd); + kfree(adev->acp.acp_cell); + + return 0; +} + +static int acp_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + /* power up on suspend */ + if (!adev->acp.acp_cell) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, false); + return 0; +} + +static int acp_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + /* power down again on resume */ + if (!adev->acp.acp_cell) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, true); + return 0; +} + +static int acp_early_init(void *handle) +{ + return 0; +} + +static bool acp_is_idle(void *handle) +{ + return true; +} + +static int acp_wait_for_idle(void *handle) +{ + return 0; +} + +static int acp_soft_reset(void *handle) +{ + return 0; +} + +static int acp_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int acp_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + bool enable = state == AMD_PG_STATE_GATE ? true : false; + + if (adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->set_powergating_by_smu) + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_ACP, enable); + + return 0; +} + +static const struct amd_ip_funcs acp_ip_funcs = { + .name = "acp_ip", + .early_init = acp_early_init, + .late_init = NULL, + .sw_init = acp_sw_init, + .sw_fini = acp_sw_fini, + .hw_init = acp_hw_init, + .hw_fini = acp_hw_fini, + .suspend = acp_suspend, + .resume = acp_resume, + .is_idle = acp_is_idle, + .wait_for_idle = acp_wait_for_idle, + .soft_reset = acp_soft_reset, + .set_clockgating_state = acp_set_clockgating_state, + .set_powergating_state = acp_set_powergating_state, +}; + +const struct amdgpu_ip_block_version acp_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_ACP, + .major = 2, + .minor = 2, + .rev = 0, + .funcs = &acp_ip_funcs, +};
diff --git a/amdgpu/amdgpu_acp.h b/amdgpu/amdgpu_acp.h new file mode 100644 index 0000000..a288ce2 --- /dev/null +++ b/amdgpu/amdgpu_acp.h
@@ -0,0 +1,42 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef __AMDGPU_ACP_H__ +#define __AMDGPU_ACP_H__ + +#include <linux/mfd/core.h> + +struct amdgpu_acp { + struct device *parent; + struct cgs_device *cgs_device; + struct amd_acp_private *private; + struct mfd_cell *acp_cell; + struct resource *acp_res; + struct acp_pm_domain *acp_genpd; +}; + +extern const struct amdgpu_ip_block_version acp_ip_block; + +#endif /* __AMDGPU_ACP_H__ */
diff --git a/amdgpu/amdgpu_acpi.c b/amdgpu/amdgpu_acpi.c new file mode 100644 index 0000000..1e41367 --- /dev/null +++ b/amdgpu/amdgpu_acpi.c
@@ -0,0 +1,889 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/pci.h> +#include <linux/acpi.h> +#include <linux/slab.h> +#include <linux/power_supply.h> +#include <linux/pm_runtime.h> +#include <acpi/video.h> + +#include <drm/drm_crtc_helper.h> +#include "amdgpu.h" +#include "amdgpu_pm.h" +#include "amdgpu_display.h" +#include "amd_acpi.h" +#include "atom.h" + +struct amdgpu_atif_notification_cfg { + bool enabled; + int command_code; +}; + +struct amdgpu_atif_notifications { + bool thermal_state; + bool forced_power_state; + bool system_power_state; + bool brightness_change; + bool dgpu_display_event; + bool gpu_package_power_limit; +}; + +struct amdgpu_atif_functions { + bool system_params; + bool sbios_requests; + bool temperature_change; + bool query_backlight_transfer_characteristics; + bool ready_to_undock; + bool external_gpu_information; +}; + +struct amdgpu_atif { + acpi_handle handle; + + struct amdgpu_atif_notifications notifications; + struct amdgpu_atif_functions functions; + struct amdgpu_atif_notification_cfg notification_cfg; + struct amdgpu_encoder *encoder_for_bl; + struct amdgpu_dm_backlight_caps backlight_caps; +}; + +/* Call the ATIF method + */ +/** + * amdgpu_atif_call - call an ATIF method + * + * @handle: acpi handle + * @function: the ATIF function to execute + * @params: ATIF function params + * + * Executes the requested ATIF function (all asics). + * Returns a pointer to the acpi output buffer. + */ +static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif, + int function, + struct acpi_buffer *params) +{ + acpi_status status; + union acpi_object atif_arg_elements[2]; + struct acpi_object_list atif_arg; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + + atif_arg.count = 2; + atif_arg.pointer = &atif_arg_elements[0]; + + atif_arg_elements[0].type = ACPI_TYPE_INTEGER; + atif_arg_elements[0].integer.value = function; + + if (params) { + atif_arg_elements[1].type = ACPI_TYPE_BUFFER; + atif_arg_elements[1].buffer.length = params->length; + atif_arg_elements[1].buffer.pointer = params->pointer; + } else { + /* We need a second fake parameter */ + atif_arg_elements[1].type = ACPI_TYPE_INTEGER; + atif_arg_elements[1].integer.value = 0; + } + + status = acpi_evaluate_object(atif->handle, NULL, &atif_arg, + &buffer); + + /* Fail only if calling the method fails and ATIF is supported */ + if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) { + DRM_DEBUG_DRIVER("failed to evaluate ATIF got %s\n", + acpi_format_exception(status)); + kfree(buffer.pointer); + return NULL; + } + + return buffer.pointer; +} + +/** + * amdgpu_atif_parse_notification - parse supported notifications + * + * @n: supported notifications struct + * @mask: supported notifications mask from ATIF + * + * Use the supported notifications mask from ATIF function + * ATIF_FUNCTION_VERIFY_INTERFACE to determine what notifications + * are supported (all asics). + */ +static void amdgpu_atif_parse_notification(struct amdgpu_atif_notifications *n, u32 mask) +{ + n->thermal_state = mask & ATIF_THERMAL_STATE_CHANGE_REQUEST_SUPPORTED; + n->forced_power_state = mask & ATIF_FORCED_POWER_STATE_CHANGE_REQUEST_SUPPORTED; + n->system_power_state = mask & ATIF_SYSTEM_POWER_SOURCE_CHANGE_REQUEST_SUPPORTED; + n->brightness_change = mask & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST_SUPPORTED; + n->dgpu_display_event = mask & ATIF_DGPU_DISPLAY_EVENT_SUPPORTED; + n->gpu_package_power_limit = mask & ATIF_GPU_PACKAGE_POWER_LIMIT_REQUEST_SUPPORTED; +} + +/** + * amdgpu_atif_parse_functions - parse supported functions + * + * @f: supported functions struct + * @mask: supported functions mask from ATIF + * + * Use the supported functions mask from ATIF function + * ATIF_FUNCTION_VERIFY_INTERFACE to determine what functions + * are supported (all asics). + */ +static void amdgpu_atif_parse_functions(struct amdgpu_atif_functions *f, u32 mask) +{ + f->system_params = mask & ATIF_GET_SYSTEM_PARAMETERS_SUPPORTED; + f->sbios_requests = mask & ATIF_GET_SYSTEM_BIOS_REQUESTS_SUPPORTED; + f->temperature_change = mask & ATIF_TEMPERATURE_CHANGE_NOTIFICATION_SUPPORTED; + f->query_backlight_transfer_characteristics = + mask & ATIF_QUERY_BACKLIGHT_TRANSFER_CHARACTERISTICS_SUPPORTED; + f->ready_to_undock = mask & ATIF_READY_TO_UNDOCK_NOTIFICATION_SUPPORTED; + f->external_gpu_information = mask & ATIF_GET_EXTERNAL_GPU_INFORMATION_SUPPORTED; +} + +/** + * amdgpu_atif_verify_interface - verify ATIF + * + * @handle: acpi handle + * @atif: amdgpu atif struct + * + * Execute the ATIF_FUNCTION_VERIFY_INTERFACE ATIF function + * to initialize ATIF and determine what features are supported + * (all asics). + * returns 0 on success, error on failure. + */ +static int amdgpu_atif_verify_interface(struct amdgpu_atif *atif) +{ + union acpi_object *info; + struct atif_verify_interface output; + size_t size; + int err = 0; + + info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL); + if (!info) + return -EIO; + + memset(&output, 0, sizeof(output)); + + size = *(u16 *) info->buffer.pointer; + if (size < 12) { + DRM_INFO("ATIF buffer is too small: %zu\n", size); + err = -EINVAL; + goto out; + } + size = min(sizeof(output), size); + + memcpy(&output, info->buffer.pointer, size); + + /* TODO: check version? */ + DRM_DEBUG_DRIVER("ATIF version %u\n", output.version); + + amdgpu_atif_parse_notification(&atif->notifications, output.notification_mask); + amdgpu_atif_parse_functions(&atif->functions, output.function_bits); + +out: + kfree(info); + return err; +} + +static acpi_handle amdgpu_atif_probe_handle(acpi_handle dhandle) +{ + acpi_handle handle = NULL; + char acpi_method_name[255] = { 0 }; + struct acpi_buffer buffer = { sizeof(acpi_method_name), acpi_method_name }; + acpi_status status; + + /* For PX/HG systems, ATIF and ATPX are in the iGPU's namespace, on dGPU only + * systems, ATIF is in the dGPU's namespace. + */ + status = acpi_get_handle(dhandle, "ATIF", &handle); + if (ACPI_SUCCESS(status)) + goto out; + + if (amdgpu_has_atpx()) { + status = acpi_get_handle(amdgpu_atpx_get_dhandle(), "ATIF", + &handle); + if (ACPI_SUCCESS(status)) + goto out; + } + + DRM_DEBUG_DRIVER("No ATIF handle found\n"); + return NULL; +out: + acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer); + DRM_DEBUG_DRIVER("Found ATIF handle %s\n", acpi_method_name); + return handle; +} + +/** + * amdgpu_atif_get_notification_params - determine notify configuration + * + * @handle: acpi handle + * @n: atif notification configuration struct + * + * Execute the ATIF_FUNCTION_GET_SYSTEM_PARAMETERS ATIF function + * to determine if a notifier is used and if so which one + * (all asics). This is either Notify(VGA, 0x81) or Notify(VGA, n) + * where n is specified in the result if a notifier is used. + * Returns 0 on success, error on failure. + */ +static int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif) +{ + union acpi_object *info; + struct amdgpu_atif_notification_cfg *n = &atif->notification_cfg; + struct atif_system_params params; + size_t size; + int err = 0; + + info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS, + NULL); + if (!info) { + err = -EIO; + goto out; + } + + size = *(u16 *) info->buffer.pointer; + if (size < 10) { + err = -EINVAL; + goto out; + } + + memset(¶ms, 0, sizeof(params)); + size = min(sizeof(params), size); + memcpy(¶ms, info->buffer.pointer, size); + + DRM_DEBUG_DRIVER("SYSTEM_PARAMS: mask = %#x, flags = %#x\n", + params.flags, params.valid_mask); + params.flags = params.flags & params.valid_mask; + + if ((params.flags & ATIF_NOTIFY_MASK) == ATIF_NOTIFY_NONE) { + n->enabled = false; + n->command_code = 0; + } else if ((params.flags & ATIF_NOTIFY_MASK) == ATIF_NOTIFY_81) { + n->enabled = true; + n->command_code = 0x81; + } else { + if (size < 11) { + err = -EINVAL; + goto out; + } + n->enabled = true; + n->command_code = params.command_code; + } + +out: + DRM_DEBUG_DRIVER("Notification %s, command code = %#x\n", + (n->enabled ? "enabled" : "disabled"), + n->command_code); + kfree(info); + return err; +} + +/** + * amdgpu_atif_query_backlight_caps - get min and max backlight input signal + * + * @handle: acpi handle + * + * Execute the QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS ATIF function + * to determine the acceptable range of backlight values + * + * Backlight_caps.caps_valid will be set to true if the query is successful + * + * The input signals are in range 0-255 + * + * This function assumes the display with backlight is the first LCD + * + * Returns 0 on success, error on failure. + */ +static int amdgpu_atif_query_backlight_caps(struct amdgpu_atif *atif) +{ + union acpi_object *info; + struct atif_qbtc_output characteristics; + struct atif_qbtc_arguments arguments; + struct acpi_buffer params; + size_t size; + int err = 0; + + arguments.size = sizeof(arguments); + arguments.requested_display = ATIF_QBTC_REQUEST_LCD1; + + params.length = sizeof(arguments); + params.pointer = (void *)&arguments; + + info = amdgpu_atif_call(atif, + ATIF_FUNCTION_QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS, + ¶ms); + if (!info) { + err = -EIO; + goto out; + } + + size = *(u16 *) info->buffer.pointer; + if (size < 10) { + err = -EINVAL; + goto out; + } + + memset(&characteristics, 0, sizeof(characteristics)); + size = min(sizeof(characteristics), size); + memcpy(&characteristics, info->buffer.pointer, size); + + atif->backlight_caps.caps_valid = true; + atif->backlight_caps.min_input_signal = + characteristics.min_input_signal; + atif->backlight_caps.max_input_signal = + characteristics.max_input_signal; +out: + kfree(info); + return err; +} + +/** + * amdgpu_atif_get_sbios_requests - get requested sbios event + * + * @handle: acpi handle + * @req: atif sbios request struct + * + * Execute the ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS ATIF function + * to determine what requests the sbios is making to the driver + * (all asics). + * Returns 0 on success, error on failure. + */ +static int amdgpu_atif_get_sbios_requests(struct amdgpu_atif *atif, + struct atif_sbios_requests *req) +{ + union acpi_object *info; + size_t size; + int count = 0; + + info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS, + NULL); + if (!info) + return -EIO; + + size = *(u16 *)info->buffer.pointer; + if (size < 0xd) { + count = -EINVAL; + goto out; + } + memset(req, 0, sizeof(*req)); + + size = min(sizeof(*req), size); + memcpy(req, info->buffer.pointer, size); + DRM_DEBUG_DRIVER("SBIOS pending requests: %#x\n", req->pending); + + count = hweight32(req->pending); + +out: + kfree(info); + return count; +} + +/** + * amdgpu_atif_handler - handle ATIF notify requests + * + * @adev: amdgpu_device pointer + * @event: atif sbios request struct + * + * Checks the acpi event and if it matches an atif event, + * handles it. + * + * Returns: + * NOTIFY_BAD or NOTIFY_DONE, depending on the event. + */ +static int amdgpu_atif_handler(struct amdgpu_device *adev, + struct acpi_bus_event *event) +{ + struct amdgpu_atif *atif = adev->atif; + int count; + + DRM_DEBUG_DRIVER("event, device_class = %s, type = %#x\n", + event->device_class, event->type); + + if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0) + return NOTIFY_DONE; + + /* Is this actually our event? */ + if (!atif || + !atif->notification_cfg.enabled || + event->type != atif->notification_cfg.command_code) { + /* These events will generate keypresses otherwise */ + if (event->type == ACPI_VIDEO_NOTIFY_PROBE) + return NOTIFY_BAD; + else + return NOTIFY_DONE; + } + + if (atif->functions.sbios_requests) { + struct atif_sbios_requests req; + + /* Check pending SBIOS requests */ + count = amdgpu_atif_get_sbios_requests(atif, &req); + + if (count <= 0) + return NOTIFY_BAD; + + DRM_DEBUG_DRIVER("ATIF: %d pending SBIOS requests\n", count); + + /* todo: add DC handling */ + if ((req.pending & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST) && + !amdgpu_device_has_dc_support(adev)) { + struct amdgpu_encoder *enc = atif->encoder_for_bl; + + if (enc) { + struct amdgpu_encoder_atom_dig *dig = enc->enc_priv; + + DRM_DEBUG_DRIVER("Changing brightness to %d\n", + req.backlight_level); + + amdgpu_display_backlight_set_level(adev, enc, req.backlight_level); + +#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) + backlight_force_update(dig->bl_dev, + BACKLIGHT_UPDATE_HOTKEY); +#endif + } + } + if (req.pending & ATIF_DGPU_DISPLAY_EVENT) { + if (adev->flags & AMD_IS_PX) { + pm_runtime_get_sync(adev->ddev->dev); + /* Just fire off a uevent and let userspace tell us what to do */ + drm_helper_hpd_irq_event(adev->ddev); + pm_runtime_mark_last_busy(adev->ddev->dev); + pm_runtime_put_autosuspend(adev->ddev->dev); + } + } + /* TODO: check other events */ + } + + /* We've handled the event, stop the notifier chain. The ACPI interface + * overloads ACPI_VIDEO_NOTIFY_PROBE, we don't want to send that to + * userspace if the event was generated only to signal a SBIOS + * request. + */ + return NOTIFY_BAD; +} + +/* Call the ATCS method + */ +/** + * amdgpu_atcs_call - call an ATCS method + * + * @handle: acpi handle + * @function: the ATCS function to execute + * @params: ATCS function params + * + * Executes the requested ATCS function (all asics). + * Returns a pointer to the acpi output buffer. + */ +static union acpi_object *amdgpu_atcs_call(acpi_handle handle, int function, + struct acpi_buffer *params) +{ + acpi_status status; + union acpi_object atcs_arg_elements[2]; + struct acpi_object_list atcs_arg; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + + atcs_arg.count = 2; + atcs_arg.pointer = &atcs_arg_elements[0]; + + atcs_arg_elements[0].type = ACPI_TYPE_INTEGER; + atcs_arg_elements[0].integer.value = function; + + if (params) { + atcs_arg_elements[1].type = ACPI_TYPE_BUFFER; + atcs_arg_elements[1].buffer.length = params->length; + atcs_arg_elements[1].buffer.pointer = params->pointer; + } else { + /* We need a second fake parameter */ + atcs_arg_elements[1].type = ACPI_TYPE_INTEGER; + atcs_arg_elements[1].integer.value = 0; + } + + status = acpi_evaluate_object(handle, "ATCS", &atcs_arg, &buffer); + + /* Fail only if calling the method fails and ATIF is supported */ + if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) { + DRM_DEBUG_DRIVER("failed to evaluate ATCS got %s\n", + acpi_format_exception(status)); + kfree(buffer.pointer); + return NULL; + } + + return buffer.pointer; +} + +/** + * amdgpu_atcs_parse_functions - parse supported functions + * + * @f: supported functions struct + * @mask: supported functions mask from ATCS + * + * Use the supported functions mask from ATCS function + * ATCS_FUNCTION_VERIFY_INTERFACE to determine what functions + * are supported (all asics). + */ +static void amdgpu_atcs_parse_functions(struct amdgpu_atcs_functions *f, u32 mask) +{ + f->get_ext_state = mask & ATCS_GET_EXTERNAL_STATE_SUPPORTED; + f->pcie_perf_req = mask & ATCS_PCIE_PERFORMANCE_REQUEST_SUPPORTED; + f->pcie_dev_rdy = mask & ATCS_PCIE_DEVICE_READY_NOTIFICATION_SUPPORTED; + f->pcie_bus_width = mask & ATCS_SET_PCIE_BUS_WIDTH_SUPPORTED; +} + +/** + * amdgpu_atcs_verify_interface - verify ATCS + * + * @handle: acpi handle + * @atcs: amdgpu atcs struct + * + * Execute the ATCS_FUNCTION_VERIFY_INTERFACE ATCS function + * to initialize ATCS and determine what features are supported + * (all asics). + * returns 0 on success, error on failure. + */ +static int amdgpu_atcs_verify_interface(acpi_handle handle, + struct amdgpu_atcs *atcs) +{ + union acpi_object *info; + struct atcs_verify_interface output; + size_t size; + int err = 0; + + info = amdgpu_atcs_call(handle, ATCS_FUNCTION_VERIFY_INTERFACE, NULL); + if (!info) + return -EIO; + + memset(&output, 0, sizeof(output)); + + size = *(u16 *) info->buffer.pointer; + if (size < 8) { + DRM_INFO("ATCS buffer is too small: %zu\n", size); + err = -EINVAL; + goto out; + } + size = min(sizeof(output), size); + + memcpy(&output, info->buffer.pointer, size); + + /* TODO: check version? */ + DRM_DEBUG_DRIVER("ATCS version %u\n", output.version); + + amdgpu_atcs_parse_functions(&atcs->functions, output.function_bits); + +out: + kfree(info); + return err; +} + +/** + * amdgpu_acpi_is_pcie_performance_request_supported + * + * @adev: amdgpu_device pointer + * + * Check if the ATCS pcie_perf_req and pcie_dev_rdy methods + * are supported (all asics). + * returns true if supported, false if not. + */ +bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev) +{ + struct amdgpu_atcs *atcs = &adev->atcs; + + if (atcs->functions.pcie_perf_req && atcs->functions.pcie_dev_rdy) + return true; + + return false; +} + +/** + * amdgpu_acpi_pcie_notify_device_ready + * + * @adev: amdgpu_device pointer + * + * Executes the PCIE_DEVICE_READY_NOTIFICATION method + * (all asics). + * returns 0 on success, error on failure. + */ +int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev) +{ + acpi_handle handle; + union acpi_object *info; + struct amdgpu_atcs *atcs = &adev->atcs; + + /* Get the device handle */ + handle = ACPI_HANDLE(&adev->pdev->dev); + if (!handle) + return -EINVAL; + + if (!atcs->functions.pcie_dev_rdy) + return -EINVAL; + + info = amdgpu_atcs_call(handle, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION, NULL); + if (!info) + return -EIO; + + kfree(info); + + return 0; +} + +/** + * amdgpu_acpi_pcie_performance_request + * + * @adev: amdgpu_device pointer + * @perf_req: requested perf level (pcie gen speed) + * @advertise: set advertise caps flag if set + * + * Executes the PCIE_PERFORMANCE_REQUEST method to + * change the pcie gen speed (all asics). + * returns 0 on success, error on failure. + */ +int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev, + u8 perf_req, bool advertise) +{ + acpi_handle handle; + union acpi_object *info; + struct amdgpu_atcs *atcs = &adev->atcs; + struct atcs_pref_req_input atcs_input; + struct atcs_pref_req_output atcs_output; + struct acpi_buffer params; + size_t size; + u32 retry = 3; + + if (amdgpu_acpi_pcie_notify_device_ready(adev)) + return -EINVAL; + + /* Get the device handle */ + handle = ACPI_HANDLE(&adev->pdev->dev); + if (!handle) + return -EINVAL; + + if (!atcs->functions.pcie_perf_req) + return -EINVAL; + + atcs_input.size = sizeof(struct atcs_pref_req_input); + /* client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num) */ + atcs_input.client_id = adev->pdev->devfn | (adev->pdev->bus->number << 8); + atcs_input.valid_flags_mask = ATCS_VALID_FLAGS_MASK; + atcs_input.flags = ATCS_WAIT_FOR_COMPLETION; + if (advertise) + atcs_input.flags |= ATCS_ADVERTISE_CAPS; + atcs_input.req_type = ATCS_PCIE_LINK_SPEED; + atcs_input.perf_req = perf_req; + + params.length = sizeof(struct atcs_pref_req_input); + params.pointer = &atcs_input; + + while (retry--) { + info = amdgpu_atcs_call(handle, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST, ¶ms); + if (!info) + return -EIO; + + memset(&atcs_output, 0, sizeof(atcs_output)); + + size = *(u16 *) info->buffer.pointer; + if (size < 3) { + DRM_INFO("ATCS buffer is too small: %zu\n", size); + kfree(info); + return -EINVAL; + } + size = min(sizeof(atcs_output), size); + + memcpy(&atcs_output, info->buffer.pointer, size); + + kfree(info); + + switch (atcs_output.ret_val) { + case ATCS_REQUEST_REFUSED: + default: + return -EINVAL; + case ATCS_REQUEST_COMPLETE: + return 0; + case ATCS_REQUEST_IN_PROGRESS: + udelay(10); + break; + } + } + + return 0; +} + +/** + * amdgpu_acpi_event - handle notify events + * + * @nb: notifier block + * @val: val + * @data: acpi event + * + * Calls relevant amdgpu functions in response to various + * acpi events. + * Returns NOTIFY code + */ +static int amdgpu_acpi_event(struct notifier_block *nb, + unsigned long val, + void *data) +{ + struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, acpi_nb); + struct acpi_bus_event *entry = (struct acpi_bus_event *)data; + + if (strcmp(entry->device_class, ACPI_AC_CLASS) == 0) { + if (power_supply_is_system_supplied() > 0) + DRM_DEBUG_DRIVER("pm: AC\n"); + else + DRM_DEBUG_DRIVER("pm: DC\n"); + + amdgpu_pm_acpi_event_handler(adev); + } + + /* Check for pending SBIOS requests */ + return amdgpu_atif_handler(adev, entry); +} + +/* Call all ACPI methods here */ +/** + * amdgpu_acpi_init - init driver acpi support + * + * @adev: amdgpu_device pointer + * + * Verifies the AMD ACPI interfaces and registers with the acpi + * notifier chain (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_acpi_init(struct amdgpu_device *adev) +{ + acpi_handle handle, atif_handle; + struct amdgpu_atif *atif; + struct amdgpu_atcs *atcs = &adev->atcs; + int ret; + + /* Get the device handle */ + handle = ACPI_HANDLE(&adev->pdev->dev); + + if (!adev->bios || !handle) + return 0; + + /* Call the ATCS method */ + ret = amdgpu_atcs_verify_interface(handle, atcs); + if (ret) { + DRM_DEBUG_DRIVER("Call to ATCS verify_interface failed: %d\n", ret); + } + + /* Probe for ATIF, and initialize it if found */ + atif_handle = amdgpu_atif_probe_handle(handle); + if (!atif_handle) + goto out; + + atif = kzalloc(sizeof(*atif), GFP_KERNEL); + if (!atif) { + DRM_WARN("Not enough memory to initialize ATIF\n"); + goto out; + } + atif->handle = atif_handle; + + /* Call the ATIF method */ + ret = amdgpu_atif_verify_interface(atif); + if (ret) { + DRM_DEBUG_DRIVER("Call to ATIF verify_interface failed: %d\n", ret); + kfree(atif); + goto out; + } + adev->atif = atif; + + if (atif->notifications.brightness_change) { + struct drm_encoder *tmp; + + /* Find the encoder controlling the brightness */ + list_for_each_entry(tmp, &adev->ddev->mode_config.encoder_list, + head) { + struct amdgpu_encoder *enc = to_amdgpu_encoder(tmp); + + if ((enc->devices & (ATOM_DEVICE_LCD_SUPPORT)) && + enc->enc_priv) { + struct amdgpu_encoder_atom_dig *dig = enc->enc_priv; + if (dig->bl_dev) { + atif->encoder_for_bl = enc; + break; + } + } + } + } + + if (atif->functions.sbios_requests && !atif->functions.system_params) { + /* XXX check this workraround, if sbios request function is + * present we have to see how it's configured in the system + * params + */ + atif->functions.system_params = true; + } + + if (atif->functions.system_params) { + ret = amdgpu_atif_get_notification_params(atif); + if (ret) { + DRM_DEBUG_DRIVER("Call to GET_SYSTEM_PARAMS failed: %d\n", + ret); + /* Disable notification */ + atif->notification_cfg.enabled = false; + } + } + + if (atif->functions.query_backlight_transfer_characteristics) { + ret = amdgpu_atif_query_backlight_caps(atif); + if (ret) { + DRM_DEBUG_DRIVER("Call to QUERY_BACKLIGHT_TRANSFER_CHARACTERISTICS failed: %d\n", + ret); + atif->backlight_caps.caps_valid = false; + } + } else { + atif->backlight_caps.caps_valid = false; + } + +out: + adev->acpi_nb.notifier_call = amdgpu_acpi_event; + register_acpi_notifier(&adev->acpi_nb); + + return ret; +} + +void amdgpu_acpi_get_backlight_caps(struct amdgpu_device *adev, + struct amdgpu_dm_backlight_caps *caps) +{ + if (!adev->atif) { + caps->caps_valid = false; + return; + } + caps->caps_valid = adev->atif->backlight_caps.caps_valid; + caps->min_input_signal = adev->atif->backlight_caps.min_input_signal; + caps->max_input_signal = adev->atif->backlight_caps.max_input_signal; +} + +/** + * amdgpu_acpi_fini - tear down driver acpi support + * + * @adev: amdgpu_device pointer + * + * Unregisters with the acpi notifier chain (all asics). + */ +void amdgpu_acpi_fini(struct amdgpu_device *adev) +{ + unregister_acpi_notifier(&adev->acpi_nb); + kfree(adev->atif); +}
diff --git a/amdgpu/amdgpu_afmt.c b/amdgpu/amdgpu_afmt.c new file mode 100644 index 0000000..a4d6597 --- /dev/null +++ b/amdgpu/amdgpu_afmt.c
@@ -0,0 +1,105 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Christian König. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ +#include <linux/hdmi.h> +#include <linux/gcd.h> + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" + +static const struct amdgpu_afmt_acr amdgpu_afmt_predefined_acr[] = { + /* 32kHz 44.1kHz 48kHz */ + /* Clock N CTS N CTS N CTS */ + { 25175, 4096, 25175, 28224, 125875, 6144, 25175 }, /* 25,20/1.001 MHz */ + { 25200, 4096, 25200, 6272, 28000, 6144, 25200 }, /* 25.20 MHz */ + { 27000, 4096, 27000, 6272, 30000, 6144, 27000 }, /* 27.00 MHz */ + { 27027, 4096, 27027, 6272, 30030, 6144, 27027 }, /* 27.00*1.001 MHz */ + { 54000, 4096, 54000, 6272, 60000, 6144, 54000 }, /* 54.00 MHz */ + { 54054, 4096, 54054, 6272, 60060, 6144, 54054 }, /* 54.00*1.001 MHz */ + { 74176, 4096, 74176, 5733, 75335, 6144, 74176 }, /* 74.25/1.001 MHz */ + { 74250, 4096, 74250, 6272, 82500, 6144, 74250 }, /* 74.25 MHz */ + { 148352, 4096, 148352, 5733, 150670, 6144, 148352 }, /* 148.50/1.001 MHz */ + { 148500, 4096, 148500, 6272, 165000, 6144, 148500 }, /* 148.50 MHz */ +}; + + +/* + * calculate CTS and N values if they are not found in the table + */ +static void amdgpu_afmt_calc_cts(uint32_t clock, int *CTS, int *N, int freq) +{ + int n, cts; + unsigned long div, mul; + + /* Safe, but overly large values */ + n = 128 * freq; + cts = clock * 1000; + + /* Smallest valid fraction */ + div = gcd(n, cts); + + n /= div; + cts /= div; + + /* + * The optimal N is 128*freq/1000. Calculate the closest larger + * value that doesn't truncate any bits. + */ + mul = ((128*freq/1000) + (n-1))/n; + + n *= mul; + cts *= mul; + + /* Check that we are in spec (not always possible) */ + if (n < (128*freq/1500)) + pr_warn("Calculated ACR N value is too small. You may experience audio problems.\n"); + if (n > (128*freq/300)) + pr_warn("Calculated ACR N value is too large. You may experience audio problems.\n"); + + *N = n; + *CTS = cts; + + DRM_DEBUG("Calculated ACR timing N=%d CTS=%d for frequency %d\n", + *N, *CTS, freq); +} + +struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock) +{ + struct amdgpu_afmt_acr res; + u8 i; + + /* Precalculated values for common clocks */ + for (i = 0; i < ARRAY_SIZE(amdgpu_afmt_predefined_acr); i++) { + if (amdgpu_afmt_predefined_acr[i].clock == clock) + return amdgpu_afmt_predefined_acr[i]; + } + + /* And odd clocks get manually calculated */ + amdgpu_afmt_calc_cts(clock, &res.cts_32khz, &res.n_32khz, 32000); + amdgpu_afmt_calc_cts(clock, &res.cts_44_1khz, &res.n_44_1khz, 44100); + amdgpu_afmt_calc_cts(clock, &res.cts_48khz, &res.n_48khz, 48000); + + return res; +}
diff --git a/amdgpu/amdgpu_amdkfd.c b/amdgpu/amdgpu_amdkfd.c new file mode 100644 index 0000000..9fa4f25 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd.c
@@ -0,0 +1,769 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "amdgpu_amdkfd.h" +#include "amd_shared.h" + +#include "amdgpu.h" +#include "amdgpu_gfx.h" +#include "amdgpu_dma_buf.h" +#include <linux/module.h> +#include <linux/dma-buf.h> +#include "amdgpu_xgmi.h" + +static const unsigned int compute_vmid_bitmap = 0xFF00; + +/* Total memory size in system memory and all GPU VRAM. Used to + * estimate worst case amount of memory to reserve for page tables + */ +uint64_t amdgpu_amdkfd_total_mem_size; + +int amdgpu_amdkfd_init(void) +{ + struct sysinfo si; + int ret; + + si_meminfo(&si); + amdgpu_amdkfd_total_mem_size = si.totalram - si.totalhigh; + amdgpu_amdkfd_total_mem_size *= si.mem_unit; + +#ifdef CONFIG_HSA_AMD + ret = kgd2kfd_init(); + amdgpu_amdkfd_gpuvm_init_mem_limits(); +#else + ret = -ENOENT; +#endif + + return ret; +} + +void amdgpu_amdkfd_fini(void) +{ + kgd2kfd_exit(); +} + +void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) +{ + const struct kfd2kgd_calls *kfd2kgd; + + switch (adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_KAVERI: + case CHIP_HAWAII: + kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions(); + break; +#endif + case CHIP_CARRIZO: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); + break; + case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: + case CHIP_RAVEN: + kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions(); + break; + case CHIP_NAVI10: + kfd2kgd = amdgpu_amdkfd_gfx_10_0_get_functions(); + break; + default: + dev_info(adev->dev, "kfd not supported on this ASIC\n"); + return; + } + + adev->kfd.dev = kgd2kfd_probe((struct kgd_dev *)adev, + adev->pdev, kfd2kgd); + + if (adev->kfd.dev) + amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; +} + +/** + * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to + * setup amdkfd + * + * @adev: amdgpu_device pointer + * @aperture_base: output returning doorbell aperture base physical address + * @aperture_size: output returning doorbell aperture size in bytes + * @start_offset: output returning # of doorbell bytes reserved for amdgpu. + * + * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up, + * takes doorbells required for its own rings and reports the setup to amdkfd. + * amdgpu reserved doorbells are at the start of the doorbell aperture. + */ +static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, + phys_addr_t *aperture_base, + size_t *aperture_size, + size_t *start_offset) +{ + /* + * The first num_doorbells are used by amdgpu. + * amdkfd takes whatever's left in the aperture. + */ + if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) { + *aperture_base = adev->doorbell.base; + *aperture_size = adev->doorbell.size; + *start_offset = adev->doorbell.num_doorbells * sizeof(u32); + } else { + *aperture_base = 0; + *aperture_size = 0; + *start_offset = 0; + } +} + +void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) +{ + int i; + int last_valid_bit; + + if (adev->kfd.dev) { + struct kgd2kfd_shared_resources gpu_resources = { + .compute_vmid_bitmap = compute_vmid_bitmap, + .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec, + .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe, + .gpuvm_size = min(adev->vm_manager.max_pfn + << AMDGPU_GPU_PAGE_SHIFT, + AMDGPU_GMC_HOLE_START), + .drm_render_minor = adev->ddev->render->index, + .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, + + }; + + /* this is going to have a few of the MSBs set that we need to + * clear + */ + bitmap_complement(gpu_resources.queue_bitmap, + adev->gfx.mec.queue_bitmap, + KGD_MAX_QUEUES); + + /* remove the KIQ bit as well */ + if (adev->gfx.kiq.ring.sched.ready) + clear_bit(amdgpu_gfx_mec_queue_to_bit(adev, + adev->gfx.kiq.ring.me - 1, + adev->gfx.kiq.ring.pipe, + adev->gfx.kiq.ring.queue), + gpu_resources.queue_bitmap); + + /* According to linux/bitmap.h we shouldn't use bitmap_clear if + * nbits is not compile time constant + */ + last_valid_bit = 1 /* only first MEC can have compute queues */ + * adev->gfx.mec.num_pipe_per_mec + * adev->gfx.mec.num_queue_per_pipe; + for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i) + clear_bit(i, gpu_resources.queue_bitmap); + + amdgpu_doorbell_get_kfd_info(adev, + &gpu_resources.doorbell_physical_address, + &gpu_resources.doorbell_aperture_size, + &gpu_resources.doorbell_start_offset); + + /* Since SOC15, BIF starts to statically use the + * lower 12 bits of doorbell addresses for routing + * based on settings in registers like + * SDMA0_DOORBELL_RANGE etc.. + * In order to route a doorbell to CP engine, the lower + * 12 bits of its address has to be outside the range + * set for SDMA, VCN, and IH blocks. + */ + if (adev->asic_type >= CHIP_VEGA10) { + gpu_resources.non_cp_doorbells_start = + adev->doorbell_index.first_non_cp; + gpu_resources.non_cp_doorbells_end = + adev->doorbell_index.last_non_cp; + } + + kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); + } +} + +void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) +{ + if (adev->kfd.dev) { + kgd2kfd_device_exit(adev->kfd.dev); + adev->kfd.dev = NULL; + } +} + +void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, + const void *ih_ring_entry) +{ + if (adev->kfd.dev) + kgd2kfd_interrupt(adev->kfd.dev, ih_ring_entry); +} + +void amdgpu_amdkfd_suspend(struct amdgpu_device *adev) +{ + if (adev->kfd.dev) + kgd2kfd_suspend(adev->kfd.dev); +} + +int amdgpu_amdkfd_resume(struct amdgpu_device *adev) +{ + int r = 0; + + if (adev->kfd.dev) + r = kgd2kfd_resume(adev->kfd.dev); + + return r; +} + +int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev) +{ + int r = 0; + + if (adev->kfd.dev) + r = kgd2kfd_pre_reset(adev->kfd.dev); + + return r; +} + +int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev) +{ + int r = 0; + + if (adev->kfd.dev) + r = kgd2kfd_post_reset(adev->kfd.dev); + + return r; +} + +void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + if (amdgpu_device_should_recover_gpu(adev)) + amdgpu_device_gpu_recover(adev, NULL); +} + +int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr, bool mqd_gfx9) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_bo *bo = NULL; + struct amdgpu_bo_param bp; + int r; + void *cpu_ptr_tmp = NULL; + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_GTT; + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + + if (mqd_gfx9) + bp.flags |= AMDGPU_GEM_CREATE_MQD_GFX9; + + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) { + dev_err(adev->dev, + "failed to allocate BO for amdkfd (%d)\n", r); + return r; + } + + /* map the buffer */ + r = amdgpu_bo_reserve(bo, true); + if (r) { + dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); + goto allocate_mem_reserve_bo_failed; + } + + r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); + if (r) { + dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); + goto allocate_mem_pin_bo_failed; + } + + r = amdgpu_ttm_alloc_gart(&bo->tbo); + if (r) { + dev_err(adev->dev, "%p bind failed\n", bo); + goto allocate_mem_kmap_bo_failed; + } + + r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp); + if (r) { + dev_err(adev->dev, + "(%d) failed to map bo to kernel for amdkfd\n", r); + goto allocate_mem_kmap_bo_failed; + } + + *mem_obj = bo; + *gpu_addr = amdgpu_bo_gpu_offset(bo); + *cpu_ptr = cpu_ptr_tmp; + + amdgpu_bo_unreserve(bo); + + return 0; + +allocate_mem_kmap_bo_failed: + amdgpu_bo_unpin(bo); +allocate_mem_pin_bo_failed: + amdgpu_bo_unreserve(bo); +allocate_mem_reserve_bo_failed: + amdgpu_bo_unref(&bo); + + return r; +} + +void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) +{ + struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj; + + amdgpu_bo_reserve(bo, true); + amdgpu_bo_kunmap(bo); + amdgpu_bo_unpin(bo); + amdgpu_bo_unreserve(bo); + amdgpu_bo_unref(&(bo)); +} + +int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, + void **mem_obj) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_bo *bo = NULL; + struct amdgpu_bo_param bp; + int r; + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = 1; + bp.domain = AMDGPU_GEM_DOMAIN_GWS; + bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + bp.type = ttm_bo_type_device; + bp.resv = NULL; + + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) { + dev_err(adev->dev, + "failed to allocate gws BO for amdkfd (%d)\n", r); + return r; + } + + *mem_obj = bo; + return 0; +} + +void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj) +{ + struct amdgpu_bo *bo = (struct amdgpu_bo *)mem_obj; + + amdgpu_bo_unref(&bo); +} + +uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd, + enum kgd_engine_type type) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + switch (type) { + case KGD_ENGINE_PFP: + return adev->gfx.pfp_fw_version; + + case KGD_ENGINE_ME: + return adev->gfx.me_fw_version; + + case KGD_ENGINE_CE: + return adev->gfx.ce_fw_version; + + case KGD_ENGINE_MEC1: + return adev->gfx.mec_fw_version; + + case KGD_ENGINE_MEC2: + return adev->gfx.mec2_fw_version; + + case KGD_ENGINE_RLC: + return adev->gfx.rlc_fw_version; + + case KGD_ENGINE_SDMA1: + return adev->sdma.instance[0].fw_version; + + case KGD_ENGINE_SDMA2: + return adev->sdma.instance[1].fw_version; + + default: + return 0; + } + + return 0; +} + +void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd, + struct kfd_local_mem_info *mem_info) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask : + ~((1ULL << 32) - 1); + resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size; + + memset(mem_info, 0, sizeof(*mem_info)); + if (!(adev->gmc.aper_base & address_mask || aper_limit & address_mask)) { + mem_info->local_mem_size_public = adev->gmc.visible_vram_size; + mem_info->local_mem_size_private = adev->gmc.real_vram_size - + adev->gmc.visible_vram_size; + } else { + mem_info->local_mem_size_public = 0; + mem_info->local_mem_size_private = adev->gmc.real_vram_size; + } + mem_info->vram_width = adev->gmc.vram_width; + + pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n", + &adev->gmc.aper_base, &aper_limit, + mem_info->local_mem_size_public, + mem_info->local_mem_size_private); + + if (amdgpu_sriov_vf(adev)) + mem_info->mem_clk_max = adev->clock.default_mclk / 100; + else if (adev->powerplay.pp_funcs) { + if (amdgpu_emu_mode == 1) + mem_info->mem_clk_max = 0; + else + mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100; + } else + mem_info->mem_clk_max = 100; +} + +uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + if (adev->gfx.funcs->get_gpu_clock_counter) + return adev->gfx.funcs->get_gpu_clock_counter(adev); + return 0; +} + +uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + /* the sclk is in quantas of 10kHz */ + if (amdgpu_sriov_vf(adev)) + return adev->clock.default_sclk / 100; + else if (adev->powerplay.pp_funcs) + return amdgpu_dpm_get_sclk(adev, false) / 100; + else + return 100; +} + +void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_cu_info acu_info = adev->gfx.cu_info; + + memset(cu_info, 0, sizeof(*cu_info)); + if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap)) + return; + + cu_info->cu_active_number = acu_info.number; + cu_info->cu_ao_mask = acu_info.ao_cu_mask; + memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0], + sizeof(acu_info.bitmap)); + cu_info->num_shader_engines = adev->gfx.config.max_shader_engines; + cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; + cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh; + cu_info->simd_per_cu = acu_info.simd_per_cu; + cu_info->max_waves_per_simd = acu_info.max_waves_per_simd; + cu_info->wave_front_size = acu_info.wave_front_size; + cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu; + cu_info->lds_size = acu_info.lds_size; +} + +int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd, + struct kgd_dev **dma_buf_kgd, + uint64_t *bo_size, void *metadata_buffer, + size_t buffer_size, uint32_t *metadata_size, + uint32_t *flags) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct dma_buf *dma_buf; + struct drm_gem_object *obj; + struct amdgpu_bo *bo; + uint64_t metadata_flags; + int r = -EINVAL; + + dma_buf = dma_buf_get(dma_buf_fd); + if (IS_ERR(dma_buf)) + return PTR_ERR(dma_buf); + + if (dma_buf->ops != &amdgpu_dmabuf_ops) + /* Can't handle non-graphics buffers */ + goto out_put; + + obj = dma_buf->priv; + if (obj->dev->driver != adev->ddev->driver) + /* Can't handle buffers from different drivers */ + goto out_put; + + adev = obj->dev->dev_private; + bo = gem_to_amdgpu_bo(obj); + if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT))) + /* Only VRAM and GTT BOs are supported */ + goto out_put; + + r = 0; + if (dma_buf_kgd) + *dma_buf_kgd = (struct kgd_dev *)adev; + if (bo_size) + *bo_size = amdgpu_bo_size(bo); + if (metadata_size) + *metadata_size = bo->metadata_size; + if (metadata_buffer) + r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size, + metadata_size, &metadata_flags); + if (flags) { + *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? + ALLOC_MEM_FLAGS_VRAM : ALLOC_MEM_FLAGS_GTT; + + if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) + *flags |= ALLOC_MEM_FLAGS_PUBLIC; + } + +out_put: + dma_buf_put(dma_buf); + return r; +} + +uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); +} + +uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return adev->gmc.xgmi.hive_id; +} +uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src) +{ + struct amdgpu_device *peer_adev = (struct amdgpu_device *)src; + struct amdgpu_device *adev = (struct amdgpu_device *)dst; + int ret = amdgpu_xgmi_get_hops_count(adev, peer_adev); + + if (ret < 0) { + DRM_ERROR("amdgpu: failed to get xgmi hops count between node %d and %d. ret = %d\n", + adev->gmc.xgmi.physical_node_id, + peer_adev->gmc.xgmi.physical_node_id, ret); + ret = 0; + } + return (uint8_t)ret; +} + +uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return adev->rmmio_remap.bus_addr; +} + +uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return adev->gds.gws_size; +} + +int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, + uint32_t vmid, uint64_t gpu_addr, + uint32_t *ib_cmd, uint32_t ib_len) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct amdgpu_ring *ring; + struct dma_fence *f = NULL; + int ret; + + switch (engine) { + case KGD_ENGINE_MEC1: + ring = &adev->gfx.compute_ring[0]; + break; + case KGD_ENGINE_SDMA1: + ring = &adev->sdma.instance[0].ring; + break; + case KGD_ENGINE_SDMA2: + ring = &adev->sdma.instance[1].ring; + break; + default: + pr_err("Invalid engine in IB submission: %d\n", engine); + ret = -EINVAL; + goto err; + } + + ret = amdgpu_job_alloc(adev, 1, &job, NULL); + if (ret) + goto err; + + ib = &job->ibs[0]; + memset(ib, 0, sizeof(struct amdgpu_ib)); + + ib->gpu_addr = gpu_addr; + ib->ptr = ib_cmd; + ib->length_dw = ib_len; + /* This works for NO_HWS. TODO: need to handle without knowing VMID */ + job->vmid = vmid; + + ret = amdgpu_ib_schedule(ring, 1, ib, job, &f); + if (ret) { + DRM_ERROR("amdgpu: failed to schedule IB.\n"); + goto err_ib_sched; + } + + ret = dma_fence_wait(f, false); + +err_ib_sched: + dma_fence_put(f); + amdgpu_job_free(job); +err: + return ret; +} + +void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + if (adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->switch_power_profile) + amdgpu_dpm_switch_power_profile(adev, + PP_SMC_POWER_PROFILE_COMPUTE, + !idle); +} + +bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid) +{ + if (adev->kfd.dev) { + if ((1 << vmid) & compute_vmid_bitmap) + return true; + } + + return false; +} + +bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return adev->have_atomics_support; +} + +#ifndef CONFIG_HSA_AMD +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) +{ + return false; +} + +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) +{ +} + +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ +} + +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) +{ + return NULL; +} + +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) +{ + return 0; +} + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void) +{ + return NULL; +} + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void) +{ + return NULL; +} + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void) +{ + return NULL; +} + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions(void) +{ + return NULL; +} + +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + const struct kfd2kgd_calls *f2g) +{ + return NULL; +} + +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources) +{ + return false; +} + +void kgd2kfd_device_exit(struct kfd_dev *kfd) +{ +} + +void kgd2kfd_exit(void) +{ +} + +void kgd2kfd_suspend(struct kfd_dev *kfd) +{ +} + +int kgd2kfd_resume(struct kfd_dev *kfd) +{ + return 0; +} + +int kgd2kfd_pre_reset(struct kfd_dev *kfd) +{ + return 0; +} + +int kgd2kfd_post_reset(struct kfd_dev *kfd) +{ + return 0; +} + +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) +{ +} + +void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) +{ +} +#endif
diff --git a/amdgpu/amdgpu_amdkfd.h b/amdgpu/amdgpu_amdkfd.h new file mode 100644 index 0000000..b6076d1 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd.h
@@ -0,0 +1,257 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */ + +#ifndef AMDGPU_AMDKFD_H_INCLUDED +#define AMDGPU_AMDKFD_H_INCLUDED + +#include <linux/types.h> +#include <linux/mm.h> +#include <linux/workqueue.h> +#include <kgd_kfd_interface.h> +#include <drm/ttm/ttm_execbuf_util.h> +#include "amdgpu_sync.h" +#include "amdgpu_vm.h" + +extern uint64_t amdgpu_amdkfd_total_mem_size; + +struct amdgpu_device; + +struct kfd_bo_va_list { + struct list_head bo_list; + struct amdgpu_bo_va *bo_va; + void *kgd_dev; + bool is_mapped; + uint64_t va; + uint64_t pte_flags; +}; + +struct kgd_mem { + struct mutex lock; + struct amdgpu_bo *bo; + struct list_head bo_va_list; + /* protected by amdkfd_process_info.lock */ + struct ttm_validate_buffer validate_list; + struct ttm_validate_buffer resv_list; + uint32_t domain; + unsigned int mapped_to_gpu_memory; + uint64_t va; + + uint32_t mapping_flags; + + atomic_t invalid; + struct amdkfd_process_info *process_info; + + struct amdgpu_sync sync; + + bool aql_queue; +}; + +/* KFD Memory Eviction */ +struct amdgpu_amdkfd_fence { + struct dma_fence base; + struct mm_struct *mm; + spinlock_t lock; + char timeline_name[TASK_COMM_LEN]; +}; + +struct amdgpu_kfd_dev { + struct kfd_dev *dev; + uint64_t vram_used; +}; + +enum kgd_engine_type { + KGD_ENGINE_PFP = 1, + KGD_ENGINE_ME, + KGD_ENGINE_CE, + KGD_ENGINE_MEC1, + KGD_ENGINE_MEC2, + KGD_ENGINE_RLC, + KGD_ENGINE_SDMA1, + KGD_ENGINE_SDMA2, + KGD_ENGINE_MAX +}; + +struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, + struct mm_struct *mm); +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); + +struct amdkfd_process_info { + /* List head of all VMs that belong to a KFD process */ + struct list_head vm_list_head; + /* List head for all KFD BOs that belong to a KFD process. */ + struct list_head kfd_bo_list; + /* List of userptr BOs that are valid or invalid */ + struct list_head userptr_valid_list; + struct list_head userptr_inval_list; + /* Lock to protect kfd_bo_list */ + struct mutex lock; + + /* Number of VMs */ + unsigned int n_vms; + /* Eviction Fence */ + struct amdgpu_amdkfd_fence *eviction_fence; + + /* MMU-notifier related fields */ + atomic_t evicted_bos; + struct delayed_work restore_userptr_work; + struct pid *pid; +}; + +int amdgpu_amdkfd_init(void); +void amdgpu_amdkfd_fini(void); + +void amdgpu_amdkfd_suspend(struct amdgpu_device *adev); +int amdgpu_amdkfd_resume(struct amdgpu_device *adev); +void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, + const void *ih_ring_entry); +void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev); +void amdgpu_amdkfd_device_init(struct amdgpu_device *adev); +void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev); + +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); +int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, + uint32_t vmid, uint64_t gpu_addr, + uint32_t *ib_cmd, uint32_t ib_len); +void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle); +bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd); + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void); +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void); +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void); +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions(void); + +bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid); + +int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev); + +int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev); + +void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd); + +/* Shared API */ +int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr, bool mqd_gfx9); +void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj); +int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, void **mem_obj); +void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj); +int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem); +int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem); +uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd, + enum kgd_engine_type type); +void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd, + struct kfd_local_mem_info *mem_info); +uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd); + +uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd); +void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info); +int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd, + struct kgd_dev **dmabuf_kgd, + uint64_t *bo_size, void *metadata_buffer, + size_t buffer_size, uint32_t *metadata_size, + uint32_t *flags); +uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd); +uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd); +uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd); +uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd); +uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src); + +#define read_user_wptr(mmptr, wptr, dst) \ + ({ \ + bool valid = false; \ + if ((mmptr) && (wptr)) { \ + if ((mmptr) == current->mm) { \ + valid = !get_user((dst), (wptr)); \ + } else if (current->mm == NULL) { \ + use_mm(mmptr); \ + valid = !get_user((dst), (wptr)); \ + unuse_mm(mmptr); \ + } \ + } \ + valid; \ + }) + +/* GPUVM API */ +int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid, + void **vm, void **process_info, + struct dma_fence **ef); +int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, + struct file *filp, unsigned int pasid, + void **vm, void **process_info, + struct dma_fence **ef); +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm); +void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm); +uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm); +int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( + struct kgd_dev *kgd, uint64_t va, uint64_t size, + void *vm, struct kgd_mem **mem, + uint64_t *offset, uint32_t flags); +int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem); +int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem, void *vm); +int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem, void *vm); +int amdgpu_amdkfd_gpuvm_sync_memory( + struct kgd_dev *kgd, struct kgd_mem *mem, bool intr); +int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, + struct kgd_mem *mem, void **kptr, uint64_t *size); +int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info, + struct dma_fence **ef); + +int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, + struct kfd_vm_fault_info *info); + +int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, + struct dma_buf *dmabuf, + uint64_t va, void *vm, + struct kgd_mem **mem, uint64_t *size, + uint64_t *mmap_offset); + +void amdgpu_amdkfd_gpuvm_init_mem_limits(void); +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); + +/* KGD2KFD callbacks */ +int kgd2kfd_init(void); +void kgd2kfd_exit(void); +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + const struct kfd2kgd_calls *f2g); +bool kgd2kfd_device_init(struct kfd_dev *kfd, + const struct kgd2kfd_shared_resources *gpu_resources); +void kgd2kfd_device_exit(struct kfd_dev *kfd); +void kgd2kfd_suspend(struct kfd_dev *kfd); +int kgd2kfd_resume(struct kfd_dev *kfd); +int kgd2kfd_pre_reset(struct kfd_dev *kfd); +int kgd2kfd_post_reset(struct kfd_dev *kfd); +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); +int kgd2kfd_quiesce_mm(struct mm_struct *mm); +int kgd2kfd_resume_mm(struct mm_struct *mm); +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, + struct dma_fence *fence); +void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd); + +#endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/amdgpu/amdgpu_amdkfd_fence.c b/amdgpu/amdgpu_amdkfd_fence.c new file mode 100644 index 0000000..3107b95 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_fence.c
@@ -0,0 +1,177 @@ +/* + * Copyright 2016-2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/dma-fence.h> +#include <linux/spinlock.h> +#include <linux/atomic.h> +#include <linux/stacktrace.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/sched/mm.h> +#include "amdgpu_amdkfd.h" + +static const struct dma_fence_ops amdkfd_fence_ops; +static atomic_t fence_seq = ATOMIC_INIT(0); + +/* Eviction Fence + * Fence helper functions to deal with KFD memory eviction. + * Big Idea - Since KFD submissions are done by user queues, a BO cannot be + * evicted unless all the user queues for that process are evicted. + * + * All the BOs in a process share an eviction fence. When process X wants + * to map VRAM memory but TTM can't find enough space, TTM will attempt to + * evict BOs from its LRU list. TTM checks if the BO is valuable to evict + * by calling ttm_bo_driver->eviction_valuable(). + * + * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs + * to process X. Otherwise, it will return true to indicate BO can be + * evicted by TTM. + * + * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue + * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move + * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler. + * + * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to + * nofity when the BO is free to move. fence_add_callback --> enable_signaling + * --> amdgpu_amdkfd_fence.enable_signaling + * + * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce + * user queues and signal fence. The work item will also start another delayed + * work item to restore BOs + */ + +struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, + struct mm_struct *mm) +{ + struct amdgpu_amdkfd_fence *fence; + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (fence == NULL) + return NULL; + + /* This reference gets released in amdkfd_fence_release */ + mmgrab(mm); + fence->mm = mm; + get_task_comm(fence->timeline_name, current); + spin_lock_init(&fence->lock); + + dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock, + context, atomic_inc_return(&fence_seq)); + + return fence; +} + +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) +{ + struct amdgpu_amdkfd_fence *fence; + + if (!f) + return NULL; + + fence = container_of(f, struct amdgpu_amdkfd_fence, base); + if (fence && f->ops == &amdkfd_fence_ops) + return fence; + + return NULL; +} + +static const char *amdkfd_fence_get_driver_name(struct dma_fence *f) +{ + return "amdgpu_amdkfd_fence"; +} + +static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f) +{ + struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); + + return fence->timeline_name; +} + +/** + * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict + * a KFD BO and schedules a job to move the BO. + * If fence is already signaled return true. + * If fence is not signaled schedule a evict KFD process work item. + */ +static bool amdkfd_fence_enable_signaling(struct dma_fence *f) +{ + struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); + + if (!fence) + return false; + + if (dma_fence_is_signaled(f)) + return true; + + if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f)) + return true; + + return false; +} + +/** + * amdkfd_fence_release - callback that fence can be freed + * + * @fence: fence + * + * This function is called when the reference count becomes zero. + * Drops the mm_struct reference and RCU schedules freeing up the fence. + */ +static void amdkfd_fence_release(struct dma_fence *f) +{ + struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); + + /* Unconditionally signal the fence. The process is getting + * terminated. + */ + if (WARN_ON(!fence)) + return; /* Not an amdgpu_amdkfd_fence */ + + mmdrop(fence->mm); + kfree_rcu(f, rcu); +} + +/** + * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f + * if same return TRUE else return FALSE. + * + * @f: [IN] fence + * @mm: [IN] mm that needs to be verified + */ +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) +{ + struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); + + if (!fence) + return false; + else if (fence->mm == mm) + return true; + + return false; +} + +static const struct dma_fence_ops amdkfd_fence_ops = { + .get_driver_name = amdkfd_fence_get_driver_name, + .get_timeline_name = amdkfd_fence_get_timeline_name, + .enable_signaling = amdkfd_fence_enable_signaling, + .release = amdkfd_fence_release, +};
diff --git a/amdgpu/amdgpu_amdkfd_gfx_v10.c b/amdgpu/amdgpu_amdkfd_gfx_v10.c new file mode 100644 index 0000000..0723f80 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -0,0 +1,975 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#undef pr_fmt +#define pr_fmt(fmt) "kfd2kgd: " fmt + +#include <linux/module.h> +#include <linux/fdtable.h> +#include <linux/uaccess.h> +#include <linux/firmware.h> +#include <linux/mmu_context.h> +#include <drm/drmP.h> +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_ucode.h" +#include "soc15_hw_ip.h" +#include "gc/gc_10_1_0_offset.h" +#include "gc/gc_10_1_0_sh_mask.h" +#include "navi10_enum.h" +#include "athub/athub_2_0_0_offset.h" +#include "athub/athub_2_0_0_sh_mask.h" +#include "oss/osssys_5_0_0_offset.h" +#include "oss/osssys_5_0_0_sh_mask.h" +#include "soc15_common.h" +#include "v10_structs.h" +#include "nv.h" +#include "nvd.h" + +enum hqd_dequeue_request_type { + NO_ACTION = 0, + DRAIN_PIPE, + RESET_WAVES, + SAVE_WAVES +}; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases); +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm); +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm); +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout); +#if 0 +static uint32_t get_watch_base_addr(struct amdgpu_device *adev); +#endif +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base); +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); + +/* Because of REG_GET_FIELD() being used, we put this function in the + * asic specific file. + */ +static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd, + struct tile_config *config) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + config->gb_addr_config = adev->gfx.config.gb_addr_config; +#if 0 +/* TODO - confirm REG_GET_FIELD x2, should be OK as is... but + * MC_ARB_RAMCFG register doesn't exist on Vega10 - initial amdgpu + * changes commented out related code, doing the same here for now but + * need to sync with Ken et al + */ + config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFBANK); + config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFRANKS); +#endif + + config->tile_config_ptr = adev->gfx.config.tile_mode_array; + config->num_tile_configs = + ARRAY_SIZE(adev->gfx.config.tile_mode_array); + config->macro_tile_config_ptr = + adev->gfx.config.macrotile_mode_array; + config->num_macro_tile_configs = + ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); + + return 0; +} + +static const struct kfd2kgd_calls kfd2kgd = { + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_dump = kgd_hqd_dump, + .hqd_sdma_dump = kgd_hqd_sdma_dump, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = + get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = + get_atc_vmid_pasid_mapping_valid, + .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, + .set_vm_context_page_table_base = set_vm_context_page_table_base, + .get_tile_config = amdgpu_amdkfd_get_tile_config, +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions() +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + mutex_lock(&adev->srbm_mutex); + nv_grbm_select(adev, mec, pipe, queue, vmid); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + nv_grbm_select(adev, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static uint32_t get_queue_mask(struct amdgpu_device *adev, + uint32_t pipe_id, uint32_t queue_id) +{ + unsigned int bit = (pipe_id * adev->gfx.mec.num_queue_per_pipe + + queue_id) & 31; + + return ((uint32_t)1) << bit; +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases); + /* APE1 no longer exists on GFX9 */ + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished + * and the SW cleared it. + * So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + pr_debug("pasid 0x%x vmid %d, reg value %x\n", pasid, vmid, pasid_mapping); + /* + * need to do this twice, once for gfx and once for mmhub + * for ATC add 16 to VMID for mmhub, for IH different registers. + * ATC_VMID0..15 registers are separate from ATC_VMID16..31. + */ + + pr_debug("ATHUB, reg %x\n", SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid); + WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid, + pasid_mapping); + +#if 0 + /* TODO: uncomment this code when the hardware support is ready. */ + while (!(RREG32(SOC15_REG_OFFSET( + ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & + (1U << vmid))) + cpu_relax(); + + pr_debug("ATHUB mapping update finished\n"); + WREG32(SOC15_REG_OFFSET(ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), + 1U << vmid); +#endif + + /* Mapping vmid to pasid also for IH block */ + pr_debug("update mapping for IH block and mmhub"); + WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid, + pasid_mapping); + + return 0; +} + +/* TODO - RING0 form of field is obsolete, seems to date back to SI + * but still works + */ + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), + CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | + CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static uint32_t get_sdma_base_addr(struct amdgpu_device *adev, + unsigned int engine_id, + unsigned int queue_id) +{ + uint32_t base[2] = { + SOC15_REG_OFFSET(SDMA0, 0, + mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL, + /* On gfx10, mmSDMA1_xxx registers are defined NOT based + * on SDMA1 base address (dw 0x1860) but based on SDMA0 + * base address (dw 0x1260). Therefore use mmSDMA0_RLC0_RB_CNTL + * instead of mmSDMA1_RLC0_RB_CNTL for the base address calc + * below + */ + SOC15_REG_OFFSET(SDMA1, 0, + mmSDMA1_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL + }; + uint32_t retval; + + retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL - + mmSDMA0_RLC0_RB_CNTL); + + pr_debug("sdma base address: 0x%x\n", retval); + + return retval; +} + +#if 0 +static uint32_t get_watch_base_addr(struct amdgpu_device *adev) +{ + uint32_t retval = SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) - + mmTCP_WATCH0_ADDR_H; + + pr_debug("kfd: reg watch base address: 0x%x\n", retval); + + return retval; +} +#endif + +static inline struct v10_compute_mqd *get_mqd(void *mqd) +{ + return (struct v10_compute_mqd *)mqd; +} + +static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd) +{ + return (struct v10_sdma_mqd *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v10_compute_mqd *m; + uint32_t *mqd_hqd; + uint32_t reg, hqd_base, data; + + m = get_mqd(mqd); + + pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id); + acquire_queue(kgd, pipe_id, queue_id); + + /* HIQ is set during driver init period with vmid set to 0*/ + if (m->cp_hqd_vmid == 0) { + uint32_t value, mec, pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", + mec, pipe, queue_id); + value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS)); + value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1, + ((mec << 5) | (pipe << 3) | queue_id | 0x80)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value); + } + + /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ + mqd_hqd = &m->cp_mqd_base_addr_lo; + hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); + + for (reg = hqd_base; + reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) + WREG32(reg, mqd_hqd[reg - hqd_base]); + + + /* Activate doorbell logic before triggering WPTR poll. */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data); + + if (wptr) { + /* Don't read wptr with get_user because the user + * context may not be accessible (if this function + * runs in a work queue). Instead trigger a one-shot + * polling read from memory in the CP. This assumes + * that wptr is GPU-accessible in the queue's VMID via + * ATC or SVM. WPTR==RPTR before starting the poll so + * the CP starts fetching new commands from the right + * place. + * + * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit + * tricky. Assume that the queue didn't overflow. The + * number of valid bits in the 32-bit RPTR depends on + * the queue size. The remaining bits are taken from + * the saved 64-bit WPTR. If the WPTR wrapped, add the + * queue size. + */ + uint32_t queue_size = + 2 << REG_GET_FIELD(m->cp_hqd_pq_control, + CP_HQD_PQ_CONTROL, QUEUE_SIZE); + uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); + + if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) + guessed_wptr += queue_size; + guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); + guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; + + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), + lower_32_bits(guessed_wptr)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), + upper_32_bits(guessed_wptr)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), + lower_32_bits((uint64_t)wptr)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), + upper_32_bits((uint64_t)wptr)); + pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__, get_queue_mask(adev, pipe_id, queue_id)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1), + get_queue_mask(adev, pipe_id, queue_id)); + } + + /* Start the EOP fetcher */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR), + REG_SET_FIELD(m->cp_hqd_eop_rptr, + CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); + + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data); + + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t i = 0, reg; +#define HQD_N_REGS 56 +#define DUMP_REG(addr) do { \ + if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ + break; \ + (*dump)[i][0] = (addr) << 2; \ + (*dump)[i++][1] = RREG32(addr); \ + } while (0) + + *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + acquire_queue(kgd, pipe_id, queue_id); + + for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); + reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) + DUMP_REG(reg); + + release_queue(kgd); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v10_sdma_mqd *m; + uint32_t sdma_base_addr, sdmax_gfx_context_cntl; + unsigned long end_jiffies; + uint32_t data; + uint64_t data64; + uint64_t __user *wptr64 = (uint64_t __user *)wptr; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + pr_debug("sdma load base addr %x for engine %d, queue %d\n", sdma_base_addr, m->sdma_engine_id, m->sdma_queue_id); + sdmax_gfx_context_cntl = m->sdma_engine_id ? + SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) : + SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); + + end_jiffies = msecs_to_jiffies(2000) + jiffies; + while (true) { + data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + data = RREG32(sdmax_gfx_context_cntl); + data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(sdmax_gfx_context_cntl, data); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET, + m->sdmax_rlcx_doorbell_offset); + + data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI, + m->sdmax_rlcx_rb_rptr_hi); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1); + if (read_user_wptr(mm, wptr64, data64)) { + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + lower_32_bits(data64)); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI, + upper_32_bits(data64)); + } else { + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + m->sdmax_rlcx_rb_rptr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI, + m->sdmax_rlcx_rb_rptr_hi); + } + WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, + m->sdmax_rlcx_rb_base_hi); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdmax_rlcx_rb_rptr_addr_lo); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdmax_rlcx_rb_rptr_addr_hi); + + data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); + + return 0; +} + +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t sdma_base_addr = get_sdma_base_addr(adev, engine_id, queue_id); + uint32_t i = 0, reg; +#undef HQD_N_REGS +#define HQD_N_REGS (19+6+7+10) + + pr_debug("sdma dump engine id %d queue_id %d\n", engine_id, queue_id); + pr_debug("sdma base addr %x\n", sdma_base_addr); + + *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; + reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; + reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++) + DUMP_REG(sdma_base_addr + reg); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) && + high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI))) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v10_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + enum hqd_dequeue_request_type type; + unsigned long end_jiffies; + uint32_t temp; + struct v10_compute_mqd *m = get_mqd(mqd); + +#if 0 + unsigned long flags; + int retry; +#endif + + acquire_queue(kgd, pipe_id, queue_id); + + if (m->cp_hqd_vmid == 0) + WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); + + switch (reset_type) { + case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: + type = DRAIN_PIPE; + break; + case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: + type = RESET_WAVES; + break; + default: + type = DRAIN_PIPE; + break; + } + +#if 0 /* Is this still needed? */ + /* Workaround: If IQ timer is active and the wait time is close to or + * equal to 0, dequeueing is not safe. Wait until either the wait time + * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is + * cleared before continuing. Also, ensure wait times are set to at + * least 0x3. + */ + local_irq_save(flags); + preempt_disable(); + retry = 5000; /* wait for 500 usecs at maximum */ + while (true) { + temp = RREG32(mmCP_HQD_IQ_TIMER); + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) { + pr_debug("HW is processing IQ\n"); + goto loop; + } + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) { + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE) + == 3) /* SEM-rearm is safe */ + break; + /* Wait time 3 is safe for CP, but our MMIO read/write + * time is close to 1 microsecond, so check for 10 to + * leave more buffer room + */ + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME) + >= 10) + break; + pr_debug("IQ timer is active\n"); + } else + break; +loop: + if (!retry) { + pr_err("CP HQD IQ timer status time out\n"); + break; + } + ndelay(100); + --retry; + } + retry = 1000; + while (true) { + temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST); + if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK)) + break; + pr_debug("Dequeue request is pending\n"); + + if (!retry) { + pr_err("CP HQD dequeue request time out\n"); + break; + } + ndelay(100); + --retry; + } + local_irq_restore(flags); + preempt_enable(); +#endif + + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type); + + end_jiffies = (utimeout * HZ / 1000) + jiffies; + while (true) { + temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); + if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + if (time_after(jiffies, end_jiffies)) { + pr_err("cp queue preemption time out.\n"); + release_queue(kgd); + return -ETIME; + } + usleep_range(500, 1000); + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v10_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t temp; + unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) | + SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); + + m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR); + m->sdmax_rlcx_rb_rptr_hi = + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI); + + return 0; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + + vmid); + return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK; +} + +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + uint32_t req = (1 << vmid) | + (0 << GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT) |/* legacy */ + GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK | + GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK | + GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK | + GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK | + GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK; + + mutex_lock(&adev->srbm_mutex); + + /* Use light weight invalidation. + * + * TODO 1: agree on the right set of invalidation registers for + * KFD use. Use the last one for now. Invalidate only GCHUB as + * SDMA is now moved to GCHUB + * + * TODO 2: support range-based invalidation, requires kfg2kgd + * interface change + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32), + 0xffffffff); + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32), + 0x0000001f); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ), req); + + while (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK)) & + (1 << vmid))) + cpu_relax(); + + mutex_unlock(&adev->srbm_mutex); +} + +static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid) +{ + signed long r; + uint32_t seq; + struct amdgpu_ring *ring = &adev->gfx.kiq.ring; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/ + amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); + amdgpu_ring_write(ring, + PACKET3_INVALIDATE_TLBS_DST_SEL(1) | + PACKET3_INVALIDATE_TLBS_PASID(pasid)); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + + r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); + if (r < 1) { + DRM_ERROR("wait for kiq fence error: %ld.\n", r); + return -ETIME; + } + + return 0; +} + +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + int vmid; + struct amdgpu_ring *ring = &adev->gfx.kiq.ring; + + if (amdgpu_emu_mode == 0 && ring->sched.ready) + return invalidate_tlbs_with_kiq(adev, pasid); + + for (vmid = 0; vmid < 16; vmid++) { + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) + continue; + if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) { + if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid) + == pasid) { + write_vmid_invalidate_request(kgd, vmid); + break; + } + } + } + + return 0; +} + +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid %d\n", vmid); + return 0; + } + + write_vmid_invalidate_request(kgd, vmid); + return 0; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data = 0; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd); + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SA_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return 0; +} + +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint64_t base = page_table_base | AMDGPU_PTE_VALID; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("trying to set page table base for wrong VMID %u\n", + vmid); + return; + } + + /* TODO: take advantage of per-process address space size. For + * now, all processes share the same address space size, like + * on GFX8 and older. + */ + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0); + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2), + lower_32_bits(adev->vm_manager.max_pfn - 1)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2), + upper_32_bits(adev->vm_manager.max_pfn - 1)); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base)); +}
diff --git a/amdgpu/amdgpu_amdkfd_gfx_v7.c b/amdgpu/amdgpu_amdkfd_gfx_v7.c new file mode 100644 index 0000000..5f459bf --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -0,0 +1,857 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/fdtable.h> +#include <linux/uaccess.h> +#include <linux/mmu_context.h> + +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "cikd.h" +#include "cik_sdma.h" +#include "gfx_v7_0.h" +#include "gca/gfx_7_2_d.h" +#include "gca/gfx_7_2_enum.h" +#include "gca/gfx_7_2_sh_mask.h" +#include "oss/oss_2_0_d.h" +#include "oss/oss_2_0_sh_mask.h" +#include "gmc/gmc_7_1_d.h" +#include "gmc/gmc_7_1_sh_mask.h" +#include "cik_structs.h" + +enum hqd_dequeue_request_type { + NO_ACTION = 0, + DRAIN_PIPE, + RESET_WAVES +}; + +enum { + MAX_TRAPID = 8, /* 3 bits in the bitfield. */ + MAX_WATCH_ADDRESSES = 4 +}; + +enum { + ADDRESS_WATCH_REG_ADDR_HI = 0, + ADDRESS_WATCH_REG_ADDR_LO, + ADDRESS_WATCH_REG_CNTL, + ADDRESS_WATCH_REG_MAX +}; + +/* not defined in the CI/KV reg file */ +enum { + ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL, + ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF, + ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000, + /* extend the mask to 26 bits to match the low address field */ + ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6, + ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF +}; + +static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = { + mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL, + mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL, + mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL, + mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL +}; + +union TCP_WATCH_CNTL_BITS { + struct { + uint32_t mask:24; + uint32_t vmid:4; + uint32_t atc:1; + uint32_t mode:2; + uint32_t valid:1; + } bitfields, bits; + uint32_t u32All; + signed int i32All; + float f32All; +}; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm); +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm); +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); + +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); + +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid); +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base); +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); +static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd); + +/* Because of REG_GET_FIELD() being used, we put this function in the + * asic specific file. + */ +static int get_tile_config(struct kgd_dev *kgd, + struct tile_config *config) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + config->gb_addr_config = adev->gfx.config.gb_addr_config; + config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFBANK); + config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFRANKS); + + config->tile_config_ptr = adev->gfx.config.tile_mode_array; + config->num_tile_configs = + ARRAY_SIZE(adev->gfx.config.tile_mode_array); + config->macro_tile_config_ptr = + adev->gfx.config.macrotile_mode_array; + config->num_macro_tile_configs = + ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); + + return 0; +} + +static const struct kfd2kgd_calls kfd2kgd = { + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_dump = kgd_hqd_dump, + .hqd_sdma_dump = kgd_hqd_sdma_dump, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, + .set_scratch_backing_va = set_scratch_backing_va, + .get_tile_config = get_tile_config, + .set_vm_context_page_table_base = set_vm_context_page_table_base, + .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, + .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg, +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void) +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); + + mutex_lock(&adev->srbm_mutex); + WREG32(mmSRBM_GFX_CNTL, value); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + WREG32(mmSRBM_GFX_CNTL, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32(mmSH_MEM_CONFIG, sh_mem_config); + WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); + WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); + WREG32(mmSH_MEM_BASES, sh_mem_bases); + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished and the + * SW cleared it. So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); + + while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) + cpu_relax(); + WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); + + return 0; +} + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | + CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) +{ + uint32_t retval; + + retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + + m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; + + pr_debug("sdma base address: 0x%x\n", retval); + + return retval; +} + +static inline struct cik_mqd *get_mqd(void *mqd) +{ + return (struct cik_mqd *)mqd; +} + +static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) +{ + return (struct cik_sdma_rlc_registers *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_mqd *m; + uint32_t *mqd_hqd; + uint32_t reg, wptr_val, data; + bool valid_wptr = false; + + m = get_mqd(mqd); + + acquire_queue(kgd, pipe_id, queue_id); + + /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */ + mqd_hqd = &m->cp_mqd_base_addr_lo; + + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++) + WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); + + /* Copy userspace write pointer value to register. + * Activate doorbell logic to monitor subsequent changes. + */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); + + /* read_user_ptr may take the mm->mmap_sem. + * release srbm_mutex to avoid circular dependency between + * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. + */ + release_queue(kgd); + valid_wptr = read_user_wptr(mm, wptr, wptr_val); + acquire_queue(kgd, pipe_id, queue_id); + if (valid_wptr) + WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); + + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(mmCP_HQD_ACTIVE, data); + + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t i = 0, reg; +#define HQD_N_REGS (35+4) +#define DUMP_REG(addr) do { \ + if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ + break; \ + (*dump)[i][0] = (addr) << 2; \ + (*dump)[i++][1] = RREG32(addr); \ + } while (0) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + acquire_queue(kgd, pipe_id, queue_id); + + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3); + + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++) + DUMP_REG(reg); + + release_queue(kgd); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + unsigned long end_jiffies; + uint32_t sdma_base_addr; + uint32_t data; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + m->sdma_rlc_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); + + end_jiffies = msecs_to_jiffies(2000) + jiffies; + while (true) { + data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + if (m->sdma_engine_id) { + data = RREG32(mmSDMA1_GFX_CONTEXT_CNTL); + data = REG_SET_FIELD(data, SDMA1_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(mmSDMA1_GFX_CONTEXT_CNTL, data); + } else { + data = RREG32(mmSDMA0_GFX_CONTEXT_CNTL); + data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data); + } + + data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdma_rlc_rb_rptr); + + if (read_user_wptr(mm, wptr, data)) + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data); + else + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + m->sdma_rlc_rb_rptr); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR, + m->sdma_rlc_virtual_addr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdma_rlc_rb_base); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, + m->sdma_rlc_rb_base_hi); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdma_rlc_rb_rptr_addr_lo); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdma_rlc_rb_rptr_addr_hi); + + data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); + + return 0; +} + +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET + + queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; + uint32_t i = 0, reg; +#undef HQD_N_REGS +#define HQD_N_REGS (19+4) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) + DUMP_REG(sdma_offset + reg); + for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK; + reg++) + DUMP_REG(sdma_offset + reg); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(mmCP_HQD_ACTIVE); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(mmCP_HQD_PQ_BASE) && + high == RREG32(mmCP_HQD_PQ_BASE_HI)) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t temp; + enum hqd_dequeue_request_type type; + unsigned long flags, end_jiffies; + int retry; + + if (adev->in_gpu_reset) + return -EIO; + + acquire_queue(kgd, pipe_id, queue_id); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0); + + switch (reset_type) { + case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: + type = DRAIN_PIPE; + break; + case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: + type = RESET_WAVES; + break; + default: + type = DRAIN_PIPE; + break; + } + + /* Workaround: If IQ timer is active and the wait time is close to or + * equal to 0, dequeueing is not safe. Wait until either the wait time + * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is + * cleared before continuing. Also, ensure wait times are set to at + * least 0x3. + */ + local_irq_save(flags); + preempt_disable(); + retry = 5000; /* wait for 500 usecs at maximum */ + while (true) { + temp = RREG32(mmCP_HQD_IQ_TIMER); + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) { + pr_debug("HW is processing IQ\n"); + goto loop; + } + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) { + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE) + == 3) /* SEM-rearm is safe */ + break; + /* Wait time 3 is safe for CP, but our MMIO read/write + * time is close to 1 microsecond, so check for 10 to + * leave more buffer room + */ + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME) + >= 10) + break; + pr_debug("IQ timer is active\n"); + } else + break; +loop: + if (!retry) { + pr_err("CP HQD IQ timer status time out\n"); + break; + } + ndelay(100); + --retry; + } + retry = 1000; + while (true) { + temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST); + if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK)) + break; + pr_debug("Dequeue request is pending\n"); + + if (!retry) { + pr_err("CP HQD dequeue request time out\n"); + break; + } + ndelay(100); + --retry; + } + local_irq_restore(flags); + preempt_enable(); + + WREG32(mmCP_HQD_DEQUEUE_REQUEST, type); + + end_jiffies = (utimeout * HZ / 1000) + jiffies; + while (true) { + temp = RREG32(mmCP_HQD_ACTIVE); + if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + if (time_after(jiffies, end_jiffies)) { + pr_err("cp queue preemption time out\n"); + release_queue(kgd); + return -ETIME; + } + usleep_range(500, 1000); + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t temp; + unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) | + SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); + + m->sdma_rlc_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR); + + return 0; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + union TCP_WATCH_CNTL_BITS cntl; + unsigned int i; + + cntl.u32All = 0; + + cntl.bitfields.valid = 0; + cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK; + cntl.bitfields.atc = 1; + + /* Turning off this address until we set all the registers */ + for (i = 0; i < MAX_WATCH_ADDRESSES; i++) + WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + union TCP_WATCH_CNTL_BITS cntl; + + cntl.u32All = cntl_val; + + /* Turning off this watch point until we set all the registers */ + cntl.bitfields.valid = 0; + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_HI], addr_hi); + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_LO], addr_lo); + + /* Enable the watch point */ + cntl.bitfields.valid = 1; + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32(mmGRBM_GFX_INDEX, gfx_index_val); + WREG32(mmSQ_CMD, sq_cmd); + + /* Restore the GRBM_GFX_INDEX register */ + + data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | + GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | + GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK; + + WREG32(mmGRBM_GFX_INDEX, data); + + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK; +} + +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + lock_srbm(kgd, 0, 0, 0, vmid); + WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va); + unlock_srbm(kgd); +} + +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("trying to set page table base for wrong VMID\n"); + return; + } + WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, + lower_32_bits(page_table_base)); +} + +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + int vmid; + unsigned int tmp; + + if (adev->in_gpu_reset) + return -EIO; + + for (vmid = 0; vmid < 16; vmid++) { + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) + continue; + + tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && + (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + RREG32(mmVM_INVALIDATE_RESPONSE); + break; + } + } + + return 0; +} + +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid\n"); + return 0; + } + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + RREG32(mmVM_INVALIDATE_RESPONSE); + return 0; +} + + /** + * read_vmid_from_vmfault_reg - read vmid from register + * + * adev: amdgpu_device pointer + * @vmid: vmid pointer + * read vmid from register (CIK). + */ +static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS); + + return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); +}
diff --git a/amdgpu/amdgpu_amdkfd_gfx_v8.c b/amdgpu/amdgpu_amdkfd_gfx_v8.c new file mode 100644 index 0000000..6d2f614 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -0,0 +1,800 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/module.h> +#include <linux/fdtable.h> +#include <linux/uaccess.h> +#include <linux/mmu_context.h> + +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "gfx_v8_0.h" +#include "gca/gfx_8_0_sh_mask.h" +#include "gca/gfx_8_0_d.h" +#include "gca/gfx_8_0_enum.h" +#include "oss/oss_3_0_sh_mask.h" +#include "oss/oss_3_0_d.h" +#include "gmc/gmc_8_1_sh_mask.h" +#include "gmc/gmc_8_1_d.h" +#include "vi_structs.h" +#include "vid.h" + +enum hqd_dequeue_request_type { + NO_ACTION = 0, + DRAIN_PIPE, + RESET_WAVES +}; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases); +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm); +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm); +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid); +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base); +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); + +/* Because of REG_GET_FIELD() being used, we put this function in the + * asic specific file. + */ +static int get_tile_config(struct kgd_dev *kgd, + struct tile_config *config) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + config->gb_addr_config = adev->gfx.config.gb_addr_config; + config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFBANK); + config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, + MC_ARB_RAMCFG, NOOFRANKS); + + config->tile_config_ptr = adev->gfx.config.tile_mode_array; + config->num_tile_configs = + ARRAY_SIZE(adev->gfx.config.tile_mode_array); + config->macro_tile_config_ptr = + adev->gfx.config.macrotile_mode_array; + config->num_macro_tile_configs = + ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); + + return 0; +} + +static const struct kfd2kgd_calls kfd2kgd = { + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_dump = kgd_hqd_dump, + .hqd_sdma_dump = kgd_hqd_sdma_dump, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = + get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = + get_atc_vmid_pasid_mapping_valid, + .set_scratch_backing_va = set_scratch_backing_va, + .get_tile_config = get_tile_config, + .set_vm_context_page_table_base = set_vm_context_page_table_base, + .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void) +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); + + mutex_lock(&adev->srbm_mutex); + WREG32(mmSRBM_GFX_CNTL, value); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + WREG32(mmSRBM_GFX_CNTL, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32(mmSH_MEM_CONFIG, sh_mem_config); + WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); + WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); + WREG32(mmSH_MEM_BASES, sh_mem_bases); + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished + * and the SW cleared it. + * So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); + + while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) + cpu_relax(); + WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); + + return 0; +} + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | + CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct vi_sdma_mqd *m) +{ + uint32_t retval; + + retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + + m->sdma_queue_id * KFD_VI_SDMA_QUEUE_OFFSET; + pr_debug("sdma base address: 0x%x\n", retval); + + return retval; +} + +static inline struct vi_mqd *get_mqd(void *mqd) +{ + return (struct vi_mqd *)mqd; +} + +static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd) +{ + return (struct vi_sdma_mqd *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct vi_mqd *m; + uint32_t *mqd_hqd; + uint32_t reg, wptr_val, data; + bool valid_wptr = false; + + m = get_mqd(mqd); + + acquire_queue(kgd, pipe_id, queue_id); + + /* HIQ is set during driver init period with vmid set to 0*/ + if (m->cp_hqd_vmid == 0) { + uint32_t value, mec, pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", + mec, pipe, queue_id); + value = RREG32(mmRLC_CP_SCHEDULERS); + value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1, + ((mec << 5) | (pipe << 3) | queue_id | 0x80)); + WREG32(mmRLC_CP_SCHEDULERS, value); + } + + /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ + mqd_hqd = &m->cp_mqd_base_addr_lo; + + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_CONTROL; reg++) + WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); + + /* Tonga errata: EOP RPTR/WPTR should be left unmodified. + * This is safe since EOP RPTR==WPTR for any inactive HQD + * on ASICs that do not support context-save. + * EOP writes/reads can start anywhere in the ring. + */ + if (get_amdgpu_device(kgd)->asic_type != CHIP_TONGA) { + WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr); + WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr); + WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem); + } + + for (reg = mmCP_HQD_EOP_EVENTS; reg <= mmCP_HQD_ERROR; reg++) + WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); + + /* Copy userspace write pointer value to register. + * Activate doorbell logic to monitor subsequent changes. + */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); + + /* read_user_ptr may take the mm->mmap_sem. + * release srbm_mutex to avoid circular dependency between + * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. + */ + release_queue(kgd); + valid_wptr = read_user_wptr(mm, wptr, wptr_val); + acquire_queue(kgd, pipe_id, queue_id); + if (valid_wptr) + WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); + + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32(mmCP_HQD_ACTIVE, data); + + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t i = 0, reg; +#define HQD_N_REGS (54+4) +#define DUMP_REG(addr) do { \ + if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ + break; \ + (*dump)[i][0] = (addr) << 2; \ + (*dump)[i++][1] = RREG32(addr); \ + } while (0) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + acquire_queue(kgd, pipe_id, queue_id); + + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2); + DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3); + + for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_DONES; reg++) + DUMP_REG(reg); + + release_queue(kgd); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct vi_sdma_mqd *m; + unsigned long end_jiffies; + uint32_t sdma_base_addr; + uint32_t data; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); + + end_jiffies = msecs_to_jiffies(2000) + jiffies; + while (true) { + data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + if (m->sdma_engine_id) { + data = RREG32(mmSDMA1_GFX_CONTEXT_CNTL); + data = REG_SET_FIELD(data, SDMA1_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(mmSDMA1_GFX_CONTEXT_CNTL, data); + } else { + data = RREG32(mmSDMA0_GFX_CONTEXT_CNTL); + data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data); + } + + data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr); + + if (read_user_wptr(mm, wptr, data)) + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data); + else + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + m->sdmax_rlcx_rb_rptr); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR, + m->sdmax_rlcx_virtual_addr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, + m->sdmax_rlcx_rb_base_hi); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdmax_rlcx_rb_rptr_addr_lo); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdmax_rlcx_rb_rptr_addr_hi); + + data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); + + return 0; +} + +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET + + queue_id * KFD_VI_SDMA_QUEUE_OFFSET; + uint32_t i = 0, reg; +#undef HQD_N_REGS +#define HQD_N_REGS (19+4+2+3+7) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) + DUMP_REG(sdma_offset + reg); + for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK; + reg++) + DUMP_REG(sdma_offset + reg); + for (reg = mmSDMA0_RLC0_CSA_ADDR_LO; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; + reg++) + DUMP_REG(sdma_offset + reg); + for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; reg <= mmSDMA0_RLC0_DUMMY_REG; + reg++) + DUMP_REG(sdma_offset + reg); + for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; reg <= mmSDMA0_RLC0_MIDCMD_CNTL; + reg++) + DUMP_REG(sdma_offset + reg); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(mmCP_HQD_ACTIVE); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(mmCP_HQD_PQ_BASE) && + high == RREG32(mmCP_HQD_PQ_BASE_HI)) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct vi_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t temp; + enum hqd_dequeue_request_type type; + unsigned long flags, end_jiffies; + int retry; + struct vi_mqd *m = get_mqd(mqd); + + if (adev->in_gpu_reset) + return -EIO; + + acquire_queue(kgd, pipe_id, queue_id); + + if (m->cp_hqd_vmid == 0) + WREG32_FIELD(RLC_CP_SCHEDULERS, scheduler1, 0); + + switch (reset_type) { + case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: + type = DRAIN_PIPE; + break; + case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: + type = RESET_WAVES; + break; + default: + type = DRAIN_PIPE; + break; + } + + /* Workaround: If IQ timer is active and the wait time is close to or + * equal to 0, dequeueing is not safe. Wait until either the wait time + * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is + * cleared before continuing. Also, ensure wait times are set to at + * least 0x3. + */ + local_irq_save(flags); + preempt_disable(); + retry = 5000; /* wait for 500 usecs at maximum */ + while (true) { + temp = RREG32(mmCP_HQD_IQ_TIMER); + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) { + pr_debug("HW is processing IQ\n"); + goto loop; + } + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) { + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE) + == 3) /* SEM-rearm is safe */ + break; + /* Wait time 3 is safe for CP, but our MMIO read/write + * time is close to 1 microsecond, so check for 10 to + * leave more buffer room + */ + if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME) + >= 10) + break; + pr_debug("IQ timer is active\n"); + } else + break; +loop: + if (!retry) { + pr_err("CP HQD IQ timer status time out\n"); + break; + } + ndelay(100); + --retry; + } + retry = 1000; + while (true) { + temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST); + if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK)) + break; + pr_debug("Dequeue request is pending\n"); + + if (!retry) { + pr_err("CP HQD dequeue request time out\n"); + break; + } + ndelay(100); + --retry; + } + local_irq_restore(flags); + preempt_enable(); + + WREG32(mmCP_HQD_DEQUEUE_REQUEST, type); + + end_jiffies = (utimeout * HZ / 1000) + jiffies; + while (true) { + temp = RREG32(mmCP_HQD_ACTIVE); + if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + if (time_after(jiffies, end_jiffies)) { + pr_err("cp queue preemption time out.\n"); + release_queue(kgd); + return -ETIME; + } + usleep_range(500, 1000); + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct vi_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t temp; + unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) | + SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); + + m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR); + + return 0; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data = 0; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32(mmGRBM_GFX_INDEX, gfx_index_val); + WREG32(mmSQ_CMD, sq_cmd); + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SH_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + WREG32(mmGRBM_GFX_INDEX, data); + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return 0; +} + +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + lock_srbm(kgd, 0, 0, 0, vmid); + WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va); + unlock_srbm(kgd); +} + +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("trying to set page table base for wrong VMID\n"); + return; + } + WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, + lower_32_bits(page_table_base)); +} + +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + int vmid; + unsigned int tmp; + + if (adev->in_gpu_reset) + return -EIO; + + for (vmid = 0; vmid < 16; vmid++) { + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) + continue; + + tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && + (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + RREG32(mmVM_INVALIDATE_RESPONSE); + break; + } + } + + return 0; +} + +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid %d\n", vmid); + return -EINVAL; + } + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); + RREG32(mmVM_INVALIDATE_RESPONSE); + return 0; +}
diff --git a/amdgpu/amdgpu_amdkfd_gfx_v9.c b/amdgpu/amdgpu_amdkfd_gfx_v9.c new file mode 100644 index 0000000..85395f2 --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -0,0 +1,890 @@ +/* + * Copyright 2014-2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#define pr_fmt(fmt) "kfd2kgd: " fmt + +#include <linux/module.h> +#include <linux/fdtable.h> +#include <linux/uaccess.h> +#include <linux/mmu_context.h> + +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "soc15_hw_ip.h" +#include "gc/gc_9_0_offset.h" +#include "gc/gc_9_0_sh_mask.h" +#include "vega10_enum.h" +#include "sdma0/sdma0_4_0_offset.h" +#include "sdma0/sdma0_4_0_sh_mask.h" +#include "sdma1/sdma1_4_0_offset.h" +#include "sdma1/sdma1_4_0_sh_mask.h" +#include "athub/athub_1_0_offset.h" +#include "athub/athub_1_0_sh_mask.h" +#include "oss/osssys_4_0_offset.h" +#include "oss/osssys_4_0_sh_mask.h" +#include "soc15_common.h" +#include "v9_structs.h" +#include "soc15.h" +#include "soc15d.h" +#include "mmhub_v1_0.h" +#include "gfxhub_v1_0.h" + + +#define V9_PIPE_PER_MEC (4) +#define V9_QUEUES_PER_PIPE_MEC (8) + +enum hqd_dequeue_request_type { + NO_ACTION = 0, + DRAIN_PIPE, + RESET_WAVES +}; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases); +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm); +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm); +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base); +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid); +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid); +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid); + +/* Because of REG_GET_FIELD() being used, we put this function in the + * asic specific file. + */ +static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd, + struct tile_config *config) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + config->gb_addr_config = adev->gfx.config.gb_addr_config; + + config->tile_config_ptr = adev->gfx.config.tile_mode_array; + config->num_tile_configs = + ARRAY_SIZE(adev->gfx.config.tile_mode_array); + config->macro_tile_config_ptr = + adev->gfx.config.macrotile_mode_array; + config->num_macro_tile_configs = + ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); + + return 0; +} + +static const struct kfd2kgd_calls kfd2kgd = { + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_dump = kgd_hqd_dump, + .hqd_sdma_dump = kgd_hqd_sdma_dump, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = + get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = + get_atc_vmid_pasid_mapping_valid, + .set_scratch_backing_va = set_scratch_backing_va, + .get_tile_config = amdgpu_amdkfd_get_tile_config, + .set_vm_context_page_table_base = set_vm_context_page_table_base, + .invalidate_tlbs = invalidate_tlbs, + .invalidate_tlbs_vmid = invalidate_tlbs_vmid, + .get_hive_id = amdgpu_amdkfd_get_hive_id, +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void) +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + mutex_lock(&adev->srbm_mutex); + soc15_grbm_select(adev, mec, pipe, queue, vmid); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + soc15_grbm_select(adev, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static uint32_t get_queue_mask(struct amdgpu_device *adev, + uint32_t pipe_id, uint32_t queue_id) +{ + unsigned int bit = (pipe_id * adev->gfx.mec.num_queue_per_pipe + + queue_id) & 31; + + return ((uint32_t)1) << bit; +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases); + /* APE1 no longer exists on GFX9 */ + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished + * and the SW cleared it. + * So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + /* + * need to do this twice, once for gfx and once for mmhub + * for ATC add 16 to VMID for mmhub, for IH different registers. + * ATC_VMID0..15 registers are separate from ATC_VMID16..31. + */ + + WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid, + pasid_mapping); + + while (!(RREG32(SOC15_REG_OFFSET( + ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & + (1U << vmid))) + cpu_relax(); + + WREG32(SOC15_REG_OFFSET(ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), + 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid, + pasid_mapping); + + WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID16_PASID_MAPPING) + vmid, + pasid_mapping); + + while (!(RREG32(SOC15_REG_OFFSET( + ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & + (1U << (vmid + 16)))) + cpu_relax(); + + WREG32(SOC15_REG_OFFSET(ATHUB, 0, + mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), + 1U << (vmid + 16)); + + /* Mapping vmid to pasid also for IH block */ + WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid, + pasid_mapping); + return 0; +} + +/* TODO - RING0 form of field is obsolete, seems to date back to SI + * but still works + */ + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), + CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | + CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static uint32_t get_sdma_base_addr(struct amdgpu_device *adev, + unsigned int engine_id, + unsigned int queue_id) +{ + uint32_t base[2] = { + SOC15_REG_OFFSET(SDMA0, 0, + mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL, + SOC15_REG_OFFSET(SDMA1, 0, + mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL + }; + uint32_t retval; + + retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL - + mmSDMA0_RLC0_RB_CNTL); + + pr_debug("sdma base address: 0x%x\n", retval); + + return retval; +} + +static inline struct v9_mqd *get_mqd(void *mqd) +{ + return (struct v9_mqd *)mqd; +} + +static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd) +{ + return (struct v9_sdma_mqd *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr, + uint32_t wptr_shift, uint32_t wptr_mask, + struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v9_mqd *m; + uint32_t *mqd_hqd; + uint32_t reg, hqd_base, data; + + m = get_mqd(mqd); + + acquire_queue(kgd, pipe_id, queue_id); + + /* HIQ is set during driver init period with vmid set to 0*/ + if (m->cp_hqd_vmid == 0) { + uint32_t value, mec, pipe; + + mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; + pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); + + pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", + mec, pipe, queue_id); + value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS)); + value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1, + ((mec << 5) | (pipe << 3) | queue_id | 0x80)); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value); + } + + /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ + mqd_hqd = &m->cp_mqd_base_addr_lo; + hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); + + for (reg = hqd_base; + reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) + WREG32_RLC(reg, mqd_hqd[reg - hqd_base]); + + + /* Activate doorbell logic before triggering WPTR poll. */ + data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, + CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data); + + if (wptr) { + /* Don't read wptr with get_user because the user + * context may not be accessible (if this function + * runs in a work queue). Instead trigger a one-shot + * polling read from memory in the CP. This assumes + * that wptr is GPU-accessible in the queue's VMID via + * ATC or SVM. WPTR==RPTR before starting the poll so + * the CP starts fetching new commands from the right + * place. + * + * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit + * tricky. Assume that the queue didn't overflow. The + * number of valid bits in the 32-bit RPTR depends on + * the queue size. The remaining bits are taken from + * the saved 64-bit WPTR. If the WPTR wrapped, add the + * queue size. + */ + uint32_t queue_size = + 2 << REG_GET_FIELD(m->cp_hqd_pq_control, + CP_HQD_PQ_CONTROL, QUEUE_SIZE); + uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); + + if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) + guessed_wptr += queue_size; + guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); + guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; + + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), + lower_32_bits(guessed_wptr)); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), + upper_32_bits(guessed_wptr)); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), + lower_32_bits((uintptr_t)wptr)); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), + upper_32_bits((uintptr_t)wptr)); + WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1), + get_queue_mask(adev, pipe_id, queue_id)); + } + + /* Start the EOP fetcher */ + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR), + REG_SET_FIELD(m->cp_hqd_eop_rptr, + CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); + + data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data); + + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_dump(struct kgd_dev *kgd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t i = 0, reg; +#define HQD_N_REGS 56 +#define DUMP_REG(addr) do { \ + if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ + break; \ + (*dump)[i][0] = (addr) << 2; \ + (*dump)[i++][1] = RREG32(addr); \ + } while (0) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + acquire_queue(kgd, pipe_id, queue_id); + + for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); + reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) + DUMP_REG(reg); + + release_queue(kgd); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, + uint32_t __user *wptr, struct mm_struct *mm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v9_sdma_mqd *m; + uint32_t sdma_base_addr, sdmax_gfx_context_cntl; + unsigned long end_jiffies; + uint32_t data; + uint64_t data64; + uint64_t __user *wptr64 = (uint64_t __user *)wptr; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + sdmax_gfx_context_cntl = m->sdma_engine_id ? + SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) : + SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); + + end_jiffies = msecs_to_jiffies(2000) + jiffies; + while (true) { + data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + data = RREG32(sdmax_gfx_context_cntl); + data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL, + RESUME_CTX, 0); + WREG32(sdmax_gfx_context_cntl, data); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET, + m->sdmax_rlcx_doorbell_offset); + + data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, + ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI, + m->sdmax_rlcx_rb_rptr_hi); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1); + if (read_user_wptr(mm, wptr64, data64)) { + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + lower_32_bits(data64)); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI, + upper_32_bits(data64)); + } else { + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, + m->sdmax_rlcx_rb_rptr); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI, + m->sdmax_rlcx_rb_rptr_hi); + } + WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, + m->sdmax_rlcx_rb_base_hi); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdmax_rlcx_rb_rptr_addr_lo); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdmax_rlcx_rb_rptr_addr_hi); + + data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, + RB_ENABLE, 1); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data); + + return 0; +} + +static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, + uint32_t engine_id, uint32_t queue_id, + uint32_t (**dump)[2], uint32_t *n_regs) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t sdma_base_addr = get_sdma_base_addr(adev, engine_id, queue_id); + uint32_t i = 0, reg; +#undef HQD_N_REGS +#define HQD_N_REGS (19+6+7+10) + + *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); + if (*dump == NULL) + return -ENOMEM; + + for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; + reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++) + DUMP_REG(sdma_base_addr + reg); + for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; + reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++) + DUMP_REG(sdma_base_addr + reg); + + WARN_ON_ONCE(i != HQD_N_REGS); + *n_regs = i; + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) && + high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI))) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v9_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, + enum kfd_preempt_type reset_type, + unsigned int utimeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + enum hqd_dequeue_request_type type; + unsigned long end_jiffies; + uint32_t temp; + struct v9_mqd *m = get_mqd(mqd); + + if (adev->in_gpu_reset) + return -EIO; + + acquire_queue(kgd, pipe_id, queue_id); + + if (m->cp_hqd_vmid == 0) + WREG32_FIELD15_RLC(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); + + switch (reset_type) { + case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: + type = DRAIN_PIPE; + break; + case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: + type = RESET_WAVES; + break; + default: + type = DRAIN_PIPE; + break; + } + + WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type); + + end_jiffies = (utimeout * HZ / 1000) + jiffies; + while (true) { + temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); + if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + if (time_after(jiffies, end_jiffies)) { + pr_err("cp queue preemption time out.\n"); + release_queue(kgd); + return -ETIME; + } + usleep_range(500, 1000); + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int utimeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct v9_sdma_mqd *m; + uint32_t sdma_base_addr; + uint32_t temp; + unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id, + m->sdma_queue_id); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) + break; + if (time_after(jiffies, end_jiffies)) + return -ETIME; + usleep_range(500, 1000); + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) | + SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); + + m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR); + m->sdmax_rlcx_rb_rptr_hi = + RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI); + + return 0; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + + vmid); + return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK; +} + +static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid, + uint32_t flush_type) +{ + signed long r; + uint32_t seq; + struct amdgpu_ring *ring = &adev->gfx.kiq.ring; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/ + amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); + amdgpu_ring_write(ring, + PACKET3_INVALIDATE_TLBS_DST_SEL(1) | + PACKET3_INVALIDATE_TLBS_ALL_HUB(1) | + PACKET3_INVALIDATE_TLBS_PASID(pasid) | + PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + + r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); + if (r < 1) { + DRM_ERROR("wait for kiq fence error: %ld.\n", r); + return -ETIME; + } + + return 0; +} + +static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + int vmid; + struct amdgpu_ring *ring = &adev->gfx.kiq.ring; + uint32_t flush_type = 0; + + if (adev->in_gpu_reset) + return -EIO; + if (adev->gmc.xgmi.num_physical_nodes && + adev->asic_type == CHIP_VEGA20) + flush_type = 2; + + if (ring->sched.ready) + return invalidate_tlbs_with_kiq(adev, pasid, flush_type); + + for (vmid = 0; vmid < 16; vmid++) { + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) + continue; + if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) { + if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid) + == pasid) { + amdgpu_gmc_flush_gpu_tlb(adev, vmid, + flush_type); + break; + } + } + } + + return 0; +} + +static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("non kfd vmid %d\n", vmid); + return 0; + } + + /* Use legacy mode tlb invalidation. + * + * Currently on Raven the code below is broken for anything but + * legacy mode due to a MMHUB power gating problem. A workaround + * is for MMHUB to wait until the condition PER_VMID_INVALIDATE_REQ + * == PER_VMID_INVALIDATE_ACK instead of simply waiting for the ack + * bit. + * + * TODO 1: agree on the right set of invalidation registers for + * KFD use. Use the last one for now. Invalidate both GC and + * MMHUB. + * + * TODO 2: support range-based invalidation, requires kfg2kgd + * interface change + */ + amdgpu_gmc_flush_gpu_tlb(adev, vmid, 0); + return 0; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data = 0; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, gfx_index_val); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd); + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SH_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return 0; +} + +static void set_scratch_backing_va(struct kgd_dev *kgd, + uint64_t va, uint32_t vmid) +{ + /* No longer needed on GFXv9. The scratch base address is + * passed to the shader by the CP. It's the user mode driver's + * responsibility. + */ +} + +static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, + uint64_t page_table_base) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { + pr_err("trying to set page table base for wrong VMID %u\n", + vmid); + return; + } + + /* TODO: take advantage of per-process address space size. For + * now, all processes share the same address space size, like + * on GFX8 and older. + */ + mmhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); + + gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); +}
diff --git a/amdgpu/amdgpu_amdkfd_gpuvm.c b/amdgpu/amdgpu_amdkfd_gpuvm.c new file mode 100644 index 0000000..6a5c96e --- /dev/null +++ b/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -0,0 +1,2185 @@ +/* + * Copyright 2014-2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#define pr_fmt(fmt) "kfd2kgd: " fmt + +#include <linux/dma-buf.h> +#include <linux/list.h> +#include <linux/pagemap.h> +#include <linux/sched/mm.h> +#include <linux/sched/task.h> + +#include "amdgpu_object.h" +#include "amdgpu_vm.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_dma_buf.h" + +/* Special VM and GART address alignment needed for VI pre-Fiji due to + * a HW bug. + */ +#define VI_BO_SIZE_ALIGN (0x8000) + +/* BO flag to indicate a KFD userptr BO */ +#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63) + +/* Userptr restore delay, just long enough to allow consecutive VM + * changes to accumulate + */ +#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 + +/* Impose limit on how much memory KFD can use */ +static struct { + uint64_t max_system_mem_limit; + uint64_t max_ttm_mem_limit; + int64_t system_mem_used; + int64_t ttm_mem_used; + spinlock_t mem_limit_lock; +} kfd_mem_limit; + +/* Struct used for amdgpu_amdkfd_bo_validate */ +struct amdgpu_vm_parser { + uint32_t domain; + bool wait; +}; + +static const char * const domain_bit_to_string[] = { + "CPU", + "GTT", + "VRAM", + "GDS", + "GWS", + "OA" +}; + +#define domain_string(domain) domain_bit_to_string[ffs(domain)-1] + +static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); + + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm, + struct kgd_mem *mem) +{ + struct kfd_bo_va_list *entry; + + list_for_each_entry(entry, &mem->bo_va_list, bo_list) + if (entry->bo_va->base.vm == avm) + return false; + + return true; +} + +/* Set memory usage limits. Current, limits are + * System (TTM + userptr) memory - 3/4th System RAM + * TTM memory - 3/8th System RAM + */ +void amdgpu_amdkfd_gpuvm_init_mem_limits(void) +{ + struct sysinfo si; + uint64_t mem; + + si_meminfo(&si); + mem = si.totalram - si.totalhigh; + mem *= si.mem_unit; + + spin_lock_init(&kfd_mem_limit.mem_limit_lock); + kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2); + kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); + pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", + (kfd_mem_limit.max_system_mem_limit >> 20), + (kfd_mem_limit.max_ttm_mem_limit >> 20)); +} + +static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, + uint64_t size, u32 domain, bool sg) +{ + size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; + uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9; + int ret = 0; + + acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, + sizeof(struct amdgpu_bo)); + + vram_needed = 0; + if (domain == AMDGPU_GEM_DOMAIN_GTT) { + /* TTM GTT memory */ + system_mem_needed = acc_size + size; + ttm_mem_needed = acc_size + size; + } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { + /* Userptr */ + system_mem_needed = acc_size + size; + ttm_mem_needed = acc_size; + } else { + /* VRAM and SG */ + system_mem_needed = acc_size; + ttm_mem_needed = acc_size; + if (domain == AMDGPU_GEM_DOMAIN_VRAM) + vram_needed = size; + } + + spin_lock(&kfd_mem_limit.mem_limit_lock); + + if ((kfd_mem_limit.system_mem_used + system_mem_needed > + kfd_mem_limit.max_system_mem_limit) || + (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > + kfd_mem_limit.max_ttm_mem_limit) || + (adev->kfd.vram_used + vram_needed > + adev->gmc.real_vram_size - reserved_for_pt)) { + ret = -ENOMEM; + } else { + kfd_mem_limit.system_mem_used += system_mem_needed; + kfd_mem_limit.ttm_mem_used += ttm_mem_needed; + adev->kfd.vram_used += vram_needed; + } + + spin_unlock(&kfd_mem_limit.mem_limit_lock); + return ret; +} + +static void unreserve_mem_limit(struct amdgpu_device *adev, + uint64_t size, u32 domain, bool sg) +{ + size_t acc_size; + + acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, + sizeof(struct amdgpu_bo)); + + spin_lock(&kfd_mem_limit.mem_limit_lock); + if (domain == AMDGPU_GEM_DOMAIN_GTT) { + kfd_mem_limit.system_mem_used -= (acc_size + size); + kfd_mem_limit.ttm_mem_used -= (acc_size + size); + } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { + kfd_mem_limit.system_mem_used -= (acc_size + size); + kfd_mem_limit.ttm_mem_used -= acc_size; + } else { + kfd_mem_limit.system_mem_used -= acc_size; + kfd_mem_limit.ttm_mem_used -= acc_size; + if (domain == AMDGPU_GEM_DOMAIN_VRAM) { + adev->kfd.vram_used -= size; + WARN_ONCE(adev->kfd.vram_used < 0, + "kfd VRAM memory accounting unbalanced"); + } + } + WARN_ONCE(kfd_mem_limit.system_mem_used < 0, + "kfd system memory accounting unbalanced"); + WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, + "kfd TTM memory accounting unbalanced"); + + spin_unlock(&kfd_mem_limit.mem_limit_lock); +} + +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + u32 domain = bo->preferred_domains; + bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU); + + if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) { + domain = AMDGPU_GEM_DOMAIN_CPU; + sg = false; + } + + unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg); +} + + +/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's + * reservation object. + * + * @bo: [IN] Remove eviction fence(s) from this BO + * @ef: [IN] This eviction fence is removed if it + * is present in the shared list. + * + * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. + */ +static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, + struct amdgpu_amdkfd_fence *ef) +{ + struct reservation_object *resv = bo->tbo.resv; + struct reservation_object_list *old, *new; + unsigned int i, j, k; + + if (!ef) + return -EINVAL; + + old = reservation_object_get_list(resv); + if (!old) + return 0; + + new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]), + GFP_KERNEL); + if (!new) + return -ENOMEM; + + /* Go through all the shared fences in the resevation object and sort + * the interesting ones to the end of the list. + */ + for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { + struct dma_fence *f; + + f = rcu_dereference_protected(old->shared[i], + reservation_object_held(resv)); + + if (f->context == ef->base.context) + RCU_INIT_POINTER(new->shared[--j], f); + else + RCU_INIT_POINTER(new->shared[k++], f); + } + new->shared_max = old->shared_max; + new->shared_count = k; + + /* Install the new fence list, seqcount provides the barriers */ + preempt_disable(); + write_seqcount_begin(&resv->seq); + RCU_INIT_POINTER(resv->fence, new); + write_seqcount_end(&resv->seq); + preempt_enable(); + + /* Drop the references to the removed fences or move them to ef_list */ + for (i = j, k = 0; i < old->shared_count; ++i) { + struct dma_fence *f; + + f = rcu_dereference_protected(new->shared[i], + reservation_object_held(resv)); + dma_fence_put(f); + } + kfree_rcu(old, rcu); + + return 0; +} + +static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, + bool wait) +{ + struct ttm_operation_ctx ctx = { false, false }; + int ret; + + if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), + "Called with userptr BO")) + return -EINVAL; + + amdgpu_bo_placement_from_domain(bo, domain); + + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (ret) + goto validate_fail; + if (wait) + amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); + +validate_fail: + return ret; +} + +static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo) +{ + struct amdgpu_vm_parser *p = param; + + return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait); +} + +/* vm_validate_pt_pd_bos - Validate page table and directory BOs + * + * Page directories are not updated here because huge page handling + * during page table updates can invalidate page directory entries + * again. Page directories are only updated after updating page + * tables. + */ +static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) +{ + struct amdgpu_bo *pd = vm->root.base.bo; + struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); + struct amdgpu_vm_parser param; + int ret; + + param.domain = AMDGPU_GEM_DOMAIN_VRAM; + param.wait = false; + + ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, + ¶m); + if (ret) { + pr_err("amdgpu: failed to validate PT BOs\n"); + return ret; + } + + ret = amdgpu_amdkfd_validate(¶m, pd); + if (ret) { + pr_err("amdgpu: failed to validate PD\n"); + return ret; + } + + vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); + + if (vm->use_cpu_for_update) { + ret = amdgpu_bo_kmap(pd, NULL); + if (ret) { + pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret); + return ret; + } + } + + return 0; +} + +static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) +{ + struct amdgpu_bo *pd = vm->root.base.bo; + struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); + int ret; + + ret = amdgpu_vm_update_directories(adev, vm); + if (ret) + return ret; + + return amdgpu_sync_fence(NULL, sync, vm->last_update, false); +} + +/* add_bo_to_vm - Add a BO to a VM + * + * Everything that needs to bo done only once when a BO is first added + * to a VM. It can later be mapped and unmapped many times without + * repeating these steps. + * + * 1. Allocate and initialize BO VA entry data structure + * 2. Add BO to the VM + * 3. Determine ASIC-specific PTE flags + * 4. Alloc page tables and directories if needed + * 4a. Validate new page tables and directories + */ +static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, + struct amdgpu_vm *vm, bool is_aql, + struct kfd_bo_va_list **p_bo_va_entry) +{ + int ret; + struct kfd_bo_va_list *bo_va_entry; + struct amdgpu_bo *bo = mem->bo; + uint64_t va = mem->va; + struct list_head *list_bo_va = &mem->bo_va_list; + unsigned long bo_size = bo->tbo.mem.size; + + if (!va) { + pr_err("Invalid VA when adding BO to VM\n"); + return -EINVAL; + } + + if (is_aql) + va += bo_size; + + bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL); + if (!bo_va_entry) + return -ENOMEM; + + pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, + va + bo_size, vm); + + /* Add BO to VM internal data structures*/ + bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo); + if (!bo_va_entry->bo_va) { + ret = -EINVAL; + pr_err("Failed to add BO object to VM. ret == %d\n", + ret); + goto err_vmadd; + } + + bo_va_entry->va = va; + bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev, + mem->mapping_flags); + bo_va_entry->kgd_dev = (void *)adev; + list_add(&bo_va_entry->bo_list, list_bo_va); + + if (p_bo_va_entry) + *p_bo_va_entry = bo_va_entry; + + /* Allocate validate page tables if needed */ + ret = vm_validate_pt_pd_bos(vm); + if (ret) { + pr_err("validate_pt_pd_bos() failed\n"); + goto err_alloc_pts; + } + + return 0; + +err_alloc_pts: + amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); + list_del(&bo_va_entry->bo_list); +err_vmadd: + kfree(bo_va_entry); + return ret; +} + +static void remove_bo_from_vm(struct amdgpu_device *adev, + struct kfd_bo_va_list *entry, unsigned long size) +{ + pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n", + entry->va, + entry->va + size, entry); + amdgpu_vm_bo_rmv(adev, entry->bo_va); + list_del(&entry->bo_list); + kfree(entry); +} + +static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, + struct amdkfd_process_info *process_info, + bool userptr) +{ + struct ttm_validate_buffer *entry = &mem->validate_list; + struct amdgpu_bo *bo = mem->bo; + + INIT_LIST_HEAD(&entry->head); + entry->num_shared = 1; + entry->bo = &bo->tbo; + mutex_lock(&process_info->lock); + if (userptr) + list_add_tail(&entry->head, &process_info->userptr_valid_list); + else + list_add_tail(&entry->head, &process_info->kfd_bo_list); + mutex_unlock(&process_info->lock); +} + +static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, + struct amdkfd_process_info *process_info) +{ + struct ttm_validate_buffer *bo_list_entry; + + bo_list_entry = &mem->validate_list; + mutex_lock(&process_info->lock); + list_del(&bo_list_entry->head); + mutex_unlock(&process_info->lock); +} + +/* Initializes user pages. It registers the MMU notifier and validates + * the userptr BO in the GTT domain. + * + * The BO must already be on the userptr_valid_list. Otherwise an + * eviction and restore may happen that leaves the new BO unmapped + * with the user mode queues running. + * + * Takes the process_info->lock to protect against concurrent restore + * workers. + * + * Returns 0 for success, negative errno for errors. + */ +static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm, + uint64_t user_addr) +{ + struct amdkfd_process_info *process_info = mem->process_info; + struct amdgpu_bo *bo = mem->bo; + struct ttm_operation_ctx ctx = { true, false }; + int ret = 0; + + mutex_lock(&process_info->lock); + + ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0); + if (ret) { + pr_err("%s: Failed to set userptr: %d\n", __func__, ret); + goto out; + } + + ret = amdgpu_mn_register(bo, user_addr); + if (ret) { + pr_err("%s: Failed to register MMU notifier: %d\n", + __func__, ret); + goto out; + } + + ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); + if (ret) { + pr_err("%s: Failed to get user pages: %d\n", __func__, ret); + goto unregister_out; + } + + ret = amdgpu_bo_reserve(bo, true); + if (ret) { + pr_err("%s: Failed to reserve BO\n", __func__); + goto release_out; + } + amdgpu_bo_placement_from_domain(bo, mem->domain); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (ret) + pr_err("%s: failed to validate BO\n", __func__); + amdgpu_bo_unreserve(bo); + +release_out: + amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); +unregister_out: + if (ret) + amdgpu_mn_unregister(bo); +out: + mutex_unlock(&process_info->lock); + return ret; +} + +/* Reserving a BO and its page table BOs must happen atomically to + * avoid deadlocks. Some operations update multiple VMs at once. Track + * all the reservation info in a context structure. Optionally a sync + * object can track VM updates. + */ +struct bo_vm_reservation_context { + struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */ + unsigned int n_vms; /* Number of VMs reserved */ + struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries */ + struct ww_acquire_ctx ticket; /* Reservation ticket */ + struct list_head list, duplicates; /* BO lists */ + struct amdgpu_sync *sync; /* Pointer to sync object */ + bool reserved; /* Whether BOs are reserved */ +}; + +enum bo_vm_match { + BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ + BO_VM_MAPPED, /* Match VMs where a BO is mapped */ + BO_VM_ALL, /* Match all VMs a BO was added to */ +}; + +/** + * reserve_bo_and_vm - reserve a BO and a VM unconditionally. + * @mem: KFD BO structure. + * @vm: the VM to reserve. + * @ctx: the struct that will be used in unreserve_bo_and_vms(). + */ +static int reserve_bo_and_vm(struct kgd_mem *mem, + struct amdgpu_vm *vm, + struct bo_vm_reservation_context *ctx) +{ + struct amdgpu_bo *bo = mem->bo; + int ret; + + WARN_ON(!vm); + + ctx->reserved = false; + ctx->n_vms = 1; + ctx->sync = &mem->sync; + + INIT_LIST_HEAD(&ctx->list); + INIT_LIST_HEAD(&ctx->duplicates); + + ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL); + if (!ctx->vm_pd) + return -ENOMEM; + + ctx->kfd_bo.priority = 0; + ctx->kfd_bo.tv.bo = &bo->tbo; + ctx->kfd_bo.tv.num_shared = 1; + list_add(&ctx->kfd_bo.tv.head, &ctx->list); + + amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]); + + ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, + false, &ctx->duplicates, true); + if (!ret) + ctx->reserved = true; + else { + pr_err("Failed to reserve buffers in ttm\n"); + kfree(ctx->vm_pd); + ctx->vm_pd = NULL; + } + + return ret; +} + +/** + * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally + * @mem: KFD BO structure. + * @vm: the VM to reserve. If NULL, then all VMs associated with the BO + * is used. Otherwise, a single VM associated with the BO. + * @map_type: the mapping status that will be used to filter the VMs. + * @ctx: the struct that will be used in unreserve_bo_and_vms(). + * + * Returns 0 for success, negative for failure. + */ +static int reserve_bo_and_cond_vms(struct kgd_mem *mem, + struct amdgpu_vm *vm, enum bo_vm_match map_type, + struct bo_vm_reservation_context *ctx) +{ + struct amdgpu_bo *bo = mem->bo; + struct kfd_bo_va_list *entry; + unsigned int i; + int ret; + + ctx->reserved = false; + ctx->n_vms = 0; + ctx->vm_pd = NULL; + ctx->sync = &mem->sync; + + INIT_LIST_HEAD(&ctx->list); + INIT_LIST_HEAD(&ctx->duplicates); + + list_for_each_entry(entry, &mem->bo_va_list, bo_list) { + if ((vm && vm != entry->bo_va->base.vm) || + (entry->is_mapped != map_type + && map_type != BO_VM_ALL)) + continue; + + ctx->n_vms++; + } + + if (ctx->n_vms != 0) { + ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), + GFP_KERNEL); + if (!ctx->vm_pd) + return -ENOMEM; + } + + ctx->kfd_bo.priority = 0; + ctx->kfd_bo.tv.bo = &bo->tbo; + ctx->kfd_bo.tv.num_shared = 1; + list_add(&ctx->kfd_bo.tv.head, &ctx->list); + + i = 0; + list_for_each_entry(entry, &mem->bo_va_list, bo_list) { + if ((vm && vm != entry->bo_va->base.vm) || + (entry->is_mapped != map_type + && map_type != BO_VM_ALL)) + continue; + + amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list, + &ctx->vm_pd[i]); + i++; + } + + ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, + false, &ctx->duplicates, true); + if (!ret) + ctx->reserved = true; + else + pr_err("Failed to reserve buffers in ttm.\n"); + + if (ret) { + kfree(ctx->vm_pd); + ctx->vm_pd = NULL; + } + + return ret; +} + +/** + * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context + * @ctx: Reservation context to unreserve + * @wait: Optionally wait for a sync object representing pending VM updates + * @intr: Whether the wait is interruptible + * + * Also frees any resources allocated in + * reserve_bo_and_(cond_)vm(s). Returns the status from + * amdgpu_sync_wait. + */ +static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, + bool wait, bool intr) +{ + int ret = 0; + + if (wait) + ret = amdgpu_sync_wait(ctx->sync, intr); + + if (ctx->reserved) + ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list); + kfree(ctx->vm_pd); + + ctx->sync = NULL; + + ctx->reserved = false; + ctx->vm_pd = NULL; + + return ret; +} + +static int unmap_bo_from_gpuvm(struct amdgpu_device *adev, + struct kfd_bo_va_list *entry, + struct amdgpu_sync *sync) +{ + struct amdgpu_bo_va *bo_va = entry->bo_va; + struct amdgpu_vm *vm = bo_va->base.vm; + + amdgpu_vm_bo_unmap(adev, bo_va, entry->va); + + amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); + + amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false); + + return 0; +} + +static int update_gpuvm_pte(struct amdgpu_device *adev, + struct kfd_bo_va_list *entry, + struct amdgpu_sync *sync) +{ + int ret; + struct amdgpu_bo_va *bo_va = entry->bo_va; + + /* Update the page tables */ + ret = amdgpu_vm_bo_update(adev, bo_va, false); + if (ret) { + pr_err("amdgpu_vm_bo_update failed\n"); + return ret; + } + + return amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false); +} + +static int map_bo_to_gpuvm(struct amdgpu_device *adev, + struct kfd_bo_va_list *entry, struct amdgpu_sync *sync, + bool no_update_pte) +{ + int ret; + + /* Set virtual address for the allocation */ + ret = amdgpu_vm_bo_map(adev, entry->bo_va, entry->va, 0, + amdgpu_bo_size(entry->bo_va->base.bo), + entry->pte_flags); + if (ret) { + pr_err("Failed to map VA 0x%llx in vm. ret %d\n", + entry->va, ret); + return ret; + } + + if (no_update_pte) + return 0; + + ret = update_gpuvm_pte(adev, entry, sync); + if (ret) { + pr_err("update_gpuvm_pte() failed\n"); + goto update_gpuvm_pte_failed; + } + + return 0; + +update_gpuvm_pte_failed: + unmap_bo_from_gpuvm(adev, entry, sync); + return ret; +} + +static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size) +{ + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); + + if (!sg) + return NULL; + if (sg_alloc_table(sg, 1, GFP_KERNEL)) { + kfree(sg); + return NULL; + } + sg->sgl->dma_address = addr; + sg->sgl->length = size; +#ifdef CONFIG_NEED_SG_DMA_LENGTH + sg->sgl->dma_length = size; +#endif + return sg; +} + +static int process_validate_vms(struct amdkfd_process_info *process_info) +{ + struct amdgpu_vm *peer_vm; + int ret; + + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) { + ret = vm_validate_pt_pd_bos(peer_vm); + if (ret) + return ret; + } + + return 0; +} + +static int process_sync_pds_resv(struct amdkfd_process_info *process_info, + struct amdgpu_sync *sync) +{ + struct amdgpu_vm *peer_vm; + int ret; + + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) { + struct amdgpu_bo *pd = peer_vm->root.base.bo; + + ret = amdgpu_sync_resv(NULL, + sync, pd->tbo.resv, + AMDGPU_FENCE_OWNER_KFD, false); + if (ret) + return ret; + } + + return 0; +} + +static int process_update_pds(struct amdkfd_process_info *process_info, + struct amdgpu_sync *sync) +{ + struct amdgpu_vm *peer_vm; + int ret; + + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) { + ret = vm_update_pds(peer_vm, sync); + if (ret) + return ret; + } + + return 0; +} + +static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, + struct dma_fence **ef) +{ + struct amdkfd_process_info *info = NULL; + int ret; + + if (!*process_info) { + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + mutex_init(&info->lock); + INIT_LIST_HEAD(&info->vm_list_head); + INIT_LIST_HEAD(&info->kfd_bo_list); + INIT_LIST_HEAD(&info->userptr_valid_list); + INIT_LIST_HEAD(&info->userptr_inval_list); + + info->eviction_fence = + amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), + current->mm); + if (!info->eviction_fence) { + pr_err("Failed to create eviction fence\n"); + ret = -ENOMEM; + goto create_evict_fence_fail; + } + + info->pid = get_task_pid(current->group_leader, PIDTYPE_PID); + atomic_set(&info->evicted_bos, 0); + INIT_DELAYED_WORK(&info->restore_userptr_work, + amdgpu_amdkfd_restore_userptr_worker); + + *process_info = info; + *ef = dma_fence_get(&info->eviction_fence->base); + } + + vm->process_info = *process_info; + + /* Validate page directory and attach eviction fence */ + ret = amdgpu_bo_reserve(vm->root.base.bo, true); + if (ret) + goto reserve_pd_fail; + ret = vm_validate_pt_pd_bos(vm); + if (ret) { + pr_err("validate_pt_pd_bos() failed\n"); + goto validate_pd_fail; + } + ret = amdgpu_bo_sync_wait(vm->root.base.bo, + AMDGPU_FENCE_OWNER_KFD, false); + if (ret) + goto wait_pd_fail; + ret = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv, 1); + if (ret) + goto reserve_shared_fail; + amdgpu_bo_fence(vm->root.base.bo, + &vm->process_info->eviction_fence->base, true); + amdgpu_bo_unreserve(vm->root.base.bo); + + /* Update process info */ + mutex_lock(&vm->process_info->lock); + list_add_tail(&vm->vm_list_node, + &(vm->process_info->vm_list_head)); + vm->process_info->n_vms++; + mutex_unlock(&vm->process_info->lock); + + return 0; + +reserve_shared_fail: +wait_pd_fail: +validate_pd_fail: + amdgpu_bo_unreserve(vm->root.base.bo); +reserve_pd_fail: + vm->process_info = NULL; + if (info) { + /* Two fence references: one in info and one in *ef */ + dma_fence_put(&info->eviction_fence->base); + dma_fence_put(*ef); + *ef = NULL; + *process_info = NULL; + put_pid(info->pid); +create_evict_fence_fail: + mutex_destroy(&info->lock); + kfree(info); + } + return ret; +} + +int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid, + void **vm, void **process_info, + struct dma_fence **ef) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *new_vm; + int ret; + + new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL); + if (!new_vm) + return -ENOMEM; + + /* Initialize AMDGPU part of the VM */ + ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, pasid); + if (ret) { + pr_err("Failed init vm ret %d\n", ret); + goto amdgpu_vm_init_fail; + } + + /* Initialize KFD part of the VM and process info */ + ret = init_kfd_vm(new_vm, process_info, ef); + if (ret) + goto init_kfd_vm_fail; + + *vm = (void *) new_vm; + + return 0; + +init_kfd_vm_fail: + amdgpu_vm_fini(adev, new_vm); +amdgpu_vm_init_fail: + kfree(new_vm); + return ret; +} + +int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, + struct file *filp, unsigned int pasid, + void **vm, void **process_info, + struct dma_fence **ef) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct drm_file *drm_priv = filp->private_data; + struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv; + struct amdgpu_vm *avm = &drv_priv->vm; + int ret; + + /* Already a compute VM? */ + if (avm->process_info) + return -EINVAL; + + /* Convert VM into a compute VM */ + ret = amdgpu_vm_make_compute(adev, avm, pasid); + if (ret) + return ret; + + /* Initialize KFD part of the VM and process info */ + ret = init_kfd_vm(avm, process_info, ef); + if (ret) + return ret; + + *vm = (void *)avm; + + return 0; +} + +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct amdkfd_process_info *process_info = vm->process_info; + struct amdgpu_bo *pd = vm->root.base.bo; + + if (!process_info) + return; + + /* Release eviction fence from PD */ + amdgpu_bo_reserve(pd, false); + amdgpu_bo_fence(pd, NULL, false); + amdgpu_bo_unreserve(pd); + + /* Update process info */ + mutex_lock(&process_info->lock); + process_info->n_vms--; + list_del(&vm->vm_list_node); + mutex_unlock(&process_info->lock); + + /* Release per-process resources when last compute VM is destroyed */ + if (!process_info->n_vms) { + WARN_ON(!list_empty(&process_info->kfd_bo_list)); + WARN_ON(!list_empty(&process_info->userptr_valid_list)); + WARN_ON(!list_empty(&process_info->userptr_inval_list)); + + dma_fence_put(&process_info->eviction_fence->base); + cancel_delayed_work_sync(&process_info->restore_userptr_work); + put_pid(process_info->pid); + mutex_destroy(&process_info->lock); + kfree(process_info); + } +} + +void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + + if (WARN_ON(!kgd || !vm)) + return; + + pr_debug("Destroying process vm %p\n", vm); + + /* Release the VM context */ + amdgpu_vm_fini(adev, avm); + kfree(vm); +} + +void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + + if (WARN_ON(!kgd || !vm)) + return; + + pr_debug("Releasing process vm %p\n", vm); + + /* The original pasid of amdgpu vm has already been + * released during making a amdgpu vm to a compute vm + * The current pasid is managed by kfd and will be + * released on kfd process destroy. Set amdgpu pasid + * to 0 to avoid duplicate release. + */ + amdgpu_vm_release_compute(adev, avm); +} + +uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm) +{ + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + struct amdgpu_bo *pd = avm->root.base.bo; + struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); + + if (adev->asic_type < CHIP_VEGA10) + return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; + return avm->pd_phys_addr; +} + +int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( + struct kgd_dev *kgd, uint64_t va, uint64_t size, + void *vm, struct kgd_mem **mem, + uint64_t *offset, uint32_t flags) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + enum ttm_bo_type bo_type = ttm_bo_type_device; + struct sg_table *sg = NULL; + uint64_t user_addr = 0; + struct amdgpu_bo *bo; + struct amdgpu_bo_param bp; + int byte_align; + u32 domain, alloc_domain; + u64 alloc_flags; + uint32_t mapping_flags; + int ret; + + /* + * Check on which domain to allocate BO + */ + if (flags & ALLOC_MEM_FLAGS_VRAM) { + domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; + alloc_flags = AMDGPU_GEM_CREATE_VRAM_CLEARED; + alloc_flags |= (flags & ALLOC_MEM_FLAGS_PUBLIC) ? + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : + AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + } else if (flags & ALLOC_MEM_FLAGS_GTT) { + domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; + alloc_flags = 0; + } else if (flags & ALLOC_MEM_FLAGS_USERPTR) { + domain = AMDGPU_GEM_DOMAIN_GTT; + alloc_domain = AMDGPU_GEM_DOMAIN_CPU; + alloc_flags = 0; + if (!offset || !*offset) + return -EINVAL; + user_addr = *offset; + } else if (flags & (ALLOC_MEM_FLAGS_DOORBELL | + ALLOC_MEM_FLAGS_MMIO_REMAP)) { + domain = AMDGPU_GEM_DOMAIN_GTT; + alloc_domain = AMDGPU_GEM_DOMAIN_CPU; + bo_type = ttm_bo_type_sg; + alloc_flags = 0; + if (size > UINT_MAX) + return -EINVAL; + sg = create_doorbell_sg(*offset, size); + if (!sg) + return -ENOMEM; + } else { + return -EINVAL; + } + + *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); + if (!*mem) { + ret = -ENOMEM; + goto err; + } + INIT_LIST_HEAD(&(*mem)->bo_va_list); + mutex_init(&(*mem)->lock); + (*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); + + /* Workaround for AQL queue wraparound bug. Map the same + * memory twice. That means we only actually allocate half + * the memory. + */ + if ((*mem)->aql_queue) + size = size >> 1; + + /* Workaround for TLB bug on older VI chips */ + byte_align = (adev->family == AMDGPU_FAMILY_VI && + adev->asic_type != CHIP_FIJI && + adev->asic_type != CHIP_POLARIS10 && + adev->asic_type != CHIP_POLARIS11 && + adev->asic_type != CHIP_POLARIS12 && + adev->asic_type != CHIP_VEGAM) ? + VI_BO_SIZE_ALIGN : 1; + + mapping_flags = AMDGPU_VM_PAGE_READABLE; + if (flags & ALLOC_MEM_FLAGS_WRITABLE) + mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; + if (flags & ALLOC_MEM_FLAGS_EXECUTABLE) + mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; + if (flags & ALLOC_MEM_FLAGS_COHERENT) + mapping_flags |= AMDGPU_VM_MTYPE_UC; + else + mapping_flags |= AMDGPU_VM_MTYPE_NC; + (*mem)->mapping_flags = mapping_flags; + + amdgpu_sync_create(&(*mem)->sync); + + ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, alloc_domain, !!sg); + if (ret) { + pr_debug("Insufficient system memory\n"); + goto err_reserve_limit; + } + + pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n", + va, size, domain_string(alloc_domain)); + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = byte_align; + bp.domain = alloc_domain; + bp.flags = alloc_flags; + bp.type = bo_type; + bp.resv = NULL; + ret = amdgpu_bo_create(adev, &bp, &bo); + if (ret) { + pr_debug("Failed to create BO on domain %s. ret %d\n", + domain_string(alloc_domain), ret); + goto err_bo_create; + } + if (bo_type == ttm_bo_type_sg) { + bo->tbo.sg = sg; + bo->tbo.ttm->sg = sg; + } + bo->kfd_bo = *mem; + (*mem)->bo = bo; + if (user_addr) + bo->flags |= AMDGPU_AMDKFD_USERPTR_BO; + + (*mem)->va = va; + (*mem)->domain = domain; + (*mem)->mapped_to_gpu_memory = 0; + (*mem)->process_info = avm->process_info; + add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); + + if (user_addr) { + ret = init_user_pages(*mem, current->mm, user_addr); + if (ret) + goto allocate_init_user_pages_failed; + } + + if (offset) + *offset = amdgpu_bo_mmap_offset(bo); + + return 0; + +allocate_init_user_pages_failed: + remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); + amdgpu_bo_unref(&bo); + /* Don't unreserve system mem limit twice */ + goto err_reserve_limit; +err_bo_create: + unreserve_mem_limit(adev, size, alloc_domain, !!sg); +err_reserve_limit: + mutex_destroy(&(*mem)->lock); + kfree(*mem); +err: + if (sg) { + sg_free_table(sg); + kfree(sg); + } + return ret; +} + +int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem) +{ + struct amdkfd_process_info *process_info = mem->process_info; + unsigned long bo_size = mem->bo->tbo.mem.size; + struct kfd_bo_va_list *entry, *tmp; + struct bo_vm_reservation_context ctx; + struct ttm_validate_buffer *bo_list_entry; + int ret; + + mutex_lock(&mem->lock); + + if (mem->mapped_to_gpu_memory > 0) { + pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", + mem->va, bo_size); + mutex_unlock(&mem->lock); + return -EBUSY; + } + + mutex_unlock(&mem->lock); + /* lock is not needed after this, since mem is unused and will + * be freed anyway + */ + + /* No more MMU notifiers */ + amdgpu_mn_unregister(mem->bo); + + /* Make sure restore workers don't access the BO any more */ + bo_list_entry = &mem->validate_list; + mutex_lock(&process_info->lock); + list_del(&bo_list_entry->head); + mutex_unlock(&process_info->lock); + + ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); + if (unlikely(ret)) + return ret; + + /* The eviction fence should be removed by the last unmap. + * TODO: Log an error condition if the bo still has the eviction fence + * attached + */ + amdgpu_amdkfd_remove_eviction_fence(mem->bo, + process_info->eviction_fence); + pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, + mem->va + bo_size * (1 + mem->aql_queue)); + + /* Remove from VM internal data structures */ + list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list) + remove_bo_from_vm((struct amdgpu_device *)entry->kgd_dev, + entry, bo_size); + + ret = unreserve_bo_and_vms(&ctx, false, false); + + /* Free the sync object */ + amdgpu_sync_free(&mem->sync); + + /* If the SG is not NULL, it's one we created for a doorbell or mmio + * remap BO. We need to free it. + */ + if (mem->bo->tbo.sg) { + sg_free_table(mem->bo->tbo.sg); + kfree(mem->bo->tbo.sg); + } + + /* Free the BO*/ + amdgpu_bo_unref(&mem->bo); + mutex_destroy(&mem->lock); + kfree(mem); + + return ret; +} + +int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem, void *vm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + int ret; + struct amdgpu_bo *bo; + uint32_t domain; + struct kfd_bo_va_list *entry; + struct bo_vm_reservation_context ctx; + struct kfd_bo_va_list *bo_va_entry = NULL; + struct kfd_bo_va_list *bo_va_entry_aql = NULL; + unsigned long bo_size; + bool is_invalid_userptr = false; + + bo = mem->bo; + if (!bo) { + pr_err("Invalid BO when mapping memory to GPU\n"); + return -EINVAL; + } + + /* Make sure restore is not running concurrently. Since we + * don't map invalid userptr BOs, we rely on the next restore + * worker to do the mapping + */ + mutex_lock(&mem->process_info->lock); + + /* Lock mmap-sem. If we find an invalid userptr BO, we can be + * sure that the MMU notifier is no longer running + * concurrently and the queues are actually stopped + */ + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { + down_write(¤t->mm->mmap_sem); + is_invalid_userptr = atomic_read(&mem->invalid); + up_write(¤t->mm->mmap_sem); + } + + mutex_lock(&mem->lock); + + domain = mem->domain; + bo_size = bo->tbo.mem.size; + + pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", + mem->va, + mem->va + bo_size * (1 + mem->aql_queue), + vm, domain_string(domain)); + + ret = reserve_bo_and_vm(mem, vm, &ctx); + if (unlikely(ret)) + goto out; + + /* Userptr can be marked as "not invalid", but not actually be + * validated yet (still in the system domain). In that case + * the queues are still stopped and we can leave mapping for + * the next restore worker + */ + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && + bo->tbo.mem.mem_type == TTM_PL_SYSTEM) + is_invalid_userptr = true; + + if (check_if_add_bo_to_vm(avm, mem)) { + ret = add_bo_to_vm(adev, mem, avm, false, + &bo_va_entry); + if (ret) + goto add_bo_to_vm_failed; + if (mem->aql_queue) { + ret = add_bo_to_vm(adev, mem, avm, + true, &bo_va_entry_aql); + if (ret) + goto add_bo_to_vm_failed_aql; + } + } else { + ret = vm_validate_pt_pd_bos(avm); + if (unlikely(ret)) + goto add_bo_to_vm_failed; + } + + if (mem->mapped_to_gpu_memory == 0 && + !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { + /* Validate BO only once. The eviction fence gets added to BO + * the first time it is mapped. Validate will wait for all + * background evictions to complete. + */ + ret = amdgpu_amdkfd_bo_validate(bo, domain, true); + if (ret) { + pr_debug("Validate failed\n"); + goto map_bo_to_gpuvm_failed; + } + } + + list_for_each_entry(entry, &mem->bo_va_list, bo_list) { + if (entry->bo_va->base.vm == vm && !entry->is_mapped) { + pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", + entry->va, entry->va + bo_size, + entry); + + ret = map_bo_to_gpuvm(adev, entry, ctx.sync, + is_invalid_userptr); + if (ret) { + pr_err("Failed to map bo to gpuvm\n"); + goto map_bo_to_gpuvm_failed; + } + + ret = vm_update_pds(vm, ctx.sync); + if (ret) { + pr_err("Failed to update page directories\n"); + goto map_bo_to_gpuvm_failed; + } + + entry->is_mapped = true; + mem->mapped_to_gpu_memory++; + pr_debug("\t INC mapping count %d\n", + mem->mapped_to_gpu_memory); + } + } + + if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count) + amdgpu_bo_fence(bo, + &avm->process_info->eviction_fence->base, + true); + ret = unreserve_bo_and_vms(&ctx, false, false); + + goto out; + +map_bo_to_gpuvm_failed: + if (bo_va_entry_aql) + remove_bo_from_vm(adev, bo_va_entry_aql, bo_size); +add_bo_to_vm_failed_aql: + if (bo_va_entry) + remove_bo_from_vm(adev, bo_va_entry, bo_size); +add_bo_to_vm_failed: + unreserve_bo_and_vms(&ctx, false, false); +out: + mutex_unlock(&mem->process_info->lock); + mutex_unlock(&mem->lock); + return ret; +} + +int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( + struct kgd_dev *kgd, struct kgd_mem *mem, void *vm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdkfd_process_info *process_info = + ((struct amdgpu_vm *)vm)->process_info; + unsigned long bo_size = mem->bo->tbo.mem.size; + struct kfd_bo_va_list *entry; + struct bo_vm_reservation_context ctx; + int ret; + + mutex_lock(&mem->lock); + + ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx); + if (unlikely(ret)) + goto out; + /* If no VMs were reserved, it means the BO wasn't actually mapped */ + if (ctx.n_vms == 0) { + ret = -EINVAL; + goto unreserve_out; + } + + ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm); + if (unlikely(ret)) + goto unreserve_out; + + pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", + mem->va, + mem->va + bo_size * (1 + mem->aql_queue), + vm); + + list_for_each_entry(entry, &mem->bo_va_list, bo_list) { + if (entry->bo_va->base.vm == vm && entry->is_mapped) { + pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", + entry->va, + entry->va + bo_size, + entry); + + ret = unmap_bo_from_gpuvm(adev, entry, ctx.sync); + if (ret == 0) { + entry->is_mapped = false; + } else { + pr_err("failed to unmap VA 0x%llx\n", + mem->va); + goto unreserve_out; + } + + mem->mapped_to_gpu_memory--; + pr_debug("\t DEC mapping count %d\n", + mem->mapped_to_gpu_memory); + } + } + + /* If BO is unmapped from all VMs, unfence it. It can be evicted if + * required. + */ + if (mem->mapped_to_gpu_memory == 0 && + !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count) + amdgpu_amdkfd_remove_eviction_fence(mem->bo, + process_info->eviction_fence); + +unreserve_out: + unreserve_bo_and_vms(&ctx, false, false); +out: + mutex_unlock(&mem->lock); + return ret; +} + +int amdgpu_amdkfd_gpuvm_sync_memory( + struct kgd_dev *kgd, struct kgd_mem *mem, bool intr) +{ + struct amdgpu_sync sync; + int ret; + + amdgpu_sync_create(&sync); + + mutex_lock(&mem->lock); + amdgpu_sync_clone(&mem->sync, &sync); + mutex_unlock(&mem->lock); + + ret = amdgpu_sync_wait(&sync, intr); + amdgpu_sync_free(&sync); + return ret; +} + +int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, + struct kgd_mem *mem, void **kptr, uint64_t *size) +{ + int ret; + struct amdgpu_bo *bo = mem->bo; + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { + pr_err("userptr can't be mapped to kernel\n"); + return -EINVAL; + } + + /* delete kgd_mem from kfd_bo_list to avoid re-validating + * this BO in BO's restoring after eviction. + */ + mutex_lock(&mem->process_info->lock); + + ret = amdgpu_bo_reserve(bo, true); + if (ret) { + pr_err("Failed to reserve bo. ret %d\n", ret); + goto bo_reserve_failed; + } + + ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); + if (ret) { + pr_err("Failed to pin bo. ret %d\n", ret); + goto pin_failed; + } + + ret = amdgpu_bo_kmap(bo, kptr); + if (ret) { + pr_err("Failed to map bo to kernel. ret %d\n", ret); + goto kmap_failed; + } + + amdgpu_amdkfd_remove_eviction_fence( + bo, mem->process_info->eviction_fence); + list_del_init(&mem->validate_list.head); + + if (size) + *size = amdgpu_bo_size(bo); + + amdgpu_bo_unreserve(bo); + + mutex_unlock(&mem->process_info->lock); + return 0; + +kmap_failed: + amdgpu_bo_unpin(bo); +pin_failed: + amdgpu_bo_unreserve(bo); +bo_reserve_failed: + mutex_unlock(&mem->process_info->lock); + + return ret; +} + +int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, + struct kfd_vm_fault_info *mem) +{ + struct amdgpu_device *adev; + + adev = (struct amdgpu_device *)kgd; + if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { + *mem = *adev->gmc.vm_fault_info; + mb(); + atomic_set(&adev->gmc.vm_fault_info_updated, 0); + } + return 0; +} + +int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, + struct dma_buf *dma_buf, + uint64_t va, void *vm, + struct kgd_mem **mem, uint64_t *size, + uint64_t *mmap_offset) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct drm_gem_object *obj; + struct amdgpu_bo *bo; + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + + if (dma_buf->ops != &amdgpu_dmabuf_ops) + /* Can't handle non-graphics buffers */ + return -EINVAL; + + obj = dma_buf->priv; + if (obj->dev->dev_private != adev) + /* Can't handle buffers from other devices */ + return -EINVAL; + + bo = gem_to_amdgpu_bo(obj); + if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT))) + /* Only VRAM and GTT BOs are supported */ + return -EINVAL; + + *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); + if (!*mem) + return -ENOMEM; + + if (size) + *size = amdgpu_bo_size(bo); + + if (mmap_offset) + *mmap_offset = amdgpu_bo_mmap_offset(bo); + + INIT_LIST_HEAD(&(*mem)->bo_va_list); + mutex_init(&(*mem)->lock); + (*mem)->mapping_flags = + AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE | + AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_NC; + + (*mem)->bo = amdgpu_bo_ref(bo); + (*mem)->va = va; + (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? + AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; + (*mem)->mapped_to_gpu_memory = 0; + (*mem)->process_info = avm->process_info; + add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); + amdgpu_sync_create(&(*mem)->sync); + + return 0; +} + +/* Evict a userptr BO by stopping the queues if necessary + * + * Runs in MMU notifier, may be in RECLAIM_FS context. This means it + * cannot do any memory allocations, and cannot take any locks that + * are held elsewhere while allocating memory. Therefore this is as + * simple as possible, using atomic counters. + * + * It doesn't do anything to the BO itself. The real work happens in + * restore, where we get updated page addresses. This function only + * ensures that GPU access to the BO is stopped. + */ +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, + struct mm_struct *mm) +{ + struct amdkfd_process_info *process_info = mem->process_info; + int invalid, evicted_bos; + int r = 0; + + invalid = atomic_inc_return(&mem->invalid); + evicted_bos = atomic_inc_return(&process_info->evicted_bos); + if (evicted_bos == 1) { + /* First eviction, stop the queues */ + r = kgd2kfd_quiesce_mm(mm); + if (r) + pr_err("Failed to quiesce KFD\n"); + schedule_delayed_work(&process_info->restore_userptr_work, + msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); + } + + return r; +} + +/* Update invalid userptr BOs + * + * Moves invalidated (evicted) userptr BOs from userptr_valid_list to + * userptr_inval_list and updates user pages for all BOs that have + * been invalidated since their last update. + */ +static int update_invalid_user_pages(struct amdkfd_process_info *process_info, + struct mm_struct *mm) +{ + struct kgd_mem *mem, *tmp_mem; + struct amdgpu_bo *bo; + struct ttm_operation_ctx ctx = { false, false }; + int invalid, ret; + + /* Move all invalidated BOs to the userptr_inval_list and + * release their user pages by migration to the CPU domain + */ + list_for_each_entry_safe(mem, tmp_mem, + &process_info->userptr_valid_list, + validate_list.head) { + if (!atomic_read(&mem->invalid)) + continue; /* BO is still valid */ + + bo = mem->bo; + + if (amdgpu_bo_reserve(bo, true)) + return -EAGAIN; + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + amdgpu_bo_unreserve(bo); + if (ret) { + pr_err("%s: Failed to invalidate userptr BO\n", + __func__); + return -EAGAIN; + } + + list_move_tail(&mem->validate_list.head, + &process_info->userptr_inval_list); + } + + if (list_empty(&process_info->userptr_inval_list)) + return 0; /* All evicted userptr BOs were freed */ + + /* Go through userptr_inval_list and update any invalid user_pages */ + list_for_each_entry(mem, &process_info->userptr_inval_list, + validate_list.head) { + invalid = atomic_read(&mem->invalid); + if (!invalid) + /* BO hasn't been invalidated since the last + * revalidation attempt. Keep its BO list. + */ + continue; + + bo = mem->bo; + + /* Get updated user pages */ + ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); + if (ret) { + pr_debug("%s: Failed to get user pages: %d\n", + __func__, ret); + + /* Return error -EBUSY or -ENOMEM, retry restore */ + return ret; + } + + amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); + + /* Mark the BO as valid unless it was invalidated + * again concurrently. + */ + if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) + return -EAGAIN; + } + + return 0; +} + +/* Validate invalid userptr BOs + * + * Validates BOs on the userptr_inval_list, and moves them back to the + * userptr_valid_list. Also updates GPUVM page tables with new page + * addresses and waits for the page table updates to complete. + */ +static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) +{ + struct amdgpu_bo_list_entry *pd_bo_list_entries; + struct list_head resv_list, duplicates; + struct ww_acquire_ctx ticket; + struct amdgpu_sync sync; + + struct amdgpu_vm *peer_vm; + struct kgd_mem *mem, *tmp_mem; + struct amdgpu_bo *bo; + struct ttm_operation_ctx ctx = { false, false }; + int i, ret; + + pd_bo_list_entries = kcalloc(process_info->n_vms, + sizeof(struct amdgpu_bo_list_entry), + GFP_KERNEL); + if (!pd_bo_list_entries) { + pr_err("%s: Failed to allocate PD BO list entries\n", __func__); + ret = -ENOMEM; + goto out_no_mem; + } + + INIT_LIST_HEAD(&resv_list); + INIT_LIST_HEAD(&duplicates); + + /* Get all the page directory BOs that need to be reserved */ + i = 0; + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) + amdgpu_vm_get_pd_bo(peer_vm, &resv_list, + &pd_bo_list_entries[i++]); + /* Add the userptr_inval_list entries to resv_list */ + list_for_each_entry(mem, &process_info->userptr_inval_list, + validate_list.head) { + list_add_tail(&mem->resv_list.head, &resv_list); + mem->resv_list.bo = mem->validate_list.bo; + mem->resv_list.num_shared = mem->validate_list.num_shared; + } + + /* Reserve all BOs and page tables for validation */ + ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates, + true); + WARN(!list_empty(&duplicates), "Duplicates should be empty"); + if (ret) + goto out_free; + + amdgpu_sync_create(&sync); + + ret = process_validate_vms(process_info); + if (ret) + goto unreserve_out; + + /* Validate BOs and update GPUVM page tables */ + list_for_each_entry_safe(mem, tmp_mem, + &process_info->userptr_inval_list, + validate_list.head) { + struct kfd_bo_va_list *bo_va_entry; + + bo = mem->bo; + + /* Validate the BO if we got user pages */ + if (bo->tbo.ttm->pages[0]) { + amdgpu_bo_placement_from_domain(bo, mem->domain); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (ret) { + pr_err("%s: failed to validate BO\n", __func__); + goto unreserve_out; + } + } + + list_move_tail(&mem->validate_list.head, + &process_info->userptr_valid_list); + + /* Update mapping. If the BO was not validated + * (because we couldn't get user pages), this will + * clear the page table entries, which will result in + * VM faults if the GPU tries to access the invalid + * memory. + */ + list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) { + if (!bo_va_entry->is_mapped) + continue; + + ret = update_gpuvm_pte((struct amdgpu_device *) + bo_va_entry->kgd_dev, + bo_va_entry, &sync); + if (ret) { + pr_err("%s: update PTE failed\n", __func__); + /* make sure this gets validated again */ + atomic_inc(&mem->invalid); + goto unreserve_out; + } + } + } + + /* Update page directories */ + ret = process_update_pds(process_info, &sync); + +unreserve_out: + ttm_eu_backoff_reservation(&ticket, &resv_list); + amdgpu_sync_wait(&sync, false); + amdgpu_sync_free(&sync); +out_free: + kfree(pd_bo_list_entries); +out_no_mem: + + return ret; +} + +/* Worker callback to restore evicted userptr BOs + * + * Tries to update and validate all userptr BOs. If successful and no + * concurrent evictions happened, the queues are restarted. Otherwise, + * reschedule for another attempt later. + */ +static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct amdkfd_process_info *process_info = + container_of(dwork, struct amdkfd_process_info, + restore_userptr_work); + struct task_struct *usertask; + struct mm_struct *mm; + int evicted_bos; + + evicted_bos = atomic_read(&process_info->evicted_bos); + if (!evicted_bos) + return; + + /* Reference task and mm in case of concurrent process termination */ + usertask = get_pid_task(process_info->pid, PIDTYPE_PID); + if (!usertask) + return; + mm = get_task_mm(usertask); + if (!mm) { + put_task_struct(usertask); + return; + } + + mutex_lock(&process_info->lock); + + if (update_invalid_user_pages(process_info, mm)) + goto unlock_out; + /* userptr_inval_list can be empty if all evicted userptr BOs + * have been freed. In that case there is nothing to validate + * and we can just restart the queues. + */ + if (!list_empty(&process_info->userptr_inval_list)) { + if (atomic_read(&process_info->evicted_bos) != evicted_bos) + goto unlock_out; /* Concurrent eviction, try again */ + + if (validate_invalid_user_pages(process_info)) + goto unlock_out; + } + /* Final check for concurrent evicton and atomic update. If + * another eviction happens after successful update, it will + * be a first eviction that calls quiesce_mm. The eviction + * reference counting inside KFD will handle this case. + */ + if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) != + evicted_bos) + goto unlock_out; + evicted_bos = 0; + if (kgd2kfd_resume_mm(mm)) { + pr_err("%s: Failed to resume KFD\n", __func__); + /* No recovery from this failure. Probably the CP is + * hanging. No point trying again. + */ + } + +unlock_out: + mutex_unlock(&process_info->lock); + mmput(mm); + put_task_struct(usertask); + + /* If validation failed, reschedule another attempt */ + if (evicted_bos) + schedule_delayed_work(&process_info->restore_userptr_work, + msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); +} + +/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given + * KFD process identified by process_info + * + * @process_info: amdkfd_process_info of the KFD process + * + * After memory eviction, restore thread calls this function. The function + * should be called when the Process is still valid. BO restore involves - + * + * 1. Release old eviction fence and create new one + * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. + * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of + * BOs that need to be reserved. + * 4. Reserve all the BOs + * 5. Validate of PD and PT BOs. + * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence + * 7. Add fence to all PD and PT BOs. + * 8. Unreserve all BOs + */ +int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) +{ + struct amdgpu_bo_list_entry *pd_bo_list; + struct amdkfd_process_info *process_info = info; + struct amdgpu_vm *peer_vm; + struct kgd_mem *mem; + struct bo_vm_reservation_context ctx; + struct amdgpu_amdkfd_fence *new_fence; + int ret = 0, i; + struct list_head duplicate_save; + struct amdgpu_sync sync_obj; + + INIT_LIST_HEAD(&duplicate_save); + INIT_LIST_HEAD(&ctx.list); + INIT_LIST_HEAD(&ctx.duplicates); + + pd_bo_list = kcalloc(process_info->n_vms, + sizeof(struct amdgpu_bo_list_entry), + GFP_KERNEL); + if (!pd_bo_list) + return -ENOMEM; + + i = 0; + mutex_lock(&process_info->lock); + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) + amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]); + + /* Reserve all BOs and page tables/directory. Add all BOs from + * kfd_bo_list to ctx.list + */ + list_for_each_entry(mem, &process_info->kfd_bo_list, + validate_list.head) { + + list_add_tail(&mem->resv_list.head, &ctx.list); + mem->resv_list.bo = mem->validate_list.bo; + mem->resv_list.num_shared = mem->validate_list.num_shared; + } + + ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list, + false, &duplicate_save, true); + if (ret) { + pr_debug("Memory eviction: TTM Reserve Failed. Try again\n"); + goto ttm_reserve_fail; + } + + amdgpu_sync_create(&sync_obj); + + /* Validate PDs and PTs */ + ret = process_validate_vms(process_info); + if (ret) + goto validate_map_fail; + + ret = process_sync_pds_resv(process_info, &sync_obj); + if (ret) { + pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); + goto validate_map_fail; + } + + /* Validate BOs and map them to GPUVM (update VM page tables). */ + list_for_each_entry(mem, &process_info->kfd_bo_list, + validate_list.head) { + + struct amdgpu_bo *bo = mem->bo; + uint32_t domain = mem->domain; + struct kfd_bo_va_list *bo_va_entry; + + ret = amdgpu_amdkfd_bo_validate(bo, domain, false); + if (ret) { + pr_debug("Memory eviction: Validate BOs failed. Try again\n"); + goto validate_map_fail; + } + ret = amdgpu_sync_fence(NULL, &sync_obj, bo->tbo.moving, false); + if (ret) { + pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); + goto validate_map_fail; + } + list_for_each_entry(bo_va_entry, &mem->bo_va_list, + bo_list) { + ret = update_gpuvm_pte((struct amdgpu_device *) + bo_va_entry->kgd_dev, + bo_va_entry, + &sync_obj); + if (ret) { + pr_debug("Memory eviction: update PTE failed. Try again\n"); + goto validate_map_fail; + } + } + } + + /* Update page directories */ + ret = process_update_pds(process_info, &sync_obj); + if (ret) { + pr_debug("Memory eviction: update PDs failed. Try again\n"); + goto validate_map_fail; + } + + /* Wait for validate and PT updates to finish */ + amdgpu_sync_wait(&sync_obj, false); + + /* Release old eviction fence and create new one, because fence only + * goes from unsignaled to signaled, fence cannot be reused. + * Use context and mm from the old fence. + */ + new_fence = amdgpu_amdkfd_fence_create( + process_info->eviction_fence->base.context, + process_info->eviction_fence->mm); + if (!new_fence) { + pr_err("Failed to create eviction fence\n"); + ret = -ENOMEM; + goto validate_map_fail; + } + dma_fence_put(&process_info->eviction_fence->base); + process_info->eviction_fence = new_fence; + *ef = dma_fence_get(&new_fence->base); + + /* Attach new eviction fence to all BOs */ + list_for_each_entry(mem, &process_info->kfd_bo_list, + validate_list.head) + amdgpu_bo_fence(mem->bo, + &process_info->eviction_fence->base, true); + + /* Attach eviction fence to PD / PT BOs */ + list_for_each_entry(peer_vm, &process_info->vm_list_head, + vm_list_node) { + struct amdgpu_bo *bo = peer_vm->root.base.bo; + + amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); + } + +validate_map_fail: + ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list); + amdgpu_sync_free(&sync_obj); +ttm_reserve_fail: + mutex_unlock(&process_info->lock); + kfree(pd_bo_list); + return ret; +} + +int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) +{ + struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; + struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; + int ret; + + if (!info || !gws) + return -EINVAL; + + *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); + if (!*mem) + return -ENOMEM; + + mutex_init(&(*mem)->lock); + (*mem)->bo = amdgpu_bo_ref(gws_bo); + (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; + (*mem)->process_info = process_info; + add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); + amdgpu_sync_create(&(*mem)->sync); + + + /* Validate gws bo the first time it is added to process */ + mutex_lock(&(*mem)->process_info->lock); + ret = amdgpu_bo_reserve(gws_bo, false); + if (unlikely(ret)) { + pr_err("Reserve gws bo failed %d\n", ret); + goto bo_reservation_failure; + } + + ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); + if (ret) { + pr_err("GWS BO validate failed %d\n", ret); + goto bo_validation_failure; + } + /* GWS resource is shared b/t amdgpu and amdkfd + * Add process eviction fence to bo so they can + * evict each other. + */ + ret = reservation_object_reserve_shared(gws_bo->tbo.resv, 1); + if (ret) + goto reserve_shared_fail; + amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true); + amdgpu_bo_unreserve(gws_bo); + mutex_unlock(&(*mem)->process_info->lock); + + return ret; + +reserve_shared_fail: +bo_validation_failure: + amdgpu_bo_unreserve(gws_bo); +bo_reservation_failure: + mutex_unlock(&(*mem)->process_info->lock); + amdgpu_sync_free(&(*mem)->sync); + remove_kgd_mem_from_kfd_bo_list(*mem, process_info); + amdgpu_bo_unref(&gws_bo); + mutex_destroy(&(*mem)->lock); + kfree(*mem); + *mem = NULL; + return ret; +} + +int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) +{ + int ret; + struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; + struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; + struct amdgpu_bo *gws_bo = kgd_mem->bo; + + /* Remove BO from process's validate list so restore worker won't touch + * it anymore + */ + remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); + + ret = amdgpu_bo_reserve(gws_bo, false); + if (unlikely(ret)) { + pr_err("Reserve gws bo failed %d\n", ret); + //TODO add BO back to validate_list? + return ret; + } + amdgpu_amdkfd_remove_eviction_fence(gws_bo, + process_info->eviction_fence); + amdgpu_bo_unreserve(gws_bo); + amdgpu_sync_free(&kgd_mem->sync); + amdgpu_bo_unref(&gws_bo); + mutex_destroy(&kgd_mem->lock); + kfree(mem); + return 0; +}
diff --git a/amdgpu/amdgpu_atombios.c b/amdgpu/amdgpu_atombios.c new file mode 100644 index 0000000..1c9d40f --- /dev/null +++ b/amdgpu/amdgpu_atombios.c
@@ -0,0 +1,2054 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "amdgpu_atomfirmware.h" +#include "amdgpu_i2c.h" +#include "amdgpu_display.h" + +#include "atom.h" +#include "atom-bits.h" +#include "atombios_encoders.h" +#include "bif/bif_4_1_d.h" + +static void amdgpu_atombios_lookup_i2c_gpio_quirks(struct amdgpu_device *adev, + ATOM_GPIO_I2C_ASSIGMENT *gpio, + u8 index) +{ + +} + +static struct amdgpu_i2c_bus_rec amdgpu_atombios_get_bus_rec_for_i2c_gpio(ATOM_GPIO_I2C_ASSIGMENT *gpio) +{ + struct amdgpu_i2c_bus_rec i2c; + + memset(&i2c, 0, sizeof(struct amdgpu_i2c_bus_rec)); + + i2c.mask_clk_reg = le16_to_cpu(gpio->usClkMaskRegisterIndex); + i2c.mask_data_reg = le16_to_cpu(gpio->usDataMaskRegisterIndex); + i2c.en_clk_reg = le16_to_cpu(gpio->usClkEnRegisterIndex); + i2c.en_data_reg = le16_to_cpu(gpio->usDataEnRegisterIndex); + i2c.y_clk_reg = le16_to_cpu(gpio->usClkY_RegisterIndex); + i2c.y_data_reg = le16_to_cpu(gpio->usDataY_RegisterIndex); + i2c.a_clk_reg = le16_to_cpu(gpio->usClkA_RegisterIndex); + i2c.a_data_reg = le16_to_cpu(gpio->usDataA_RegisterIndex); + i2c.mask_clk_mask = (1 << gpio->ucClkMaskShift); + i2c.mask_data_mask = (1 << gpio->ucDataMaskShift); + i2c.en_clk_mask = (1 << gpio->ucClkEnShift); + i2c.en_data_mask = (1 << gpio->ucDataEnShift); + i2c.y_clk_mask = (1 << gpio->ucClkY_Shift); + i2c.y_data_mask = (1 << gpio->ucDataY_Shift); + i2c.a_clk_mask = (1 << gpio->ucClkA_Shift); + i2c.a_data_mask = (1 << gpio->ucDataA_Shift); + + if (gpio->sucI2cId.sbfAccess.bfHW_Capable) + i2c.hw_capable = true; + else + i2c.hw_capable = false; + + if (gpio->sucI2cId.ucAccess == 0xa0) + i2c.mm_i2c = true; + else + i2c.mm_i2c = false; + + i2c.i2c_id = gpio->sucI2cId.ucAccess; + + if (i2c.mask_clk_reg) + i2c.valid = true; + else + i2c.valid = false; + + return i2c; +} + +struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device *adev, + uint8_t id) +{ + struct atom_context *ctx = adev->mode_info.atom_context; + ATOM_GPIO_I2C_ASSIGMENT *gpio; + struct amdgpu_i2c_bus_rec i2c; + int index = GetIndexIntoMasterTable(DATA, GPIO_I2C_Info); + struct _ATOM_GPIO_I2C_INFO *i2c_info; + uint16_t data_offset, size; + int i, num_indices; + + memset(&i2c, 0, sizeof(struct amdgpu_i2c_bus_rec)); + i2c.valid = false; + + if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { + i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); + + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_GPIO_I2C_ASSIGMENT); + + gpio = &i2c_info->asGPIO_Info[0]; + for (i = 0; i < num_indices; i++) { + + amdgpu_atombios_lookup_i2c_gpio_quirks(adev, gpio, i); + + if (gpio->sucI2cId.ucAccess == id) { + i2c = amdgpu_atombios_get_bus_rec_for_i2c_gpio(gpio); + break; + } + gpio = (ATOM_GPIO_I2C_ASSIGMENT *) + ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT)); + } + } + + return i2c; +} + +void amdgpu_atombios_i2c_init(struct amdgpu_device *adev) +{ + struct atom_context *ctx = adev->mode_info.atom_context; + ATOM_GPIO_I2C_ASSIGMENT *gpio; + struct amdgpu_i2c_bus_rec i2c; + int index = GetIndexIntoMasterTable(DATA, GPIO_I2C_Info); + struct _ATOM_GPIO_I2C_INFO *i2c_info; + uint16_t data_offset, size; + int i, num_indices; + char stmp[32]; + + if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { + i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); + + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_GPIO_I2C_ASSIGMENT); + + gpio = &i2c_info->asGPIO_Info[0]; + for (i = 0; i < num_indices; i++) { + amdgpu_atombios_lookup_i2c_gpio_quirks(adev, gpio, i); + + i2c = amdgpu_atombios_get_bus_rec_for_i2c_gpio(gpio); + + if (i2c.valid) { + sprintf(stmp, "0x%x", i2c.i2c_id); + adev->i2c_bus[i] = amdgpu_i2c_create(adev->ddev, &i2c, stmp); + } + gpio = (ATOM_GPIO_I2C_ASSIGMENT *) + ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT)); + } + } +} + +struct amdgpu_gpio_rec +amdgpu_atombios_lookup_gpio(struct amdgpu_device *adev, + u8 id) +{ + struct atom_context *ctx = adev->mode_info.atom_context; + struct amdgpu_gpio_rec gpio; + int index = GetIndexIntoMasterTable(DATA, GPIO_Pin_LUT); + struct _ATOM_GPIO_PIN_LUT *gpio_info; + ATOM_GPIO_PIN_ASSIGNMENT *pin; + u16 data_offset, size; + int i, num_indices; + + memset(&gpio, 0, sizeof(struct amdgpu_gpio_rec)); + gpio.valid = false; + + if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { + gpio_info = (struct _ATOM_GPIO_PIN_LUT *)(ctx->bios + data_offset); + + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_GPIO_PIN_ASSIGNMENT); + + pin = gpio_info->asGPIO_Pin; + for (i = 0; i < num_indices; i++) { + if (id == pin->ucGPIO_ID) { + gpio.id = pin->ucGPIO_ID; + gpio.reg = le16_to_cpu(pin->usGpioPin_AIndex); + gpio.shift = pin->ucGpioPinBitShift; + gpio.mask = (1 << pin->ucGpioPinBitShift); + gpio.valid = true; + break; + } + pin = (ATOM_GPIO_PIN_ASSIGNMENT *) + ((u8 *)pin + sizeof(ATOM_GPIO_PIN_ASSIGNMENT)); + } + } + + return gpio; +} + +static struct amdgpu_hpd +amdgpu_atombios_get_hpd_info_from_gpio(struct amdgpu_device *adev, + struct amdgpu_gpio_rec *gpio) +{ + struct amdgpu_hpd hpd; + u32 reg; + + memset(&hpd, 0, sizeof(struct amdgpu_hpd)); + + reg = amdgpu_display_hpd_get_gpio_reg(adev); + + hpd.gpio = *gpio; + if (gpio->reg == reg) { + switch(gpio->mask) { + case (1 << 0): + hpd.hpd = AMDGPU_HPD_1; + break; + case (1 << 8): + hpd.hpd = AMDGPU_HPD_2; + break; + case (1 << 16): + hpd.hpd = AMDGPU_HPD_3; + break; + case (1 << 24): + hpd.hpd = AMDGPU_HPD_4; + break; + case (1 << 26): + hpd.hpd = AMDGPU_HPD_5; + break; + case (1 << 28): + hpd.hpd = AMDGPU_HPD_6; + break; + default: + hpd.hpd = AMDGPU_HPD_NONE; + break; + } + } else + hpd.hpd = AMDGPU_HPD_NONE; + return hpd; +} + +static const int object_connector_convert[] = { + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_DVII, + DRM_MODE_CONNECTOR_DVII, + DRM_MODE_CONNECTOR_DVID, + DRM_MODE_CONNECTOR_DVID, + DRM_MODE_CONNECTOR_VGA, + DRM_MODE_CONNECTOR_Composite, + DRM_MODE_CONNECTOR_SVIDEO, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_9PinDIN, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_HDMIA, + DRM_MODE_CONNECTOR_HDMIB, + DRM_MODE_CONNECTOR_LVDS, + DRM_MODE_CONNECTOR_9PinDIN, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_Unknown, + DRM_MODE_CONNECTOR_DisplayPort, + DRM_MODE_CONNECTOR_eDP, + DRM_MODE_CONNECTOR_Unknown +}; + +bool amdgpu_atombios_has_dce_engine_info(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + struct atom_context *ctx = mode_info->atom_context; + int index = GetIndexIntoMasterTable(DATA, Object_Header); + u16 size, data_offset; + u8 frev, crev; + ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj; + ATOM_OBJECT_HEADER *obj_header; + + if (!amdgpu_atom_parse_data_header(ctx, index, &size, &frev, &crev, &data_offset)) + return false; + + if (crev < 2) + return false; + + obj_header = (ATOM_OBJECT_HEADER *) (ctx->bios + data_offset); + path_obj = (ATOM_DISPLAY_OBJECT_PATH_TABLE *) + (ctx->bios + data_offset + + le16_to_cpu(obj_header->usDisplayPathTableOffset)); + + if (path_obj->ucNumOfDispPath) + return true; + else + return false; +} + +bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + struct atom_context *ctx = mode_info->atom_context; + int index = GetIndexIntoMasterTable(DATA, Object_Header); + u16 size, data_offset; + u8 frev, crev; + ATOM_CONNECTOR_OBJECT_TABLE *con_obj; + ATOM_ENCODER_OBJECT_TABLE *enc_obj; + ATOM_OBJECT_TABLE *router_obj; + ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj; + ATOM_OBJECT_HEADER *obj_header; + int i, j, k, path_size, device_support; + int connector_type; + u16 conn_id, connector_object_id; + struct amdgpu_i2c_bus_rec ddc_bus; + struct amdgpu_router router; + struct amdgpu_gpio_rec gpio; + struct amdgpu_hpd hpd; + + if (!amdgpu_atom_parse_data_header(ctx, index, &size, &frev, &crev, &data_offset)) + return false; + + if (crev < 2) + return false; + + obj_header = (ATOM_OBJECT_HEADER *) (ctx->bios + data_offset); + path_obj = (ATOM_DISPLAY_OBJECT_PATH_TABLE *) + (ctx->bios + data_offset + + le16_to_cpu(obj_header->usDisplayPathTableOffset)); + con_obj = (ATOM_CONNECTOR_OBJECT_TABLE *) + (ctx->bios + data_offset + + le16_to_cpu(obj_header->usConnectorObjectTableOffset)); + enc_obj = (ATOM_ENCODER_OBJECT_TABLE *) + (ctx->bios + data_offset + + le16_to_cpu(obj_header->usEncoderObjectTableOffset)); + router_obj = (ATOM_OBJECT_TABLE *) + (ctx->bios + data_offset + + le16_to_cpu(obj_header->usRouterObjectTableOffset)); + device_support = le16_to_cpu(obj_header->usDeviceSupport); + + path_size = 0; + for (i = 0; i < path_obj->ucNumOfDispPath; i++) { + uint8_t *addr = (uint8_t *) path_obj->asDispPath; + ATOM_DISPLAY_OBJECT_PATH *path; + addr += path_size; + path = (ATOM_DISPLAY_OBJECT_PATH *) addr; + path_size += le16_to_cpu(path->usSize); + + if (device_support & le16_to_cpu(path->usDeviceTag)) { + uint8_t con_obj_id, con_obj_num, con_obj_type; + + con_obj_id = + (le16_to_cpu(path->usConnObjectId) & OBJECT_ID_MASK) + >> OBJECT_ID_SHIFT; + con_obj_num = + (le16_to_cpu(path->usConnObjectId) & ENUM_ID_MASK) + >> ENUM_ID_SHIFT; + con_obj_type = + (le16_to_cpu(path->usConnObjectId) & + OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT; + + /* Skip TV/CV support */ + if ((le16_to_cpu(path->usDeviceTag) == + ATOM_DEVICE_TV1_SUPPORT) || + (le16_to_cpu(path->usDeviceTag) == + ATOM_DEVICE_CV_SUPPORT)) + continue; + + if (con_obj_id >= ARRAY_SIZE(object_connector_convert)) { + DRM_ERROR("invalid con_obj_id %d for device tag 0x%04x\n", + con_obj_id, le16_to_cpu(path->usDeviceTag)); + continue; + } + + connector_type = + object_connector_convert[con_obj_id]; + connector_object_id = con_obj_id; + + if (connector_type == DRM_MODE_CONNECTOR_Unknown) + continue; + + router.ddc_valid = false; + router.cd_valid = false; + for (j = 0; j < ((le16_to_cpu(path->usSize) - 8) / 2); j++) { + uint8_t grph_obj_id, grph_obj_num, grph_obj_type; + + grph_obj_id = + (le16_to_cpu(path->usGraphicObjIds[j]) & + OBJECT_ID_MASK) >> OBJECT_ID_SHIFT; + grph_obj_num = + (le16_to_cpu(path->usGraphicObjIds[j]) & + ENUM_ID_MASK) >> ENUM_ID_SHIFT; + grph_obj_type = + (le16_to_cpu(path->usGraphicObjIds[j]) & + OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT; + + if (grph_obj_type == GRAPH_OBJECT_TYPE_ENCODER) { + for (k = 0; k < enc_obj->ucNumberOfObjects; k++) { + u16 encoder_obj = le16_to_cpu(enc_obj->asObjects[k].usObjectID); + if (le16_to_cpu(path->usGraphicObjIds[j]) == encoder_obj) { + ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *) + (ctx->bios + data_offset + + le16_to_cpu(enc_obj->asObjects[k].usRecordOffset)); + ATOM_ENCODER_CAP_RECORD *cap_record; + u16 caps = 0; + + while (record->ucRecordSize > 0 && + record->ucRecordType > 0 && + record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) { + switch (record->ucRecordType) { + case ATOM_ENCODER_CAP_RECORD_TYPE: + cap_record =(ATOM_ENCODER_CAP_RECORD *) + record; + caps = le16_to_cpu(cap_record->usEncoderCap); + break; + } + record = (ATOM_COMMON_RECORD_HEADER *) + ((char *)record + record->ucRecordSize); + } + amdgpu_display_add_encoder(adev, encoder_obj, + le16_to_cpu(path->usDeviceTag), + caps); + } + } + } else if (grph_obj_type == GRAPH_OBJECT_TYPE_ROUTER) { + for (k = 0; k < router_obj->ucNumberOfObjects; k++) { + u16 router_obj_id = le16_to_cpu(router_obj->asObjects[k].usObjectID); + if (le16_to_cpu(path->usGraphicObjIds[j]) == router_obj_id) { + ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *) + (ctx->bios + data_offset + + le16_to_cpu(router_obj->asObjects[k].usRecordOffset)); + ATOM_I2C_RECORD *i2c_record; + ATOM_I2C_ID_CONFIG_ACCESS *i2c_config; + ATOM_ROUTER_DDC_PATH_SELECT_RECORD *ddc_path; + ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD *cd_path; + ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT *router_src_dst_table = + (ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT *) + (ctx->bios + data_offset + + le16_to_cpu(router_obj->asObjects[k].usSrcDstTableOffset)); + u8 *num_dst_objs = (u8 *) + ((u8 *)router_src_dst_table + 1 + + (router_src_dst_table->ucNumberOfSrc * 2)); + u16 *dst_objs = (u16 *)(num_dst_objs + 1); + int enum_id; + + router.router_id = router_obj_id; + for (enum_id = 0; enum_id < (*num_dst_objs); enum_id++) { + if (le16_to_cpu(path->usConnObjectId) == + le16_to_cpu(dst_objs[enum_id])) + break; + } + + while (record->ucRecordSize > 0 && + record->ucRecordType > 0 && + record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) { + switch (record->ucRecordType) { + case ATOM_I2C_RECORD_TYPE: + i2c_record = + (ATOM_I2C_RECORD *) + record; + i2c_config = + (ATOM_I2C_ID_CONFIG_ACCESS *) + &i2c_record->sucI2cId; + router.i2c_info = + amdgpu_atombios_lookup_i2c_gpio(adev, + i2c_config-> + ucAccess); + router.i2c_addr = i2c_record->ucI2CAddr >> 1; + break; + case ATOM_ROUTER_DDC_PATH_SELECT_RECORD_TYPE: + ddc_path = (ATOM_ROUTER_DDC_PATH_SELECT_RECORD *) + record; + router.ddc_valid = true; + router.ddc_mux_type = ddc_path->ucMuxType; + router.ddc_mux_control_pin = ddc_path->ucMuxControlPin; + router.ddc_mux_state = ddc_path->ucMuxState[enum_id]; + break; + case ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD_TYPE: + cd_path = (ATOM_ROUTER_DATA_CLOCK_PATH_SELECT_RECORD *) + record; + router.cd_valid = true; + router.cd_mux_type = cd_path->ucMuxType; + router.cd_mux_control_pin = cd_path->ucMuxControlPin; + router.cd_mux_state = cd_path->ucMuxState[enum_id]; + break; + } + record = (ATOM_COMMON_RECORD_HEADER *) + ((char *)record + record->ucRecordSize); + } + } + } + } + } + + /* look up gpio for ddc, hpd */ + ddc_bus.valid = false; + hpd.hpd = AMDGPU_HPD_NONE; + if ((le16_to_cpu(path->usDeviceTag) & + (ATOM_DEVICE_TV_SUPPORT | ATOM_DEVICE_CV_SUPPORT)) == 0) { + for (j = 0; j < con_obj->ucNumberOfObjects; j++) { + if (le16_to_cpu(path->usConnObjectId) == + le16_to_cpu(con_obj->asObjects[j]. + usObjectID)) { + ATOM_COMMON_RECORD_HEADER + *record = + (ATOM_COMMON_RECORD_HEADER + *) + (ctx->bios + data_offset + + le16_to_cpu(con_obj-> + asObjects[j]. + usRecordOffset)); + ATOM_I2C_RECORD *i2c_record; + ATOM_HPD_INT_RECORD *hpd_record; + ATOM_I2C_ID_CONFIG_ACCESS *i2c_config; + + while (record->ucRecordSize > 0 && + record->ucRecordType > 0 && + record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) { + switch (record->ucRecordType) { + case ATOM_I2C_RECORD_TYPE: + i2c_record = + (ATOM_I2C_RECORD *) + record; + i2c_config = + (ATOM_I2C_ID_CONFIG_ACCESS *) + &i2c_record->sucI2cId; + ddc_bus = amdgpu_atombios_lookup_i2c_gpio(adev, + i2c_config-> + ucAccess); + break; + case ATOM_HPD_INT_RECORD_TYPE: + hpd_record = + (ATOM_HPD_INT_RECORD *) + record; + gpio = amdgpu_atombios_lookup_gpio(adev, + hpd_record->ucHPDIntGPIOID); + hpd = amdgpu_atombios_get_hpd_info_from_gpio(adev, &gpio); + hpd.plugged_state = hpd_record->ucPlugged_PinState; + break; + } + record = + (ATOM_COMMON_RECORD_HEADER + *) ((char *)record + + + record-> + ucRecordSize); + } + break; + } + } + } + + /* needed for aux chan transactions */ + ddc_bus.hpd = hpd.hpd; + + conn_id = le16_to_cpu(path->usConnObjectId); + + amdgpu_display_add_connector(adev, + conn_id, + le16_to_cpu(path->usDeviceTag), + connector_type, &ddc_bus, + connector_object_id, + &hpd, + &router); + + } + } + + amdgpu_link_encoder_connector(adev->ddev); + + return true; +} + +union firmware_info { + ATOM_FIRMWARE_INFO info; + ATOM_FIRMWARE_INFO_V1_2 info_12; + ATOM_FIRMWARE_INFO_V1_3 info_13; + ATOM_FIRMWARE_INFO_V1_4 info_14; + ATOM_FIRMWARE_INFO_V2_1 info_21; + ATOM_FIRMWARE_INFO_V2_2 info_22; +}; + +int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, FirmwareInfo); + uint8_t frev, crev; + uint16_t data_offset; + int ret = -EINVAL; + + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + int i; + struct amdgpu_pll *ppll = &adev->clock.ppll[0]; + struct amdgpu_pll *spll = &adev->clock.spll; + struct amdgpu_pll *mpll = &adev->clock.mpll; + union firmware_info *firmware_info = + (union firmware_info *)(mode_info->atom_context->bios + + data_offset); + /* pixel clocks */ + ppll->reference_freq = + le16_to_cpu(firmware_info->info.usReferenceClock); + ppll->reference_div = 0; + + ppll->pll_out_min = + le32_to_cpu(firmware_info->info_12.ulMinPixelClockPLL_Output); + ppll->pll_out_max = + le32_to_cpu(firmware_info->info.ulMaxPixelClockPLL_Output); + + ppll->lcd_pll_out_min = + le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100; + if (ppll->lcd_pll_out_min == 0) + ppll->lcd_pll_out_min = ppll->pll_out_min; + ppll->lcd_pll_out_max = + le16_to_cpu(firmware_info->info_14.usLcdMaxPixelClockPLL_Output) * 100; + if (ppll->lcd_pll_out_max == 0) + ppll->lcd_pll_out_max = ppll->pll_out_max; + + if (ppll->pll_out_min == 0) + ppll->pll_out_min = 64800; + + ppll->pll_in_min = + le16_to_cpu(firmware_info->info.usMinPixelClockPLL_Input); + ppll->pll_in_max = + le16_to_cpu(firmware_info->info.usMaxPixelClockPLL_Input); + + ppll->min_post_div = 2; + ppll->max_post_div = 0x7f; + ppll->min_frac_feedback_div = 0; + ppll->max_frac_feedback_div = 9; + ppll->min_ref_div = 2; + ppll->max_ref_div = 0x3ff; + ppll->min_feedback_div = 4; + ppll->max_feedback_div = 0xfff; + ppll->best_vco = 0; + + for (i = 1; i < AMDGPU_MAX_PPLL; i++) + adev->clock.ppll[i] = *ppll; + + /* system clock */ + spll->reference_freq = + le16_to_cpu(firmware_info->info_21.usCoreReferenceClock); + spll->reference_div = 0; + + spll->pll_out_min = + le16_to_cpu(firmware_info->info.usMinEngineClockPLL_Output); + spll->pll_out_max = + le32_to_cpu(firmware_info->info.ulMaxEngineClockPLL_Output); + + /* ??? */ + if (spll->pll_out_min == 0) + spll->pll_out_min = 64800; + + spll->pll_in_min = + le16_to_cpu(firmware_info->info.usMinEngineClockPLL_Input); + spll->pll_in_max = + le16_to_cpu(firmware_info->info.usMaxEngineClockPLL_Input); + + spll->min_post_div = 1; + spll->max_post_div = 1; + spll->min_ref_div = 2; + spll->max_ref_div = 0xff; + spll->min_feedback_div = 4; + spll->max_feedback_div = 0xff; + spll->best_vco = 0; + + /* memory clock */ + mpll->reference_freq = + le16_to_cpu(firmware_info->info_21.usMemoryReferenceClock); + mpll->reference_div = 0; + + mpll->pll_out_min = + le16_to_cpu(firmware_info->info.usMinMemoryClockPLL_Output); + mpll->pll_out_max = + le32_to_cpu(firmware_info->info.ulMaxMemoryClockPLL_Output); + + /* ??? */ + if (mpll->pll_out_min == 0) + mpll->pll_out_min = 64800; + + mpll->pll_in_min = + le16_to_cpu(firmware_info->info.usMinMemoryClockPLL_Input); + mpll->pll_in_max = + le16_to_cpu(firmware_info->info.usMaxMemoryClockPLL_Input); + + adev->clock.default_sclk = + le32_to_cpu(firmware_info->info.ulDefaultEngineClock); + adev->clock.default_mclk = + le32_to_cpu(firmware_info->info.ulDefaultMemoryClock); + + mpll->min_post_div = 1; + mpll->max_post_div = 1; + mpll->min_ref_div = 2; + mpll->max_ref_div = 0xff; + mpll->min_feedback_div = 4; + mpll->max_feedback_div = 0xff; + mpll->best_vco = 0; + + /* disp clock */ + adev->clock.default_dispclk = + le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq); + /* set a reasonable default for DP */ + if (adev->clock.default_dispclk < 53900) { + DRM_DEBUG("Changing default dispclk from %dMhz to 600Mhz\n", + adev->clock.default_dispclk / 100); + adev->clock.default_dispclk = 60000; + } else if (adev->clock.default_dispclk <= 60000) { + DRM_DEBUG("Changing default dispclk from %dMhz to 625Mhz\n", + adev->clock.default_dispclk / 100); + adev->clock.default_dispclk = 62500; + } + adev->clock.dp_extclk = + le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq); + adev->clock.current_dispclk = adev->clock.default_dispclk; + + adev->clock.max_pixel_clock = le16_to_cpu(firmware_info->info.usMaxPixelClock); + if (adev->clock.max_pixel_clock == 0) + adev->clock.max_pixel_clock = 40000; + + /* not technically a clock, but... */ + adev->mode_info.firmware_flags = + le16_to_cpu(firmware_info->info.usFirmwareCapability.susAccess); + + ret = 0; + } + + adev->pm.current_sclk = adev->clock.default_sclk; + adev->pm.current_mclk = adev->clock.default_mclk; + + return ret; +} + +union gfx_info { + ATOM_GFX_INFO_V2_1 info; +}; + +int amdgpu_atombios_get_gfx_info(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, GFX_Info); + uint8_t frev, crev; + uint16_t data_offset; + int ret = -EINVAL; + + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + union gfx_info *gfx_info = (union gfx_info *) + (mode_info->atom_context->bios + data_offset); + + adev->gfx.config.max_shader_engines = gfx_info->info.max_shader_engines; + adev->gfx.config.max_tile_pipes = gfx_info->info.max_tile_pipes; + adev->gfx.config.max_cu_per_sh = gfx_info->info.max_cu_per_sh; + adev->gfx.config.max_sh_per_se = gfx_info->info.max_sh_per_se; + adev->gfx.config.max_backends_per_se = gfx_info->info.max_backends_per_se; + adev->gfx.config.max_texture_channel_caches = + gfx_info->info.max_texture_channel_caches; + + ret = 0; + } + return ret; +} + +union igp_info { + struct _ATOM_INTEGRATED_SYSTEM_INFO info; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_8 info_8; + struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_9 info_9; +}; + +/* + * Return vram width from integrated system info table, if available, + * or 0 if not. + */ +int amdgpu_atombios_get_vram_width(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + u16 data_offset, size; + union igp_info *igp_info; + u8 frev, crev; + + /* get any igp specific overrides */ + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 8: + case 9: + return igp_info->info_8.ucUMAChannelNumber * 64; + default: + return 0; + } + } + + return 0; +} + +static void amdgpu_atombios_get_igp_ss_overrides(struct amdgpu_device *adev, + struct amdgpu_atom_ss *ss, + int id) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo); + u16 data_offset, size; + union igp_info *igp_info; + u8 frev, crev; + u16 percentage = 0, rate = 0; + + /* get any igp specific overrides */ + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size, + &frev, &crev, &data_offset)) { + igp_info = (union igp_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 6: + switch (id) { + case ASIC_INTERNAL_SS_ON_TMDS: + percentage = le16_to_cpu(igp_info->info_6.usDVISSPercentage); + rate = le16_to_cpu(igp_info->info_6.usDVISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_HDMI: + percentage = le16_to_cpu(igp_info->info_6.usHDMISSPercentage); + rate = le16_to_cpu(igp_info->info_6.usHDMISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_LVDS: + percentage = le16_to_cpu(igp_info->info_6.usLvdsSSPercentage); + rate = le16_to_cpu(igp_info->info_6.usLvdsSSpreadRateIn10Hz); + break; + } + break; + case 7: + switch (id) { + case ASIC_INTERNAL_SS_ON_TMDS: + percentage = le16_to_cpu(igp_info->info_7.usDVISSPercentage); + rate = le16_to_cpu(igp_info->info_7.usDVISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_HDMI: + percentage = le16_to_cpu(igp_info->info_7.usHDMISSPercentage); + rate = le16_to_cpu(igp_info->info_7.usHDMISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_LVDS: + percentage = le16_to_cpu(igp_info->info_7.usLvdsSSPercentage); + rate = le16_to_cpu(igp_info->info_7.usLvdsSSpreadRateIn10Hz); + break; + } + break; + case 8: + switch (id) { + case ASIC_INTERNAL_SS_ON_TMDS: + percentage = le16_to_cpu(igp_info->info_8.usDVISSPercentage); + rate = le16_to_cpu(igp_info->info_8.usDVISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_HDMI: + percentage = le16_to_cpu(igp_info->info_8.usHDMISSPercentage); + rate = le16_to_cpu(igp_info->info_8.usHDMISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_LVDS: + percentage = le16_to_cpu(igp_info->info_8.usLvdsSSPercentage); + rate = le16_to_cpu(igp_info->info_8.usLvdsSSpreadRateIn10Hz); + break; + } + break; + case 9: + switch (id) { + case ASIC_INTERNAL_SS_ON_TMDS: + percentage = le16_to_cpu(igp_info->info_9.usDVISSPercentage); + rate = le16_to_cpu(igp_info->info_9.usDVISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_HDMI: + percentage = le16_to_cpu(igp_info->info_9.usHDMISSPercentage); + rate = le16_to_cpu(igp_info->info_9.usHDMISSpreadRateIn10Hz); + break; + case ASIC_INTERNAL_SS_ON_LVDS: + percentage = le16_to_cpu(igp_info->info_9.usLvdsSSPercentage); + rate = le16_to_cpu(igp_info->info_9.usLvdsSSpreadRateIn10Hz); + break; + } + break; + default: + DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev); + break; + } + if (percentage) + ss->percentage = percentage; + if (rate) + ss->rate = rate; + } +} + +union asic_ss_info { + struct _ATOM_ASIC_INTERNAL_SS_INFO info; + struct _ATOM_ASIC_INTERNAL_SS_INFO_V2 info_2; + struct _ATOM_ASIC_INTERNAL_SS_INFO_V3 info_3; +}; + +union asic_ss_assignment { + struct _ATOM_ASIC_SS_ASSIGNMENT v1; + struct _ATOM_ASIC_SS_ASSIGNMENT_V2 v2; + struct _ATOM_ASIC_SS_ASSIGNMENT_V3 v3; +}; + +bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev, + struct amdgpu_atom_ss *ss, + int id, u32 clock) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info); + uint16_t data_offset, size; + union asic_ss_info *ss_info; + union asic_ss_assignment *ss_assign; + uint8_t frev, crev; + int i, num_indices; + + if (id == ASIC_INTERNAL_MEMORY_SS) { + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_MEMORY_CLOCK_SS_SUPPORT)) + return false; + } + if (id == ASIC_INTERNAL_ENGINE_SS) { + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_ENGINE_CLOCK_SS_SUPPORT)) + return false; + } + + memset(ss, 0, sizeof(struct amdgpu_atom_ss)); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size, + &frev, &crev, &data_offset)) { + + ss_info = + (union asic_ss_info *)(mode_info->atom_context->bios + data_offset); + + switch (frev) { + case 1: + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_ASIC_SS_ASSIGNMENT); + + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info.asSpreadSpectrum[0]); + for (i = 0; i < num_indices; i++) { + if ((ss_assign->v1.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v1.ulTargetClockRange))) { + ss->percentage = + le16_to_cpu(ss_assign->v1.usSpreadSpectrumPercentage); + ss->type = ss_assign->v1.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v1.usSpreadRateInKhz); + ss->percentage_divider = 100; + return true; + } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT)); + } + break; + case 2: + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V2); + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info_2.asSpreadSpectrum[0]); + for (i = 0; i < num_indices; i++) { + if ((ss_assign->v2.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v2.ulTargetClockRange))) { + ss->percentage = + le16_to_cpu(ss_assign->v2.usSpreadSpectrumPercentage); + ss->type = ss_assign->v2.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v2.usSpreadRateIn10Hz); + ss->percentage_divider = 100; + if ((crev == 2) && + ((id == ASIC_INTERNAL_ENGINE_SS) || + (id == ASIC_INTERNAL_MEMORY_SS))) + ss->rate /= 100; + return true; + } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V2)); + } + break; + case 3: + num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V3); + ss_assign = (union asic_ss_assignment *)((u8 *)&ss_info->info_3.asSpreadSpectrum[0]); + for (i = 0; i < num_indices; i++) { + if ((ss_assign->v3.ucClockIndication == id) && + (clock <= le32_to_cpu(ss_assign->v3.ulTargetClockRange))) { + ss->percentage = + le16_to_cpu(ss_assign->v3.usSpreadSpectrumPercentage); + ss->type = ss_assign->v3.ucSpreadSpectrumMode; + ss->rate = le16_to_cpu(ss_assign->v3.usSpreadRateIn10Hz); + if (ss_assign->v3.ucSpreadSpectrumMode & + SS_MODE_V3_PERCENTAGE_DIV_BY_1000_MASK) + ss->percentage_divider = 1000; + else + ss->percentage_divider = 100; + if ((id == ASIC_INTERNAL_ENGINE_SS) || + (id == ASIC_INTERNAL_MEMORY_SS)) + ss->rate /= 100; + if (adev->flags & AMD_IS_APU) + amdgpu_atombios_get_igp_ss_overrides(adev, ss, id); + return true; + } + ss_assign = (union asic_ss_assignment *) + ((u8 *)ss_assign + sizeof(ATOM_ASIC_SS_ASSIGNMENT_V3)); + } + break; + default: + DRM_ERROR("Unsupported ASIC_InternalSS_Info table: %d %d\n", frev, crev); + break; + } + + } + return false; +} + +union get_clock_dividers { + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS v1; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V2 v2; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3 v3; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 v4; + struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5 v5; + struct _COMPUTE_GPU_CLOCK_INPUT_PARAMETERS_V1_6 v6_in; + struct _COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 v6_out; +}; + +int amdgpu_atombios_get_clock_dividers(struct amdgpu_device *adev, + u8 clock_type, + u32 clock, + bool strobe_mode, + struct atom_clock_dividers *dividers) +{ + union get_clock_dividers args; + int index = GetIndexIntoMasterTable(COMMAND, ComputeMemoryEnginePLL); + u8 frev, crev; + + memset(&args, 0, sizeof(args)); + memset(dividers, 0, sizeof(struct atom_clock_dividers)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 2: + case 3: + case 5: + /* r6xx, r7xx, evergreen, ni, si. + * TODO: add support for asic_type <= CHIP_RV770*/ + if (clock_type == COMPUTE_ENGINE_PLL_PARAM) { + args.v3.ulClockParams = cpu_to_le32((clock_type << 24) | clock); + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v3.ucPostDiv; + dividers->enable_post_div = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false; + dividers->enable_dithen = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true; + dividers->whole_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v3.ucRefDiv; + dividers->vco_mode = (args.v3.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0; + } else { + /* for SI we use ComputeMemoryClockParam for memory plls */ + if (adev->asic_type >= CHIP_TAHITI) + return -EINVAL; + args.v5.ulClockParams = cpu_to_le32((clock_type << 24) | clock); + if (strobe_mode) + args.v5.ucInputFlag = ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_div = args.v5.ucPostDiv; + dividers->enable_post_div = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false; + dividers->enable_dithen = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true; + dividers->whole_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v5.ucRefDiv; + dividers->vco_mode = (args.v5.ucCntlFlag & + ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0; + } + break; + case 4: + /* fusion */ + args.v4.ulClock = cpu_to_le32(clock); /* 10 khz */ + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->post_divider = dividers->post_div = args.v4.ucPostDiv; + dividers->real_clock = le32_to_cpu(args.v4.ulClock); + break; + case 6: + /* CI */ + /* COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, COMPUTE_GPUCLK_INPUT_FLAG_SCLK */ + args.v6_in.ulClock.ulComputeClockFlag = clock_type; + args.v6_in.ulClock.ulClockFreq = cpu_to_le32(clock); /* 10 khz */ + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + dividers->whole_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDiv); + dividers->frac_fb_div = le16_to_cpu(args.v6_out.ulFbDiv.usFbDivFrac); + dividers->ref_div = args.v6_out.ucPllRefDiv; + dividers->post_div = args.v6_out.ucPllPostDiv; + dividers->flags = args.v6_out.ucPllCntlFlag; + dividers->real_clock = le32_to_cpu(args.v6_out.ulClock.ulClock); + dividers->post_divider = args.v6_out.ulClock.ucPostDiv; + break; + default: + return -EINVAL; + } + return 0; +} + +int amdgpu_atombios_get_memory_pll_dividers(struct amdgpu_device *adev, + u32 clock, + bool strobe_mode, + struct atom_mpll_param *mpll_param) +{ + COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 args; + int index = GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam); + u8 frev, crev; + + memset(&args, 0, sizeof(args)); + memset(mpll_param, 0, sizeof(struct atom_mpll_param)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (frev) { + case 2: + switch (crev) { + case 1: + /* SI */ + args.ulClock = cpu_to_le32(clock); /* 10 khz */ + args.ucInputFlag = 0; + if (strobe_mode) + args.ucInputFlag |= MPLL_INPUT_FLAG_STROBE_MODE_EN; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + mpll_param->clkfrac = le16_to_cpu(args.ulFbDiv.usFbDivFrac); + mpll_param->clkf = le16_to_cpu(args.ulFbDiv.usFbDiv); + mpll_param->post_div = args.ucPostDiv; + mpll_param->dll_speed = args.ucDllSpeed; + mpll_param->bwcntl = args.ucBWCntl; + mpll_param->vco_mode = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_VCO_MODE_MASK); + mpll_param->yclk_sel = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_BYPASS_DQ_PLL) ? 1 : 0; + mpll_param->qdr = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_QDR_ENABLE) ? 1 : 0; + mpll_param->half_rate = + (args.ucPllCntlFlag & MPLL_CNTL_FLAG_AD_HALF_RATE) ? 1 : 0; + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + return 0; +} + +void amdgpu_atombios_set_engine_dram_timings(struct amdgpu_device *adev, + u32 eng_clock, u32 mem_clock) +{ + SET_ENGINE_CLOCK_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings); + u32 tmp; + + memset(&args, 0, sizeof(args)); + + tmp = eng_clock & SET_CLOCK_FREQ_MASK; + tmp |= (COMPUTE_ENGINE_PLL_PARAM << 24); + + args.ulTargetEngineClock = cpu_to_le32(tmp); + if (mem_clock) + args.sReserved.ulClock = cpu_to_le32(mem_clock & SET_CLOCK_FREQ_MASK); + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_get_default_voltages(struct amdgpu_device *adev, + u16 *vddc, u16 *vddci, u16 *mvdd) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, FirmwareInfo); + u8 frev, crev; + u16 data_offset; + union firmware_info *firmware_info; + + *vddc = 0; + *vddci = 0; + *mvdd = 0; + + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + firmware_info = + (union firmware_info *)(mode_info->atom_context->bios + + data_offset); + *vddc = le16_to_cpu(firmware_info->info_14.usBootUpVDDCVoltage); + if ((frev == 2) && (crev >= 2)) { + *vddci = le16_to_cpu(firmware_info->info_22.usBootUpVDDCIVoltage); + *mvdd = le16_to_cpu(firmware_info->info_22.usBootUpMVDDCVoltage); + } + } +} + +union set_voltage { + struct _SET_VOLTAGE_PS_ALLOCATION alloc; + struct _SET_VOLTAGE_PARAMETERS v1; + struct _SET_VOLTAGE_PARAMETERS_V2 v2; + struct _SET_VOLTAGE_PARAMETERS_V1_3 v3; +}; + +int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type, + u16 voltage_id, u16 *voltage) +{ + union set_voltage args; + int index = GetIndexIntoMasterTable(COMMAND, SetVoltage); + u8 frev, crev; + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 1: + return -EINVAL; + case 2: + args.v2.ucVoltageType = SET_VOLTAGE_GET_MAX_VOLTAGE; + args.v2.ucVoltageMode = 0; + args.v2.usVoltageLevel = 0; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + *voltage = le16_to_cpu(args.v2.usVoltageLevel); + break; + case 3: + args.v3.ucVoltageType = voltage_type; + args.v3.ucVoltageMode = ATOM_GET_VOLTAGE_LEVEL; + args.v3.usVoltageLevel = cpu_to_le16(voltage_id); + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + *voltage = le16_to_cpu(args.v3.usVoltageLevel); + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +int amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(struct amdgpu_device *adev, + u16 *voltage, + u16 leakage_idx) +{ + return amdgpu_atombios_get_max_vddc(adev, VOLTAGE_TYPE_VDDC, leakage_idx, voltage); +} + +int amdgpu_atombios_get_leakage_id_from_vbios(struct amdgpu_device *adev, + u16 *leakage_id) +{ + union set_voltage args; + int index = GetIndexIntoMasterTable(COMMAND, SetVoltage); + u8 frev, crev; + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return -EINVAL; + + switch (crev) { + case 3: + case 4: + args.v3.ucVoltageType = 0; + args.v3.ucVoltageMode = ATOM_GET_LEAKAGE_ID; + args.v3.usVoltageLevel = 0; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + *leakage_id = le16_to_cpu(args.v3.usVoltageLevel); + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +int amdgpu_atombios_get_leakage_vddc_based_on_leakage_params(struct amdgpu_device *adev, + u16 *vddc, u16 *vddci, + u16 virtual_voltage_id, + u16 vbios_voltage_id) +{ + int index = GetIndexIntoMasterTable(DATA, ASIC_ProfilingInfo); + u8 frev, crev; + u16 data_offset, size; + int i, j; + ATOM_ASIC_PROFILING_INFO_V2_1 *profile; + u16 *leakage_bin, *vddc_id_buf, *vddc_buf, *vddci_id_buf, *vddci_buf; + + *vddc = 0; + *vddci = 0; + + if (!amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) + return -EINVAL; + + profile = (ATOM_ASIC_PROFILING_INFO_V2_1 *) + (adev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 1: + return -EINVAL; + case 2: + switch (crev) { + case 1: + if (size < sizeof(ATOM_ASIC_PROFILING_INFO_V2_1)) + return -EINVAL; + leakage_bin = (u16 *) + (adev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usLeakageBinArrayOffset)); + vddc_id_buf = (u16 *) + (adev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDC_IdArrayOffset)); + vddc_buf = (u16 *) + (adev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDC_LevelArrayOffset)); + vddci_id_buf = (u16 *) + (adev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDCI_IdArrayOffset)); + vddci_buf = (u16 *) + (adev->mode_info.atom_context->bios + data_offset + + le16_to_cpu(profile->usElbVDDCI_LevelArrayOffset)); + + if (profile->ucElbVDDC_Num > 0) { + for (i = 0; i < profile->ucElbVDDC_Num; i++) { + if (vddc_id_buf[i] == virtual_voltage_id) { + for (j = 0; j < profile->ucLeakageBinNum; j++) { + if (vbios_voltage_id <= leakage_bin[j]) { + *vddc = vddc_buf[j * profile->ucElbVDDC_Num + i]; + break; + } + } + break; + } + } + } + if (profile->ucElbVDDCI_Num > 0) { + for (i = 0; i < profile->ucElbVDDCI_Num; i++) { + if (vddci_id_buf[i] == virtual_voltage_id) { + for (j = 0; j < profile->ucLeakageBinNum; j++) { + if (vbios_voltage_id <= leakage_bin[j]) { + *vddci = vddci_buf[j * profile->ucElbVDDCI_Num + i]; + break; + } + } + break; + } + } + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + + return 0; +} + +union get_voltage_info { + struct _GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 in; + struct _GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 evv_out; +}; + +int amdgpu_atombios_get_voltage_evv(struct amdgpu_device *adev, + u16 virtual_voltage_id, + u16 *voltage) +{ + int index = GetIndexIntoMasterTable(COMMAND, GetVoltageInfo); + u32 entry_id; + u32 count = adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; + union get_voltage_info args; + + for (entry_id = 0; entry_id < count; entry_id++) { + if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[entry_id].v == + virtual_voltage_id) + break; + } + + if (entry_id >= count) + return -EINVAL; + + args.in.ucVoltageType = VOLTAGE_TYPE_VDDC; + args.in.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE; + args.in.usVoltageLevel = cpu_to_le16(virtual_voltage_id); + args.in.ulSCLKFreq = + cpu_to_le32(adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[entry_id].clk); + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + *voltage = le16_to_cpu(args.evv_out.usVoltageLevel); + + return 0; +} + +union voltage_object_info { + struct _ATOM_VOLTAGE_OBJECT_INFO v1; + struct _ATOM_VOLTAGE_OBJECT_INFO_V2 v2; + struct _ATOM_VOLTAGE_OBJECT_INFO_V3_1 v3; +}; + +union voltage_object { + struct _ATOM_VOLTAGE_OBJECT v1; + struct _ATOM_VOLTAGE_OBJECT_V2 v2; + union _ATOM_VOLTAGE_OBJECT_V3 v3; +}; + + +static ATOM_VOLTAGE_OBJECT_V3 *amdgpu_atombios_lookup_voltage_object_v3(ATOM_VOLTAGE_OBJECT_INFO_V3_1 *v3, + u8 voltage_type, u8 voltage_mode) +{ + u32 size = le16_to_cpu(v3->sHeader.usStructureSize); + u32 offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]); + u8 *start = (u8*)v3; + + while (offset < size) { + ATOM_VOLTAGE_OBJECT_V3 *vo = (ATOM_VOLTAGE_OBJECT_V3 *)(start + offset); + if ((vo->asGpioVoltageObj.sHeader.ucVoltageType == voltage_type) && + (vo->asGpioVoltageObj.sHeader.ucVoltageMode == voltage_mode)) + return vo; + offset += le16_to_cpu(vo->asGpioVoltageObj.sHeader.usSize); + } + return NULL; +} + +int amdgpu_atombios_get_svi2_info(struct amdgpu_device *adev, + u8 voltage_type, + u8 *svd_gpio_id, u8 *svc_gpio_id) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (adev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 3: + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + amdgpu_atombios_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, + VOLTAGE_OBJ_SVID2); + if (voltage_object) { + *svd_gpio_id = voltage_object->v3.asSVID2Obj.ucSVDGpioId; + *svc_gpio_id = voltage_object->v3.asSVID2Obj.ucSVCGpioId; + } else { + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + + } + return 0; +} + +bool +amdgpu_atombios_is_voltage_gpio(struct amdgpu_device *adev, + u8 voltage_type, u8 voltage_mode) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + union voltage_object_info *voltage_info; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (adev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 3: + switch (crev) { + case 1: + if (amdgpu_atombios_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, voltage_mode)) + return true; + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return false; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return false; + } + + } + return false; +} + +int amdgpu_atombios_get_voltage_table(struct amdgpu_device *adev, + u8 voltage_type, u8 voltage_mode, + struct atom_voltage_table *voltage_table) +{ + int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo); + u8 frev, crev; + u16 data_offset, size; + int i; + union voltage_object_info *voltage_info; + union voltage_object *voltage_object = NULL; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + voltage_info = (union voltage_object_info *) + (adev->mode_info.atom_context->bios + data_offset); + + switch (frev) { + case 3: + switch (crev) { + case 1: + voltage_object = (union voltage_object *) + amdgpu_atombios_lookup_voltage_object_v3(&voltage_info->v3, + voltage_type, voltage_mode); + if (voltage_object) { + ATOM_GPIO_VOLTAGE_OBJECT_V3 *gpio = + &voltage_object->v3.asGpioVoltageObj; + VOLTAGE_LUT_ENTRY_V2 *lut; + if (gpio->ucGpioEntryNum > MAX_VOLTAGE_ENTRIES) + return -EINVAL; + lut = &gpio->asVolGpioLut[0]; + for (i = 0; i < gpio->ucGpioEntryNum; i++) { + voltage_table->entries[i].value = + le16_to_cpu(lut->usVoltageValue); + voltage_table->entries[i].smio_low = + le32_to_cpu(lut->ulVoltageId); + lut = (VOLTAGE_LUT_ENTRY_V2 *) + ((u8 *)lut + sizeof(VOLTAGE_LUT_ENTRY_V2)); + } + voltage_table->mask_low = le32_to_cpu(gpio->ulGpioMaskVal); + voltage_table->count = gpio->ucGpioEntryNum; + voltage_table->phase_delay = gpio->ucPhaseDelay; + return 0; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + break; + default: + DRM_ERROR("unknown voltage object table\n"); + return -EINVAL; + } + } + return -EINVAL; +} + +union vram_info { + struct _ATOM_VRAM_INFO_V3 v1_3; + struct _ATOM_VRAM_INFO_V4 v1_4; + struct _ATOM_VRAM_INFO_HEADER_V2_1 v2_1; +}; + +#define MEM_ID_MASK 0xff000000 +#define MEM_ID_SHIFT 24 +#define CLOCK_RANGE_MASK 0x00ffffff +#define CLOCK_RANGE_SHIFT 0 +#define LOW_NIBBLE_MASK 0xf +#define DATA_EQU_PREV 0 +#define DATA_FROM_TABLE 4 + +int amdgpu_atombios_init_mc_reg_table(struct amdgpu_device *adev, + u8 module_index, + struct atom_mc_reg_table *reg_table) +{ + int index = GetIndexIntoMasterTable(DATA, VRAM_Info); + u8 frev, crev, num_entries, t_mem_id, num_ranges = 0; + u32 i = 0, j; + u16 data_offset, size; + union vram_info *vram_info; + + memset(reg_table, 0, sizeof(struct atom_mc_reg_table)); + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) { + vram_info = (union vram_info *) + (adev->mode_info.atom_context->bios + data_offset); + switch (frev) { + case 1: + DRM_ERROR("old table version %d, %d\n", frev, crev); + return -EINVAL; + case 2: + switch (crev) { + case 1: + if (module_index < vram_info->v2_1.ucNumOfVRAMModule) { + ATOM_INIT_REG_BLOCK *reg_block = + (ATOM_INIT_REG_BLOCK *) + ((u8 *)vram_info + le16_to_cpu(vram_info->v2_1.usMemClkPatchTblOffset)); + ATOM_MEMORY_SETTING_DATA_BLOCK *reg_data = + (ATOM_MEMORY_SETTING_DATA_BLOCK *) + ((u8 *)reg_block + (2 * sizeof(u16)) + + le16_to_cpu(reg_block->usRegIndexTblSize)); + ATOM_INIT_REG_INDEX_FORMAT *format = ®_block->asRegIndexBuf[0]; + num_entries = (u8)((le16_to_cpu(reg_block->usRegIndexTblSize)) / + sizeof(ATOM_INIT_REG_INDEX_FORMAT)) - 1; + if (num_entries > VBIOS_MC_REGISTER_ARRAY_SIZE) + return -EINVAL; + while (i < num_entries) { + if (format->ucPreRegDataLength & ACCESS_PLACEHOLDER) + break; + reg_table->mc_reg_address[i].s1 = + (u16)(le16_to_cpu(format->usRegIndex)); + reg_table->mc_reg_address[i].pre_reg_data = + (u8)(format->ucPreRegDataLength); + i++; + format = (ATOM_INIT_REG_INDEX_FORMAT *) + ((u8 *)format + sizeof(ATOM_INIT_REG_INDEX_FORMAT)); + } + reg_table->last = i; + while ((le32_to_cpu(*(u32 *)reg_data) != END_OF_REG_DATA_BLOCK) && + (num_ranges < VBIOS_MAX_AC_TIMING_ENTRIES)) { + t_mem_id = (u8)((le32_to_cpu(*(u32 *)reg_data) & MEM_ID_MASK) + >> MEM_ID_SHIFT); + if (module_index == t_mem_id) { + reg_table->mc_reg_table_entry[num_ranges].mclk_max = + (u32)((le32_to_cpu(*(u32 *)reg_data) & CLOCK_RANGE_MASK) + >> CLOCK_RANGE_SHIFT); + for (i = 0, j = 1; i < reg_table->last; i++) { + if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_FROM_TABLE) { + reg_table->mc_reg_table_entry[num_ranges].mc_data[i] = + (u32)le32_to_cpu(*((u32 *)reg_data + j)); + j++; + } else if ((reg_table->mc_reg_address[i].pre_reg_data & LOW_NIBBLE_MASK) == DATA_EQU_PREV) { + reg_table->mc_reg_table_entry[num_ranges].mc_data[i] = + reg_table->mc_reg_table_entry[num_ranges].mc_data[i - 1]; + } + } + num_ranges++; + } + reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *) + ((u8 *)reg_data + le16_to_cpu(reg_block->usRegDataBlkSize)); + } + if (le32_to_cpu(*(u32 *)reg_data) != END_OF_REG_DATA_BLOCK) + return -EINVAL; + reg_table->num_entries = num_ranges; + } else + return -EINVAL; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + return -EINVAL; + } + return 0; + } + return -EINVAL; +} + +bool amdgpu_atombios_has_gpu_virtualization_table(struct amdgpu_device *adev) +{ + int index = GetIndexIntoMasterTable(DATA, GPUVirtualizationInfo); + u8 frev, crev; + u16 data_offset, size; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size, + &frev, &crev, &data_offset)) + return true; + + return false; +} + +void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock) +{ + uint32_t bios_6_scratch; + + bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6); + + if (lock) { + bios_6_scratch |= ATOM_S6_CRITICAL_STATE; + bios_6_scratch &= ~ATOM_S6_ACC_MODE; + } else { + bios_6_scratch &= ~ATOM_S6_CRITICAL_STATE; + bios_6_scratch |= ATOM_S6_ACC_MODE; + } + + WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch); +} + +static void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev) +{ + uint32_t bios_2_scratch, bios_6_scratch; + + adev->bios_scratch_reg_offset = mmBIOS_SCRATCH_0; + + bios_2_scratch = RREG32(adev->bios_scratch_reg_offset + 2); + bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6); + + /* let the bios control the backlight */ + bios_2_scratch &= ~ATOM_S2_VRI_BRIGHT_ENABLE; + + /* tell the bios not to handle mode switching */ + bios_6_scratch |= ATOM_S6_ACC_BLOCK_DISPLAY_SWITCH; + + /* clear the vbios dpms state */ + bios_2_scratch &= ~ATOM_S2_DEVICE_DPMS_STATE; + + WREG32(adev->bios_scratch_reg_offset + 2, bios_2_scratch); + WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch); +} + +void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev, + bool hung) +{ + u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3); + + if (hung) + tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG; + else + tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG; + + WREG32(adev->bios_scratch_reg_offset + 3, tmp); +} + +bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev) +{ + u32 tmp = RREG32(adev->bios_scratch_reg_offset + 7); + + if (tmp & ATOM_S7_ASIC_INIT_COMPLETE_MASK) + return false; + else + return true; +} + +/* Atom needs data in little endian format so swap as appropriate when copying + * data to or from atom. Note that atom operates on dw units. + * + * Use to_le=true when sending data to atom and provide at least + * ALIGN(num_bytes,4) bytes in the dst buffer. + * + * Use to_le=false when receiving data from atom and provide ALIGN(num_bytes,4) + * byes in the src buffer. + */ +void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le) +{ +#ifdef __BIG_ENDIAN + u32 src_tmp[5], dst_tmp[5]; + int i; + u8 align_num_bytes = ALIGN(num_bytes, 4); + + if (to_le) { + memcpy(src_tmp, src, num_bytes); + for (i = 0; i < align_num_bytes / 4; i++) + dst_tmp[i] = cpu_to_le32(src_tmp[i]); + memcpy(dst, dst_tmp, align_num_bytes); + } else { + memcpy(src_tmp, src, align_num_bytes); + for (i = 0; i < align_num_bytes / 4; i++) + dst_tmp[i] = le32_to_cpu(src_tmp[i]); + memcpy(dst, dst_tmp, num_bytes); + } +#else + memcpy(dst, src, num_bytes); +#endif +} + +static int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev) +{ + struct atom_context *ctx = adev->mode_info.atom_context; + int index = GetIndexIntoMasterTable(DATA, VRAM_UsageByFirmware); + uint16_t data_offset; + int usage_bytes = 0; + struct _ATOM_VRAM_USAGE_BY_FIRMWARE *firmware_usage; + u64 start_addr; + u64 size; + + if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) { + firmware_usage = (struct _ATOM_VRAM_USAGE_BY_FIRMWARE *)(ctx->bios + data_offset); + + DRM_DEBUG("atom firmware requested %08x %dkb\n", + le32_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware), + le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb)); + + start_addr = firmware_usage->asFirmwareVramReserveInfo[0].ulStartAddrUsedByFirmware; + size = firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb; + + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) == + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { + /* Firmware request VRAM reservation for SR-IOV */ + adev->fw_vram_usage.start_offset = (start_addr & + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; + adev->fw_vram_usage.size = size << 10; + /* Use the default scratch size */ + usage_bytes = 0; + } else { + usage_bytes = le16_to_cpu(firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb) * 1024; + } + } + ctx->scratch_size_bytes = 0; + if (usage_bytes == 0) + usage_bytes = 20 * 1024; + /* allocate some scratch memory */ + ctx->scratch = kzalloc(usage_bytes, GFP_KERNEL); + if (!ctx->scratch) + return -ENOMEM; + ctx->scratch_size_bytes = usage_bytes; + return 0; +} + +/* ATOM accessor methods */ +/* + * ATOM is an interpreted byte code stored in tables in the vbios. The + * driver registers callbacks to access registers and the interpreter + * in the driver parses the tables and executes then to program specific + * actions (set display modes, asic init, etc.). See amdgpu_atombios.c, + * atombios.h, and atom.c + */ + +/** + * cail_pll_read - read PLL register + * + * @info: atom card_info pointer + * @reg: PLL register offset + * + * Provides a PLL register accessor for the atom interpreter (r4xx+). + * Returns the value of the PLL register. + */ +static uint32_t cail_pll_read(struct card_info *info, uint32_t reg) +{ + return 0; +} + +/** + * cail_pll_write - write PLL register + * + * @info: atom card_info pointer + * @reg: PLL register offset + * @val: value to write to the pll register + * + * Provides a PLL register accessor for the atom interpreter (r4xx+). + */ +static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val) +{ + +} + +/** + * cail_mc_read - read MC (Memory Controller) register + * + * @info: atom card_info pointer + * @reg: MC register offset + * + * Provides an MC register accessor for the atom interpreter (r4xx+). + * Returns the value of the MC register. + */ +static uint32_t cail_mc_read(struct card_info *info, uint32_t reg) +{ + return 0; +} + +/** + * cail_mc_write - write MC (Memory Controller) register + * + * @info: atom card_info pointer + * @reg: MC register offset + * @val: value to write to the pll register + * + * Provides a MC register accessor for the atom interpreter (r4xx+). + */ +static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val) +{ + +} + +/** + * cail_reg_write - write MMIO register + * + * @info: atom card_info pointer + * @reg: MMIO register offset + * @val: value to write to the pll register + * + * Provides a MMIO register accessor for the atom interpreter (r4xx+). + */ +static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val) +{ + struct amdgpu_device *adev = info->dev->dev_private; + + WREG32(reg, val); +} + +/** + * cail_reg_read - read MMIO register + * + * @info: atom card_info pointer + * @reg: MMIO register offset + * + * Provides an MMIO register accessor for the atom interpreter (r4xx+). + * Returns the value of the MMIO register. + */ +static uint32_t cail_reg_read(struct card_info *info, uint32_t reg) +{ + struct amdgpu_device *adev = info->dev->dev_private; + uint32_t r; + + r = RREG32(reg); + return r; +} + +/** + * cail_ioreg_write - write IO register + * + * @info: atom card_info pointer + * @reg: IO register offset + * @val: value to write to the pll register + * + * Provides a IO register accessor for the atom interpreter (r4xx+). + */ +static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val) +{ + struct amdgpu_device *adev = info->dev->dev_private; + + WREG32_IO(reg, val); +} + +/** + * cail_ioreg_read - read IO register + * + * @info: atom card_info pointer + * @reg: IO register offset + * + * Provides an IO register accessor for the atom interpreter (r4xx+). + * Returns the value of the IO register. + */ +static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg) +{ + struct amdgpu_device *adev = info->dev->dev_private; + uint32_t r; + + r = RREG32_IO(reg); + return r; +} + +static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + struct atom_context *ctx = adev->mode_info.atom_context; + + return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version); +} + +static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version, + NULL); + +/** + * amdgpu_atombios_fini - free the driver info and callbacks for atombios + * + * @adev: amdgpu_device pointer + * + * Frees the driver info and register access callbacks for the ATOM + * interpreter (r4xx+). + * Called at driver shutdown. + */ +void amdgpu_atombios_fini(struct amdgpu_device *adev) +{ + if (adev->mode_info.atom_context) { + kfree(adev->mode_info.atom_context->scratch); + kfree(adev->mode_info.atom_context->iio); + } + kfree(adev->mode_info.atom_context); + adev->mode_info.atom_context = NULL; + kfree(adev->mode_info.atom_card_info); + adev->mode_info.atom_card_info = NULL; + device_remove_file(adev->dev, &dev_attr_vbios_version); +} + +/** + * amdgpu_atombios_init - init the driver info and callbacks for atombios + * + * @adev: amdgpu_device pointer + * + * Initializes the driver info and register access callbacks for the + * ATOM interpreter (r4xx+). + * Returns 0 on sucess, -ENOMEM on failure. + * Called at driver startup. + */ +int amdgpu_atombios_init(struct amdgpu_device *adev) +{ + struct card_info *atom_card_info = + kzalloc(sizeof(struct card_info), GFP_KERNEL); + int ret; + + if (!atom_card_info) + return -ENOMEM; + + adev->mode_info.atom_card_info = atom_card_info; + atom_card_info->dev = adev->ddev; + atom_card_info->reg_read = cail_reg_read; + atom_card_info->reg_write = cail_reg_write; + /* needed for iio ops */ + if (adev->rio_mem) { + atom_card_info->ioreg_read = cail_ioreg_read; + atom_card_info->ioreg_write = cail_ioreg_write; + } else { + DRM_DEBUG("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n"); + atom_card_info->ioreg_read = cail_reg_read; + atom_card_info->ioreg_write = cail_reg_write; + } + atom_card_info->mc_read = cail_mc_read; + atom_card_info->mc_write = cail_mc_write; + atom_card_info->pll_read = cail_pll_read; + atom_card_info->pll_write = cail_pll_write; + + adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios); + if (!adev->mode_info.atom_context) { + amdgpu_atombios_fini(adev); + return -ENOMEM; + } + + mutex_init(&adev->mode_info.atom_context->mutex); + if (adev->is_atom_fw) { + amdgpu_atomfirmware_scratch_regs_init(adev); + amdgpu_atomfirmware_allocate_fb_scratch(adev); + } else { + amdgpu_atombios_scratch_regs_init(adev); + amdgpu_atombios_allocate_fb_scratch(adev); + } + + ret = device_create_file(adev->dev, &dev_attr_vbios_version); + if (ret) { + DRM_ERROR("Failed to create device file for VBIOS version\n"); + return ret; + } + + return 0; +} +
diff --git a/amdgpu/amdgpu_atombios.h b/amdgpu/amdgpu_atombios.h new file mode 100644 index 0000000..fd8f180 --- /dev/null +++ b/amdgpu/amdgpu_atombios.h
@@ -0,0 +1,222 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_ATOMBIOS_H__ +#define __AMDGPU_ATOMBIOS_H__ + +struct atom_clock_dividers { + u32 post_div; + union { + struct { +#ifdef __BIG_ENDIAN + u32 reserved : 6; + u32 whole_fb_div : 12; + u32 frac_fb_div : 14; +#else + u32 frac_fb_div : 14; + u32 whole_fb_div : 12; + u32 reserved : 6; +#endif + }; + u32 fb_div; + }; + u32 ref_div; + bool enable_post_div; + bool enable_dithen; + u32 vco_mode; + u32 real_clock; + /* added for CI */ + u32 post_divider; + u32 flags; +}; + +struct atom_mpll_param { + union { + struct { +#ifdef __BIG_ENDIAN + u32 reserved : 8; + u32 clkfrac : 12; + u32 clkf : 12; +#else + u32 clkf : 12; + u32 clkfrac : 12; + u32 reserved : 8; +#endif + }; + u32 fb_div; + }; + u32 post_div; + u32 bwcntl; + u32 dll_speed; + u32 vco_mode; + u32 yclk_sel; + u32 qdr; + u32 half_rate; +}; + +#define MEM_TYPE_GDDR5 0x50 +#define MEM_TYPE_GDDR4 0x40 +#define MEM_TYPE_GDDR3 0x30 +#define MEM_TYPE_DDR2 0x20 +#define MEM_TYPE_GDDR1 0x10 +#define MEM_TYPE_DDR3 0xb0 +#define MEM_TYPE_MASK 0xf0 + +struct atom_memory_info { + u8 mem_vendor; + u8 mem_type; +}; + +#define MAX_AC_TIMING_ENTRIES 16 + +struct atom_memory_clock_range_table +{ + u8 num_entries; + u8 rsv[3]; + u32 mclk[MAX_AC_TIMING_ENTRIES]; +}; + +#define VBIOS_MC_REGISTER_ARRAY_SIZE 32 +#define VBIOS_MAX_AC_TIMING_ENTRIES 20 + +struct atom_mc_reg_entry { + u32 mclk_max; + u32 mc_data[VBIOS_MC_REGISTER_ARRAY_SIZE]; +}; + +struct atom_mc_register_address { + u16 s1; + u8 pre_reg_data; +}; + +struct atom_mc_reg_table { + u8 last; + u8 num_entries; + struct atom_mc_reg_entry mc_reg_table_entry[VBIOS_MAX_AC_TIMING_ENTRIES]; + struct atom_mc_register_address mc_reg_address[VBIOS_MC_REGISTER_ARRAY_SIZE]; +}; + +#define MAX_VOLTAGE_ENTRIES 32 + +struct atom_voltage_table_entry +{ + u16 value; + u32 smio_low; +}; + +struct atom_voltage_table +{ + u32 count; + u32 mask_low; + u32 phase_delay; + struct atom_voltage_table_entry entries[MAX_VOLTAGE_ENTRIES]; +}; + +struct amdgpu_gpio_rec +amdgpu_atombios_lookup_gpio(struct amdgpu_device *adev, + u8 id); + +struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device *adev, + uint8_t id); +void amdgpu_atombios_i2c_init(struct amdgpu_device *adev); + +bool amdgpu_atombios_has_dce_engine_info(struct amdgpu_device *adev); + +bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device *adev); + +int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev); + +int amdgpu_atombios_get_gfx_info(struct amdgpu_device *adev); + +int amdgpu_atombios_get_vram_width(struct amdgpu_device *adev); + +bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev, + struct amdgpu_atom_ss *ss, + int id, u32 clock); + +int amdgpu_atombios_get_clock_dividers(struct amdgpu_device *adev, + u8 clock_type, + u32 clock, + bool strobe_mode, + struct atom_clock_dividers *dividers); + +int amdgpu_atombios_get_memory_pll_dividers(struct amdgpu_device *adev, + u32 clock, + bool strobe_mode, + struct atom_mpll_param *mpll_param); + +void amdgpu_atombios_set_engine_dram_timings(struct amdgpu_device *adev, + u32 eng_clock, u32 mem_clock); + +int amdgpu_atombios_get_leakage_id_from_vbios(struct amdgpu_device *adev, + u16 *leakage_id); + +int amdgpu_atombios_get_leakage_vddc_based_on_leakage_params(struct amdgpu_device *adev, + u16 *vddc, u16 *vddci, + u16 virtual_voltage_id, + u16 vbios_voltage_id); + +int amdgpu_atombios_get_voltage_evv(struct amdgpu_device *adev, + u16 virtual_voltage_id, + u16 *voltage); + +bool +amdgpu_atombios_is_voltage_gpio(struct amdgpu_device *adev, + u8 voltage_type, u8 voltage_mode); + +int amdgpu_atombios_get_voltage_table(struct amdgpu_device *adev, + u8 voltage_type, u8 voltage_mode, + struct atom_voltage_table *voltage_table); + +int amdgpu_atombios_init_mc_reg_table(struct amdgpu_device *adev, + u8 module_index, + struct atom_mc_reg_table *reg_table); + +bool amdgpu_atombios_has_gpu_virtualization_table(struct amdgpu_device *adev); + +void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock); +void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev, + bool hung); +bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev); + +void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le); +int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type, + u16 voltage_id, u16 *voltage); +int amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(struct amdgpu_device *adev, + u16 *voltage, + u16 leakage_idx); +void amdgpu_atombios_get_default_voltages(struct amdgpu_device *adev, + u16 *vddc, u16 *vddci, u16 *mvdd); +int amdgpu_atombios_get_clock_dividers(struct amdgpu_device *adev, + u8 clock_type, + u32 clock, + bool strobe_mode, + struct atom_clock_dividers *dividers); +int amdgpu_atombios_get_svi2_info(struct amdgpu_device *adev, + u8 voltage_type, + u8 *svd_gpio_id, u8 *svc_gpio_id); + +void amdgpu_atombios_fini(struct amdgpu_device *adev); +int amdgpu_atombios_init(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_atomfirmware.c b/amdgpu/amdgpu_atomfirmware.c new file mode 100644 index 0000000..daf6874 --- /dev/null +++ b/amdgpu/amdgpu_atomfirmware.c
@@ -0,0 +1,466 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atomfirmware.h" +#include "amdgpu_atomfirmware.h" +#include "atom.h" +#include "atombios.h" + +bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev) +{ + int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + firmwareinfo); + uint16_t data_offset; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, NULL, + NULL, NULL, &data_offset)) { + struct atom_firmware_info_v3_1 *firmware_info = + (struct atom_firmware_info_v3_1 *)(adev->mode_info.atom_context->bios + + data_offset); + + if (le32_to_cpu(firmware_info->firmware_capability) & + ATOM_FIRMWARE_CAP_GPU_VIRTUALIZATION) + return true; + } + return false; +} + +void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev) +{ + int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + firmwareinfo); + uint16_t data_offset; + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, NULL, + NULL, NULL, &data_offset)) { + struct atom_firmware_info_v3_1 *firmware_info = + (struct atom_firmware_info_v3_1 *)(adev->mode_info.atom_context->bios + + data_offset); + + adev->bios_scratch_reg_offset = + le32_to_cpu(firmware_info->bios_scratch_reg_startaddr); + } +} + +int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) +{ + struct atom_context *ctx = adev->mode_info.atom_context; + int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + vram_usagebyfirmware); + struct vram_usagebyfirmware_v2_1 * firmware_usage; + uint32_t start_addr, size; + uint16_t data_offset; + int usage_bytes = 0; + + if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) { + firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset); + DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n", + le32_to_cpu(firmware_usage->start_address_in_kb), + le16_to_cpu(firmware_usage->used_by_firmware_in_kb), + le16_to_cpu(firmware_usage->used_by_driver_in_kb)); + + start_addr = le32_to_cpu(firmware_usage->start_address_in_kb); + size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb); + + if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) == + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { + /* Firmware request VRAM reservation for SR-IOV */ + adev->fw_vram_usage.start_offset = (start_addr & + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; + adev->fw_vram_usage.size = size << 10; + /* Use the default scratch size */ + usage_bytes = 0; + } else { + usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10; + } + } + ctx->scratch_size_bytes = 0; + if (usage_bytes == 0) + usage_bytes = 20 * 1024; + /* allocate some scratch memory */ + ctx->scratch = kzalloc(usage_bytes, GFP_KERNEL); + if (!ctx->scratch) + return -ENOMEM; + ctx->scratch_size_bytes = usage_bytes; + return 0; +} + +union igp_info { + struct atom_integrated_system_info_v1_11 v11; +}; + +union umc_info { + struct atom_umc_info_v3_1 v31; +}; + +union vram_info { + struct atom_vram_info_header_v2_3 v23; + struct atom_vram_info_header_v2_4 v24; +}; +/* + * Return vram width from integrated system info table, if available, + * or 0 if not. + */ +int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + u16 data_offset, size; + union igp_info *igp_info; + union vram_info *vram_info; + u32 mem_channel_number; + u32 mem_channel_width; + u8 frev, crev; + + if (adev->flags & AMD_IS_APU) + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + integratedsysteminfo); + else + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + vram_info); + + /* get any igp specific overrides */ + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size, + &frev, &crev, &data_offset)) { + if (adev->flags & AMD_IS_APU) { + igp_info = (union igp_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 11: + mem_channel_number = igp_info->v11.umachannelnumber; + /* channel width is 64 */ + return mem_channel_number * 64; + default: + return 0; + } + } else { + vram_info = (union vram_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 3: + mem_channel_number = vram_info->v23.vram_module[0].channel_num; + mem_channel_width = vram_info->v23.vram_module[0].channel_width; + return mem_channel_number * (1 << mem_channel_width); + case 4: + mem_channel_number = vram_info->v24.vram_module[0].channel_num; + mem_channel_width = vram_info->v24.vram_module[0].channel_width; + return mem_channel_number * (1 << mem_channel_width); + default: + return 0; + } + } + } + + return 0; +} + +static int convert_atom_mem_type_to_vram_type (struct amdgpu_device *adev, + int atom_mem_type) +{ + int vram_type; + + if (adev->flags & AMD_IS_APU) { + switch (atom_mem_type) { + case Ddr2MemType: + case LpDdr2MemType: + vram_type = AMDGPU_VRAM_TYPE_DDR2; + break; + case Ddr3MemType: + case LpDdr3MemType: + vram_type = AMDGPU_VRAM_TYPE_DDR3; + break; + case Ddr4MemType: + case LpDdr4MemType: + vram_type = AMDGPU_VRAM_TYPE_DDR4; + break; + default: + vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; + break; + } + } else { + switch (atom_mem_type) { + case ATOM_DGPU_VRAM_TYPE_GDDR5: + vram_type = AMDGPU_VRAM_TYPE_GDDR5; + break; + case ATOM_DGPU_VRAM_TYPE_HBM2: + vram_type = AMDGPU_VRAM_TYPE_HBM; + break; + case ATOM_DGPU_VRAM_TYPE_GDDR6: + vram_type = AMDGPU_VRAM_TYPE_GDDR6; + break; + default: + vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; + break; + } + } + + return vram_type; +} +/* + * Return vram type from either integrated system info table + * or umc info table, if available, or 0 (TYPE_UNKNOWN) if not + */ +int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + u16 data_offset, size; + union igp_info *igp_info; + union vram_info *vram_info; + u8 frev, crev; + u8 mem_type; + + if (adev->flags & AMD_IS_APU) + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + integratedsysteminfo); + else + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + vram_info); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, + index, &size, + &frev, &crev, &data_offset)) { + if (adev->flags & AMD_IS_APU) { + igp_info = (union igp_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 11: + mem_type = igp_info->v11.memorytype; + return convert_atom_mem_type_to_vram_type(adev, mem_type); + default: + return 0; + } + } else { + vram_info = (union vram_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 3: + mem_type = vram_info->v23.vram_module[0].memory_type; + return convert_atom_mem_type_to_vram_type(adev, mem_type); + case 4: + mem_type = vram_info->v24.vram_module[0].memory_type; + return convert_atom_mem_type_to_vram_type(adev, mem_type); + default: + return 0; + } + } + } + + return 0; +} + +/* + * Return true if vbios enabled ecc by default, if umc info table is available + * or false if ecc is not enabled or umc info table is not available + */ +bool amdgpu_atomfirmware_mem_ecc_supported(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + u16 data_offset, size; + union umc_info *umc_info; + u8 frev, crev; + bool ecc_default_enabled = false; + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + umc_info); + + if (amdgpu_atom_parse_data_header(mode_info->atom_context, + index, &size, &frev, &crev, &data_offset)) { + /* support umc_info 3.1+ */ + if ((frev == 3 && crev >= 1) || (frev > 3)) { + umc_info = (union umc_info *) + (mode_info->atom_context->bios + data_offset); + ecc_default_enabled = + (le32_to_cpu(umc_info->v31.umc_config) & + UMC_CONFIG__DEFAULT_MEM_ECC_ENABLE) ? true : false; + } + } + + return ecc_default_enabled; +} + +union firmware_info { + struct atom_firmware_info_v3_1 v31; +}; + +/* + * Return true if vbios supports sram ecc or false if not + */ +bool amdgpu_atomfirmware_sram_ecc_supported(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + u16 data_offset, size; + union firmware_info *firmware_info; + u8 frev, crev; + bool sram_ecc_supported = false; + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + firmwareinfo); + + if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, + index, &size, &frev, &crev, &data_offset)) { + /* support firmware_info 3.1 + */ + if ((frev == 3 && crev >=1) || (frev > 3)) { + firmware_info = (union firmware_info *) + (mode_info->atom_context->bios + data_offset); + sram_ecc_supported = + (le32_to_cpu(firmware_info->v31.firmware_capability) & + ATOM_FIRMWARE_CAP_SRAM_ECC) ? true : false; + } + } + + return sram_ecc_supported; +} + +union smu_info { + struct atom_smu_info_v3_1 v31; +}; + +int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + struct amdgpu_pll *spll = &adev->clock.spll; + struct amdgpu_pll *mpll = &adev->clock.mpll; + uint8_t frev, crev; + uint16_t data_offset; + int ret = -EINVAL, index; + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + firmwareinfo); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + union firmware_info *firmware_info = + (union firmware_info *)(mode_info->atom_context->bios + + data_offset); + + adev->clock.default_sclk = + le32_to_cpu(firmware_info->v31.bootup_sclk_in10khz); + adev->clock.default_mclk = + le32_to_cpu(firmware_info->v31.bootup_mclk_in10khz); + + adev->pm.current_sclk = adev->clock.default_sclk; + adev->pm.current_mclk = adev->clock.default_mclk; + + /* not technically a clock, but... */ + adev->mode_info.firmware_flags = + le32_to_cpu(firmware_info->v31.firmware_capability); + + ret = 0; + } + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + smu_info); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + union smu_info *smu_info = + (union smu_info *)(mode_info->atom_context->bios + + data_offset); + + /* system clock */ + spll->reference_freq = le32_to_cpu(smu_info->v31.core_refclk_10khz); + + spll->reference_div = 0; + spll->min_post_div = 1; + spll->max_post_div = 1; + spll->min_ref_div = 2; + spll->max_ref_div = 0xff; + spll->min_feedback_div = 4; + spll->max_feedback_div = 0xff; + spll->best_vco = 0; + + ret = 0; + } + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + umc_info); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + union umc_info *umc_info = + (union umc_info *)(mode_info->atom_context->bios + + data_offset); + + /* memory clock */ + mpll->reference_freq = le32_to_cpu(umc_info->v31.mem_refclk_10khz); + + mpll->reference_div = 0; + mpll->min_post_div = 1; + mpll->max_post_div = 1; + mpll->min_ref_div = 2; + mpll->max_ref_div = 0xff; + mpll->min_feedback_div = 4; + mpll->max_feedback_div = 0xff; + mpll->best_vco = 0; + + ret = 0; + } + + return ret; +} + +union gfx_info { + struct atom_gfx_info_v2_4 v24; +}; + +int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index; + uint8_t frev, crev; + uint16_t data_offset; + + index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, + gfx_info); + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + union gfx_info *gfx_info = (union gfx_info *) + (mode_info->atom_context->bios + data_offset); + switch (crev) { + case 4: + adev->gfx.config.max_shader_engines = gfx_info->v24.max_shader_engines; + adev->gfx.config.max_cu_per_sh = gfx_info->v24.max_cu_per_sh; + adev->gfx.config.max_sh_per_se = gfx_info->v24.max_sh_per_se; + adev->gfx.config.max_backends_per_se = gfx_info->v24.max_backends_per_se; + adev->gfx.config.max_texture_channel_caches = gfx_info->v24.max_texture_channel_caches; + adev->gfx.config.max_gprs = le16_to_cpu(gfx_info->v24.gc_num_gprs); + adev->gfx.config.max_gs_threads = gfx_info->v24.gc_num_max_gs_thds; + adev->gfx.config.gs_vgt_table_depth = gfx_info->v24.gc_gs_table_depth; + adev->gfx.config.gs_prim_buffer_depth = + le16_to_cpu(gfx_info->v24.gc_gsprim_buff_depth); + adev->gfx.config.double_offchip_lds_buf = + gfx_info->v24.gc_double_offchip_lds_buffer; + adev->gfx.cu_info.wave_front_size = le16_to_cpu(gfx_info->v24.gc_wave_size); + adev->gfx.cu_info.max_waves_per_simd = le16_to_cpu(gfx_info->v24.gc_max_waves_per_simd); + adev->gfx.cu_info.max_scratch_slots_per_cu = gfx_info->v24.gc_max_scratch_slots_per_cu; + adev->gfx.cu_info.lds_size = le16_to_cpu(gfx_info->v24.gc_lds_size); + return 0; + default: + return -EINVAL; + } + + } + return -EINVAL; +}
diff --git a/amdgpu/amdgpu_atomfirmware.h b/amdgpu/amdgpu_atomfirmware.h new file mode 100644 index 0000000..5ec6f92 --- /dev/null +++ b/amdgpu/amdgpu_atomfirmware.h
@@ -0,0 +1,39 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_ATOMFIRMWARE_H__ +#define __AMDGPU_ATOMFIRMWARE_H__ + +#define get_index_into_master_table(master_table, table_name) (offsetof(struct master_table, table_name) / sizeof(uint16_t)) + +bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev); +void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev); +int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev); +int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev); +int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev); +int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev); +int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev); +bool amdgpu_atomfirmware_mem_ecc_supported(struct amdgpu_device *adev); +bool amdgpu_atomfirmware_sram_ecc_supported(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_benchmark.c b/amdgpu/amdgpu_benchmark.c new file mode 100644 index 0000000..649e68c --- /dev/null +++ b/amdgpu/amdgpu_benchmark.c
@@ -0,0 +1,244 @@ +/* + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Jerome Glisse + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" + +#define AMDGPU_BENCHMARK_ITERATIONS 1024 +#define AMDGPU_BENCHMARK_COMMON_MODES_N 17 + +static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size, + uint64_t saddr, uint64_t daddr, int n) +{ + unsigned long start_jiffies; + unsigned long end_jiffies; + struct dma_fence *fence = NULL; + int i, r; + + start_jiffies = jiffies; + for (i = 0; i < n; i++) { + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence, + false, false); + if (r) + goto exit_do_move; + r = dma_fence_wait(fence, false); + if (r) + goto exit_do_move; + dma_fence_put(fence); + } + end_jiffies = jiffies; + r = jiffies_to_msecs(end_jiffies - start_jiffies); + +exit_do_move: + if (fence) + dma_fence_put(fence); + return r; +} + + +static void amdgpu_benchmark_log_results(int n, unsigned size, + unsigned int time, + unsigned sdomain, unsigned ddomain, + char *kind) +{ + unsigned int throughput = (n * (size >> 10)) / time; + DRM_INFO("amdgpu: %s %u bo moves of %u kB from" + " %d to %d in %u ms, throughput: %u Mb/s or %u MB/s\n", + kind, n, size >> 10, sdomain, ddomain, time, + throughput * 8, throughput); +} + +static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size, + unsigned sdomain, unsigned ddomain) +{ + struct amdgpu_bo *dobj = NULL; + struct amdgpu_bo *sobj = NULL; + struct amdgpu_bo_param bp; + uint64_t saddr, daddr; + int r, n; + int time; + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = PAGE_SIZE; + bp.domain = sdomain; + bp.flags = 0; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + n = AMDGPU_BENCHMARK_ITERATIONS; + r = amdgpu_bo_create(adev, &bp, &sobj); + if (r) { + goto out_cleanup; + } + r = amdgpu_bo_reserve(sobj, false); + if (unlikely(r != 0)) + goto out_cleanup; + r = amdgpu_bo_pin(sobj, sdomain); + if (r) { + amdgpu_bo_unreserve(sobj); + goto out_cleanup; + } + r = amdgpu_ttm_alloc_gart(&sobj->tbo); + amdgpu_bo_unreserve(sobj); + if (r) { + goto out_cleanup; + } + saddr = amdgpu_bo_gpu_offset(sobj); + bp.domain = ddomain; + r = amdgpu_bo_create(adev, &bp, &dobj); + if (r) { + goto out_cleanup; + } + r = amdgpu_bo_reserve(dobj, false); + if (unlikely(r != 0)) + goto out_cleanup; + r = amdgpu_bo_pin(dobj, ddomain); + if (r) { + amdgpu_bo_unreserve(sobj); + goto out_cleanup; + } + r = amdgpu_ttm_alloc_gart(&dobj->tbo); + amdgpu_bo_unreserve(dobj); + if (r) { + goto out_cleanup; + } + daddr = amdgpu_bo_gpu_offset(dobj); + + if (adev->mman.buffer_funcs) { + time = amdgpu_benchmark_do_move(adev, size, saddr, daddr, n); + if (time < 0) + goto out_cleanup; + if (time > 0) + amdgpu_benchmark_log_results(n, size, time, + sdomain, ddomain, "dma"); + } + +out_cleanup: + /* Check error value now. The value can be overwritten when clean up.*/ + if (r) { + DRM_ERROR("Error while benchmarking BO move.\n"); + } + + if (sobj) { + r = amdgpu_bo_reserve(sobj, true); + if (likely(r == 0)) { + amdgpu_bo_unpin(sobj); + amdgpu_bo_unreserve(sobj); + } + amdgpu_bo_unref(&sobj); + } + if (dobj) { + r = amdgpu_bo_reserve(dobj, true); + if (likely(r == 0)) { + amdgpu_bo_unpin(dobj); + amdgpu_bo_unreserve(dobj); + } + amdgpu_bo_unref(&dobj); + } +} + +void amdgpu_benchmark(struct amdgpu_device *adev, int test_number) +{ + int i; + static const int common_modes[AMDGPU_BENCHMARK_COMMON_MODES_N] = { + 640 * 480 * 4, + 720 * 480 * 4, + 800 * 600 * 4, + 848 * 480 * 4, + 1024 * 768 * 4, + 1152 * 768 * 4, + 1280 * 720 * 4, + 1280 * 800 * 4, + 1280 * 854 * 4, + 1280 * 960 * 4, + 1280 * 1024 * 4, + 1440 * 900 * 4, + 1400 * 1050 * 4, + 1680 * 1050 * 4, + 1600 * 1200 * 4, + 1920 * 1080 * 4, + 1920 * 1200 * 4 + }; + + switch (test_number) { + case 1: + /* simple test, VRAM to GTT and GTT to VRAM */ + amdgpu_benchmark_move(adev, 1024*1024, AMDGPU_GEM_DOMAIN_GTT, + AMDGPU_GEM_DOMAIN_VRAM); + amdgpu_benchmark_move(adev, 1024*1024, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT); + break; + case 2: + /* simple test, VRAM to VRAM */ + amdgpu_benchmark_move(adev, 1024*1024, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_VRAM); + break; + case 3: + /* GTT to VRAM, buffer size sweep, powers of 2 */ + for (i = 1; i <= 16384; i <<= 1) + amdgpu_benchmark_move(adev, i * AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, + AMDGPU_GEM_DOMAIN_VRAM); + break; + case 4: + /* VRAM to GTT, buffer size sweep, powers of 2 */ + for (i = 1; i <= 16384; i <<= 1) + amdgpu_benchmark_move(adev, i * AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT); + break; + case 5: + /* VRAM to VRAM, buffer size sweep, powers of 2 */ + for (i = 1; i <= 16384; i <<= 1) + amdgpu_benchmark_move(adev, i * AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_VRAM); + break; + case 6: + /* GTT to VRAM, buffer size sweep, common modes */ + for (i = 0; i < AMDGPU_BENCHMARK_COMMON_MODES_N; i++) + amdgpu_benchmark_move(adev, common_modes[i], + AMDGPU_GEM_DOMAIN_GTT, + AMDGPU_GEM_DOMAIN_VRAM); + break; + case 7: + /* VRAM to GTT, buffer size sweep, common modes */ + for (i = 0; i < AMDGPU_BENCHMARK_COMMON_MODES_N; i++) + amdgpu_benchmark_move(adev, common_modes[i], + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT); + break; + case 8: + /* VRAM to VRAM, buffer size sweep, common modes */ + for (i = 0; i < AMDGPU_BENCHMARK_COMMON_MODES_N; i++) + amdgpu_benchmark_move(adev, common_modes[i], + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_VRAM); + break; + + default: + DRM_ERROR("Unknown benchmark\n"); + } +}
diff --git a/amdgpu/amdgpu_bios.c b/amdgpu/amdgpu_bios.c new file mode 100644 index 0000000..50dff69 --- /dev/null +++ b/amdgpu/amdgpu_bios.c
@@ -0,0 +1,442 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ + +#include "amdgpu.h" +#include "atom.h" + +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/acpi.h> +/* + * BIOS. + */ + +#define AMD_VBIOS_SIGNATURE " 761295520" +#define AMD_VBIOS_SIGNATURE_OFFSET 0x30 +#define AMD_VBIOS_SIGNATURE_SIZE sizeof(AMD_VBIOS_SIGNATURE) +#define AMD_VBIOS_SIGNATURE_END (AMD_VBIOS_SIGNATURE_OFFSET + AMD_VBIOS_SIGNATURE_SIZE) +#define AMD_IS_VALID_VBIOS(p) ((p)[0] == 0x55 && (p)[1] == 0xAA) +#define AMD_VBIOS_LENGTH(p) ((p)[2] << 9) + +/* Check if current bios is an ATOM BIOS. + * Return true if it is ATOM BIOS. Otherwise, return false. + */ +static bool check_atom_bios(uint8_t *bios, size_t size) +{ + uint16_t tmp, bios_header_start; + + if (!bios || size < 0x49) { + DRM_INFO("vbios mem is null or mem size is wrong\n"); + return false; + } + + if (!AMD_IS_VALID_VBIOS(bios)) { + DRM_INFO("BIOS signature incorrect %x %x\n", bios[0], bios[1]); + return false; + } + + bios_header_start = bios[0x48] | (bios[0x49] << 8); + if (!bios_header_start) { + DRM_INFO("Can't locate bios header\n"); + return false; + } + + tmp = bios_header_start + 4; + if (size < tmp) { + DRM_INFO("BIOS header is broken\n"); + return false; + } + + if (!memcmp(bios + tmp, "ATOM", 4) || + !memcmp(bios + tmp, "MOTA", 4)) { + DRM_DEBUG("ATOMBIOS detected\n"); + return true; + } + + return false; +} + +/* If you boot an IGP board with a discrete card as the primary, + * the IGP rom is not accessible via the rom bar as the IGP rom is + * part of the system bios. On boot, the system bios puts a + * copy of the igp rom at the start of vram if a discrete card is + * present. + */ +static bool igp_read_bios_from_vram(struct amdgpu_device *adev) +{ + uint8_t __iomem *bios; + resource_size_t vram_base; + resource_size_t size = 256 * 1024; /* ??? */ + + if (!(adev->flags & AMD_IS_APU)) + if (amdgpu_device_need_post(adev)) + return false; + + adev->bios = NULL; + vram_base = pci_resource_start(adev->pdev, 0); + bios = ioremap_wc(vram_base, size); + if (!bios) { + return false; + } + + adev->bios = kmalloc(size, GFP_KERNEL); + if (!adev->bios) { + iounmap(bios); + return false; + } + adev->bios_size = size; + memcpy_fromio(adev->bios, bios, size); + iounmap(bios); + + if (!check_atom_bios(adev->bios, size)) { + kfree(adev->bios); + return false; + } + + return true; +} + +bool amdgpu_read_bios(struct amdgpu_device *adev) +{ + uint8_t __iomem *bios; + size_t size; + + adev->bios = NULL; + /* XXX: some cards may return 0 for rom size? ddx has a workaround */ + bios = pci_map_rom(adev->pdev, &size); + if (!bios) { + return false; + } + + adev->bios = kzalloc(size, GFP_KERNEL); + if (adev->bios == NULL) { + pci_unmap_rom(adev->pdev, bios); + return false; + } + adev->bios_size = size; + memcpy_fromio(adev->bios, bios, size); + pci_unmap_rom(adev->pdev, bios); + + if (!check_atom_bios(adev->bios, size)) { + kfree(adev->bios); + return false; + } + + return true; +} + +static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev) +{ + u8 header[AMD_VBIOS_SIGNATURE_END+1] = {0}; + int len; + + if (!adev->asic_funcs->read_bios_from_rom) + return false; + + /* validate VBIOS signature */ + if (amdgpu_asic_read_bios_from_rom(adev, &header[0], sizeof(header)) == false) + return false; + header[AMD_VBIOS_SIGNATURE_END] = 0; + + if ((!AMD_IS_VALID_VBIOS(header)) || + 0 != memcmp((char *)&header[AMD_VBIOS_SIGNATURE_OFFSET], + AMD_VBIOS_SIGNATURE, + strlen(AMD_VBIOS_SIGNATURE))) + return false; + + /* valid vbios, go on */ + len = AMD_VBIOS_LENGTH(header); + len = ALIGN(len, 4); + adev->bios = kmalloc(len, GFP_KERNEL); + if (!adev->bios) { + DRM_ERROR("no memory to allocate for BIOS\n"); + return false; + } + adev->bios_size = len; + + /* read complete BIOS */ + amdgpu_asic_read_bios_from_rom(adev, adev->bios, len); + + if (!check_atom_bios(adev->bios, len)) { + kfree(adev->bios); + return false; + } + + return true; +} + +static bool amdgpu_read_platform_bios(struct amdgpu_device *adev) +{ + uint8_t __iomem *bios; + size_t size; + + adev->bios = NULL; + + bios = pci_platform_rom(adev->pdev, &size); + if (!bios) { + return false; + } + + adev->bios = kzalloc(size, GFP_KERNEL); + if (adev->bios == NULL) + return false; + + memcpy_fromio(adev->bios, bios, size); + + if (!check_atom_bios(adev->bios, size)) { + kfree(adev->bios); + return false; + } + + adev->bios_size = size; + + return true; +} + +#ifdef CONFIG_ACPI +/* ATRM is used to get the BIOS on the discrete cards in + * dual-gpu systems. + */ +/* retrieve the ROM in 4k blocks */ +#define ATRM_BIOS_PAGE 4096 +/** + * amdgpu_atrm_call - fetch a chunk of the vbios + * + * @atrm_handle: acpi ATRM handle + * @bios: vbios image pointer + * @offset: offset of vbios image data to fetch + * @len: length of vbios image data to fetch + * + * Executes ATRM to fetch a chunk of the discrete + * vbios image on PX systems (all asics). + * Returns the length of the buffer fetched. + */ +static int amdgpu_atrm_call(acpi_handle atrm_handle, uint8_t *bios, + int offset, int len) +{ + acpi_status status; + union acpi_object atrm_arg_elements[2], *obj; + struct acpi_object_list atrm_arg; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL}; + + atrm_arg.count = 2; + atrm_arg.pointer = &atrm_arg_elements[0]; + + atrm_arg_elements[0].type = ACPI_TYPE_INTEGER; + atrm_arg_elements[0].integer.value = offset; + + atrm_arg_elements[1].type = ACPI_TYPE_INTEGER; + atrm_arg_elements[1].integer.value = len; + + status = acpi_evaluate_object(atrm_handle, NULL, &atrm_arg, &buffer); + if (ACPI_FAILURE(status)) { + printk("failed to evaluate ATRM got %s\n", acpi_format_exception(status)); + return -ENODEV; + } + + obj = (union acpi_object *)buffer.pointer; + memcpy(bios+offset, obj->buffer.pointer, obj->buffer.length); + len = obj->buffer.length; + kfree(buffer.pointer); + return len; +} + +static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev) +{ + int ret; + int size = 256 * 1024; + int i; + struct pci_dev *pdev = NULL; + acpi_handle dhandle, atrm_handle; + acpi_status status; + bool found = false; + + /* ATRM is for the discrete card only */ + if (adev->flags & AMD_IS_APU) + return false; + + while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) { + dhandle = ACPI_HANDLE(&pdev->dev); + if (!dhandle) + continue; + + status = acpi_get_handle(dhandle, "ATRM", &atrm_handle); + if (!ACPI_FAILURE(status)) { + found = true; + break; + } + } + + if (!found) { + while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_OTHER << 8, pdev)) != NULL) { + dhandle = ACPI_HANDLE(&pdev->dev); + if (!dhandle) + continue; + + status = acpi_get_handle(dhandle, "ATRM", &atrm_handle); + if (!ACPI_FAILURE(status)) { + found = true; + break; + } + } + } + + if (!found) + return false; + + adev->bios = kmalloc(size, GFP_KERNEL); + if (!adev->bios) { + DRM_ERROR("Unable to allocate bios\n"); + return false; + } + + for (i = 0; i < size / ATRM_BIOS_PAGE; i++) { + ret = amdgpu_atrm_call(atrm_handle, + adev->bios, + (i * ATRM_BIOS_PAGE), + ATRM_BIOS_PAGE); + if (ret < ATRM_BIOS_PAGE) + break; + } + + if (!check_atom_bios(adev->bios, size)) { + kfree(adev->bios); + return false; + } + adev->bios_size = size; + return true; +} +#else +static inline bool amdgpu_atrm_get_bios(struct amdgpu_device *adev) +{ + return false; +} +#endif + +static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) +{ + if (adev->flags & AMD_IS_APU) + return igp_read_bios_from_vram(adev); + else + return amdgpu_asic_read_disabled_bios(adev); +} + +#ifdef CONFIG_ACPI +static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) +{ + struct acpi_table_header *hdr; + acpi_size tbl_size; + UEFI_ACPI_VFCT *vfct; + unsigned offset; + + if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr))) + return false; + tbl_size = hdr->length; + if (tbl_size < sizeof(UEFI_ACPI_VFCT)) { + DRM_ERROR("ACPI VFCT table present but broken (too short #1)\n"); + return false; + } + + vfct = (UEFI_ACPI_VFCT *)hdr; + offset = vfct->VBIOSImageOffset; + + while (offset < tbl_size) { + GOP_VBIOS_CONTENT *vbios = (GOP_VBIOS_CONTENT *)((char *)hdr + offset); + VFCT_IMAGE_HEADER *vhdr = &vbios->VbiosHeader; + + offset += sizeof(VFCT_IMAGE_HEADER); + if (offset > tbl_size) { + DRM_ERROR("ACPI VFCT image header truncated\n"); + return false; + } + + offset += vhdr->ImageLength; + if (offset > tbl_size) { + DRM_ERROR("ACPI VFCT image truncated\n"); + return false; + } + + if (vhdr->ImageLength && + vhdr->PCIBus == adev->pdev->bus->number && + vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) && + vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) && + vhdr->VendorID == adev->pdev->vendor && + vhdr->DeviceID == adev->pdev->device) { + adev->bios = kmemdup(&vbios->VbiosContent, + vhdr->ImageLength, + GFP_KERNEL); + + if (!check_atom_bios(adev->bios, vhdr->ImageLength)) { + kfree(adev->bios); + return false; + } + adev->bios_size = vhdr->ImageLength; + return true; + } + } + + DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n"); + return false; +} +#else +static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) +{ + return false; +} +#endif + +bool amdgpu_get_bios(struct amdgpu_device *adev) +{ + if (amdgpu_atrm_get_bios(adev)) + goto success; + + if (amdgpu_acpi_vfct_bios(adev)) + goto success; + + if (igp_read_bios_from_vram(adev)) + goto success; + + if (amdgpu_read_bios(adev)) + goto success; + + if (amdgpu_read_bios_from_rom(adev)) + goto success; + + if (amdgpu_read_disabled_bios(adev)) + goto success; + + if (amdgpu_read_platform_bios(adev)) + goto success; + + DRM_ERROR("Unable to locate a BIOS ROM\n"); + return false; + +success: + adev->is_atom_fw = (adev->asic_type >= CHIP_VEGA10) ? true : false; + return true; +}
diff --git a/amdgpu/amdgpu_bo_list.c b/amdgpu/amdgpu_bo_list.c new file mode 100644 index 0000000..7bcf86c --- /dev/null +++ b/amdgpu/amdgpu_bo_list.c
@@ -0,0 +1,332 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König <deathsimple@vodafone.de> + */ + +#include <linux/uaccess.h> + +#include "amdgpu.h" +#include "amdgpu_trace.h" + +#define AMDGPU_BO_LIST_MAX_PRIORITY 32u +#define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1) + +static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu) +{ + struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list, + rhead); + + kvfree(list); +} + +static void amdgpu_bo_list_free(struct kref *ref) +{ + struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list, + refcount); + struct amdgpu_bo_list_entry *e; + + amdgpu_bo_list_for_each_entry(e, list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + + amdgpu_bo_unref(&bo); + } + + call_rcu(&list->rhead, amdgpu_bo_list_free_rcu); +} + +int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp, + struct drm_amdgpu_bo_list_entry *info, + unsigned num_entries, struct amdgpu_bo_list **result) +{ + unsigned last_entry = 0, first_userptr = num_entries; + struct amdgpu_bo_list_entry *array; + struct amdgpu_bo_list *list; + uint64_t total_size = 0; + size_t size; + unsigned i; + int r; + + if (num_entries > (SIZE_MAX - sizeof(struct amdgpu_bo_list)) + / sizeof(struct amdgpu_bo_list_entry)) + return -EINVAL; + + size = sizeof(struct amdgpu_bo_list); + size += num_entries * sizeof(struct amdgpu_bo_list_entry); + list = kvmalloc(size, GFP_KERNEL); + if (!list) + return -ENOMEM; + + kref_init(&list->refcount); + list->gds_obj = NULL; + list->gws_obj = NULL; + list->oa_obj = NULL; + + array = amdgpu_bo_list_array_entry(list, 0); + memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); + + for (i = 0; i < num_entries; ++i) { + struct amdgpu_bo_list_entry *entry; + struct drm_gem_object *gobj; + struct amdgpu_bo *bo; + struct mm_struct *usermm; + + gobj = drm_gem_object_lookup(filp, info[i].bo_handle); + if (!gobj) { + r = -ENOENT; + goto error_free; + } + + bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); + drm_gem_object_put_unlocked(gobj); + + usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); + if (usermm) { + if (usermm != current->mm) { + amdgpu_bo_unref(&bo); + r = -EPERM; + goto error_free; + } + entry = &array[--first_userptr]; + } else { + entry = &array[last_entry++]; + } + + entry->priority = min(info[i].bo_priority, + AMDGPU_BO_LIST_MAX_PRIORITY); + entry->tv.bo = &bo->tbo; + + if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GDS) + list->gds_obj = bo; + if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GWS) + list->gws_obj = bo; + if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_OA) + list->oa_obj = bo; + + total_size += amdgpu_bo_size(bo); + trace_amdgpu_bo_list_set(list, bo); + } + + list->first_userptr = first_userptr; + list->num_entries = num_entries; + + trace_amdgpu_cs_bo_status(list->num_entries, total_size); + + *result = list; + return 0; + +error_free: + while (i--) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(array[i].tv.bo); + + amdgpu_bo_unref(&bo); + } + kvfree(list); + return r; + +} + +static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id) +{ + struct amdgpu_bo_list *list; + + mutex_lock(&fpriv->bo_list_lock); + list = idr_remove(&fpriv->bo_list_handles, id); + mutex_unlock(&fpriv->bo_list_lock); + if (list) + kref_put(&list->refcount, amdgpu_bo_list_free); +} + +int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id, + struct amdgpu_bo_list **result) +{ + rcu_read_lock(); + *result = idr_find(&fpriv->bo_list_handles, id); + + if (*result && kref_get_unless_zero(&(*result)->refcount)) { + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + return -ENOENT; +} + +void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list, + struct list_head *validated) +{ + /* This is based on the bucket sort with O(n) time complexity. + * An item with priority "i" is added to bucket[i]. The lists are then + * concatenated in descending order. + */ + struct list_head bucket[AMDGPU_BO_LIST_NUM_BUCKETS]; + struct amdgpu_bo_list_entry *e; + unsigned i; + + for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++) + INIT_LIST_HEAD(&bucket[i]); + + /* Since buffers which appear sooner in the relocation list are + * likely to be used more often than buffers which appear later + * in the list, the sort mustn't change the ordering of buffers + * with the same priority, i.e. it must be stable. + */ + amdgpu_bo_list_for_each_entry(e, list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + unsigned priority = e->priority; + + if (!bo->parent) + list_add_tail(&e->tv.head, &bucket[priority]); + + e->user_pages = NULL; + } + + /* Connect the sorted buckets in the output list. */ + for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++) + list_splice(&bucket[i], validated); +} + +void amdgpu_bo_list_put(struct amdgpu_bo_list *list) +{ + kref_put(&list->refcount, amdgpu_bo_list_free); +} + +int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, + struct drm_amdgpu_bo_list_entry **info_param) +{ + const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr); + const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry); + struct drm_amdgpu_bo_list_entry *info; + int r; + + info = kvmalloc_array(in->bo_number, info_size, GFP_KERNEL); + if (!info) + return -ENOMEM; + + /* copy the handle array from userspace to a kernel buffer */ + r = -EFAULT; + if (likely(info_size == in->bo_info_size)) { + unsigned long bytes = in->bo_number * + in->bo_info_size; + + if (copy_from_user(info, uptr, bytes)) + goto error_free; + + } else { + unsigned long bytes = min(in->bo_info_size, info_size); + unsigned i; + + memset(info, 0, in->bo_number * info_size); + for (i = 0; i < in->bo_number; ++i) { + if (copy_from_user(&info[i], uptr, bytes)) + goto error_free; + + uptr += in->bo_info_size; + } + } + + *info_param = info; + return 0; + +error_free: + kvfree(info); + return r; +} + +int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + union drm_amdgpu_bo_list *args = data; + uint32_t handle = args->in.list_handle; + struct drm_amdgpu_bo_list_entry *info = NULL; + struct amdgpu_bo_list *list, *old; + int r; + + r = amdgpu_bo_create_list_entry_array(&args->in, &info); + if (r) + goto error_free; + + switch (args->in.operation) { + case AMDGPU_BO_LIST_OP_CREATE: + r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number, + &list); + if (r) + goto error_free; + + mutex_lock(&fpriv->bo_list_lock); + r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL); + mutex_unlock(&fpriv->bo_list_lock); + if (r < 0) { + amdgpu_bo_list_put(list); + return r; + } + + handle = r; + break; + + case AMDGPU_BO_LIST_OP_DESTROY: + amdgpu_bo_list_destroy(fpriv, handle); + handle = 0; + break; + + case AMDGPU_BO_LIST_OP_UPDATE: + r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number, + &list); + if (r) + goto error_free; + + mutex_lock(&fpriv->bo_list_lock); + old = idr_replace(&fpriv->bo_list_handles, list, handle); + mutex_unlock(&fpriv->bo_list_lock); + + if (IS_ERR(old)) { + amdgpu_bo_list_put(list); + r = PTR_ERR(old); + goto error_free; + } + + amdgpu_bo_list_put(old); + break; + + default: + r = -EINVAL; + goto error_free; + } + + memset(args, 0, sizeof(*args)); + args->out.list_handle = handle; + kvfree(info); + + return 0; + +error_free: + if (info) + kvfree(info); + return r; +}
diff --git a/amdgpu/amdgpu_bo_list.h b/amdgpu/amdgpu_bo_list.h new file mode 100644 index 0000000..a130e76 --- /dev/null +++ b/amdgpu/amdgpu_bo_list.h
@@ -0,0 +1,84 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_BO_LIST_H__ +#define __AMDGPU_BO_LIST_H__ + +#include <drm/ttm/ttm_execbuf_util.h> +#include <drm/amdgpu_drm.h> + +struct amdgpu_device; +struct amdgpu_bo; +struct amdgpu_bo_va; +struct amdgpu_fpriv; + +struct amdgpu_bo_list_entry { + struct ttm_validate_buffer tv; + struct amdgpu_bo_va *bo_va; + uint32_t priority; + struct page **user_pages; + bool user_invalidated; +}; + +struct amdgpu_bo_list { + struct rcu_head rhead; + struct kref refcount; + struct amdgpu_bo *gds_obj; + struct amdgpu_bo *gws_obj; + struct amdgpu_bo *oa_obj; + unsigned first_userptr; + unsigned num_entries; +}; + +int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id, + struct amdgpu_bo_list **result); +void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list, + struct list_head *validated); +void amdgpu_bo_list_put(struct amdgpu_bo_list *list); +int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, + struct drm_amdgpu_bo_list_entry **info_param); + +int amdgpu_bo_list_create(struct amdgpu_device *adev, + struct drm_file *filp, + struct drm_amdgpu_bo_list_entry *info, + unsigned num_entries, + struct amdgpu_bo_list **list); + +static inline struct amdgpu_bo_list_entry * +amdgpu_bo_list_array_entry(struct amdgpu_bo_list *list, unsigned index) +{ + struct amdgpu_bo_list_entry *array = (void *)&list[1]; + + return &array[index]; +} + +#define amdgpu_bo_list_for_each_entry(e, list) \ + for (e = amdgpu_bo_list_array_entry(list, 0); \ + e != amdgpu_bo_list_array_entry(list, (list)->num_entries); \ + ++e) + +#define amdgpu_bo_list_for_each_userptr_entry(e, list) \ + for (e = amdgpu_bo_list_array_entry(list, (list)->first_userptr); \ + e != amdgpu_bo_list_array_entry(list, (list)->num_entries); \ + ++e) + +#endif
diff --git a/amdgpu/amdgpu_cgs.c b/amdgpu/amdgpu_cgs.c new file mode 100644 index 0000000..031b094 --- /dev/null +++ b/amdgpu/amdgpu_cgs.c
@@ -0,0 +1,505 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include <linux/list.h> +#include <linux/pci.h> +#include <linux/slab.h> + +#include <linux/firmware.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atom.h" +#include "amdgpu_ucode.h" + +struct amdgpu_cgs_device { + struct cgs_device base; + struct amdgpu_device *adev; +}; + +#define CGS_FUNC_ADEV \ + struct amdgpu_device *adev = \ + ((struct amdgpu_cgs_device *)cgs_device)->adev + + +static uint32_t amdgpu_cgs_read_register(struct cgs_device *cgs_device, unsigned offset) +{ + CGS_FUNC_ADEV; + return RREG32(offset); +} + +static void amdgpu_cgs_write_register(struct cgs_device *cgs_device, unsigned offset, + uint32_t value) +{ + CGS_FUNC_ADEV; + WREG32(offset, value); +} + +static uint32_t amdgpu_cgs_read_ind_register(struct cgs_device *cgs_device, + enum cgs_ind_reg space, + unsigned index) +{ + CGS_FUNC_ADEV; + switch (space) { + case CGS_IND_REG__MMIO: + return RREG32_IDX(index); + case CGS_IND_REG__PCIE: + return RREG32_PCIE(index); + case CGS_IND_REG__SMC: + return RREG32_SMC(index); + case CGS_IND_REG__UVD_CTX: + return RREG32_UVD_CTX(index); + case CGS_IND_REG__DIDT: + return RREG32_DIDT(index); + case CGS_IND_REG_GC_CAC: + return RREG32_GC_CAC(index); + case CGS_IND_REG_SE_CAC: + return RREG32_SE_CAC(index); + case CGS_IND_REG__AUDIO_ENDPT: + DRM_ERROR("audio endpt register access not implemented.\n"); + return 0; + } + WARN(1, "Invalid indirect register space"); + return 0; +} + +static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device, + enum cgs_ind_reg space, + unsigned index, uint32_t value) +{ + CGS_FUNC_ADEV; + switch (space) { + case CGS_IND_REG__MMIO: + return WREG32_IDX(index, value); + case CGS_IND_REG__PCIE: + return WREG32_PCIE(index, value); + case CGS_IND_REG__SMC: + return WREG32_SMC(index, value); + case CGS_IND_REG__UVD_CTX: + return WREG32_UVD_CTX(index, value); + case CGS_IND_REG__DIDT: + return WREG32_DIDT(index, value); + case CGS_IND_REG_GC_CAC: + return WREG32_GC_CAC(index, value); + case CGS_IND_REG_SE_CAC: + return WREG32_SE_CAC(index, value); + case CGS_IND_REG__AUDIO_ENDPT: + DRM_ERROR("audio endpt register access not implemented.\n"); + return; + } + WARN(1, "Invalid indirect register space"); +} + +static uint32_t fw_type_convert(struct cgs_device *cgs_device, uint32_t fw_type) +{ + CGS_FUNC_ADEV; + enum AMDGPU_UCODE_ID result = AMDGPU_UCODE_ID_MAXIMUM; + + switch (fw_type) { + case CGS_UCODE_ID_SDMA0: + result = AMDGPU_UCODE_ID_SDMA0; + break; + case CGS_UCODE_ID_SDMA1: + result = AMDGPU_UCODE_ID_SDMA1; + break; + case CGS_UCODE_ID_CP_CE: + result = AMDGPU_UCODE_ID_CP_CE; + break; + case CGS_UCODE_ID_CP_PFP: + result = AMDGPU_UCODE_ID_CP_PFP; + break; + case CGS_UCODE_ID_CP_ME: + result = AMDGPU_UCODE_ID_CP_ME; + break; + case CGS_UCODE_ID_CP_MEC: + case CGS_UCODE_ID_CP_MEC_JT1: + result = AMDGPU_UCODE_ID_CP_MEC1; + break; + case CGS_UCODE_ID_CP_MEC_JT2: + /* for VI. JT2 should be the same as JT1, because: + 1, MEC2 and MEC1 use exactly same FW. + 2, JT2 is not pached but JT1 is. + */ + if (adev->asic_type >= CHIP_TOPAZ) + result = AMDGPU_UCODE_ID_CP_MEC1; + else + result = AMDGPU_UCODE_ID_CP_MEC2; + break; + case CGS_UCODE_ID_RLC_G: + result = AMDGPU_UCODE_ID_RLC_G; + break; + case CGS_UCODE_ID_STORAGE: + result = AMDGPU_UCODE_ID_STORAGE; + break; + default: + DRM_ERROR("Firmware type not supported\n"); + } + return result; +} + +static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device, + enum cgs_ucode_id type) +{ + CGS_FUNC_ADEV; + uint16_t fw_version = 0; + + switch (type) { + case CGS_UCODE_ID_SDMA0: + fw_version = adev->sdma.instance[0].fw_version; + break; + case CGS_UCODE_ID_SDMA1: + fw_version = adev->sdma.instance[1].fw_version; + break; + case CGS_UCODE_ID_CP_CE: + fw_version = adev->gfx.ce_fw_version; + break; + case CGS_UCODE_ID_CP_PFP: + fw_version = adev->gfx.pfp_fw_version; + break; + case CGS_UCODE_ID_CP_ME: + fw_version = adev->gfx.me_fw_version; + break; + case CGS_UCODE_ID_CP_MEC: + fw_version = adev->gfx.mec_fw_version; + break; + case CGS_UCODE_ID_CP_MEC_JT1: + fw_version = adev->gfx.mec_fw_version; + break; + case CGS_UCODE_ID_CP_MEC_JT2: + fw_version = adev->gfx.mec_fw_version; + break; + case CGS_UCODE_ID_RLC_G: + fw_version = adev->gfx.rlc_fw_version; + break; + case CGS_UCODE_ID_STORAGE: + break; + default: + DRM_ERROR("firmware type %d do not have version\n", type); + break; + } + return fw_version; +} + +static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device, + enum cgs_ucode_id type, + struct cgs_firmware_info *info) +{ + CGS_FUNC_ADEV; + + if ((CGS_UCODE_ID_SMU != type) && (CGS_UCODE_ID_SMU_SK != type)) { + uint64_t gpu_addr; + uint32_t data_size; + const struct gfx_firmware_header_v1_0 *header; + enum AMDGPU_UCODE_ID id; + struct amdgpu_firmware_info *ucode; + + id = fw_type_convert(cgs_device, type); + ucode = &adev->firmware.ucode[id]; + if (ucode->fw == NULL) + return -EINVAL; + + gpu_addr = ucode->mc_addr; + header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + data_size = le32_to_cpu(header->header.ucode_size_bytes); + + if ((type == CGS_UCODE_ID_CP_MEC_JT1) || + (type == CGS_UCODE_ID_CP_MEC_JT2)) { + gpu_addr += ALIGN(le32_to_cpu(header->header.ucode_size_bytes), PAGE_SIZE); + data_size = le32_to_cpu(header->jt_size) << 2; + } + + info->kptr = ucode->kaddr; + info->image_size = data_size; + info->mc_addr = gpu_addr; + info->version = (uint16_t)le32_to_cpu(header->header.ucode_version); + + if (CGS_UCODE_ID_CP_MEC == type) + info->image_size = le32_to_cpu(header->jt_offset) << 2; + + info->fw_version = amdgpu_get_firmware_version(cgs_device, type); + info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version); + } else { + char fw_name[30] = {0}; + int err = 0; + uint32_t ucode_size; + uint32_t ucode_start_address; + const uint8_t *src; + const struct smc_firmware_header_v1_0 *hdr; + const struct common_firmware_header *header; + struct amdgpu_firmware_info *ucode = NULL; + + if (!adev->pm.fw) { + switch (adev->asic_type) { + case CHIP_TAHITI: + strcpy(fw_name, "radeon/tahiti_smc.bin"); + break; + case CHIP_PITCAIRN: + if ((adev->pdev->revision == 0x81) && + ((adev->pdev->device == 0x6810) || + (adev->pdev->device == 0x6811))) { + info->is_kicker = true; + strcpy(fw_name, "radeon/pitcairn_k_smc.bin"); + } else { + strcpy(fw_name, "radeon/pitcairn_smc.bin"); + } + break; + case CHIP_VERDE: + if (((adev->pdev->device == 0x6820) && + ((adev->pdev->revision == 0x81) || + (adev->pdev->revision == 0x83))) || + ((adev->pdev->device == 0x6821) && + ((adev->pdev->revision == 0x83) || + (adev->pdev->revision == 0x87))) || + ((adev->pdev->revision == 0x87) && + ((adev->pdev->device == 0x6823) || + (adev->pdev->device == 0x682b)))) { + info->is_kicker = true; + strcpy(fw_name, "radeon/verde_k_smc.bin"); + } else { + strcpy(fw_name, "radeon/verde_smc.bin"); + } + break; + case CHIP_OLAND: + if (((adev->pdev->revision == 0x81) && + ((adev->pdev->device == 0x6600) || + (adev->pdev->device == 0x6604) || + (adev->pdev->device == 0x6605) || + (adev->pdev->device == 0x6610))) || + ((adev->pdev->revision == 0x83) && + (adev->pdev->device == 0x6610))) { + info->is_kicker = true; + strcpy(fw_name, "radeon/oland_k_smc.bin"); + } else { + strcpy(fw_name, "radeon/oland_smc.bin"); + } + break; + case CHIP_HAINAN: + if (((adev->pdev->revision == 0x81) && + (adev->pdev->device == 0x6660)) || + ((adev->pdev->revision == 0x83) && + ((adev->pdev->device == 0x6660) || + (adev->pdev->device == 0x6663) || + (adev->pdev->device == 0x6665) || + (adev->pdev->device == 0x6667)))) { + info->is_kicker = true; + strcpy(fw_name, "radeon/hainan_k_smc.bin"); + } else if ((adev->pdev->revision == 0xc3) && + (adev->pdev->device == 0x6665)) { + info->is_kicker = true; + strcpy(fw_name, "radeon/banks_k_2_smc.bin"); + } else { + strcpy(fw_name, "radeon/hainan_smc.bin"); + } + break; + case CHIP_BONAIRE: + if ((adev->pdev->revision == 0x80) || + (adev->pdev->revision == 0x81) || + (adev->pdev->device == 0x665f)) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/bonaire_k_smc.bin"); + } else { + strcpy(fw_name, "amdgpu/bonaire_smc.bin"); + } + break; + case CHIP_HAWAII: + if (adev->pdev->revision == 0x80) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/hawaii_k_smc.bin"); + } else { + strcpy(fw_name, "amdgpu/hawaii_smc.bin"); + } + break; + case CHIP_TOPAZ: + if (((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x81)) || + ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0x83)) || + ((adev->pdev->device == 0x6907) && (adev->pdev->revision == 0x87)) || + ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0xD1)) || + ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0xD3))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/topaz_k_smc.bin"); + } else + strcpy(fw_name, "amdgpu/topaz_smc.bin"); + break; + case CHIP_TONGA: + if (((adev->pdev->device == 0x6939) && (adev->pdev->revision == 0xf1)) || + ((adev->pdev->device == 0x6938) && (adev->pdev->revision == 0xf1))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/tonga_k_smc.bin"); + } else + strcpy(fw_name, "amdgpu/tonga_smc.bin"); + break; + case CHIP_FIJI: + strcpy(fw_name, "amdgpu/fiji_smc.bin"); + break; + case CHIP_POLARIS11: + if (type == CGS_UCODE_ID_SMU) { + if (((adev->pdev->device == 0x67ef) && + ((adev->pdev->revision == 0xe0) || + (adev->pdev->revision == 0xe5))) || + ((adev->pdev->device == 0x67ff) && + ((adev->pdev->revision == 0xcf) || + (adev->pdev->revision == 0xef) || + (adev->pdev->revision == 0xff)))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/polaris11_k_smc.bin"); + } else if ((adev->pdev->device == 0x67ef) && + (adev->pdev->revision == 0xe2)) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/polaris11_k2_smc.bin"); + } else { + strcpy(fw_name, "amdgpu/polaris11_smc.bin"); + } + } else if (type == CGS_UCODE_ID_SMU_SK) { + strcpy(fw_name, "amdgpu/polaris11_smc_sk.bin"); + } + break; + case CHIP_POLARIS10: + if (type == CGS_UCODE_ID_SMU) { + if (((adev->pdev->device == 0x67df) && + ((adev->pdev->revision == 0xe0) || + (adev->pdev->revision == 0xe3) || + (adev->pdev->revision == 0xe4) || + (adev->pdev->revision == 0xe5) || + (adev->pdev->revision == 0xe7) || + (adev->pdev->revision == 0xef))) || + ((adev->pdev->device == 0x6fdf) && + ((adev->pdev->revision == 0xef) || + (adev->pdev->revision == 0xff)))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/polaris10_k_smc.bin"); + } else if ((adev->pdev->device == 0x67df) && + ((adev->pdev->revision == 0xe1) || + (adev->pdev->revision == 0xf7))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/polaris10_k2_smc.bin"); + } else { + strcpy(fw_name, "amdgpu/polaris10_smc.bin"); + } + } else if (type == CGS_UCODE_ID_SMU_SK) { + strcpy(fw_name, "amdgpu/polaris10_smc_sk.bin"); + } + break; + case CHIP_POLARIS12: + if (((adev->pdev->device == 0x6987) && + ((adev->pdev->revision == 0xc0) || + (adev->pdev->revision == 0xc3))) || + ((adev->pdev->device == 0x6981) && + ((adev->pdev->revision == 0x00) || + (adev->pdev->revision == 0x01) || + (adev->pdev->revision == 0x10)))) { + info->is_kicker = true; + strcpy(fw_name, "amdgpu/polaris12_k_smc.bin"); + } else { + strcpy(fw_name, "amdgpu/polaris12_smc.bin"); + } + break; + case CHIP_VEGAM: + strcpy(fw_name, "amdgpu/vegam_smc.bin"); + break; + case CHIP_VEGA10: + if ((adev->pdev->device == 0x687f) && + ((adev->pdev->revision == 0xc0) || + (adev->pdev->revision == 0xc1) || + (adev->pdev->revision == 0xc3))) + strcpy(fw_name, "amdgpu/vega10_acg_smc.bin"); + else + strcpy(fw_name, "amdgpu/vega10_smc.bin"); + break; + case CHIP_VEGA12: + strcpy(fw_name, "amdgpu/vega12_smc.bin"); + break; + case CHIP_VEGA20: + strcpy(fw_name, "amdgpu/vega20_smc.bin"); + break; + default: + DRM_ERROR("SMC firmware not supported\n"); + return -EINVAL; + } + + err = request_firmware(&adev->pm.fw, fw_name, adev->dev); + if (err) { + DRM_ERROR("Failed to request firmware\n"); + return err; + } + + err = amdgpu_ucode_validate(adev->pm.fw); + if (err) { + DRM_ERROR("Failed to load firmware \"%s\"", fw_name); + release_firmware(adev->pm.fw); + adev->pm.fw = NULL; + return err; + } + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC]; + ucode->ucode_id = AMDGPU_UCODE_ID_SMC; + ucode->fw = adev->pm.fw; + header = (const struct common_firmware_header *)ucode->fw->data; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); + } + } + + hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data; + amdgpu_ucode_print_smc_hdr(&hdr->header); + adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); + ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); + src = (const uint8_t *)(adev->pm.fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + info->version = adev->pm.fw_version; + info->image_size = ucode_size; + info->ucode_start_address = ucode_start_address; + info->kptr = (void *)src; + } + return 0; +} + +static const struct cgs_ops amdgpu_cgs_ops = { + .read_register = amdgpu_cgs_read_register, + .write_register = amdgpu_cgs_write_register, + .read_ind_register = amdgpu_cgs_read_ind_register, + .write_ind_register = amdgpu_cgs_write_ind_register, + .get_firmware_info = amdgpu_cgs_get_firmware_info, +}; + +struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev) +{ + struct amdgpu_cgs_device *cgs_device = + kmalloc(sizeof(*cgs_device), GFP_KERNEL); + + if (!cgs_device) { + DRM_ERROR("Couldn't allocate CGS device structure\n"); + return NULL; + } + + cgs_device->base.ops = &amdgpu_cgs_ops; + cgs_device->adev = adev; + + return (struct cgs_device *)cgs_device; +} + +void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device) +{ + kfree(cgs_device); +}
diff --git a/amdgpu/amdgpu_connectors.c b/amdgpu/amdgpu_connectors.c new file mode 100644 index 0000000..73b2ede --- /dev/null +++ b/amdgpu/amdgpu_connectors.c
@@ -0,0 +1,1898 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <drm/drm_edid.h> +#include <drm/drm_fb_helper.h> +#include <drm/drm_probe_helper.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atom.h" +#include "atombios_encoders.h" +#include "atombios_dp.h" +#include "amdgpu_connectors.h" +#include "amdgpu_i2c.h" +#include "amdgpu_display.h" + +#include <linux/pm_runtime.h> + +void amdgpu_connector_hotplug(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + /* bail if the connector does not have hpd pin, e.g., + * VGA, TV, etc. + */ + if (amdgpu_connector->hpd.hpd == AMDGPU_HPD_NONE) + return; + + amdgpu_display_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd); + + /* if the connector is already off, don't turn it back on */ + if (connector->dpms != DRM_MODE_DPMS_ON) + return; + + /* just deal with DP (not eDP) here. */ + if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) { + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + + /* if existing sink type was not DP no need to retrain */ + if (dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_DISPLAYPORT) + return; + + /* first get sink type as it may be reset after (un)plug */ + dig_connector->dp_sink_type = amdgpu_atombios_dp_get_sinktype(amdgpu_connector); + /* don't do anything if sink is not display port, i.e., + * passive dp->(dvi|hdmi) adaptor + */ + if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT && + amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd) && + amdgpu_atombios_dp_needs_link_train(amdgpu_connector)) { + /* Don't start link training before we have the DPCD */ + if (amdgpu_atombios_dp_get_dpcd(amdgpu_connector)) + return; + + /* Turn the connector off and back on immediately, which + * will trigger link training + */ + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + } + } +} + +static void amdgpu_connector_property_change_mode(struct drm_encoder *encoder) +{ + struct drm_crtc *crtc = encoder->crtc; + + if (crtc && crtc->enabled) { + drm_crtc_helper_set_mode(crtc, &crtc->mode, + crtc->x, crtc->y, crtc->primary->fb); + } +} + +int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector; + int bpc = 8; + unsigned mode_clock, max_tmds_clock; + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_HDMIB: + if (amdgpu_connector->use_digital) { + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + if (connector->display_info.bpc) + bpc = connector->display_info.bpc; + } + } + break; + case DRM_MODE_CONNECTOR_DVID: + case DRM_MODE_CONNECTOR_HDMIA: + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + if (connector->display_info.bpc) + bpc = connector->display_info.bpc; + } + break; + case DRM_MODE_CONNECTOR_DisplayPort: + dig_connector = amdgpu_connector->con_priv; + if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || + (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) || + drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + if (connector->display_info.bpc) + bpc = connector->display_info.bpc; + } + break; + case DRM_MODE_CONNECTOR_eDP: + case DRM_MODE_CONNECTOR_LVDS: + if (connector->display_info.bpc) + bpc = connector->display_info.bpc; + else { + const struct drm_connector_helper_funcs *connector_funcs = + connector->helper_private; + struct drm_encoder *encoder = connector_funcs->best_encoder(connector); + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + + if (dig->lcd_misc & ATOM_PANEL_MISC_V13_6BIT_PER_COLOR) + bpc = 6; + else if (dig->lcd_misc & ATOM_PANEL_MISC_V13_8BIT_PER_COLOR) + bpc = 8; + } + break; + } + + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + /* + * Pre DCE-8 hw can't handle > 12 bpc, and more than 12 bpc doesn't make + * much sense without support for > 12 bpc framebuffers. RGB 4:4:4 at + * 12 bpc is always supported on hdmi deep color sinks, as this is + * required by the HDMI-1.3 spec. Clamp to a safe 12 bpc maximum. + */ + if (bpc > 12) { + DRM_DEBUG("%s: HDMI deep color %d bpc unsupported. Using 12 bpc.\n", + connector->name, bpc); + bpc = 12; + } + + /* Any defined maximum tmds clock limit we must not exceed? */ + if (connector->display_info.max_tmds_clock > 0) { + /* mode_clock is clock in kHz for mode to be modeset on this connector */ + mode_clock = amdgpu_connector->pixelclock_for_modeset; + + /* Maximum allowable input clock in kHz */ + max_tmds_clock = connector->display_info.max_tmds_clock; + + DRM_DEBUG("%s: hdmi mode dotclock %d kHz, max tmds input clock %d kHz.\n", + connector->name, mode_clock, max_tmds_clock); + + /* Check if bpc is within clock limit. Try to degrade gracefully otherwise */ + if ((bpc == 12) && (mode_clock * 3/2 > max_tmds_clock)) { + if ((connector->display_info.edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30) && + (mode_clock * 5/4 <= max_tmds_clock)) + bpc = 10; + else + bpc = 8; + + DRM_DEBUG("%s: HDMI deep color 12 bpc exceeds max tmds clock. Using %d bpc.\n", + connector->name, bpc); + } + + if ((bpc == 10) && (mode_clock * 5/4 > max_tmds_clock)) { + bpc = 8; + DRM_DEBUG("%s: HDMI deep color 10 bpc exceeds max tmds clock. Using %d bpc.\n", + connector->name, bpc); + } + } else if (bpc > 8) { + /* max_tmds_clock missing, but hdmi spec mandates it for deep color. */ + DRM_DEBUG("%s: Required max tmds clock for HDMI deep color missing. Using 8 bpc.\n", + connector->name); + bpc = 8; + } + } + + if ((amdgpu_deep_color == 0) && (bpc > 8)) { + DRM_DEBUG("%s: Deep color disabled. Set amdgpu module param deep_color=1 to enable.\n", + connector->name); + bpc = 8; + } + + DRM_DEBUG("%s: Display bpc=%d, returned bpc=%d\n", + connector->name, connector->display_info.bpc, bpc); + + return bpc; +} + +static void +amdgpu_connector_update_scratch_regs(struct drm_connector *connector, + enum drm_connector_status status) +{ + struct drm_encoder *best_encoder; + struct drm_encoder *encoder; + const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; + bool connected; + int i; + + best_encoder = connector_funcs->best_encoder(connector); + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + if ((encoder == best_encoder) && (status == connector_status_connected)) + connected = true; + else + connected = false; + + amdgpu_atombios_encoder_set_bios_scratch_regs(connector, encoder, connected); + } +} + +static struct drm_encoder * +amdgpu_connector_find_encoder(struct drm_connector *connector, + int encoder_type) +{ + struct drm_encoder *encoder; + int i; + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + if (encoder->encoder_type == encoder_type) + return encoder; + } + + return NULL; +} + +struct edid *amdgpu_connector_edid(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_property_blob *edid_blob = connector->edid_blob_ptr; + + if (amdgpu_connector->edid) { + return amdgpu_connector->edid; + } else if (edid_blob) { + struct edid *edid = kmemdup(edid_blob->data, edid_blob->length, GFP_KERNEL); + if (edid) + amdgpu_connector->edid = edid; + } + return amdgpu_connector->edid; +} + +static struct edid * +amdgpu_connector_get_hardcoded_edid(struct amdgpu_device *adev) +{ + struct edid *edid; + + if (adev->mode_info.bios_hardcoded_edid) { + edid = kmalloc(adev->mode_info.bios_hardcoded_edid_size, GFP_KERNEL); + if (edid) { + memcpy((unsigned char *)edid, + (unsigned char *)adev->mode_info.bios_hardcoded_edid, + adev->mode_info.bios_hardcoded_edid_size); + return edid; + } + } + return NULL; +} + +static void amdgpu_connector_get_edid(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + if (amdgpu_connector->edid) + return; + + /* on hw with routers, select right port */ + if (amdgpu_connector->router.ddc_valid) + amdgpu_i2c_router_select_ddc_port(amdgpu_connector); + + if ((amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) != + ENCODER_OBJECT_ID_NONE) && + amdgpu_connector->ddc_bus->has_aux) { + amdgpu_connector->edid = drm_get_edid(connector, + &amdgpu_connector->ddc_bus->aux.ddc); + } else if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) || + (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) { + struct amdgpu_connector_atom_dig *dig = amdgpu_connector->con_priv; + + if ((dig->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT || + dig->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) && + amdgpu_connector->ddc_bus->has_aux) + amdgpu_connector->edid = drm_get_edid(connector, + &amdgpu_connector->ddc_bus->aux.ddc); + else if (amdgpu_connector->ddc_bus) + amdgpu_connector->edid = drm_get_edid(connector, + &amdgpu_connector->ddc_bus->adapter); + } else if (amdgpu_connector->ddc_bus) { + amdgpu_connector->edid = drm_get_edid(connector, + &amdgpu_connector->ddc_bus->adapter); + } + + if (!amdgpu_connector->edid) { + /* some laptops provide a hardcoded edid in rom for LCDs */ + if (((connector->connector_type == DRM_MODE_CONNECTOR_LVDS) || + (connector->connector_type == DRM_MODE_CONNECTOR_eDP))) + amdgpu_connector->edid = amdgpu_connector_get_hardcoded_edid(adev); + } +} + +static void amdgpu_connector_free_edid(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + kfree(amdgpu_connector->edid); + amdgpu_connector->edid = NULL; +} + +static int amdgpu_connector_ddc_get_modes(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + int ret; + + if (amdgpu_connector->edid) { + drm_connector_update_edid_property(connector, amdgpu_connector->edid); + ret = drm_add_edid_modes(connector, amdgpu_connector->edid); + return ret; + } + drm_connector_update_edid_property(connector, NULL); + return 0; +} + +static struct drm_encoder * +amdgpu_connector_best_single_encoder(struct drm_connector *connector) +{ + struct drm_encoder *encoder; + int i; + + /* pick the first one */ + drm_connector_for_each_possible_encoder(connector, encoder, i) + return encoder; + + return NULL; +} + +static void amdgpu_get_native_mode(struct drm_connector *connector) +{ + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + struct amdgpu_encoder *amdgpu_encoder; + + if (encoder == NULL) + return; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (!list_empty(&connector->probed_modes)) { + struct drm_display_mode *preferred_mode = + list_first_entry(&connector->probed_modes, + struct drm_display_mode, head); + + amdgpu_encoder->native_mode = *preferred_mode; + } else { + amdgpu_encoder->native_mode.clock = 0; + } +} + +static struct drm_display_mode * +amdgpu_connector_lcd_native_mode(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *mode = NULL; + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + + if (native_mode->hdisplay != 0 && + native_mode->vdisplay != 0 && + native_mode->clock != 0) { + mode = drm_mode_duplicate(dev, native_mode); + mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER; + drm_mode_set_name(mode); + + DRM_DEBUG_KMS("Adding native panel mode %s\n", mode->name); + } else if (native_mode->hdisplay != 0 && + native_mode->vdisplay != 0) { + /* mac laptops without an edid */ + /* Note that this is not necessarily the exact panel mode, + * but an approximation based on the cvt formula. For these + * systems we should ideally read the mode info out of the + * registers or add a mode table, but this works and is much + * simpler. + */ + mode = drm_cvt_mode(dev, native_mode->hdisplay, native_mode->vdisplay, 60, true, false, false); + mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER; + DRM_DEBUG_KMS("Adding cvt approximation of native panel mode %s\n", mode->name); + } + return mode; +} + +static void amdgpu_connector_add_common_modes(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *mode = NULL; + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + int i; + static const struct mode_size { + int w; + int h; + } common_modes[17] = { + { 640, 480}, + { 720, 480}, + { 800, 600}, + { 848, 480}, + {1024, 768}, + {1152, 768}, + {1280, 720}, + {1280, 800}, + {1280, 854}, + {1280, 960}, + {1280, 1024}, + {1440, 900}, + {1400, 1050}, + {1680, 1050}, + {1600, 1200}, + {1920, 1080}, + {1920, 1200} + }; + + for (i = 0; i < 17; i++) { + if (amdgpu_encoder->devices & (ATOM_DEVICE_TV_SUPPORT)) { + if (common_modes[i].w > 1024 || + common_modes[i].h > 768) + continue; + } + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + if (common_modes[i].w > native_mode->hdisplay || + common_modes[i].h > native_mode->vdisplay || + (common_modes[i].w == native_mode->hdisplay && + common_modes[i].h == native_mode->vdisplay)) + continue; + } + if (common_modes[i].w < 320 || common_modes[i].h < 200) + continue; + + mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false); + drm_mode_probed_add(connector, mode); + } +} + +static int amdgpu_connector_set_property(struct drm_connector *connector, + struct drm_property *property, + uint64_t val) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + + if (property == adev->mode_info.coherent_mode_property) { + struct amdgpu_encoder_atom_dig *dig; + bool new_coherent_mode; + + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (!amdgpu_encoder->enc_priv) + return 0; + + dig = amdgpu_encoder->enc_priv; + new_coherent_mode = val ? true : false; + if (dig->coherent_mode != new_coherent_mode) { + dig->coherent_mode = new_coherent_mode; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.audio_property) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (amdgpu_connector->audio != val) { + amdgpu_connector->audio = val; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.dither_property) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (amdgpu_connector->dither != val) { + amdgpu_connector->dither = val; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.underscan_property) { + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (amdgpu_encoder->underscan_type != val) { + amdgpu_encoder->underscan_type = val; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.underscan_hborder_property) { + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (amdgpu_encoder->underscan_hborder != val) { + amdgpu_encoder->underscan_hborder = val; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.underscan_vborder_property) { + /* need to find digital encoder on connector */ + encoder = amdgpu_connector_find_encoder(connector, DRM_MODE_ENCODER_TMDS); + if (!encoder) + return 0; + + amdgpu_encoder = to_amdgpu_encoder(encoder); + + if (amdgpu_encoder->underscan_vborder != val) { + amdgpu_encoder->underscan_vborder = val; + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + } + + if (property == adev->mode_info.load_detect_property) { + struct amdgpu_connector *amdgpu_connector = + to_amdgpu_connector(connector); + + if (val == 0) + amdgpu_connector->dac_load_detect = false; + else + amdgpu_connector->dac_load_detect = true; + } + + if (property == dev->mode_config.scaling_mode_property) { + enum amdgpu_rmx_type rmx_type; + + if (connector->encoder) { + amdgpu_encoder = to_amdgpu_encoder(connector->encoder); + } else { + const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; + amdgpu_encoder = to_amdgpu_encoder(connector_funcs->best_encoder(connector)); + } + + switch (val) { + default: + case DRM_MODE_SCALE_NONE: rmx_type = RMX_OFF; break; + case DRM_MODE_SCALE_CENTER: rmx_type = RMX_CENTER; break; + case DRM_MODE_SCALE_ASPECT: rmx_type = RMX_ASPECT; break; + case DRM_MODE_SCALE_FULLSCREEN: rmx_type = RMX_FULL; break; + } + if (amdgpu_encoder->rmx_type == rmx_type) + return 0; + + if ((rmx_type != DRM_MODE_SCALE_NONE) && + (amdgpu_encoder->native_mode.clock == 0)) + return 0; + + amdgpu_encoder->rmx_type = rmx_type; + + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + } + + return 0; +} + +static void +amdgpu_connector_fixup_lcd_native_mode(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + struct drm_display_mode *t, *mode; + + /* If the EDID preferred mode doesn't match the native mode, use it */ + list_for_each_entry_safe(mode, t, &connector->probed_modes, head) { + if (mode->type & DRM_MODE_TYPE_PREFERRED) { + if (mode->hdisplay != native_mode->hdisplay || + mode->vdisplay != native_mode->vdisplay) + memcpy(native_mode, mode, sizeof(*mode)); + } + } + + /* Try to get native mode details from EDID if necessary */ + if (!native_mode->clock) { + list_for_each_entry_safe(mode, t, &connector->probed_modes, head) { + if (mode->hdisplay == native_mode->hdisplay && + mode->vdisplay == native_mode->vdisplay) { + *native_mode = *mode; + drm_mode_set_crtcinfo(native_mode, CRTC_INTERLACE_HALVE_V); + DRM_DEBUG_KMS("Determined LVDS native mode details from EDID\n"); + break; + } + } + } + + if (!native_mode->clock) { + DRM_DEBUG_KMS("No LVDS native mode details, disabling RMX\n"); + amdgpu_encoder->rmx_type = RMX_OFF; + } +} + +static int amdgpu_connector_lvds_get_modes(struct drm_connector *connector) +{ + struct drm_encoder *encoder; + int ret = 0; + struct drm_display_mode *mode; + + amdgpu_connector_get_edid(connector); + ret = amdgpu_connector_ddc_get_modes(connector); + if (ret > 0) { + encoder = amdgpu_connector_best_single_encoder(connector); + if (encoder) { + amdgpu_connector_fixup_lcd_native_mode(encoder, connector); + /* add scaled modes */ + amdgpu_connector_add_common_modes(encoder, connector); + } + return ret; + } + + encoder = amdgpu_connector_best_single_encoder(connector); + if (!encoder) + return 0; + + /* we have no EDID modes */ + mode = amdgpu_connector_lcd_native_mode(encoder); + if (mode) { + ret = 1; + drm_mode_probed_add(connector, mode); + /* add the width/height from vbios tables if available */ + connector->display_info.width_mm = mode->width_mm; + connector->display_info.height_mm = mode->height_mm; + /* add scaled modes */ + amdgpu_connector_add_common_modes(encoder, connector); + } + + return ret; +} + +static enum drm_mode_status amdgpu_connector_lvds_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + + if ((mode->hdisplay < 320) || (mode->vdisplay < 240)) + return MODE_PANEL; + + if (encoder) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + + /* AVIVO hardware supports downscaling modes larger than the panel + * to the panel size, but I'm not sure this is desirable. + */ + if ((mode->hdisplay > native_mode->hdisplay) || + (mode->vdisplay > native_mode->vdisplay)) + return MODE_PANEL; + + /* if scaling is disabled, block non-native modes */ + if (amdgpu_encoder->rmx_type == RMX_OFF) { + if ((mode->hdisplay != native_mode->hdisplay) || + (mode->vdisplay != native_mode->vdisplay)) + return MODE_PANEL; + } + } + + return MODE_OK; +} + +static enum drm_connector_status +amdgpu_connector_lvds_detect(struct drm_connector *connector, bool force) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + enum drm_connector_status ret = connector_status_disconnected; + int r; + + if (!drm_kms_helper_is_poll_worker()) { + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + } + + if (encoder) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + + /* check if panel is valid */ + if (native_mode->hdisplay >= 320 && native_mode->vdisplay >= 240) + ret = connector_status_connected; + + } + + /* check for edid as well */ + amdgpu_connector_get_edid(connector); + if (amdgpu_connector->edid) + ret = connector_status_connected; + /* check acpi lid status ??? */ + + amdgpu_connector_update_scratch_regs(connector, ret); + + if (!drm_kms_helper_is_poll_worker()) { + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + } + + return ret; +} + +static void amdgpu_connector_unregister(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + if (amdgpu_connector->ddc_bus && amdgpu_connector->ddc_bus->has_aux) { + drm_dp_aux_unregister(&amdgpu_connector->ddc_bus->aux); + amdgpu_connector->ddc_bus->has_aux = false; + } +} + +static void amdgpu_connector_destroy(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + amdgpu_connector_free_edid(connector); + kfree(amdgpu_connector->con_priv); + drm_connector_unregister(connector); + drm_connector_cleanup(connector); + kfree(connector); +} + +static int amdgpu_connector_set_lcd_property(struct drm_connector *connector, + struct drm_property *property, + uint64_t value) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_encoder *amdgpu_encoder; + enum amdgpu_rmx_type rmx_type; + + DRM_DEBUG_KMS("\n"); + if (property != dev->mode_config.scaling_mode_property) + return 0; + + if (connector->encoder) + amdgpu_encoder = to_amdgpu_encoder(connector->encoder); + else { + const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; + amdgpu_encoder = to_amdgpu_encoder(connector_funcs->best_encoder(connector)); + } + + switch (value) { + case DRM_MODE_SCALE_NONE: rmx_type = RMX_OFF; break; + case DRM_MODE_SCALE_CENTER: rmx_type = RMX_CENTER; break; + case DRM_MODE_SCALE_ASPECT: rmx_type = RMX_ASPECT; break; + default: + case DRM_MODE_SCALE_FULLSCREEN: rmx_type = RMX_FULL; break; + } + if (amdgpu_encoder->rmx_type == rmx_type) + return 0; + + amdgpu_encoder->rmx_type = rmx_type; + + amdgpu_connector_property_change_mode(&amdgpu_encoder->base); + return 0; +} + + +static const struct drm_connector_helper_funcs amdgpu_connector_lvds_helper_funcs = { + .get_modes = amdgpu_connector_lvds_get_modes, + .mode_valid = amdgpu_connector_lvds_mode_valid, + .best_encoder = amdgpu_connector_best_single_encoder, +}; + +static const struct drm_connector_funcs amdgpu_connector_lvds_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = amdgpu_connector_lvds_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .early_unregister = amdgpu_connector_unregister, + .destroy = amdgpu_connector_destroy, + .set_property = amdgpu_connector_set_lcd_property, +}; + +static int amdgpu_connector_vga_get_modes(struct drm_connector *connector) +{ + int ret; + + amdgpu_connector_get_edid(connector); + ret = amdgpu_connector_ddc_get_modes(connector); + + return ret; +} + +static enum drm_mode_status amdgpu_connector_vga_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + + /* XXX check mode bandwidth */ + + if ((mode->clock / 10) > adev->clock.max_pixel_clock) + return MODE_CLOCK_HIGH; + + return MODE_OK; +} + +static enum drm_connector_status +amdgpu_connector_vga_detect(struct drm_connector *connector, bool force) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_encoder *encoder; + const struct drm_encoder_helper_funcs *encoder_funcs; + bool dret = false; + enum drm_connector_status ret = connector_status_disconnected; + int r; + + if (!drm_kms_helper_is_poll_worker()) { + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + } + + encoder = amdgpu_connector_best_single_encoder(connector); + if (!encoder) + ret = connector_status_disconnected; + + if (amdgpu_connector->ddc_bus) + dret = amdgpu_display_ddc_probe(amdgpu_connector, false); + if (dret) { + amdgpu_connector->detected_by_load = false; + amdgpu_connector_free_edid(connector); + amdgpu_connector_get_edid(connector); + + if (!amdgpu_connector->edid) { + DRM_ERROR("%s: probed a monitor but no|invalid EDID\n", + connector->name); + ret = connector_status_connected; + } else { + amdgpu_connector->use_digital = + !!(amdgpu_connector->edid->input & DRM_EDID_INPUT_DIGITAL); + + /* some oems have boards with separate digital and analog connectors + * with a shared ddc line (often vga + hdmi) + */ + if (amdgpu_connector->use_digital && amdgpu_connector->shared_ddc) { + amdgpu_connector_free_edid(connector); + ret = connector_status_disconnected; + } else { + ret = connector_status_connected; + } + } + } else { + + /* if we aren't forcing don't do destructive polling */ + if (!force) { + /* only return the previous status if we last + * detected a monitor via load. + */ + if (amdgpu_connector->detected_by_load) + ret = connector->status; + goto out; + } + + if (amdgpu_connector->dac_load_detect && encoder) { + encoder_funcs = encoder->helper_private; + ret = encoder_funcs->detect(encoder, connector); + if (ret != connector_status_disconnected) + amdgpu_connector->detected_by_load = true; + } + } + + amdgpu_connector_update_scratch_regs(connector, ret); + +out: + if (!drm_kms_helper_is_poll_worker()) { + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + } + + return ret; +} + +static const struct drm_connector_helper_funcs amdgpu_connector_vga_helper_funcs = { + .get_modes = amdgpu_connector_vga_get_modes, + .mode_valid = amdgpu_connector_vga_mode_valid, + .best_encoder = amdgpu_connector_best_single_encoder, +}; + +static const struct drm_connector_funcs amdgpu_connector_vga_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = amdgpu_connector_vga_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .early_unregister = amdgpu_connector_unregister, + .destroy = amdgpu_connector_destroy, + .set_property = amdgpu_connector_set_property, +}; + +static bool +amdgpu_connector_check_hpd_status_unchanged(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + enum drm_connector_status status; + + if (amdgpu_connector->hpd.hpd != AMDGPU_HPD_NONE) { + if (amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) + status = connector_status_connected; + else + status = connector_status_disconnected; + if (connector->status == status) + return true; + } + + return false; +} + +/* + * DVI is complicated + * Do a DDC probe, if DDC probe passes, get the full EDID so + * we can do analog/digital monitor detection at this point. + * If the monitor is an analog monitor or we got no DDC, + * we need to find the DAC encoder object for this connector. + * If we got no DDC, we do load detection on the DAC encoder object. + * If we got analog DDC or load detection passes on the DAC encoder + * we have to check if this analog encoder is shared with anyone else (TV) + * if its shared we have to set the other connector to disconnected. + */ +static enum drm_connector_status +amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + const struct drm_encoder_helper_funcs *encoder_funcs; + int r; + enum drm_connector_status ret = connector_status_disconnected; + bool dret = false, broken_edid = false; + + if (!drm_kms_helper_is_poll_worker()) { + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + } + + if (!force && amdgpu_connector_check_hpd_status_unchanged(connector)) { + ret = connector->status; + goto exit; + } + + if (amdgpu_connector->ddc_bus) + dret = amdgpu_display_ddc_probe(amdgpu_connector, false); + if (dret) { + amdgpu_connector->detected_by_load = false; + amdgpu_connector_free_edid(connector); + amdgpu_connector_get_edid(connector); + + if (!amdgpu_connector->edid) { + DRM_ERROR("%s: probed a monitor but no|invalid EDID\n", + connector->name); + ret = connector_status_connected; + broken_edid = true; /* defer use_digital to later */ + } else { + amdgpu_connector->use_digital = + !!(amdgpu_connector->edid->input & DRM_EDID_INPUT_DIGITAL); + + /* some oems have boards with separate digital and analog connectors + * with a shared ddc line (often vga + hdmi) + */ + if ((!amdgpu_connector->use_digital) && amdgpu_connector->shared_ddc) { + amdgpu_connector_free_edid(connector); + ret = connector_status_disconnected; + } else { + ret = connector_status_connected; + } + + /* This gets complicated. We have boards with VGA + HDMI with a + * shared DDC line and we have boards with DVI-D + HDMI with a shared + * DDC line. The latter is more complex because with DVI<->HDMI adapters + * you don't really know what's connected to which port as both are digital. + */ + if (amdgpu_connector->shared_ddc && (ret == connector_status_connected)) { + struct drm_connector *list_connector; + struct amdgpu_connector *list_amdgpu_connector; + list_for_each_entry(list_connector, &dev->mode_config.connector_list, head) { + if (connector == list_connector) + continue; + list_amdgpu_connector = to_amdgpu_connector(list_connector); + if (list_amdgpu_connector->shared_ddc && + (list_amdgpu_connector->ddc_bus->rec.i2c_id == + amdgpu_connector->ddc_bus->rec.i2c_id)) { + /* cases where both connectors are digital */ + if (list_connector->connector_type != DRM_MODE_CONNECTOR_VGA) { + /* hpd is our only option in this case */ + if (!amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) { + amdgpu_connector_free_edid(connector); + ret = connector_status_disconnected; + } + } + } + } + } + } + } + + if ((ret == connector_status_connected) && (amdgpu_connector->use_digital == true)) + goto out; + + /* DVI-D and HDMI-A are digital only */ + if ((connector->connector_type == DRM_MODE_CONNECTOR_DVID) || + (connector->connector_type == DRM_MODE_CONNECTOR_HDMIA)) + goto out; + + /* if we aren't forcing don't do destructive polling */ + if (!force) { + /* only return the previous status if we last + * detected a monitor via load. + */ + if (amdgpu_connector->detected_by_load) + ret = connector->status; + goto out; + } + + /* find analog encoder */ + if (amdgpu_connector->dac_load_detect) { + struct drm_encoder *encoder; + int i; + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + if (encoder->encoder_type != DRM_MODE_ENCODER_DAC && + encoder->encoder_type != DRM_MODE_ENCODER_TVDAC) + continue; + + encoder_funcs = encoder->helper_private; + if (encoder_funcs->detect) { + if (!broken_edid) { + if (ret != connector_status_connected) { + /* deal with analog monitors without DDC */ + ret = encoder_funcs->detect(encoder, connector); + if (ret == connector_status_connected) { + amdgpu_connector->use_digital = false; + } + if (ret != connector_status_disconnected) + amdgpu_connector->detected_by_load = true; + } + } else { + enum drm_connector_status lret; + /* assume digital unless load detected otherwise */ + amdgpu_connector->use_digital = true; + lret = encoder_funcs->detect(encoder, connector); + DRM_DEBUG_KMS("load_detect %x returned: %x\n",encoder->encoder_type,lret); + if (lret == connector_status_connected) + amdgpu_connector->use_digital = false; + } + break; + } + } + } + +out: + /* updated in get modes as well since we need to know if it's analog or digital */ + amdgpu_connector_update_scratch_regs(connector, ret); + +exit: + if (!drm_kms_helper_is_poll_worker()) { + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + } + + return ret; +} + +/* okay need to be smart in here about which encoder to pick */ +static struct drm_encoder * +amdgpu_connector_dvi_encoder(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_encoder *encoder; + int i; + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + if (amdgpu_connector->use_digital == true) { + if (encoder->encoder_type == DRM_MODE_ENCODER_TMDS) + return encoder; + } else { + if (encoder->encoder_type == DRM_MODE_ENCODER_DAC || + encoder->encoder_type == DRM_MODE_ENCODER_TVDAC) + return encoder; + } + } + + /* see if we have a default encoder TODO */ + + /* then check use digitial */ + /* pick the first one */ + drm_connector_for_each_possible_encoder(connector, encoder, i) + return encoder; + + return NULL; +} + +static void amdgpu_connector_dvi_force(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + if (connector->force == DRM_FORCE_ON) + amdgpu_connector->use_digital = false; + if (connector->force == DRM_FORCE_ON_DIGITAL) + amdgpu_connector->use_digital = true; +} + +static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + /* XXX check mode bandwidth */ + + if (amdgpu_connector->use_digital && (mode->clock > 165000)) { + if ((amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_I) || + (amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D) || + (amdgpu_connector->connector_object_id == CONNECTOR_OBJECT_ID_HDMI_TYPE_B)) { + return MODE_OK; + } else if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + /* HDMI 1.3+ supports max clock of 340 Mhz */ + if (mode->clock > 340000) + return MODE_CLOCK_HIGH; + else + return MODE_OK; + } else { + return MODE_CLOCK_HIGH; + } + } + + /* check against the max pixel clock */ + if ((mode->clock / 10) > adev->clock.max_pixel_clock) + return MODE_CLOCK_HIGH; + + return MODE_OK; +} + +static const struct drm_connector_helper_funcs amdgpu_connector_dvi_helper_funcs = { + .get_modes = amdgpu_connector_vga_get_modes, + .mode_valid = amdgpu_connector_dvi_mode_valid, + .best_encoder = amdgpu_connector_dvi_encoder, +}; + +static const struct drm_connector_funcs amdgpu_connector_dvi_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = amdgpu_connector_dvi_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .set_property = amdgpu_connector_set_property, + .early_unregister = amdgpu_connector_unregister, + .destroy = amdgpu_connector_destroy, + .force = amdgpu_connector_dvi_force, +}; + +static int amdgpu_connector_dp_get_modes(struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *amdgpu_dig_connector = amdgpu_connector->con_priv; + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + int ret; + + if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) || + (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) { + struct drm_display_mode *mode; + + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { + if (!amdgpu_dig_connector->edp_on) + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_ON); + amdgpu_connector_get_edid(connector); + ret = amdgpu_connector_ddc_get_modes(connector); + if (!amdgpu_dig_connector->edp_on) + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_OFF); + } else { + /* need to setup ddc on the bridge */ + if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) != + ENCODER_OBJECT_ID_NONE) { + if (encoder) + amdgpu_atombios_encoder_setup_ext_encoder_ddc(encoder); + } + amdgpu_connector_get_edid(connector); + ret = amdgpu_connector_ddc_get_modes(connector); + } + + if (ret > 0) { + if (encoder) { + amdgpu_connector_fixup_lcd_native_mode(encoder, connector); + /* add scaled modes */ + amdgpu_connector_add_common_modes(encoder, connector); + } + return ret; + } + + if (!encoder) + return 0; + + /* we have no EDID modes */ + mode = amdgpu_connector_lcd_native_mode(encoder); + if (mode) { + ret = 1; + drm_mode_probed_add(connector, mode); + /* add the width/height from vbios tables if available */ + connector->display_info.width_mm = mode->width_mm; + connector->display_info.height_mm = mode->height_mm; + /* add scaled modes */ + amdgpu_connector_add_common_modes(encoder, connector); + } + } else { + /* need to setup ddc on the bridge */ + if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) != + ENCODER_OBJECT_ID_NONE) { + if (encoder) + amdgpu_atombios_encoder_setup_ext_encoder_ddc(encoder); + } + amdgpu_connector_get_edid(connector); + ret = amdgpu_connector_ddc_get_modes(connector); + + amdgpu_get_native_mode(connector); + } + + return ret; +} + +u16 amdgpu_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector) +{ + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + int i; + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + amdgpu_encoder = to_amdgpu_encoder(encoder); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_TRAVIS: + case ENCODER_OBJECT_ID_NUTMEG: + return amdgpu_encoder->encoder_id; + default: + break; + } + } + + return ENCODER_OBJECT_ID_NONE; +} + +static bool amdgpu_connector_encoder_is_hbr2(struct drm_connector *connector) +{ + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + int i; + bool found = false; + + drm_connector_for_each_possible_encoder(connector, encoder, i) { + amdgpu_encoder = to_amdgpu_encoder(encoder); + if (amdgpu_encoder->caps & ATOM_ENCODER_CAP_RECORD_HBR2) + found = true; + } + + return found; +} + +bool amdgpu_connector_is_dp12_capable(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + + if ((adev->clock.default_dispclk >= 53900) && + amdgpu_connector_encoder_is_hbr2(connector)) { + return true; + } + + return false; +} + +static enum drm_connector_status +amdgpu_connector_dp_detect(struct drm_connector *connector, bool force) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + enum drm_connector_status ret = connector_status_disconnected; + struct amdgpu_connector_atom_dig *amdgpu_dig_connector = amdgpu_connector->con_priv; + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + int r; + + if (!drm_kms_helper_is_poll_worker()) { + r = pm_runtime_get_sync(connector->dev->dev); + if (r < 0) + return connector_status_disconnected; + } + + if (!force && amdgpu_connector_check_hpd_status_unchanged(connector)) { + ret = connector->status; + goto out; + } + + amdgpu_connector_free_edid(connector); + + if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) || + (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) { + if (encoder) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + + /* check if panel is valid */ + if (native_mode->hdisplay >= 320 && native_mode->vdisplay >= 240) + ret = connector_status_connected; + } + /* eDP is always DP */ + amdgpu_dig_connector->dp_sink_type = CONNECTOR_OBJECT_ID_DISPLAYPORT; + if (!amdgpu_dig_connector->edp_on) + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_ON); + if (!amdgpu_atombios_dp_get_dpcd(amdgpu_connector)) + ret = connector_status_connected; + if (!amdgpu_dig_connector->edp_on) + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_OFF); + } else if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) != + ENCODER_OBJECT_ID_NONE) { + /* DP bridges are always DP */ + amdgpu_dig_connector->dp_sink_type = CONNECTOR_OBJECT_ID_DISPLAYPORT; + /* get the DPCD from the bridge */ + amdgpu_atombios_dp_get_dpcd(amdgpu_connector); + + if (encoder) { + /* setup ddc on the bridge */ + amdgpu_atombios_encoder_setup_ext_encoder_ddc(encoder); + /* bridge chips are always aux */ + /* try DDC */ + if (amdgpu_display_ddc_probe(amdgpu_connector, true)) + ret = connector_status_connected; + else if (amdgpu_connector->dac_load_detect) { /* try load detection */ + const struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private; + ret = encoder_funcs->detect(encoder, connector); + } + } + } else { + amdgpu_dig_connector->dp_sink_type = + amdgpu_atombios_dp_get_sinktype(amdgpu_connector); + if (amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) { + ret = connector_status_connected; + if (amdgpu_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) + amdgpu_atombios_dp_get_dpcd(amdgpu_connector); + } else { + if (amdgpu_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) { + if (!amdgpu_atombios_dp_get_dpcd(amdgpu_connector)) + ret = connector_status_connected; + } else { + /* try non-aux ddc (DP to DVI/HDMI/etc. adapter) */ + if (amdgpu_display_ddc_probe(amdgpu_connector, + false)) + ret = connector_status_connected; + } + } + } + + amdgpu_connector_update_scratch_regs(connector, ret); +out: + if (!drm_kms_helper_is_poll_worker()) { + pm_runtime_mark_last_busy(connector->dev->dev); + pm_runtime_put_autosuspend(connector->dev->dev); + } + + return ret; +} + +static enum drm_mode_status amdgpu_connector_dp_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *amdgpu_dig_connector = amdgpu_connector->con_priv; + + /* XXX check mode bandwidth */ + + if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) || + (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) { + struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector); + + if ((mode->hdisplay < 320) || (mode->vdisplay < 240)) + return MODE_PANEL; + + if (encoder) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + + /* AVIVO hardware supports downscaling modes larger than the panel + * to the panel size, but I'm not sure this is desirable. + */ + if ((mode->hdisplay > native_mode->hdisplay) || + (mode->vdisplay > native_mode->vdisplay)) + return MODE_PANEL; + + /* if scaling is disabled, block non-native modes */ + if (amdgpu_encoder->rmx_type == RMX_OFF) { + if ((mode->hdisplay != native_mode->hdisplay) || + (mode->vdisplay != native_mode->vdisplay)) + return MODE_PANEL; + } + } + return MODE_OK; + } else { + if ((amdgpu_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || + (amdgpu_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { + return amdgpu_atombios_dp_mode_valid_helper(connector, mode); + } else { + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + /* HDMI 1.3+ supports max clock of 340 Mhz */ + if (mode->clock > 340000) + return MODE_CLOCK_HIGH; + } else { + if (mode->clock > 165000) + return MODE_CLOCK_HIGH; + } + } + } + + return MODE_OK; +} + +static const struct drm_connector_helper_funcs amdgpu_connector_dp_helper_funcs = { + .get_modes = amdgpu_connector_dp_get_modes, + .mode_valid = amdgpu_connector_dp_mode_valid, + .best_encoder = amdgpu_connector_dvi_encoder, +}; + +static const struct drm_connector_funcs amdgpu_connector_dp_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = amdgpu_connector_dp_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .set_property = amdgpu_connector_set_property, + .early_unregister = amdgpu_connector_unregister, + .destroy = amdgpu_connector_destroy, + .force = amdgpu_connector_dvi_force, +}; + +static const struct drm_connector_funcs amdgpu_connector_edp_funcs = { + .dpms = drm_helper_connector_dpms, + .detect = amdgpu_connector_dp_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .set_property = amdgpu_connector_set_lcd_property, + .early_unregister = amdgpu_connector_unregister, + .destroy = amdgpu_connector_destroy, + .force = amdgpu_connector_dvi_force, +}; + +void +amdgpu_connector_add(struct amdgpu_device *adev, + uint32_t connector_id, + uint32_t supported_device, + int connector_type, + struct amdgpu_i2c_bus_rec *i2c_bus, + uint16_t connector_object_id, + struct amdgpu_hpd *hpd, + struct amdgpu_router *router) +{ + struct drm_device *dev = adev->ddev; + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + struct amdgpu_connector_atom_dig *amdgpu_dig_connector; + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + uint32_t subpixel_order = SubPixelNone; + bool shared_ddc = false; + bool is_dp_bridge = false; + bool has_aux = false; + + if (connector_type == DRM_MODE_CONNECTOR_Unknown) + return; + + /* see if we already added it */ + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + amdgpu_connector = to_amdgpu_connector(connector); + if (amdgpu_connector->connector_id == connector_id) { + amdgpu_connector->devices |= supported_device; + return; + } + if (amdgpu_connector->ddc_bus && i2c_bus->valid) { + if (amdgpu_connector->ddc_bus->rec.i2c_id == i2c_bus->i2c_id) { + amdgpu_connector->shared_ddc = true; + shared_ddc = true; + } + if (amdgpu_connector->router_bus && router->ddc_valid && + (amdgpu_connector->router.router_id == router->router_id)) { + amdgpu_connector->shared_ddc = false; + shared_ddc = false; + } + } + } + + /* check if it's a dp bridge */ + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + amdgpu_encoder = to_amdgpu_encoder(encoder); + if (amdgpu_encoder->devices & supported_device) { + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_TRAVIS: + case ENCODER_OBJECT_ID_NUTMEG: + is_dp_bridge = true; + break; + default: + break; + } + } + } + + amdgpu_connector = kzalloc(sizeof(struct amdgpu_connector), GFP_KERNEL); + if (!amdgpu_connector) + return; + + connector = &amdgpu_connector->base; + + amdgpu_connector->connector_id = connector_id; + amdgpu_connector->devices = supported_device; + amdgpu_connector->shared_ddc = shared_ddc; + amdgpu_connector->connector_object_id = connector_object_id; + amdgpu_connector->hpd = *hpd; + + amdgpu_connector->router = *router; + if (router->ddc_valid || router->cd_valid) { + amdgpu_connector->router_bus = amdgpu_i2c_lookup(adev, &router->i2c_info); + if (!amdgpu_connector->router_bus) + DRM_ERROR("Failed to assign router i2c bus! Check dmesg for i2c errors.\n"); + } + + if (is_dp_bridge) { + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (amdgpu_connector->ddc_bus) + has_aux = true; + else + DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + switch (connector_type) { + case DRM_MODE_CONNECTOR_VGA: + case DRM_MODE_CONNECTOR_DVIA: + default: + drm_connector_init(dev, &amdgpu_connector->base, + &amdgpu_connector_dp_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, + &amdgpu_connector_dp_helper_funcs); + connector->interlace_allowed = true; + connector->doublescan_allowed = true; + amdgpu_connector->dac_load_detect = true; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.load_detect_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + break; + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_DVID: + case DRM_MODE_CONNECTOR_HDMIA: + case DRM_MODE_CONNECTOR_HDMIB: + case DRM_MODE_CONNECTOR_DisplayPort: + drm_connector_init(dev, &amdgpu_connector->base, + &amdgpu_connector_dp_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, + &amdgpu_connector_dp_helper_funcs); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_property, + UNDERSCAN_OFF); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_hborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_vborder_property, + 0); + + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.dither_property, + AMDGPU_FMT_DITHER_DISABLE); + + if (amdgpu_audio != 0) + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.audio_property, + AMDGPU_AUDIO_AUTO); + + subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = true; + if (connector_type == DRM_MODE_CONNECTOR_HDMIB) + connector->doublescan_allowed = true; + else + connector->doublescan_allowed = false; + if (connector_type == DRM_MODE_CONNECTOR_DVII) { + amdgpu_connector->dac_load_detect = true; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.load_detect_property, + 1); + } + break; + case DRM_MODE_CONNECTOR_LVDS: + case DRM_MODE_CONNECTOR_eDP: + drm_connector_init(dev, &amdgpu_connector->base, + &amdgpu_connector_edp_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, + &amdgpu_connector_dp_helper_funcs); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_FULLSCREEN); + subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = false; + connector->doublescan_allowed = false; + break; + } + } else { + switch (connector_type) { + case DRM_MODE_CONNECTOR_VGA: + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_vga_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_vga_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (!amdgpu_connector->ddc_bus) + DRM_ERROR("VGA: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + amdgpu_connector->dac_load_detect = true; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.load_detect_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + /* no HPD on analog connectors */ + amdgpu_connector->hpd.hpd = AMDGPU_HPD_NONE; + connector->interlace_allowed = true; + connector->doublescan_allowed = true; + break; + case DRM_MODE_CONNECTOR_DVIA: + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_vga_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_vga_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (!amdgpu_connector->ddc_bus) + DRM_ERROR("DVIA: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + amdgpu_connector->dac_load_detect = true; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.load_detect_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + /* no HPD on analog connectors */ + amdgpu_connector->hpd.hpd = AMDGPU_HPD_NONE; + connector->interlace_allowed = true; + connector->doublescan_allowed = true; + break; + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_DVID: + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_dvi_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_dvi_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (!amdgpu_connector->ddc_bus) + DRM_ERROR("DVI: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + subpixel_order = SubPixelHorizontalRGB; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.coherent_mode_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_property, + UNDERSCAN_OFF); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_hborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_vborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + + if (amdgpu_audio != 0) { + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.audio_property, + AMDGPU_AUDIO_AUTO); + } + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.dither_property, + AMDGPU_FMT_DITHER_DISABLE); + if (connector_type == DRM_MODE_CONNECTOR_DVII) { + amdgpu_connector->dac_load_detect = true; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.load_detect_property, + 1); + } + connector->interlace_allowed = true; + if (connector_type == DRM_MODE_CONNECTOR_DVII) + connector->doublescan_allowed = true; + else + connector->doublescan_allowed = false; + break; + case DRM_MODE_CONNECTOR_HDMIA: + case DRM_MODE_CONNECTOR_HDMIB: + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_dvi_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_dvi_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (!amdgpu_connector->ddc_bus) + DRM_ERROR("HDMI: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.coherent_mode_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_property, + UNDERSCAN_OFF); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_hborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_vborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + if (amdgpu_audio != 0) { + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.audio_property, + AMDGPU_AUDIO_AUTO); + } + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.dither_property, + AMDGPU_FMT_DITHER_DISABLE); + subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = true; + if (connector_type == DRM_MODE_CONNECTOR_HDMIB) + connector->doublescan_allowed = true; + else + connector->doublescan_allowed = false; + break; + case DRM_MODE_CONNECTOR_DisplayPort: + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_dp_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_dp_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (amdgpu_connector->ddc_bus) + has_aux = true; + else + DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + subpixel_order = SubPixelHorizontalRGB; + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.coherent_mode_property, + 1); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_property, + UNDERSCAN_OFF); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_hborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.underscan_vborder_property, + 0); + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_NONE); + if (amdgpu_audio != 0) { + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.audio_property, + AMDGPU_AUDIO_AUTO); + } + drm_object_attach_property(&amdgpu_connector->base.base, + adev->mode_info.dither_property, + AMDGPU_FMT_DITHER_DISABLE); + connector->interlace_allowed = true; + /* in theory with a DP to VGA converter... */ + connector->doublescan_allowed = false; + break; + case DRM_MODE_CONNECTOR_eDP: + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_edp_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_dp_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (amdgpu_connector->ddc_bus) + has_aux = true; + else + DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_FULLSCREEN); + subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = false; + connector->doublescan_allowed = false; + break; + case DRM_MODE_CONNECTOR_LVDS: + amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL); + if (!amdgpu_dig_connector) + goto failed; + amdgpu_connector->con_priv = amdgpu_dig_connector; + drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_lvds_funcs, connector_type); + drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_lvds_helper_funcs); + if (i2c_bus->valid) { + amdgpu_connector->ddc_bus = amdgpu_i2c_lookup(adev, i2c_bus); + if (!amdgpu_connector->ddc_bus) + DRM_ERROR("LVDS: Failed to assign ddc bus! Check dmesg for i2c errors.\n"); + } + drm_object_attach_property(&amdgpu_connector->base.base, + dev->mode_config.scaling_mode_property, + DRM_MODE_SCALE_FULLSCREEN); + subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = false; + connector->doublescan_allowed = false; + break; + } + } + + if (amdgpu_connector->hpd.hpd == AMDGPU_HPD_NONE) { + if (i2c_bus->valid) { + connector->polled = DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT; + } + } else + connector->polled = DRM_CONNECTOR_POLL_HPD; + + connector->display_info.subpixel_order = subpixel_order; + drm_connector_register(connector); + + if (has_aux) + amdgpu_atombios_dp_aux_init(amdgpu_connector); + + return; + +failed: + drm_connector_cleanup(connector); + kfree(connector); +}
diff --git a/amdgpu/amdgpu_connectors.h b/amdgpu/amdgpu_connectors.h new file mode 100644 index 0000000..61fcef1 --- /dev/null +++ b/amdgpu/amdgpu_connectors.h
@@ -0,0 +1,42 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_CONNECTORS_H__ +#define __AMDGPU_CONNECTORS_H__ + +struct edid *amdgpu_connector_edid(struct drm_connector *connector); +void amdgpu_connector_hotplug(struct drm_connector *connector); +int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector); +u16 amdgpu_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector); +bool amdgpu_connector_is_dp12_capable(struct drm_connector *connector); +void +amdgpu_connector_add(struct amdgpu_device *adev, + uint32_t connector_id, + uint32_t supported_device, + int connector_type, + struct amdgpu_i2c_bus_rec *i2c_bus, + uint16_t connector_object_id, + struct amdgpu_hpd *hpd, + struct amdgpu_router *router); + +#endif
diff --git a/amdgpu/amdgpu_cs.c b/amdgpu/amdgpu_cs.c new file mode 100644 index 0000000..8b26c97 --- /dev/null +++ b/amdgpu/amdgpu_cs.c
@@ -0,0 +1,1747 @@ +/* + * Copyright 2008 Jerome Glisse. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Jerome Glisse <glisse@freedesktop.org> + */ + +#include <linux/file.h> +#include <linux/pagemap.h> +#include <linux/sync_file.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_syncobj.h> +#include "amdgpu.h" +#include "amdgpu_trace.h" +#include "amdgpu_gmc.h" +#include "amdgpu_gem.h" + +static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, + struct drm_amdgpu_cs_chunk_fence *data, + uint32_t *offset) +{ + struct drm_gem_object *gobj; + struct amdgpu_bo *bo; + unsigned long size; + int r; + + gobj = drm_gem_object_lookup(p->filp, data->handle); + if (gobj == NULL) + return -EINVAL; + + bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); + p->uf_entry.priority = 0; + p->uf_entry.tv.bo = &bo->tbo; + /* One for TTM and one for the CS job */ + p->uf_entry.tv.num_shared = 2; + + drm_gem_object_put_unlocked(gobj); + + size = amdgpu_bo_size(bo); + if (size != PAGE_SIZE || (data->offset + 8) > size) { + r = -EINVAL; + goto error_unref; + } + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { + r = -EINVAL; + goto error_unref; + } + + *offset = data->offset; + + return 0; + +error_unref: + amdgpu_bo_unref(&bo); + return r; +} + +static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, + struct drm_amdgpu_bo_list_in *data) +{ + int r; + struct drm_amdgpu_bo_list_entry *info = NULL; + + r = amdgpu_bo_create_list_entry_array(data, &info); + if (r) + return r; + + r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, + &p->bo_list); + if (r) + goto error_free; + + kvfree(info); + return 0; + +error_free: + if (info) + kvfree(info); + + return r; +} + +static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) +{ + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + uint64_t *chunk_array_user; + uint64_t *chunk_array; + unsigned size, num_ibs = 0; + uint32_t uf_offset = 0; + int i; + int ret; + + if (cs->in.num_chunks == 0) + return 0; + + chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); + if (!chunk_array) + return -ENOMEM; + + p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); + if (!p->ctx) { + ret = -EINVAL; + goto free_chunk; + } + + mutex_lock(&p->ctx->lock); + + /* skip guilty context job */ + if (atomic_read(&p->ctx->guilty) == 1) { + ret = -ECANCELED; + goto free_chunk; + } + + /* get chunks */ + chunk_array_user = u64_to_user_ptr(cs->in.chunks); + if (copy_from_user(chunk_array, chunk_array_user, + sizeof(uint64_t)*cs->in.num_chunks)) { + ret = -EFAULT; + goto free_chunk; + } + + p->nchunks = cs->in.num_chunks; + p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), + GFP_KERNEL); + if (!p->chunks) { + ret = -ENOMEM; + goto free_chunk; + } + + for (i = 0; i < p->nchunks; i++) { + struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; + struct drm_amdgpu_cs_chunk user_chunk; + uint32_t __user *cdata; + + chunk_ptr = u64_to_user_ptr(chunk_array[i]); + if (copy_from_user(&user_chunk, chunk_ptr, + sizeof(struct drm_amdgpu_cs_chunk))) { + ret = -EFAULT; + i--; + goto free_partial_kdata; + } + p->chunks[i].chunk_id = user_chunk.chunk_id; + p->chunks[i].length_dw = user_chunk.length_dw; + + size = p->chunks[i].length_dw; + cdata = u64_to_user_ptr(user_chunk.chunk_data); + + p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); + if (p->chunks[i].kdata == NULL) { + ret = -ENOMEM; + i--; + goto free_partial_kdata; + } + size *= sizeof(uint32_t); + if (copy_from_user(p->chunks[i].kdata, cdata, size)) { + ret = -EFAULT; + goto free_partial_kdata; + } + + switch (p->chunks[i].chunk_id) { + case AMDGPU_CHUNK_ID_IB: + ++num_ibs; + break; + + case AMDGPU_CHUNK_ID_FENCE: + size = sizeof(struct drm_amdgpu_cs_chunk_fence); + if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { + ret = -EINVAL; + goto free_partial_kdata; + } + + ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, + &uf_offset); + if (ret) + goto free_partial_kdata; + + break; + + case AMDGPU_CHUNK_ID_BO_HANDLES: + size = sizeof(struct drm_amdgpu_bo_list_in); + if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { + ret = -EINVAL; + goto free_partial_kdata; + } + + ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); + if (ret) + goto free_partial_kdata; + + break; + + case AMDGPU_CHUNK_ID_DEPENDENCIES: + case AMDGPU_CHUNK_ID_SYNCOBJ_IN: + case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: + case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: + break; + + default: + ret = -EINVAL; + goto free_partial_kdata; + } + } + + ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); + if (ret) + goto free_all_kdata; + + if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { + ret = -ECANCELED; + goto free_all_kdata; + } + + if (p->uf_entry.tv.bo) + p->job->uf_addr = uf_offset; + kfree(chunk_array); + + /* Use this opportunity to fill in task info for the vm */ + amdgpu_vm_set_task_info(vm); + + return 0; + +free_all_kdata: + i = p->nchunks - 1; +free_partial_kdata: + for (; i >= 0; i--) + kvfree(p->chunks[i].kdata); + kfree(p->chunks); + p->chunks = NULL; + p->nchunks = 0; +free_chunk: + kfree(chunk_array); + + return ret; +} + +/* Convert microseconds to bytes. */ +static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) +{ + if (us <= 0 || !adev->mm_stats.log2_max_MBps) + return 0; + + /* Since accum_us is incremented by a million per second, just + * multiply it by the number of MB/s to get the number of bytes. + */ + return us << adev->mm_stats.log2_max_MBps; +} + +static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) +{ + if (!adev->mm_stats.log2_max_MBps) + return 0; + + return bytes >> adev->mm_stats.log2_max_MBps; +} + +/* Returns how many bytes TTM can move right now. If no bytes can be moved, + * it returns 0. If it returns non-zero, it's OK to move at least one buffer, + * which means it can go over the threshold once. If that happens, the driver + * will be in debt and no other buffer migrations can be done until that debt + * is repaid. + * + * This approach allows moving a buffer of any size (it's important to allow + * that). + * + * The currency is simply time in microseconds and it increases as the clock + * ticks. The accumulated microseconds (us) are converted to bytes and + * returned. + */ +static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, + u64 *max_bytes, + u64 *max_vis_bytes) +{ + s64 time_us, increment_us; + u64 free_vram, total_vram, used_vram; + + /* Allow a maximum of 200 accumulated ms. This is basically per-IB + * throttling. + * + * It means that in order to get full max MBps, at least 5 IBs per + * second must be submitted and not more than 200ms apart from each + * other. + */ + const s64 us_upper_bound = 200000; + + if (!adev->mm_stats.log2_max_MBps) { + *max_bytes = 0; + *max_vis_bytes = 0; + return; + } + + total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); + used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; + + spin_lock(&adev->mm_stats.lock); + + /* Increase the amount of accumulated us. */ + time_us = ktime_to_us(ktime_get()); + increment_us = time_us - adev->mm_stats.last_update_us; + adev->mm_stats.last_update_us = time_us; + adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, + us_upper_bound); + + /* This prevents the short period of low performance when the VRAM + * usage is low and the driver is in debt or doesn't have enough + * accumulated us to fill VRAM quickly. + * + * The situation can occur in these cases: + * - a lot of VRAM is freed by userspace + * - the presence of a big buffer causes a lot of evictions + * (solution: split buffers into smaller ones) + * + * If 128 MB or 1/8th of VRAM is free, start filling it now by setting + * accum_us to a positive number. + */ + if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { + s64 min_us; + + /* Be more aggresive on dGPUs. Try to fill a portion of free + * VRAM now. + */ + if (!(adev->flags & AMD_IS_APU)) + min_us = bytes_to_us(adev, free_vram / 4); + else + min_us = 0; /* Reset accum_us on APUs. */ + + adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); + } + + /* This is set to 0 if the driver is in debt to disallow (optional) + * buffer moves. + */ + *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); + + /* Do the same for visible VRAM if half of it is free */ + if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { + u64 total_vis_vram = adev->gmc.visible_vram_size; + u64 used_vis_vram = + amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + + if (used_vis_vram < total_vis_vram) { + u64 free_vis_vram = total_vis_vram - used_vis_vram; + adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + + increment_us, us_upper_bound); + + if (free_vis_vram >= total_vis_vram / 2) + adev->mm_stats.accum_us_vis = + max(bytes_to_us(adev, free_vis_vram / 2), + adev->mm_stats.accum_us_vis); + } + + *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); + } else { + *max_vis_bytes = 0; + } + + spin_unlock(&adev->mm_stats.lock); +} + +/* Report how many bytes have really been moved for the last command + * submission. This can result in a debt that can stop buffer migrations + * temporarily. + */ +void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, + u64 num_vis_bytes) +{ + spin_lock(&adev->mm_stats.lock); + adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); + adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); + spin_unlock(&adev->mm_stats.lock); +} + +static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, + struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + .resv = bo->tbo.resv, + .flags = 0 + }; + uint32_t domain; + int r; + + if (bo->pin_count) + return 0; + + /* Don't move this buffer if we have depleted our allowance + * to move it. Don't move anything if the threshold is zero. + */ + if (p->bytes_moved < p->bytes_moved_threshold) { + if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { + /* And don't move a CPU_ACCESS_REQUIRED BO to limited + * visible VRAM if we've depleted our allowance to do + * that. + */ + if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) + domain = bo->preferred_domains; + else + domain = bo->allowed_domains; + } else { + domain = bo->preferred_domains; + } + } else { + domain = bo->allowed_domains; + } + +retry: + amdgpu_bo_placement_from_domain(bo, domain); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + + p->bytes_moved += ctx.bytes_moved; + if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + amdgpu_bo_in_cpu_visible_vram(bo)) + p->bytes_moved_vis += ctx.bytes_moved; + + if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { + domain = bo->allowed_domains; + goto retry; + } + + return r; +} + +/* Last resort, try to evict something from the current working set */ +static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p, + struct amdgpu_bo *validated) +{ + uint32_t domain = validated->allowed_domains; + struct ttm_operation_ctx ctx = { true, false }; + int r; + + if (!p->evictable) + return false; + + for (;&p->evictable->tv.head != &p->validated; + p->evictable = list_prev_entry(p->evictable, tv.head)) { + + struct amdgpu_bo_list_entry *candidate = p->evictable; + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(candidate->tv.bo); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + bool update_bytes_moved_vis; + uint32_t other; + + /* If we reached our current BO we can forget it */ + if (bo == validated) + break; + + /* We can't move pinned BOs here */ + if (bo->pin_count) + continue; + + other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); + + /* Check if this BO is in one of the domains we need space for */ + if (!(other & domain)) + continue; + + /* Check if we can move this BO somewhere else */ + other = bo->allowed_domains & ~domain; + if (!other) + continue; + + /* Good we can try to move this BO somewhere else */ + update_bytes_moved_vis = + !amdgpu_gmc_vram_full_visible(&adev->gmc) && + amdgpu_bo_in_cpu_visible_vram(bo); + amdgpu_bo_placement_from_domain(bo, other); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + p->bytes_moved += ctx.bytes_moved; + if (update_bytes_moved_vis) + p->bytes_moved_vis += ctx.bytes_moved; + + if (unlikely(r)) + break; + + p->evictable = list_prev_entry(p->evictable, tv.head); + list_move(&candidate->tv.head, &p->validated); + + return true; + } + + return false; +} + +static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo) +{ + struct amdgpu_cs_parser *p = param; + int r; + + do { + r = amdgpu_cs_bo_validate(p, bo); + } while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo)); + if (r) + return r; + + if (bo->shadow) + r = amdgpu_cs_bo_validate(p, bo->shadow); + + return r; +} + +static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, + struct list_head *validated) +{ + struct ttm_operation_ctx ctx = { true, false }; + struct amdgpu_bo_list_entry *lobj; + int r; + + list_for_each_entry(lobj, validated, tv.head) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); + bool binding_userptr = false; + struct mm_struct *usermm; + + usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); + if (usermm && usermm != current->mm) + return -EPERM; + + if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && + lobj->user_invalidated && lobj->user_pages) { + amdgpu_bo_placement_from_domain(bo, + AMDGPU_GEM_DOMAIN_CPU); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (r) + return r; + + amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, + lobj->user_pages); + binding_userptr = true; + } + + if (p->evictable == lobj) + p->evictable = NULL; + + r = amdgpu_cs_validate(p, bo); + if (r) + return r; + + if (binding_userptr) { + kvfree(lobj->user_pages); + lobj->user_pages = NULL; + } + } + return 0; +} + +static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, + union drm_amdgpu_cs *cs) +{ + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_list_entry *e; + struct list_head duplicates; + struct amdgpu_bo *gds; + struct amdgpu_bo *gws; + struct amdgpu_bo *oa; + int r; + + INIT_LIST_HEAD(&p->validated); + + /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ + if (cs->in.bo_list_handle) { + if (p->bo_list) + return -EINVAL; + + r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, + &p->bo_list); + if (r) + return r; + } else if (!p->bo_list) { + /* Create a empty bo_list when no handle is provided */ + r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, + &p->bo_list); + if (r) + return r; + } + + /* One for TTM and one for the CS job */ + amdgpu_bo_list_for_each_entry(e, p->bo_list) + e->tv.num_shared = 2; + + amdgpu_bo_list_get_list(p->bo_list, &p->validated); + if (p->bo_list->first_userptr != p->bo_list->num_entries) + p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX); + + INIT_LIST_HEAD(&duplicates); + amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); + + if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) + list_add(&p->uf_entry.tv.head, &p->validated); + + /* Get userptr backing pages. If pages are updated after registered + * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do + * amdgpu_ttm_backend_bind() to flush and invalidate new pages + */ + amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + bool userpage_invalidated = false; + int i; + + e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, + sizeof(struct page *), + GFP_KERNEL | __GFP_ZERO); + if (!e->user_pages) { + DRM_ERROR("calloc failure\n"); + return -ENOMEM; + } + + r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); + if (r) { + kvfree(e->user_pages); + e->user_pages = NULL; + return r; + } + + for (i = 0; i < bo->tbo.ttm->num_pages; i++) { + if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { + userpage_invalidated = true; + break; + } + } + e->user_invalidated = userpage_invalidated; + } + + r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, + &duplicates, false); + if (unlikely(r != 0)) { + if (r != -ERESTARTSYS) + DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); + goto out; + } + + amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, + &p->bytes_moved_vis_threshold); + p->bytes_moved = 0; + p->bytes_moved_vis = 0; + p->evictable = list_last_entry(&p->validated, + struct amdgpu_bo_list_entry, + tv.head); + + r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, + amdgpu_cs_validate, p); + if (r) { + DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); + goto error_validate; + } + + r = amdgpu_cs_list_validate(p, &duplicates); + if (r) + goto error_validate; + + r = amdgpu_cs_list_validate(p, &p->validated); + if (r) + goto error_validate; + + amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, + p->bytes_moved_vis); + + gds = p->bo_list->gds_obj; + gws = p->bo_list->gws_obj; + oa = p->bo_list->oa_obj; + + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + + /* Make sure we use the exclusive slot for shared BOs */ + if (bo->prime_shared_count) + e->tv.num_shared = 0; + e->bo_va = amdgpu_vm_bo_find(vm, bo); + } + + if (gds) { + p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; + p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; + } + if (gws) { + p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; + p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; + } + if (oa) { + p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; + p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; + } + + if (!r && p->uf_entry.tv.bo) { + struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); + + r = amdgpu_ttm_alloc_gart(&uf->tbo); + p->job->uf_addr += amdgpu_bo_gpu_offset(uf); + } + +error_validate: + if (r) + ttm_eu_backoff_reservation(&p->ticket, &p->validated); +out: + return r; +} + +static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) +{ + struct amdgpu_bo_list_entry *e; + int r; + + list_for_each_entry(e, &p->validated, tv.head) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + struct reservation_object *resv = bo->tbo.resv; + + r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp, + amdgpu_bo_explicit_sync(bo)); + + if (r) + return r; + } + return 0; +} + +/** + * cs_parser_fini() - clean parser states + * @parser: parser structure holding parsing context. + * @error: error number + * + * If error is set than unvalidate buffer, otherwise just free memory + * used by parsing context. + **/ +static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, + bool backoff) +{ + unsigned i; + + if (error && backoff) + ttm_eu_backoff_reservation(&parser->ticket, + &parser->validated); + + for (i = 0; i < parser->num_post_deps; i++) { + drm_syncobj_put(parser->post_deps[i].syncobj); + kfree(parser->post_deps[i].chain); + } + kfree(parser->post_deps); + + dma_fence_put(parser->fence); + + if (parser->ctx) { + mutex_unlock(&parser->ctx->lock); + amdgpu_ctx_put(parser->ctx); + } + if (parser->bo_list) + amdgpu_bo_list_put(parser->bo_list); + + for (i = 0; i < parser->nchunks; i++) + kvfree(parser->chunks[i].kdata); + kfree(parser->chunks); + if (parser->job) + amdgpu_job_free(parser->job); + if (parser->uf_entry.tv.bo) { + struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); + + amdgpu_bo_unref(&uf); + } +} + +static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_device *adev = p->adev; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_list_entry *e; + struct amdgpu_bo_va *bo_va; + struct amdgpu_bo *bo; + int r; + + /* Only for UVD/VCE VM emulation */ + if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) { + unsigned i, j; + + for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { + struct drm_amdgpu_cs_chunk_ib *chunk_ib; + struct amdgpu_bo_va_mapping *m; + struct amdgpu_bo *aobj = NULL; + struct amdgpu_cs_chunk *chunk; + uint64_t offset, va_start; + struct amdgpu_ib *ib; + uint8_t *kptr; + + chunk = &p->chunks[i]; + ib = &p->job->ibs[j]; + chunk_ib = chunk->kdata; + + if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) + continue; + + va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK; + r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); + if (r) { + DRM_ERROR("IB va_start is invalid\n"); + return r; + } + + if ((va_start + chunk_ib->ib_bytes) > + (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { + DRM_ERROR("IB va_start+ib_bytes is invalid\n"); + return -EINVAL; + } + + /* the IB should be reserved at this point */ + r = amdgpu_bo_kmap(aobj, (void **)&kptr); + if (r) { + return r; + } + + offset = m->start * AMDGPU_GPU_PAGE_SIZE; + kptr += va_start - offset; + + if (ring->funcs->parse_cs) { + memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); + amdgpu_bo_kunmap(aobj); + + r = amdgpu_ring_parse_cs(ring, p, j); + if (r) + return r; + } else { + ib->ptr = (uint32_t *)kptr; + r = amdgpu_ring_patch_cs_in_place(ring, p, j); + amdgpu_bo_kunmap(aobj); + if (r) + return r; + } + + j++; + } + } + + if (!p->job->vm) + return amdgpu_cs_sync_rings(p); + + + r = amdgpu_vm_clear_freed(adev, vm, NULL); + if (r) + return r; + + r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); + if (r) + return r; + + r = amdgpu_sync_fence(adev, &p->job->sync, + fpriv->prt_va->last_pt_update, false); + if (r) + return r; + + if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + struct dma_fence *f; + + bo_va = fpriv->csa_va; + BUG_ON(!bo_va); + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + return r; + + f = bo_va->last_pt_update; + r = amdgpu_sync_fence(adev, &p->job->sync, f, false); + if (r) + return r; + } + + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + struct dma_fence *f; + + /* ignore duplicates */ + bo = ttm_to_amdgpu_bo(e->tv.bo); + if (!bo) + continue; + + bo_va = e->bo_va; + if (bo_va == NULL) + continue; + + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + return r; + + f = bo_va->last_pt_update; + r = amdgpu_sync_fence(adev, &p->job->sync, f, false); + if (r) + return r; + } + + r = amdgpu_vm_handle_moved(adev, vm); + if (r) + return r; + + r = amdgpu_vm_update_directories(adev, vm); + if (r) + return r; + + r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false); + if (r) + return r; + + p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); + + if (amdgpu_vm_debug) { + /* Invalidate all BOs to test for userspace bugs */ + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + + /* ignore duplicates */ + if (!bo) + continue; + + amdgpu_vm_bo_invalidate(adev, bo, false); + } + } + + return amdgpu_cs_sync_rings(p); +} + +static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, + struct amdgpu_cs_parser *parser) +{ + struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + int r, ce_preempt = 0, de_preempt = 0; + struct amdgpu_ring *ring; + int i, j; + + for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { + struct amdgpu_cs_chunk *chunk; + struct amdgpu_ib *ib; + struct drm_amdgpu_cs_chunk_ib *chunk_ib; + struct drm_sched_entity *entity; + + chunk = &parser->chunks[i]; + ib = &parser->job->ibs[j]; + chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; + + if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) + continue; + + if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && + (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { + if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { + if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) + ce_preempt++; + else + de_preempt++; + } + + /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ + if (ce_preempt > 1 || de_preempt > 1) + return -EINVAL; + } + + r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type, + chunk_ib->ip_instance, chunk_ib->ring, + &entity); + if (r) + return r; + + if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) + parser->job->preamble_status |= + AMDGPU_PREAMBLE_IB_PRESENT; + + if (parser->entity && parser->entity != entity) + return -EINVAL; + + parser->entity = entity; + + ring = to_amdgpu_ring(entity->rq->sched); + r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ? + chunk_ib->ib_bytes : 0, ib); + if (r) { + DRM_ERROR("Failed to get ib !\n"); + return r; + } + + ib->gpu_addr = chunk_ib->va_start; + ib->length_dw = chunk_ib->ib_bytes / 4; + ib->flags = chunk_ib->flags; + + j++; + } + + /* MM engine doesn't support user fences */ + ring = to_amdgpu_ring(parser->entity->rq->sched); + if (parser->job->uf_addr && ring->funcs->no_user_fence) + return -EINVAL; + + return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->entity); +} + +static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + unsigned num_deps; + int i, r; + struct drm_amdgpu_cs_chunk_dep *deps; + + deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_dep); + + for (i = 0; i < num_deps; ++i) { + struct amdgpu_ctx *ctx; + struct drm_sched_entity *entity; + struct dma_fence *fence; + + ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); + if (ctx == NULL) + return -EINVAL; + + r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, + deps[i].ip_instance, + deps[i].ring, &entity); + if (r) { + amdgpu_ctx_put(ctx); + return r; + } + + fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); + amdgpu_ctx_put(ctx); + + if (IS_ERR(fence)) + return PTR_ERR(fence); + else if (!fence) + continue; + + if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { + struct drm_sched_fence *s_fence; + struct dma_fence *old = fence; + + s_fence = to_drm_sched_fence(fence); + fence = dma_fence_get(&s_fence->scheduled); + dma_fence_put(old); + } + + r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true); + dma_fence_put(fence); + if (r) + return r; + } + return 0; +} + +static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, + uint32_t handle, u64 point, + u64 flags) +{ + struct dma_fence *fence; + int r; + + r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); + if (r) { + DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", + handle, point, r); + return r; + } + + r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true); + dma_fence_put(fence); + + return r; +} + +static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_sem *deps; + unsigned num_deps; + int i, r; + + deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_sem); + for (i = 0; i < num_deps; ++i) { + r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle, + 0, 0); + if (r) + return r; + } + + return 0; +} + + +static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; + unsigned num_deps; + int i, r; + + syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_syncobj); + for (i = 0; i < num_deps; ++i) { + r = amdgpu_syncobj_lookup_and_add_to_sync(p, + syncobj_deps[i].handle, + syncobj_deps[i].point, + syncobj_deps[i].flags); + if (r) + return r; + } + + return 0; +} + +static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_sem *deps; + unsigned num_deps; + int i; + + deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_sem); + + if (p->post_deps) + return -EINVAL; + + p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), + GFP_KERNEL); + p->num_post_deps = 0; + + if (!p->post_deps) + return -ENOMEM; + + + for (i = 0; i < num_deps; ++i) { + p->post_deps[i].syncobj = + drm_syncobj_find(p->filp, deps[i].handle); + if (!p->post_deps[i].syncobj) + return -EINVAL; + p->post_deps[i].chain = NULL; + p->post_deps[i].point = 0; + p->num_post_deps++; + } + + return 0; +} + + +static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; + unsigned num_deps; + int i; + + syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_syncobj); + + if (p->post_deps) + return -EINVAL; + + p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), + GFP_KERNEL); + p->num_post_deps = 0; + + if (!p->post_deps) + return -ENOMEM; + + for (i = 0; i < num_deps; ++i) { + struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; + + dep->chain = NULL; + if (syncobj_deps[i].point) { + dep->chain = kmalloc(sizeof(*dep->chain), GFP_KERNEL); + if (!dep->chain) + return -ENOMEM; + } + + dep->syncobj = drm_syncobj_find(p->filp, + syncobj_deps[i].handle); + if (!dep->syncobj) { + kfree(dep->chain); + return -EINVAL; + } + dep->point = syncobj_deps[i].point; + p->num_post_deps++; + } + + return 0; +} + +static int amdgpu_cs_dependencies(struct amdgpu_device *adev, + struct amdgpu_cs_parser *p) +{ + int i, r; + + for (i = 0; i < p->nchunks; ++i) { + struct amdgpu_cs_chunk *chunk; + + chunk = &p->chunks[i]; + + switch (chunk->chunk_id) { + case AMDGPU_CHUNK_ID_DEPENDENCIES: + case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: + r = amdgpu_cs_process_fence_dep(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_IN: + r = amdgpu_cs_process_syncobj_in_dep(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: + r = amdgpu_cs_process_syncobj_out_dep(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: + r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: + r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk); + if (r) + return r; + break; + } + } + + return 0; +} + +static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) +{ + int i; + + for (i = 0; i < p->num_post_deps; ++i) { + if (p->post_deps[i].chain && p->post_deps[i].point) { + drm_syncobj_add_point(p->post_deps[i].syncobj, + p->post_deps[i].chain, + p->fence, p->post_deps[i].point); + p->post_deps[i].chain = NULL; + } else { + drm_syncobj_replace_fence(p->post_deps[i].syncobj, + p->fence); + } + } +} + +static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, + union drm_amdgpu_cs *cs) +{ + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct drm_sched_entity *entity = p->entity; + enum drm_sched_priority priority; + struct amdgpu_ring *ring; + struct amdgpu_bo_list_entry *e; + struct amdgpu_job *job; + uint64_t seq; + int r; + + job = p->job; + p->job = NULL; + + r = drm_sched_job_init(&job->base, entity, p->filp); + if (r) + goto error_unlock; + + /* No memory allocation is allowed while holding the mn lock. + * p->mn is hold until amdgpu_cs_submit is finished and fence is added + * to BOs. + */ + amdgpu_mn_lock(p->mn); + + /* If userptr are invalidated after amdgpu_cs_parser_bos(), return + * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. + */ + amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + + r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); + } + if (r) { + r = -EAGAIN; + goto error_abort; + } + + job->owner = p->filp; + p->fence = dma_fence_get(&job->base.s_fence->finished); + + amdgpu_ctx_add_fence(p->ctx, entity, p->fence, &seq); + amdgpu_cs_post_dependencies(p); + + if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && + !p->ctx->preamble_presented) { + job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; + p->ctx->preamble_presented = true; + } + + cs->out.handle = seq; + job->uf_sequence = seq; + + amdgpu_job_free_resources(job); + + trace_amdgpu_cs_ioctl(job); + amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); + priority = job->base.s_priority; + drm_sched_entity_push_job(&job->base, entity); + + ring = to_amdgpu_ring(entity->rq->sched); + amdgpu_ring_priority_get(ring, priority); + + amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); + + ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); + amdgpu_mn_unlock(p->mn); + + return 0; + +error_abort: + drm_sched_job_cleanup(&job->base); + amdgpu_mn_unlock(p->mn); + +error_unlock: + amdgpu_job_free(job); + return r; +} + +int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + union drm_amdgpu_cs *cs = data; + struct amdgpu_cs_parser parser = {}; + bool reserved_buffers = false; + int i, r; + + if (!adev->accel_working) + return -EBUSY; + + parser.adev = adev; + parser.filp = filp; + + r = amdgpu_cs_parser_init(&parser, data); + if (r) { + DRM_ERROR("Failed to initialize parser %d!\n", r); + goto out; + } + + r = amdgpu_cs_ib_fill(adev, &parser); + if (r) + goto out; + + r = amdgpu_cs_dependencies(adev, &parser); + if (r) { + DRM_ERROR("Failed in the dependencies handling %d!\n", r); + goto out; + } + + r = amdgpu_cs_parser_bos(&parser, data); + if (r) { + if (r == -ENOMEM) + DRM_ERROR("Not enough memory for command submission!\n"); + else if (r != -ERESTARTSYS && r != -EAGAIN) + DRM_ERROR("Failed to process the buffer list %d!\n", r); + goto out; + } + + reserved_buffers = true; + + for (i = 0; i < parser.job->num_ibs; i++) + trace_amdgpu_cs(&parser, i); + + r = amdgpu_cs_vm_handling(&parser); + if (r) + goto out; + + r = amdgpu_cs_submit(&parser, cs); + +out: + amdgpu_cs_parser_fini(&parser, r, reserved_buffers); + + return r; +} + +/** + * amdgpu_cs_wait_ioctl - wait for a command submission to finish + * + * @dev: drm device + * @data: data from userspace + * @filp: file private + * + * Wait for the command submission identified by handle to finish. + */ +int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + union drm_amdgpu_wait_cs *wait = data; + unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); + struct drm_sched_entity *entity; + struct amdgpu_ctx *ctx; + struct dma_fence *fence; + long r; + + ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); + if (ctx == NULL) + return -EINVAL; + + r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, + wait->in.ring, &entity); + if (r) { + amdgpu_ctx_put(ctx); + return r; + } + + fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); + if (IS_ERR(fence)) + r = PTR_ERR(fence); + else if (fence) { + r = dma_fence_wait_timeout(fence, true, timeout); + if (r > 0 && fence->error) + r = fence->error; + dma_fence_put(fence); + } else + r = 1; + + amdgpu_ctx_put(ctx); + if (r < 0) + return r; + + memset(wait, 0, sizeof(*wait)); + wait->out.status = (r == 0); + + return 0; +} + +/** + * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence + * + * @adev: amdgpu device + * @filp: file private + * @user: drm_amdgpu_fence copied from user space + */ +static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, + struct drm_file *filp, + struct drm_amdgpu_fence *user) +{ + struct drm_sched_entity *entity; + struct amdgpu_ctx *ctx; + struct dma_fence *fence; + int r; + + ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); + if (ctx == NULL) + return ERR_PTR(-EINVAL); + + r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, + user->ring, &entity); + if (r) { + amdgpu_ctx_put(ctx); + return ERR_PTR(r); + } + + fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); + amdgpu_ctx_put(ctx); + + return fence; +} + +int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + union drm_amdgpu_fence_to_handle *info = data; + struct dma_fence *fence; + struct drm_syncobj *syncobj; + struct sync_file *sync_file; + int fd, r; + + fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + if (!fence) + fence = dma_fence_get_stub(); + + switch (info->in.what) { + case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: + r = drm_syncobj_create(&syncobj, 0, fence); + dma_fence_put(fence); + if (r) + return r; + r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); + drm_syncobj_put(syncobj); + return r; + + case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: + r = drm_syncobj_create(&syncobj, 0, fence); + dma_fence_put(fence); + if (r) + return r; + r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle); + drm_syncobj_put(syncobj); + return r; + + case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + dma_fence_put(fence); + return fd; + } + + sync_file = sync_file_create(fence); + dma_fence_put(fence); + if (!sync_file) { + put_unused_fd(fd); + return -ENOMEM; + } + + fd_install(fd, sync_file->file); + info->out.handle = fd; + return 0; + + default: + return -EINVAL; + } +} + +/** + * amdgpu_cs_wait_all_fence - wait on all fences to signal + * + * @adev: amdgpu device + * @filp: file private + * @wait: wait parameters + * @fences: array of drm_amdgpu_fence + */ +static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, + struct drm_file *filp, + union drm_amdgpu_wait_fences *wait, + struct drm_amdgpu_fence *fences) +{ + uint32_t fence_count = wait->in.fence_count; + unsigned int i; + long r = 1; + + for (i = 0; i < fence_count; i++) { + struct dma_fence *fence; + unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); + + fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); + if (IS_ERR(fence)) + return PTR_ERR(fence); + else if (!fence) + continue; + + r = dma_fence_wait_timeout(fence, true, timeout); + dma_fence_put(fence); + if (r < 0) + return r; + + if (r == 0) + break; + + if (fence->error) + return fence->error; + } + + memset(wait, 0, sizeof(*wait)); + wait->out.status = (r > 0); + + return 0; +} + +/** + * amdgpu_cs_wait_any_fence - wait on any fence to signal + * + * @adev: amdgpu device + * @filp: file private + * @wait: wait parameters + * @fences: array of drm_amdgpu_fence + */ +static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, + struct drm_file *filp, + union drm_amdgpu_wait_fences *wait, + struct drm_amdgpu_fence *fences) +{ + unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); + uint32_t fence_count = wait->in.fence_count; + uint32_t first = ~0; + struct dma_fence **array; + unsigned int i; + long r; + + /* Prepare the fence array */ + array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); + + if (array == NULL) + return -ENOMEM; + + for (i = 0; i < fence_count; i++) { + struct dma_fence *fence; + + fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); + if (IS_ERR(fence)) { + r = PTR_ERR(fence); + goto err_free_fence_array; + } else if (fence) { + array[i] = fence; + } else { /* NULL, the fence has been already signaled */ + r = 1; + first = i; + goto out; + } + } + + r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, + &first); + if (r < 0) + goto err_free_fence_array; + +out: + memset(wait, 0, sizeof(*wait)); + wait->out.status = (r > 0); + wait->out.first_signaled = first; + + if (first < fence_count && array[first]) + r = array[first]->error; + else + r = 0; + +err_free_fence_array: + for (i = 0; i < fence_count; i++) + dma_fence_put(array[i]); + kfree(array); + + return r; +} + +/** + * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish + * + * @dev: drm device + * @data: data from userspace + * @filp: file private + */ +int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + union drm_amdgpu_wait_fences *wait = data; + uint32_t fence_count = wait->in.fence_count; + struct drm_amdgpu_fence *fences_user; + struct drm_amdgpu_fence *fences; + int r; + + /* Get the fences from userspace */ + fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), + GFP_KERNEL); + if (fences == NULL) + return -ENOMEM; + + fences_user = u64_to_user_ptr(wait->in.fences); + if (copy_from_user(fences, fences_user, + sizeof(struct drm_amdgpu_fence) * fence_count)) { + r = -EFAULT; + goto err_free_fences; + } + + if (wait->in.wait_all) + r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); + else + r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); + +err_free_fences: + kfree(fences); + + return r; +} + +/** + * amdgpu_cs_find_bo_va - find bo_va for VM address + * + * @parser: command submission parser context + * @addr: VM address + * @bo: resulting BO of the mapping found + * + * Search the buffer objects in the command submission context for a certain + * virtual memory address. Returns allocation structure when found, NULL + * otherwise. + */ +int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, + uint64_t addr, struct amdgpu_bo **bo, + struct amdgpu_bo_va_mapping **map) +{ + struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_va_mapping *mapping; + int r; + + addr /= AMDGPU_GPU_PAGE_SIZE; + + mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); + if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) + return -EINVAL; + + *bo = mapping->bo_va->base.bo; + *map = mapping; + + /* Double check that the BO is reserved by this CS */ + if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket) + return -EINVAL; + + if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { + (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; + amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); + r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); + if (r) + return r; + } + + return amdgpu_ttm_alloc_gart(&(*bo)->tbo); +}
diff --git a/amdgpu/amdgpu_csa.c b/amdgpu/amdgpu_csa.c new file mode 100644 index 0000000..35a8d3c --- /dev/null +++ b/amdgpu/amdgpu_csa.c
@@ -0,0 +1,109 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + + * * Author: Monk.liu@amd.com + */ + +#include "amdgpu.h" + +uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev) +{ + uint64_t addr = adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT; + + addr -= AMDGPU_VA_RESERVED_SIZE; + addr = amdgpu_gmc_sign_extend(addr); + + return addr; +} + +int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo, + u32 domain, uint32_t size) +{ + int r; + void *ptr; + + r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, + domain, bo, + NULL, &ptr); + if (!*bo) + return -ENOMEM; + + memset(ptr, 0, size); + adev->virt.csa_cpu_addr = ptr; + return 0; +} + +void amdgpu_free_static_csa(struct amdgpu_bo **bo) +{ + amdgpu_bo_free_kernel(bo, NULL, NULL); +} + +/* + * amdgpu_map_static_csa should be called during amdgpu_vm_init + * it maps virtual address amdgpu_csa_vaddr() to this VM, and each command + * submission of GFX should use this virtual address within META_DATA init + * package to support SRIOV gfx preemption. + */ +int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va, + uint64_t csa_addr, uint32_t size) +{ + struct ww_acquire_ctx ticket; + struct list_head list; + struct amdgpu_bo_list_entry pd; + struct ttm_validate_buffer csa_tv; + int r; + + INIT_LIST_HEAD(&list); + INIT_LIST_HEAD(&csa_tv.head); + csa_tv.bo = &bo->tbo; + csa_tv.num_shared = 1; + + list_add(&csa_tv.head, &list); + amdgpu_vm_get_pd_bo(vm, &list, &pd); + + r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL, false); + if (r) { + DRM_ERROR("failed to reserve CSA,PD BOs: err=%d\n", r); + return r; + } + + *bo_va = amdgpu_vm_bo_add(adev, vm, bo); + if (!*bo_va) { + ttm_eu_backoff_reservation(&ticket, &list); + DRM_ERROR("failed to create bo_va for static CSA\n"); + return -ENOMEM; + } + + r = amdgpu_vm_bo_map(adev, *bo_va, csa_addr, 0, size, + AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | + AMDGPU_PTE_EXECUTABLE); + + if (r) { + DRM_ERROR("failed to do bo_map on static CSA, err=%d\n", r); + amdgpu_vm_bo_rmv(adev, *bo_va); + ttm_eu_backoff_reservation(&ticket, &list); + return r; + } + + ttm_eu_backoff_reservation(&ticket, &list); + return 0; +}
diff --git a/amdgpu/amdgpu_csa.h b/amdgpu/amdgpu_csa.h new file mode 100644 index 0000000..524b443 --- /dev/null +++ b/amdgpu/amdgpu_csa.h
@@ -0,0 +1,39 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Monk.liu@amd.com + */ + +#ifndef AMDGPU_CSA_MANAGER_H +#define AMDGPU_CSA_MANAGER_H + +#define AMDGPU_CSA_SIZE (128 * 1024) + +uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev); +uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev); +int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo, + u32 domain, uint32_t size); +int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va, + uint64_t csa_addr, uint32_t size); +void amdgpu_free_static_csa(struct amdgpu_bo **bo); + +#endif
diff --git a/amdgpu/amdgpu_ctx.c b/amdgpu/amdgpu_ctx.c new file mode 100644 index 0000000..7398b48 --- /dev/null +++ b/amdgpu/amdgpu_ctx.c
@@ -0,0 +1,622 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: monk liu <monk.liu@amd.com> + */ + +#include <drm/drm_auth.h> +#include "amdgpu.h" +#include "amdgpu_sched.h" +#include "amdgpu_ras.h" + +#define to_amdgpu_ctx_entity(e) \ + container_of((e), struct amdgpu_ctx_entity, entity) + +const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { + [AMDGPU_HW_IP_GFX] = 1, + [AMDGPU_HW_IP_COMPUTE] = 4, + [AMDGPU_HW_IP_DMA] = 2, + [AMDGPU_HW_IP_UVD] = 1, + [AMDGPU_HW_IP_VCE] = 1, + [AMDGPU_HW_IP_UVD_ENC] = 1, + [AMDGPU_HW_IP_VCN_DEC] = 1, + [AMDGPU_HW_IP_VCN_ENC] = 1, + [AMDGPU_HW_IP_VCN_JPEG] = 1, +}; + +static int amdgput_ctx_total_num_entities(void) +{ + unsigned i, num_entities = 0; + + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) + num_entities += amdgpu_ctx_num_entities[i]; + + return num_entities; +} + +static int amdgpu_ctx_priority_permit(struct drm_file *filp, + enum drm_sched_priority priority) +{ + /* NORMAL and below are accessible by everyone */ + if (priority <= DRM_SCHED_PRIORITY_NORMAL) + return 0; + + if (capable(CAP_SYS_NICE)) + return 0; + + if (drm_is_current_master(filp)) + return 0; + + return -EACCES; +} + +static int amdgpu_ctx_init(struct amdgpu_device *adev, + enum drm_sched_priority priority, + struct drm_file *filp, + struct amdgpu_ctx *ctx) +{ + unsigned num_entities = amdgput_ctx_total_num_entities(); + unsigned i, j; + int r; + + if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) + return -EINVAL; + + r = amdgpu_ctx_priority_permit(filp, priority); + if (r) + return r; + + memset(ctx, 0, sizeof(*ctx)); + ctx->adev = adev; + + ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities, + sizeof(struct dma_fence*), GFP_KERNEL); + if (!ctx->fences) + return -ENOMEM; + + ctx->entities[0] = kcalloc(num_entities, + sizeof(struct amdgpu_ctx_entity), + GFP_KERNEL); + if (!ctx->entities[0]) { + r = -ENOMEM; + goto error_free_fences; + } + + for (i = 0; i < num_entities; ++i) { + struct amdgpu_ctx_entity *entity = &ctx->entities[0][i]; + + entity->sequence = 1; + entity->fences = &ctx->fences[amdgpu_sched_jobs * i]; + } + for (i = 1; i < AMDGPU_HW_IP_NUM; ++i) + ctx->entities[i] = ctx->entities[i - 1] + + amdgpu_ctx_num_entities[i - 1]; + + kref_init(&ctx->refcount); + spin_lock_init(&ctx->ring_lock); + mutex_init(&ctx->lock); + + ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); + ctx->reset_counter_query = ctx->reset_counter; + ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); + ctx->init_priority = priority; + ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; + + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; + struct drm_sched_rq *rqs[AMDGPU_MAX_RINGS]; + unsigned num_rings; + unsigned num_rqs = 0; + + switch (i) { + case AMDGPU_HW_IP_GFX: + rings[0] = &adev->gfx.gfx_ring[0]; + num_rings = 1; + break; + case AMDGPU_HW_IP_COMPUTE: + for (j = 0; j < adev->gfx.num_compute_rings; ++j) + rings[j] = &adev->gfx.compute_ring[j]; + num_rings = adev->gfx.num_compute_rings; + break; + case AMDGPU_HW_IP_DMA: + for (j = 0; j < adev->sdma.num_instances; ++j) + rings[j] = &adev->sdma.instance[j].ring; + num_rings = adev->sdma.num_instances; + break; + case AMDGPU_HW_IP_UVD: + rings[0] = &adev->uvd.inst[0].ring; + num_rings = 1; + break; + case AMDGPU_HW_IP_VCE: + rings[0] = &adev->vce.ring[0]; + num_rings = 1; + break; + case AMDGPU_HW_IP_UVD_ENC: + rings[0] = &adev->uvd.inst[0].ring_enc[0]; + num_rings = 1; + break; + case AMDGPU_HW_IP_VCN_DEC: + rings[0] = &adev->vcn.ring_dec; + num_rings = 1; + break; + case AMDGPU_HW_IP_VCN_ENC: + rings[0] = &adev->vcn.ring_enc[0]; + num_rings = 1; + break; + case AMDGPU_HW_IP_VCN_JPEG: + rings[0] = &adev->vcn.ring_jpeg; + num_rings = 1; + break; + } + + for (j = 0; j < num_rings; ++j) { + if (!rings[j]->adev) + continue; + + rqs[num_rqs++] = &rings[j]->sched.sched_rq[priority]; + } + + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) + r = drm_sched_entity_init(&ctx->entities[i][j].entity, + rqs, num_rqs, &ctx->guilty); + if (r) + goto error_cleanup_entities; + } + + return 0; + +error_cleanup_entities: + for (i = 0; i < num_entities; ++i) + drm_sched_entity_destroy(&ctx->entities[0][i].entity); + kfree(ctx->entities[0]); + +error_free_fences: + kfree(ctx->fences); + ctx->fences = NULL; + return r; +} + +static void amdgpu_ctx_fini(struct kref *ref) +{ + struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); + unsigned num_entities = amdgput_ctx_total_num_entities(); + struct amdgpu_device *adev = ctx->adev; + unsigned i, j; + + if (!adev) + return; + + for (i = 0; i < num_entities; ++i) + for (j = 0; j < amdgpu_sched_jobs; ++j) + dma_fence_put(ctx->entities[0][i].fences[j]); + kfree(ctx->fences); + kfree(ctx->entities[0]); + + mutex_destroy(&ctx->lock); + + kfree(ctx); +} + +int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, + u32 ring, struct drm_sched_entity **entity) +{ + if (hw_ip >= AMDGPU_HW_IP_NUM) { + DRM_ERROR("unknown HW IP type: %d\n", hw_ip); + return -EINVAL; + } + + /* Right now all IPs have only one instance - multiple rings. */ + if (instance != 0) { + DRM_DEBUG("invalid ip instance: %d\n", instance); + return -EINVAL; + } + + if (ring >= amdgpu_ctx_num_entities[hw_ip]) { + DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring); + return -EINVAL; + } + + *entity = &ctx->entities[hw_ip][ring].entity; + return 0; +} + +static int amdgpu_ctx_alloc(struct amdgpu_device *adev, + struct amdgpu_fpriv *fpriv, + struct drm_file *filp, + enum drm_sched_priority priority, + uint32_t *id) +{ + struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx *ctx; + int r; + + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + mutex_lock(&mgr->lock); + r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL); + if (r < 0) { + mutex_unlock(&mgr->lock); + kfree(ctx); + return r; + } + + *id = (uint32_t)r; + r = amdgpu_ctx_init(adev, priority, filp, ctx); + if (r) { + idr_remove(&mgr->ctx_handles, *id); + *id = 0; + kfree(ctx); + } + mutex_unlock(&mgr->lock); + return r; +} + +static void amdgpu_ctx_do_release(struct kref *ref) +{ + struct amdgpu_ctx *ctx; + unsigned num_entities; + u32 i; + + ctx = container_of(ref, struct amdgpu_ctx, refcount); + + num_entities = 0; + for (i = 0; i < AMDGPU_HW_IP_NUM; i++) + num_entities += amdgpu_ctx_num_entities[i]; + + for (i = 0; i < num_entities; i++) + drm_sched_entity_destroy(&ctx->entities[0][i].entity); + + amdgpu_ctx_fini(ref); +} + +static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) +{ + struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx *ctx; + + mutex_lock(&mgr->lock); + ctx = idr_remove(&mgr->ctx_handles, id); + if (ctx) + kref_put(&ctx->refcount, amdgpu_ctx_do_release); + mutex_unlock(&mgr->lock); + return ctx ? 0 : -EINVAL; +} + +static int amdgpu_ctx_query(struct amdgpu_device *adev, + struct amdgpu_fpriv *fpriv, uint32_t id, + union drm_amdgpu_ctx_out *out) +{ + struct amdgpu_ctx *ctx; + struct amdgpu_ctx_mgr *mgr; + unsigned reset_counter; + + if (!fpriv) + return -EINVAL; + + mgr = &fpriv->ctx_mgr; + mutex_lock(&mgr->lock); + ctx = idr_find(&mgr->ctx_handles, id); + if (!ctx) { + mutex_unlock(&mgr->lock); + return -EINVAL; + } + + /* TODO: these two are always zero */ + out->state.flags = 0x0; + out->state.hangs = 0x0; + + /* determine if a GPU reset has occured since the last call */ + reset_counter = atomic_read(&adev->gpu_reset_counter); + /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ + if (ctx->reset_counter_query == reset_counter) + out->state.reset_status = AMDGPU_CTX_NO_RESET; + else + out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; + ctx->reset_counter_query = reset_counter; + + mutex_unlock(&mgr->lock); + return 0; +} + +static int amdgpu_ctx_query2(struct amdgpu_device *adev, + struct amdgpu_fpriv *fpriv, uint32_t id, + union drm_amdgpu_ctx_out *out) +{ + struct amdgpu_ctx *ctx; + struct amdgpu_ctx_mgr *mgr; + uint32_t ras_counter; + + if (!fpriv) + return -EINVAL; + + mgr = &fpriv->ctx_mgr; + mutex_lock(&mgr->lock); + ctx = idr_find(&mgr->ctx_handles, id); + if (!ctx) { + mutex_unlock(&mgr->lock); + return -EINVAL; + } + + out->state.flags = 0x0; + out->state.hangs = 0x0; + + if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) + out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; + + if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) + out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; + + if (atomic_read(&ctx->guilty)) + out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; + + /*query ue count*/ + ras_counter = amdgpu_ras_query_error_count(adev, false); + /*ras counter is monotonic increasing*/ + if (ras_counter != ctx->ras_counter_ue) { + out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE; + ctx->ras_counter_ue = ras_counter; + } + + /*query ce count*/ + ras_counter = amdgpu_ras_query_error_count(adev, true); + if (ras_counter != ctx->ras_counter_ce) { + out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE; + ctx->ras_counter_ce = ras_counter; + } + + mutex_unlock(&mgr->lock); + return 0; +} + +int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + int r; + uint32_t id; + enum drm_sched_priority priority; + + union drm_amdgpu_ctx *args = data; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + + r = 0; + id = args->in.ctx_id; + priority = amdgpu_to_sched_priority(args->in.priority); + + /* For backwards compatibility reasons, we need to accept + * ioctls with garbage in the priority field */ + if (priority == DRM_SCHED_PRIORITY_INVALID) + priority = DRM_SCHED_PRIORITY_NORMAL; + + switch (args->in.op) { + case AMDGPU_CTX_OP_ALLOC_CTX: + r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); + args->out.alloc.ctx_id = id; + break; + case AMDGPU_CTX_OP_FREE_CTX: + r = amdgpu_ctx_free(fpriv, id); + break; + case AMDGPU_CTX_OP_QUERY_STATE: + r = amdgpu_ctx_query(adev, fpriv, id, &args->out); + break; + case AMDGPU_CTX_OP_QUERY_STATE2: + r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); + break; + default: + return -EINVAL; + } + + return r; +} + +struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) +{ + struct amdgpu_ctx *ctx; + struct amdgpu_ctx_mgr *mgr; + + if (!fpriv) + return NULL; + + mgr = &fpriv->ctx_mgr; + + mutex_lock(&mgr->lock); + ctx = idr_find(&mgr->ctx_handles, id); + if (ctx) + kref_get(&ctx->refcount); + mutex_unlock(&mgr->lock); + return ctx; +} + +int amdgpu_ctx_put(struct amdgpu_ctx *ctx) +{ + if (ctx == NULL) + return -EINVAL; + + kref_put(&ctx->refcount, amdgpu_ctx_do_release); + return 0; +} + +void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity, + struct dma_fence *fence, uint64_t* handle) +{ + struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); + uint64_t seq = centity->sequence; + struct dma_fence *other = NULL; + unsigned idx = 0; + + idx = seq & (amdgpu_sched_jobs - 1); + other = centity->fences[idx]; + if (other) + BUG_ON(!dma_fence_is_signaled(other)); + + dma_fence_get(fence); + + spin_lock(&ctx->ring_lock); + centity->fences[idx] = fence; + centity->sequence++; + spin_unlock(&ctx->ring_lock); + + dma_fence_put(other); + if (handle) + *handle = seq; +} + +struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity, + uint64_t seq) +{ + struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); + struct dma_fence *fence; + + spin_lock(&ctx->ring_lock); + + if (seq == ~0ull) + seq = centity->sequence - 1; + + if (seq >= centity->sequence) { + spin_unlock(&ctx->ring_lock); + return ERR_PTR(-EINVAL); + } + + + if (seq + amdgpu_sched_jobs < centity->sequence) { + spin_unlock(&ctx->ring_lock); + return NULL; + } + + fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]); + spin_unlock(&ctx->ring_lock); + + return fence; +} + +void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, + enum drm_sched_priority priority) +{ + unsigned num_entities = amdgput_ctx_total_num_entities(); + enum drm_sched_priority ctx_prio; + unsigned i; + + ctx->override_priority = priority; + + ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? + ctx->init_priority : ctx->override_priority; + + for (i = 0; i < num_entities; i++) { + struct drm_sched_entity *entity = &ctx->entities[0][i].entity; + + drm_sched_entity_set_priority(entity, ctx_prio); + } +} + +int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity) +{ + struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); + struct dma_fence *other; + unsigned idx; + long r; + + spin_lock(&ctx->ring_lock); + idx = centity->sequence & (amdgpu_sched_jobs - 1); + other = dma_fence_get(centity->fences[idx]); + spin_unlock(&ctx->ring_lock); + + if (!other) + return 0; + + r = dma_fence_wait(other, true); + if (r < 0 && r != -ERESTARTSYS) + DRM_ERROR("Error (%ld) waiting for fence!\n", r); + + dma_fence_put(other); + return r; +} + +void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) +{ + mutex_init(&mgr->lock); + idr_init(&mgr->ctx_handles); +} + +long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) +{ + unsigned num_entities = amdgput_ctx_total_num_entities(); + struct amdgpu_ctx *ctx; + struct idr *idp; + uint32_t id, i; + + idp = &mgr->ctx_handles; + + mutex_lock(&mgr->lock); + idr_for_each_entry(idp, ctx, id) { + for (i = 0; i < num_entities; i++) { + struct drm_sched_entity *entity; + + entity = &ctx->entities[0][i].entity; + timeout = drm_sched_entity_flush(entity, timeout); + } + } + mutex_unlock(&mgr->lock); + return timeout; +} + +void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) +{ + unsigned num_entities = amdgput_ctx_total_num_entities(); + struct amdgpu_ctx *ctx; + struct idr *idp; + uint32_t id, i; + + idp = &mgr->ctx_handles; + + idr_for_each_entry(idp, ctx, id) { + if (kref_read(&ctx->refcount) != 1) { + DRM_ERROR("ctx %p is still alive\n", ctx); + continue; + } + + for (i = 0; i < num_entities; i++) + drm_sched_entity_fini(&ctx->entities[0][i].entity); + } +} + +void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) +{ + struct amdgpu_ctx *ctx; + struct idr *idp; + uint32_t id; + + amdgpu_ctx_mgr_entity_fini(mgr); + + idp = &mgr->ctx_handles; + + idr_for_each_entry(idp, ctx, id) { + if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1) + DRM_ERROR("ctx %p is still alive\n", ctx); + } + + idr_destroy(&mgr->ctx_handles); + mutex_destroy(&mgr->lock); +}
diff --git a/amdgpu/amdgpu_ctx.h b/amdgpu/amdgpu_ctx.h new file mode 100644 index 0000000..5f1b54c --- /dev/null +++ b/amdgpu/amdgpu_ctx.h
@@ -0,0 +1,90 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_CTX_H__ +#define __AMDGPU_CTX_H__ + +#include "amdgpu_ring.h" + +struct drm_device; +struct drm_file; +struct amdgpu_fpriv; + +struct amdgpu_ctx_entity { + uint64_t sequence; + struct dma_fence **fences; + struct drm_sched_entity entity; +}; + +struct amdgpu_ctx { + struct kref refcount; + struct amdgpu_device *adev; + unsigned reset_counter; + unsigned reset_counter_query; + uint32_t vram_lost_counter; + spinlock_t ring_lock; + struct dma_fence **fences; + struct amdgpu_ctx_entity *entities[AMDGPU_HW_IP_NUM]; + bool preamble_presented; + enum drm_sched_priority init_priority; + enum drm_sched_priority override_priority; + struct mutex lock; + atomic_t guilty; + uint32_t ras_counter_ce; + uint32_t ras_counter_ue; +}; + +struct amdgpu_ctx_mgr { + struct amdgpu_device *adev; + struct mutex lock; + /* protected by lock */ + struct idr ctx_handles; +}; + +extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM]; + +struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); +int amdgpu_ctx_put(struct amdgpu_ctx *ctx); + +int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, + u32 ring, struct drm_sched_entity **entity); +void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity, + struct dma_fence *fence, uint64_t *seq); +struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity, + uint64_t seq); +void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, + enum drm_sched_priority priority); + +int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, + struct drm_sched_entity *entity); + +void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr); +void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr); +long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout); +void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr); + +#endif
diff --git a/amdgpu/amdgpu_debugfs.c b/amdgpu/amdgpu_debugfs.c new file mode 100644 index 0000000..5652cc7 --- /dev/null +++ b/amdgpu/amdgpu_debugfs.c
@@ -0,0 +1,1121 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/kthread.h> +#include <linux/pci.h> +#include <linux/uaccess.h> + +#include <drm/drm_debugfs.h> + +#include "amdgpu.h" + +/** + * amdgpu_debugfs_add_files - Add simple debugfs entries + * + * @adev: Device to attach debugfs entries to + * @files: Array of function callbacks that respond to reads + * @nfiles: Number of callbacks to register + * + */ +int amdgpu_debugfs_add_files(struct amdgpu_device *adev, + const struct drm_info_list *files, + unsigned nfiles) +{ + unsigned i; + + for (i = 0; i < adev->debugfs_count; i++) { + if (adev->debugfs[i].files == files) { + /* Already registered */ + return 0; + } + } + + i = adev->debugfs_count + 1; + if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) { + DRM_ERROR("Reached maximum number of debugfs components.\n"); + DRM_ERROR("Report so we increase " + "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n"); + return -EINVAL; + } + adev->debugfs[adev->debugfs_count].files = files; + adev->debugfs[adev->debugfs_count].num_files = nfiles; + adev->debugfs_count = i; +#if defined(CONFIG_DEBUG_FS) + drm_debugfs_create_files(files, nfiles, + adev->ddev->primary->debugfs_root, + adev->ddev->primary); +#endif + return 0; +} + +#if defined(CONFIG_DEBUG_FS) + +/** + * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes + * + * @read: True if reading + * @f: open file handle + * @buf: User buffer to write/read to + * @size: Number of bytes to write/read + * @pos: Offset to seek to + * + * This debugfs entry has special meaning on the offset being sought. + * Various bits have different meanings: + * + * Bit 62: Indicates a GRBM bank switch is needed + * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is + * zero) + * Bits 24..33: The SE or ME selector if needed + * Bits 34..43: The SH (or SA) or PIPE selector if needed + * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed + * + * Bit 23: Indicates that the PM power gating lock should be held + * This is necessary to read registers that might be + * unreliable during a power gating transistion. + * + * The lower bits are the BYTE offset of the register to read. This + * allows reading multiple registers in a single call and having + * the returned size reflect that. + */ +static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, + char __user *buf, size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + bool pm_pg_lock, use_bank, use_ring; + unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid; + + pm_pg_lock = use_bank = use_ring = false; + instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0; + + if (size & 0x3 || *pos & 0x3 || + ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61)))) + return -EINVAL; + + /* are we reading registers for which a PG lock is necessary? */ + pm_pg_lock = (*pos >> 23) & 1; + + if (*pos & (1ULL << 62)) { + se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24; + sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34; + instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44; + + if (se_bank == 0x3FF) + se_bank = 0xFFFFFFFF; + if (sh_bank == 0x3FF) + sh_bank = 0xFFFFFFFF; + if (instance_bank == 0x3FF) + instance_bank = 0xFFFFFFFF; + use_bank = 1; + } else if (*pos & (1ULL << 61)) { + + me = (*pos & GENMASK_ULL(33, 24)) >> 24; + pipe = (*pos & GENMASK_ULL(43, 34)) >> 34; + queue = (*pos & GENMASK_ULL(53, 44)) >> 44; + vmid = (*pos & GENMASK_ULL(58, 54)) >> 54; + + use_ring = 1; + } else { + use_bank = use_ring = 0; + } + + *pos &= (1UL << 22) - 1; + + if (use_bank) { + if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) || + (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) + return -EINVAL; + mutex_lock(&adev->grbm_idx_mutex); + amdgpu_gfx_select_se_sh(adev, se_bank, + sh_bank, instance_bank); + } else if (use_ring) { + mutex_lock(&adev->srbm_mutex); + amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid); + } + + if (pm_pg_lock) + mutex_lock(&adev->pm.mutex); + + while (size) { + uint32_t value; + + if (read) { + value = RREG32(*pos >> 2); + r = put_user(value, (uint32_t *)buf); + } else { + r = get_user(value, (uint32_t *)buf); + if (!r) + WREG32(*pos >> 2, value); + } + if (r) { + result = r; + goto end; + } + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + +end: + if (use_bank) { + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + mutex_unlock(&adev->grbm_idx_mutex); + } else if (use_ring) { + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + } + + if (pm_pg_lock) + mutex_unlock(&adev->pm.mutex); + + return result; +} + +/** + * amdgpu_debugfs_regs_read - Callback for reading MMIO registers + */ +static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos); +} + +/** + * amdgpu_debugfs_regs_write - Callback for writing MMIO registers + */ +static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos); +} + + +/** + * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to read. This + * allows reading multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + value = RREG32_PCIE(*pos >> 2); + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register + * + * @f: open file handle + * @buf: User buffer to write data from + * @size: Number of bytes to write + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to write. This + * allows writing multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + r = get_user(value, (uint32_t *)buf); + if (r) + return r; + + WREG32_PCIE(*pos >> 2, value); + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_regs_didt_read - Read from a DIDT register + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to read. This + * allows reading multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + value = RREG32_DIDT(*pos >> 2); + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_regs_didt_write - Write to a DIDT register + * + * @f: open file handle + * @buf: User buffer to write data from + * @size: Number of bytes to write + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to write. This + * allows writing multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + r = get_user(value, (uint32_t *)buf); + if (r) + return r; + + WREG32_DIDT(*pos >> 2, value); + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_regs_smc_read - Read from a SMC register + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to read. This + * allows reading multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + value = RREG32_SMC(*pos); + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_regs_smc_write - Write to a SMC register + * + * @f: open file handle + * @buf: User buffer to write data from + * @size: Number of bytes to write + * @pos: Offset to seek to + * + * The lower bits are the BYTE offset of the register to write. This + * allows writing multiple registers in a single call and having + * the returned size reflect that. + */ +static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + while (size) { + uint32_t value; + + r = get_user(value, (uint32_t *)buf); + if (r) + return r; + + WREG32_SMC(*pos, value); + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_debugfs_gca_config_read - Read from gfx config data + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * This file is used to access configuration data in a somewhat + * stable fashion. The format is a series of DWORDs with the first + * indicating which revision it is. New content is appended to the + * end so that older software can still read the data. + */ + +static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + uint32_t *config, no_regs = 0; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + config = kmalloc_array(256, sizeof(*config), GFP_KERNEL); + if (!config) + return -ENOMEM; + + /* version, increment each time something is added */ + config[no_regs++] = 3; + config[no_regs++] = adev->gfx.config.max_shader_engines; + config[no_regs++] = adev->gfx.config.max_tile_pipes; + config[no_regs++] = adev->gfx.config.max_cu_per_sh; + config[no_regs++] = adev->gfx.config.max_sh_per_se; + config[no_regs++] = adev->gfx.config.max_backends_per_se; + config[no_regs++] = adev->gfx.config.max_texture_channel_caches; + config[no_regs++] = adev->gfx.config.max_gprs; + config[no_regs++] = adev->gfx.config.max_gs_threads; + config[no_regs++] = adev->gfx.config.max_hw_contexts; + config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend; + config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend; + config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size; + config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size; + config[no_regs++] = adev->gfx.config.num_tile_pipes; + config[no_regs++] = adev->gfx.config.backend_enable_mask; + config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes; + config[no_regs++] = adev->gfx.config.mem_row_size_in_kb; + config[no_regs++] = adev->gfx.config.shader_engine_tile_size; + config[no_regs++] = adev->gfx.config.num_gpus; + config[no_regs++] = adev->gfx.config.multi_gpu_tile_size; + config[no_regs++] = adev->gfx.config.mc_arb_ramcfg; + config[no_regs++] = adev->gfx.config.gb_addr_config; + config[no_regs++] = adev->gfx.config.num_rbs; + + /* rev==1 */ + config[no_regs++] = adev->rev_id; + config[no_regs++] = adev->pg_flags; + config[no_regs++] = adev->cg_flags; + + /* rev==2 */ + config[no_regs++] = adev->family; + config[no_regs++] = adev->external_rev_id; + + /* rev==3 */ + config[no_regs++] = adev->pdev->device; + config[no_regs++] = adev->pdev->revision; + config[no_regs++] = adev->pdev->subsystem_device; + config[no_regs++] = adev->pdev->subsystem_vendor; + + while (size && (*pos < no_regs * 4)) { + uint32_t value; + + value = config[*pos >> 2]; + r = put_user(value, (uint32_t *)buf); + if (r) { + kfree(config); + return r; + } + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + kfree(config); + return result; +} + +/** + * amdgpu_debugfs_sensor_read - Read from the powerplay sensors + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The offset is treated as the BYTE address of one of the sensors + * enumerated in amd/include/kgd_pp_interface.h under the + * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK + * you would use the offset 3 * 4 = 12. + */ +static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + int idx, x, outsize, r, valuesize; + uint32_t values[16]; + + if (size & 3 || *pos & 0x3) + return -EINVAL; + + if (!adev->pm.dpm_enabled) + return -EINVAL; + + /* convert offset to sensor number */ + idx = *pos >> 2; + + valuesize = sizeof(values); + r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize); + if (r) + return r; + + if (size > valuesize) + return -EINVAL; + + outsize = 0; + x = 0; + if (!r) { + while (size) { + r = put_user(values[x++], (int32_t *)buf); + buf += 4; + size -= 4; + outsize += 4; + } + } + + return !r ? outsize : r; +} + +/** amdgpu_debugfs_wave_read - Read WAVE STATUS data + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The offset being sought changes which wave that the status data + * will be returned for. The bits are used as follows: + * + * Bits 0..6: Byte offset into data + * Bits 7..14: SE selector + * Bits 15..22: SH/SA selector + * Bits 23..30: CU/{WGP+SIMD} selector + * Bits 31..36: WAVE ID selector + * Bits 37..44: SIMD ID selector + * + * The returned data begins with one DWORD of version information + * Followed by WAVE STATUS registers relevant to the GFX IP version + * being used. See gfx_v8_0_read_wave_data() for an example output. + */ +static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = f->f_inode->i_private; + int r, x; + ssize_t result=0; + uint32_t offset, se, sh, cu, wave, simd, data[32]; + + if (size & 3 || *pos & 3) + return -EINVAL; + + /* decode offset */ + offset = (*pos & GENMASK_ULL(6, 0)); + se = (*pos & GENMASK_ULL(14, 7)) >> 7; + sh = (*pos & GENMASK_ULL(22, 15)) >> 15; + cu = (*pos & GENMASK_ULL(30, 23)) >> 23; + wave = (*pos & GENMASK_ULL(36, 31)) >> 31; + simd = (*pos & GENMASK_ULL(44, 37)) >> 37; + + /* switch to the specific se/sh/cu */ + mutex_lock(&adev->grbm_idx_mutex); + amdgpu_gfx_select_se_sh(adev, se, sh, cu); + + x = 0; + if (adev->gfx.funcs->read_wave_data) + adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x); + + amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); + mutex_unlock(&adev->grbm_idx_mutex); + + if (!x) + return -EINVAL; + + while (size && (offset < x * 4)) { + uint32_t value; + + value = data[offset >> 2]; + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + offset += 4; + size -= 4; + } + + return result; +} + +/** amdgpu_debugfs_gpr_read - Read wave gprs + * + * @f: open file handle + * @buf: User buffer to store read data in + * @size: Number of bytes to read + * @pos: Offset to seek to + * + * The offset being sought changes which wave that the status data + * will be returned for. The bits are used as follows: + * + * Bits 0..11: Byte offset into data + * Bits 12..19: SE selector + * Bits 20..27: SH/SA selector + * Bits 28..35: CU/{WGP+SIMD} selector + * Bits 36..43: WAVE ID selector + * Bits 37..44: SIMD ID selector + * Bits 52..59: Thread selector + * Bits 60..61: Bank selector (VGPR=0,SGPR=1) + * + * The return data comes from the SGPR or VGPR register bank for + * the selected operational unit. + */ +static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = f->f_inode->i_private; + int r; + ssize_t result = 0; + uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data; + + if (size & 3 || *pos & 3) + return -EINVAL; + + /* decode offset */ + offset = *pos & GENMASK_ULL(11, 0); + se = (*pos & GENMASK_ULL(19, 12)) >> 12; + sh = (*pos & GENMASK_ULL(27, 20)) >> 20; + cu = (*pos & GENMASK_ULL(35, 28)) >> 28; + wave = (*pos & GENMASK_ULL(43, 36)) >> 36; + simd = (*pos & GENMASK_ULL(51, 44)) >> 44; + thread = (*pos & GENMASK_ULL(59, 52)) >> 52; + bank = (*pos & GENMASK_ULL(61, 60)) >> 60; + + data = kcalloc(1024, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + /* switch to the specific se/sh/cu */ + mutex_lock(&adev->grbm_idx_mutex); + amdgpu_gfx_select_se_sh(adev, se, sh, cu); + + if (bank == 0) { + if (adev->gfx.funcs->read_wave_vgprs) + adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data); + } else { + if (adev->gfx.funcs->read_wave_sgprs) + adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data); + } + + amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); + mutex_unlock(&adev->grbm_idx_mutex); + + while (size) { + uint32_t value; + + value = data[offset++]; + r = put_user(value, (uint32_t *)buf); + if (r) { + result = r; + goto err; + } + + result += 4; + buf += 4; + size -= 4; + } + +err: + kfree(data); + return result; +} + +static const struct file_operations amdgpu_debugfs_regs_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_regs_read, + .write = amdgpu_debugfs_regs_write, + .llseek = default_llseek +}; +static const struct file_operations amdgpu_debugfs_regs_didt_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_regs_didt_read, + .write = amdgpu_debugfs_regs_didt_write, + .llseek = default_llseek +}; +static const struct file_operations amdgpu_debugfs_regs_pcie_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_regs_pcie_read, + .write = amdgpu_debugfs_regs_pcie_write, + .llseek = default_llseek +}; +static const struct file_operations amdgpu_debugfs_regs_smc_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_regs_smc_read, + .write = amdgpu_debugfs_regs_smc_write, + .llseek = default_llseek +}; + +static const struct file_operations amdgpu_debugfs_gca_config_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_gca_config_read, + .llseek = default_llseek +}; + +static const struct file_operations amdgpu_debugfs_sensors_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_sensor_read, + .llseek = default_llseek +}; + +static const struct file_operations amdgpu_debugfs_wave_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_wave_read, + .llseek = default_llseek +}; +static const struct file_operations amdgpu_debugfs_gpr_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_gpr_read, + .llseek = default_llseek +}; + +static const struct file_operations *debugfs_regs[] = { + &amdgpu_debugfs_regs_fops, + &amdgpu_debugfs_regs_didt_fops, + &amdgpu_debugfs_regs_pcie_fops, + &amdgpu_debugfs_regs_smc_fops, + &amdgpu_debugfs_gca_config_fops, + &amdgpu_debugfs_sensors_fops, + &amdgpu_debugfs_wave_fops, + &amdgpu_debugfs_gpr_fops, +}; + +static const char *debugfs_regs_names[] = { + "amdgpu_regs", + "amdgpu_regs_didt", + "amdgpu_regs_pcie", + "amdgpu_regs_smc", + "amdgpu_gca_config", + "amdgpu_sensors", + "amdgpu_wave", + "amdgpu_gpr", +}; + +/** + * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide + * register access. + * + * @adev: The device to attach the debugfs entries to + */ +int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) +{ + struct drm_minor *minor = adev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { + ent = debugfs_create_file(debugfs_regs_names[i], + S_IFREG | S_IRUGO, root, + adev, debugfs_regs[i]); + if (!i && !IS_ERR_OR_NULL(ent)) + i_size_write(ent->d_inode, adev->rmmio_size); + adev->debugfs_regs[i] = ent; + } + + return 0; +} + +void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) +{ + unsigned i; + + for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { + if (adev->debugfs_regs[i]) { + debugfs_remove(adev->debugfs_regs[i]); + adev->debugfs_regs[i] = NULL; + } + } +} + +static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + int r = 0, i; + + /* hold on the scheduler */ + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + kthread_park(ring->sched.thread); + } + + seq_printf(m, "run ib test:\n"); + r = amdgpu_ib_ring_tests(adev); + if (r) + seq_printf(m, "ib ring tests failed (%d).\n", r); + else + seq_printf(m, "ib ring tests passed.\n"); + + /* go on the scheduler */ + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + kthread_unpark(ring->sched.thread); + } + + return 0; +} + +static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + + seq_write(m, adev->bios, adev->bios_size); + return 0; +} + +static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + + seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev)); + return 0; +} + +static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + + seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT)); + return 0; +} + +static const struct drm_info_list amdgpu_debugfs_list[] = { + {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump}, + {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}, + {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram}, + {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt}, +}; + +static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring, + struct dma_fence **fences) +{ + struct amdgpu_fence_driver *drv = &ring->fence_drv; + uint32_t sync_seq, last_seq; + + last_seq = atomic_read(&ring->fence_drv.last_seq); + sync_seq = ring->fence_drv.sync_seq; + + last_seq &= drv->num_fences_mask; + sync_seq &= drv->num_fences_mask; + + do { + struct dma_fence *fence, **ptr; + + ++last_seq; + last_seq &= drv->num_fences_mask; + ptr = &drv->fences[last_seq]; + + fence = rcu_dereference_protected(*ptr, 1); + RCU_INIT_POINTER(*ptr, NULL); + + if (!fence) + continue; + + fences[last_seq] = fence; + + } while (last_seq != sync_seq); +} + +static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences, + int length) +{ + int i; + struct dma_fence *fence; + + for (i = 0; i < length; i++) { + fence = fences[i]; + if (!fence) + continue; + dma_fence_signal(fence); + dma_fence_put(fence); + } +} + +static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job; + struct dma_fence *fence; + + spin_lock(&sched->job_list_lock); + list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + fence = sched->ops->run_job(s_job); + dma_fence_put(fence); + } + spin_unlock(&sched->job_list_lock); +} + +static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) +{ + struct amdgpu_job *job; + struct drm_sched_job *s_job; + uint32_t preempt_seq; + struct dma_fence *fence, **ptr; + struct amdgpu_fence_driver *drv = &ring->fence_drv; + struct drm_gpu_scheduler *sched = &ring->sched; + + if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) + return; + + preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2)); + if (preempt_seq <= atomic_read(&drv->last_seq)) + return; + + preempt_seq &= drv->num_fences_mask; + ptr = &drv->fences[preempt_seq]; + fence = rcu_dereference_protected(*ptr, 1); + + spin_lock(&sched->job_list_lock); + list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + job = to_amdgpu_job(s_job); + if (job->fence == fence) + /* mark the job as preempted */ + job->preemption_status |= AMDGPU_IB_PREEMPTED; + } + spin_unlock(&sched->job_list_lock); +} + +static int amdgpu_debugfs_ib_preempt(void *data, u64 val) +{ + int r, resched, length; + struct amdgpu_ring *ring; + struct dma_fence **fences = NULL; + struct amdgpu_device *adev = (struct amdgpu_device *)data; + + if (val >= AMDGPU_MAX_RINGS) + return -EINVAL; + + ring = adev->rings[val]; + + if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread) + return -EINVAL; + + /* the last preemption failed */ + if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr)) + return -EBUSY; + + length = ring->fence_drv.num_fences_mask + 1; + fences = kcalloc(length, sizeof(void *), GFP_KERNEL); + if (!fences) + return -ENOMEM; + + /* stop the scheduler */ + kthread_park(ring->sched.thread); + + resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); + + /* preempt the IB */ + r = amdgpu_ring_preempt_ib(ring); + if (r) { + DRM_WARN("failed to preempt ring %d\n", ring->idx); + goto failure; + } + + amdgpu_fence_process(ring); + + if (atomic_read(&ring->fence_drv.last_seq) != + ring->fence_drv.sync_seq) { + DRM_INFO("ring %d was preempted\n", ring->idx); + + amdgpu_ib_preempt_mark_partial_job(ring); + + /* swap out the old fences */ + amdgpu_ib_preempt_fences_swap(ring, fences); + + amdgpu_fence_driver_force_completion(ring); + + /* resubmit unfinished jobs */ + amdgpu_ib_preempt_job_recovery(&ring->sched); + + /* wait for jobs finished */ + amdgpu_fence_wait_empty(ring); + + /* signal the old fences */ + amdgpu_ib_preempt_signal_fences(fences, length); + } + +failure: + /* restart the scheduler */ + kthread_unpark(ring->sched.thread); + + ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched); + + if (fences) + kfree(fences); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL, + amdgpu_debugfs_ib_preempt, "%llu\n"); + +int amdgpu_debugfs_init(struct amdgpu_device *adev) +{ + adev->debugfs_preempt = + debugfs_create_file("amdgpu_preempt_ib", 0600, + adev->ddev->primary->debugfs_root, + (void *)adev, &fops_ib_preempt); + if (!(adev->debugfs_preempt)) { + DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n"); + return -EIO; + } + + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list, + ARRAY_SIZE(amdgpu_debugfs_list)); +} + +void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev) +{ + if (adev->debugfs_preempt) + debugfs_remove(adev->debugfs_preempt); +} + +#else +int amdgpu_debugfs_init(struct amdgpu_device *adev) +{ + return 0; +} +void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev) { } +int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) +{ + return 0; +} +void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { } +#endif
diff --git a/amdgpu/amdgpu_debugfs.h b/amdgpu/amdgpu_debugfs.h new file mode 100644 index 0000000..f289d28 --- /dev/null +++ b/amdgpu/amdgpu_debugfs.h
@@ -0,0 +1,43 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +/* + * Debugfs + */ +struct amdgpu_debugfs { + const struct drm_info_list *files; + unsigned num_files; +}; + +int amdgpu_debugfs_regs_init(struct amdgpu_device *adev); +void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev); +int amdgpu_debugfs_init(struct amdgpu_device *adev); +void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev); +int amdgpu_debugfs_add_files(struct amdgpu_device *adev, + const struct drm_info_list *files, + unsigned nfiles); +int amdgpu_debugfs_fence_init(struct amdgpu_device *adev); +int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev); +int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
diff --git a/amdgpu/amdgpu_device.c b/amdgpu/amdgpu_device.c new file mode 100644 index 0000000..5a7f893 --- /dev/null +++ b/amdgpu/amdgpu_device.c
@@ -0,0 +1,3977 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#include <linux/power_supply.h> +#include <linux/kthread.h> +#include <linux/module.h> +#include <linux/console.h> +#include <linux/slab.h> + +#include <drm/drm_atomic_helper.h> +#include <drm/drm_probe_helper.h> +#include <drm/amdgpu_drm.h> +#include <linux/vgaarb.h> +#include <linux/vga_switcheroo.h> +#include <linux/efi.h> +#include "amdgpu.h" +#include "amdgpu_trace.h" +#include "amdgpu_i2c.h" +#include "atom.h" +#include "amdgpu_atombios.h" +#include "amdgpu_atomfirmware.h" +#include "amd_pcie.h" +#ifdef CONFIG_DRM_AMDGPU_SI +#include "si.h" +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK +#include "cik.h" +#endif +#include "vi.h" +#include "soc15.h" +#include "nv.h" +#include "bif/bif_4_1_d.h" +#include <linux/pci.h> +#include <linux/firmware.h> +#include "amdgpu_vf_error.h" + +#include "amdgpu_amdkfd.h" +#include "amdgpu_pm.h" + +#include "amdgpu_xgmi.h" +#include "amdgpu_ras.h" +#include "amdgpu_pmu.h" + +MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); +MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin"); + +#define AMDGPU_RESUME_MS 2000 + +static const char *amdgpu_asic_name[] = { + "TAHITI", + "PITCAIRN", + "VERDE", + "OLAND", + "HAINAN", + "BONAIRE", + "KAVERI", + "KABINI", + "HAWAII", + "MULLINS", + "TOPAZ", + "TONGA", + "FIJI", + "CARRIZO", + "STONEY", + "POLARIS10", + "POLARIS11", + "POLARIS12", + "VEGAM", + "VEGA10", + "VEGA12", + "VEGA20", + "RAVEN", + "NAVI10", + "LAST", +}; + +/** + * DOC: pcie_replay_count + * + * The amdgpu driver provides a sysfs API for reporting the total number + * of PCIe replays (NAKs) + * The file pcie_replay_count is used for this and returns the total + * number of replays as a sum of the NAKs generated and NAKs received + */ + +static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); + + return snprintf(buf, PAGE_SIZE, "%llu\n", cnt); +} + +static DEVICE_ATTR(pcie_replay_count, S_IRUGO, + amdgpu_device_get_pcie_replay_count, NULL); + +static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); + +/** + * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control + * + * @dev: drm_device pointer + * + * Returns true if the device is a dGPU with HG/PX power control, + * otherwise return false. + */ +bool amdgpu_device_is_px(struct drm_device *dev) +{ + struct amdgpu_device *adev = dev->dev_private; + + if (adev->flags & AMD_IS_PX) + return true; + return false; +} + +/* + * MMIO register access helper functions. + */ +/** + * amdgpu_mm_rreg - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @acc_flags: access flags which require special behavior + * + * Returns the 32 bit value from the offset specified. + */ +uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, + uint32_t acc_flags) +{ + uint32_t ret; + + if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) + return amdgpu_virt_kiq_rreg(adev, reg); + + if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) + ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); + else { + unsigned long flags; + + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); + ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + } + trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); + return ret; +} + +/* + * MMIO register read with bytes helper functions + * @offset:bytes offset from MMIO start + * +*/ + +/** + * amdgpu_mm_rreg8 - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @offset: byte aligned register offset + * + * Returns the 8 bit value from the offset specified. + */ +uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) { + if (offset < adev->rmmio_size) + return (readb(adev->rmmio + offset)); + BUG(); +} + +/* + * MMIO register write with bytes helper functions + * @offset:bytes offset from MMIO start + * @value: the value want to be written to the register + * +*/ +/** + * amdgpu_mm_wreg8 - read a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @offset: byte aligned register offset + * @value: 8 bit value to write + * + * Writes the value specified to the offset specified. + */ +void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) { + if (offset < adev->rmmio_size) + writeb(value, adev->rmmio + offset); + else + BUG(); +} + +/** + * amdgpu_mm_wreg - write to a memory mapped IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @v: 32 bit value to write to the register + * @acc_flags: access flags which require special behavior + * + * Writes the value specified to the offset specified. + */ +void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, + uint32_t acc_flags) +{ + trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); + + if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { + adev->last_mm_index = v; + } + + if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) + return amdgpu_virt_kiq_wreg(adev, reg, v); + + if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) + writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); + else { + unsigned long flags; + + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); + writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + } + + if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { + udelay(500); + } +} + +/** + * amdgpu_io_rreg - read an IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * + * Returns the 32 bit value from the offset specified. + */ +u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) +{ + if ((reg * 4) < adev->rio_mem_size) + return ioread32(adev->rio_mem + (reg * 4)); + else { + iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); + return ioread32(adev->rio_mem + (mmMM_DATA * 4)); + } +} + +/** + * amdgpu_io_wreg - write to an IO register + * + * @adev: amdgpu_device pointer + * @reg: dword aligned register offset + * @v: 32 bit value to write to the register + * + * Writes the value specified to the offset specified. + */ +void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { + adev->last_mm_index = v; + } + + if ((reg * 4) < adev->rio_mem_size) + iowrite32(v, adev->rio_mem + (reg * 4)); + else { + iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); + iowrite32(v, adev->rio_mem + (mmMM_DATA * 4)); + } + + if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { + udelay(500); + } +} + +/** + * amdgpu_mm_rdoorbell - read a doorbell dword + * + * @adev: amdgpu_device pointer + * @index: doorbell index + * + * Returns the value in the doorbell aperture at the + * requested doorbell index (CIK). + */ +u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) +{ + if (index < adev->doorbell.num_doorbells) { + return readl(adev->doorbell.ptr + index); + } else { + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); + return 0; + } +} + +/** + * amdgpu_mm_wdoorbell - write a doorbell dword + * + * @adev: amdgpu_device pointer + * @index: doorbell index + * @v: value to write + * + * Writes @v to the doorbell aperture at the + * requested doorbell index (CIK). + */ +void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) +{ + if (index < adev->doorbell.num_doorbells) { + writel(v, adev->doorbell.ptr + index); + } else { + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); + } +} + +/** + * amdgpu_mm_rdoorbell64 - read a doorbell Qword + * + * @adev: amdgpu_device pointer + * @index: doorbell index + * + * Returns the value in the doorbell aperture at the + * requested doorbell index (VEGA10+). + */ +u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) +{ + if (index < adev->doorbell.num_doorbells) { + return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); + } else { + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); + return 0; + } +} + +/** + * amdgpu_mm_wdoorbell64 - write a doorbell Qword + * + * @adev: amdgpu_device pointer + * @index: doorbell index + * @v: value to write + * + * Writes @v to the doorbell aperture at the + * requested doorbell index (VEGA10+). + */ +void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) +{ + if (index < adev->doorbell.num_doorbells) { + atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); + } else { + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); + } +} + +/** + * amdgpu_invalid_rreg - dummy reg read function + * + * @adev: amdgpu device pointer + * @reg: offset of register + * + * Dummy register read function. Used for register blocks + * that certain asics don't have (all asics). + * Returns the value in the register. + */ +static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) +{ + DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); + BUG(); + return 0; +} + +/** + * amdgpu_invalid_wreg - dummy reg write function + * + * @adev: amdgpu device pointer + * @reg: offset of register + * @v: value to write to the register + * + * Dummy register read function. Used for register blocks + * that certain asics don't have (all asics). + */ +static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) +{ + DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", + reg, v); + BUG(); +} + +/** + * amdgpu_block_invalid_rreg - dummy reg read function + * + * @adev: amdgpu device pointer + * @block: offset of instance + * @reg: offset of register + * + * Dummy register read function. Used for register blocks + * that certain asics don't have (all asics). + * Returns the value in the register. + */ +static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, + uint32_t block, uint32_t reg) +{ + DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", + reg, block); + BUG(); + return 0; +} + +/** + * amdgpu_block_invalid_wreg - dummy reg write function + * + * @adev: amdgpu device pointer + * @block: offset of instance + * @reg: offset of register + * @v: value to write to the register + * + * Dummy register read function. Used for register blocks + * that certain asics don't have (all asics). + */ +static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, + uint32_t block, + uint32_t reg, uint32_t v) +{ + DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", + reg, block, v); + BUG(); +} + +/** + * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page + * + * @adev: amdgpu device pointer + * + * Allocates a scratch page of VRAM for use by various things in the + * driver. + */ +static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) +{ + return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &adev->vram_scratch.robj, + &adev->vram_scratch.gpu_addr, + (void **)&adev->vram_scratch.ptr); +} + +/** + * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page + * + * @adev: amdgpu device pointer + * + * Frees the VRAM scratch page. + */ +static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) +{ + amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); +} + +/** + * amdgpu_device_program_register_sequence - program an array of registers. + * + * @adev: amdgpu_device pointer + * @registers: pointer to the register array + * @array_size: size of the register array + * + * Programs an array or registers with and and or masks. + * This is a helper for setting golden registers. + */ +void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, + const u32 *registers, + const u32 array_size) +{ + u32 tmp, reg, and_mask, or_mask; + int i; + + if (array_size % 3) + return; + + for (i = 0; i < array_size; i +=3) { + reg = registers[i + 0]; + and_mask = registers[i + 1]; + or_mask = registers[i + 2]; + + if (and_mask == 0xffffffff) { + tmp = or_mask; + } else { + tmp = RREG32(reg); + tmp &= ~and_mask; + if (adev->family >= AMDGPU_FAMILY_AI) + tmp |= (or_mask & and_mask); + else + tmp |= or_mask; + } + WREG32(reg, tmp); + } +} + +/** + * amdgpu_device_pci_config_reset - reset the GPU + * + * @adev: amdgpu_device pointer + * + * Resets the GPU using the pci config reset sequence. + * Only applicable to asics prior to vega10. + */ +void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) +{ + pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); +} + +/* + * GPU doorbell aperture helpers function. + */ +/** + * amdgpu_device_doorbell_init - Init doorbell driver information. + * + * @adev: amdgpu_device pointer + * + * Init doorbell driver information (CIK) + * Returns 0 on success, error on failure. + */ +static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) +{ + + /* No doorbell on SI hardware generation */ + if (adev->asic_type < CHIP_BONAIRE) { + adev->doorbell.base = 0; + adev->doorbell.size = 0; + adev->doorbell.num_doorbells = 0; + adev->doorbell.ptr = NULL; + return 0; + } + + if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) + return -EINVAL; + + amdgpu_asic_init_doorbell_index(adev); + + /* doorbell bar mapping */ + adev->doorbell.base = pci_resource_start(adev->pdev, 2); + adev->doorbell.size = pci_resource_len(adev->pdev, 2); + + adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), + adev->doorbell_index.max_assignment+1); + if (adev->doorbell.num_doorbells == 0) + return -EINVAL; + + /* For Vega, reserve and map two pages on doorbell BAR since SDMA + * paging queue doorbell use the second page. The + * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the + * doorbells are in the first page. So with paging queue enabled, + * the max num_doorbells should + 1 page (0x400 in dword) + */ + if (adev->asic_type >= CHIP_VEGA10) + adev->doorbell.num_doorbells += 0x400; + + adev->doorbell.ptr = ioremap(adev->doorbell.base, + adev->doorbell.num_doorbells * + sizeof(u32)); + if (adev->doorbell.ptr == NULL) + return -ENOMEM; + + return 0; +} + +/** + * amdgpu_device_doorbell_fini - Tear down doorbell driver information. + * + * @adev: amdgpu_device pointer + * + * Tear down doorbell driver information (CIK) + */ +static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) +{ + iounmap(adev->doorbell.ptr); + adev->doorbell.ptr = NULL; +} + + + +/* + * amdgpu_device_wb_*() + * Writeback is the method by which the GPU updates special pages in memory + * with the status of certain GPU events (fences, ring pointers,etc.). + */ + +/** + * amdgpu_device_wb_fini - Disable Writeback and free memory + * + * @adev: amdgpu_device pointer + * + * Disables Writeback and frees the Writeback memory (all asics). + * Used at driver shutdown. + */ +static void amdgpu_device_wb_fini(struct amdgpu_device *adev) +{ + if (adev->wb.wb_obj) { + amdgpu_bo_free_kernel(&adev->wb.wb_obj, + &adev->wb.gpu_addr, + (void **)&adev->wb.wb); + adev->wb.wb_obj = NULL; + } +} + +/** + * amdgpu_device_wb_init- Init Writeback driver info and allocate memory + * + * @adev: amdgpu_device pointer + * + * Initializes writeback and allocates writeback memory (all asics). + * Used at driver startup. + * Returns 0 on success or an -error on failure. + */ +static int amdgpu_device_wb_init(struct amdgpu_device *adev) +{ + int r; + + if (adev->wb.wb_obj == NULL) { + /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ + r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, + &adev->wb.wb_obj, &adev->wb.gpu_addr, + (void **)&adev->wb.wb); + if (r) { + dev_warn(adev->dev, "(%d) create WB bo failed\n", r); + return r; + } + + adev->wb.num_wb = AMDGPU_MAX_WB; + memset(&adev->wb.used, 0, sizeof(adev->wb.used)); + + /* clear wb memory */ + memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); + } + + return 0; +} + +/** + * amdgpu_device_wb_get - Allocate a wb entry + * + * @adev: amdgpu_device pointer + * @wb: wb index + * + * Allocate a wb slot for use by the driver (all asics). + * Returns 0 on success or -EINVAL on failure. + */ +int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) +{ + unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); + + if (offset < adev->wb.num_wb) { + __set_bit(offset, adev->wb.used); + *wb = offset << 3; /* convert to dw offset */ + return 0; + } else { + return -EINVAL; + } +} + +/** + * amdgpu_device_wb_free - Free a wb entry + * + * @adev: amdgpu_device pointer + * @wb: wb index + * + * Free a wb slot allocated for use by the driver (all asics) + */ +void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) +{ + wb >>= 3; + if (wb < adev->wb.num_wb) + __clear_bit(wb, adev->wb.used); +} + +/** + * amdgpu_device_resize_fb_bar - try to resize FB BAR + * + * @adev: amdgpu_device pointer + * + * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not + * to fail, but if any of the BARs is not accessible after the size we abort + * driver loading by returning -ENODEV. + */ +int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) +{ + u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); + u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; + struct pci_bus *root; + struct resource *res; + unsigned i; + u16 cmd; + int r; + + /* Bypass for VF */ + if (amdgpu_sriov_vf(adev)) + return 0; + + /* Check if the root BUS has 64bit memory resources */ + root = adev->pdev->bus; + while (root->parent) + root = root->parent; + + pci_bus_for_each_resource(root, res, i) { + if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && + res->start > 0x100000000ull) + break; + } + + /* Trying to resize is pointless without a root hub window above 4GB */ + if (!res) + return 0; + + /* Disable memory decoding while we change the BAR addresses and size */ + pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); + pci_write_config_word(adev->pdev, PCI_COMMAND, + cmd & ~PCI_COMMAND_MEMORY); + + /* Free the VRAM and doorbell BAR, we most likely need to move both. */ + amdgpu_device_doorbell_fini(adev); + if (adev->asic_type >= CHIP_BONAIRE) + pci_release_resource(adev->pdev, 2); + + pci_release_resource(adev->pdev, 0); + + r = pci_resize_resource(adev->pdev, 0, rbar_size); + if (r == -ENOSPC) + DRM_INFO("Not enough PCI address space for a large BAR."); + else if (r && r != -ENOTSUPP) + DRM_ERROR("Problem resizing BAR0 (%d).", r); + + pci_assign_unassigned_bus_resources(adev->pdev->bus); + + /* When the doorbell or fb BAR isn't available we have no chance of + * using the device. + */ + r = amdgpu_device_doorbell_init(adev); + if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) + return -ENODEV; + + pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); + + return 0; +} + +/* + * GPU helpers function. + */ +/** + * amdgpu_device_need_post - check if the hw need post or not + * + * @adev: amdgpu_device pointer + * + * Check if the asic has been initialized (all asics) at driver startup + * or post is needed if hw reset is performed. + * Returns true if need or false if not. + */ +bool amdgpu_device_need_post(struct amdgpu_device *adev) +{ + uint32_t reg; + + if (amdgpu_sriov_vf(adev)) + return false; + + if (amdgpu_passthrough(adev)) { + /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot + * some old smc fw still need driver do vPost otherwise gpu hang, while + * those smc fw version above 22.15 doesn't have this flaw, so we force + * vpost executed for smc version below 22.15 + */ + if (adev->asic_type == CHIP_FIJI) { + int err; + uint32_t fw_ver; + err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); + /* force vPost if error occured */ + if (err) + return true; + + fw_ver = *((uint32_t *)adev->pm.fw->data + 69); + if (fw_ver < 0x00160e00) + return true; + } + } + + if (adev->has_hw_reset) { + adev->has_hw_reset = false; + return true; + } + + /* bios scratch used on CIK+ */ + if (adev->asic_type >= CHIP_BONAIRE) + return amdgpu_atombios_scratch_need_asic_init(adev); + + /* check MEM_SIZE for older asics */ + reg = amdgpu_asic_get_config_memsize(adev); + + if ((reg != 0) && (reg != 0xffffffff)) + return false; + + return true; +} + +/* if we get transitioned to only one device, take VGA back */ +/** + * amdgpu_device_vga_set_decode - enable/disable vga decode + * + * @cookie: amdgpu_device pointer + * @state: enable/disable vga decode + * + * Enable/disable vga decode (all asics). + * Returns VGA resource flags. + */ +static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state) +{ + struct amdgpu_device *adev = cookie; + amdgpu_asic_set_vga_state(adev, state); + if (state) + return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | + VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; + else + return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; +} + +/** + * amdgpu_device_check_block_size - validate the vm block size + * + * @adev: amdgpu_device pointer + * + * Validates the vm block size specified via module parameter. + * The vm block size defines number of bits in page table versus page directory, + * a page is 4KB so we have 12 bits offset, minimum 9 bits in the + * page table and the remaining bits are in the page directory. + */ +static void amdgpu_device_check_block_size(struct amdgpu_device *adev) +{ + /* defines number of bits in page table versus page directory, + * a page is 4KB so we have 12 bits offset, minimum 9 bits in the + * page table and the remaining bits are in the page directory */ + if (amdgpu_vm_block_size == -1) + return; + + if (amdgpu_vm_block_size < 9) { + dev_warn(adev->dev, "VM page table size (%d) too small\n", + amdgpu_vm_block_size); + amdgpu_vm_block_size = -1; + } +} + +/** + * amdgpu_device_check_vm_size - validate the vm size + * + * @adev: amdgpu_device pointer + * + * Validates the vm size in GB specified via module parameter. + * The VM size is the size of the GPU virtual memory space in GB. + */ +static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) +{ + /* no need to check the default value */ + if (amdgpu_vm_size == -1) + return; + + if (amdgpu_vm_size < 1) { + dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", + amdgpu_vm_size); + amdgpu_vm_size = -1; + } +} + +static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) +{ + struct sysinfo si; + bool is_os_64 = (sizeof(void *) == 8) ? true : false; + uint64_t total_memory; + uint64_t dram_size_seven_GB = 0x1B8000000; + uint64_t dram_size_three_GB = 0xB8000000; + + if (amdgpu_smu_memory_pool_size == 0) + return; + + if (!is_os_64) { + DRM_WARN("Not 64-bit OS, feature not supported\n"); + goto def_value; + } + si_meminfo(&si); + total_memory = (uint64_t)si.totalram * si.mem_unit; + + if ((amdgpu_smu_memory_pool_size == 1) || + (amdgpu_smu_memory_pool_size == 2)) { + if (total_memory < dram_size_three_GB) + goto def_value1; + } else if ((amdgpu_smu_memory_pool_size == 4) || + (amdgpu_smu_memory_pool_size == 8)) { + if (total_memory < dram_size_seven_GB) + goto def_value1; + } else { + DRM_WARN("Smu memory pool size not supported\n"); + goto def_value; + } + adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; + + return; + +def_value1: + DRM_WARN("No enough system memory\n"); +def_value: + adev->pm.smu_prv_buffer_size = 0; +} + +/** + * amdgpu_device_check_arguments - validate module params + * + * @adev: amdgpu_device pointer + * + * Validates certain module parameters and updates + * the associated values used by the driver (all asics). + */ +static int amdgpu_device_check_arguments(struct amdgpu_device *adev) +{ + int ret = 0; + + if (amdgpu_sched_jobs < 4) { + dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", + amdgpu_sched_jobs); + amdgpu_sched_jobs = 4; + } else if (!is_power_of_2(amdgpu_sched_jobs)){ + dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", + amdgpu_sched_jobs); + amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); + } + + if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { + /* gart size must be greater or equal to 32M */ + dev_warn(adev->dev, "gart size (%d) too small\n", + amdgpu_gart_size); + amdgpu_gart_size = -1; + } + + if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { + /* gtt size must be greater or equal to 32M */ + dev_warn(adev->dev, "gtt size (%d) too small\n", + amdgpu_gtt_size); + amdgpu_gtt_size = -1; + } + + /* valid range is between 4 and 9 inclusive */ + if (amdgpu_vm_fragment_size != -1 && + (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { + dev_warn(adev->dev, "valid range is between 4 and 9\n"); + amdgpu_vm_fragment_size = -1; + } + + amdgpu_device_check_smu_prv_buffer_size(adev); + + amdgpu_device_check_vm_size(adev); + + amdgpu_device_check_block_size(adev); + + ret = amdgpu_device_get_job_timeout_settings(adev); + if (ret) { + dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); + return ret; + } + + adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); + + return ret; +} + +/** + * amdgpu_switcheroo_set_state - set switcheroo state + * + * @pdev: pci dev pointer + * @state: vga_switcheroo state + * + * Callback for the switcheroo driver. Suspends or resumes the + * the asics before or after it is powered up using ACPI methods. + */ +static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + + if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF) + return; + + if (state == VGA_SWITCHEROO_ON) { + pr_info("amdgpu: switched on\n"); + /* don't suspend or resume card normally */ + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + + amdgpu_device_resume(dev, true, true); + + dev->switch_power_state = DRM_SWITCH_POWER_ON; + drm_kms_helper_poll_enable(dev); + } else { + pr_info("amdgpu: switched off\n"); + drm_kms_helper_poll_disable(dev); + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + amdgpu_device_suspend(dev, true, true); + dev->switch_power_state = DRM_SWITCH_POWER_OFF; + } +} + +/** + * amdgpu_switcheroo_can_switch - see if switcheroo state can change + * + * @pdev: pci dev pointer + * + * Callback for the switcheroo driver. Check of the switcheroo + * state can be changed. + * Returns true if the state can be changed, false if not. + */ +static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + + /* + * FIXME: open_count is protected by drm_global_mutex but that would lead to + * locking inversion with the driver load path. And the access here is + * completely racy anyway. So don't bother with locking for now. + */ + return dev->open_count == 0; +} + +static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { + .set_gpu_state = amdgpu_switcheroo_set_state, + .reprobe = NULL, + .can_switch = amdgpu_switcheroo_can_switch, +}; + +/** + * amdgpu_device_ip_set_clockgating_state - set the CG state + * + * @dev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * @state: clockgating state (gate or ungate) + * + * Sets the requested clockgating state for all instances of + * the hardware IP specified. + * Returns the error code from the last instance. + */ +int amdgpu_device_ip_set_clockgating_state(void *dev, + enum amd_ip_block_type block_type, + enum amd_clockgating_state state) +{ + struct amdgpu_device *adev = dev; + int i, r = 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type != block_type) + continue; + if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) + continue; + r = adev->ip_blocks[i].version->funcs->set_clockgating_state( + (void *)adev, state); + if (r) + DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + return r; +} + +/** + * amdgpu_device_ip_set_powergating_state - set the PG state + * + * @dev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * @state: powergating state (gate or ungate) + * + * Sets the requested powergating state for all instances of + * the hardware IP specified. + * Returns the error code from the last instance. + */ +int amdgpu_device_ip_set_powergating_state(void *dev, + enum amd_ip_block_type block_type, + enum amd_powergating_state state) +{ + struct amdgpu_device *adev = dev; + int i, r = 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type != block_type) + continue; + if (!adev->ip_blocks[i].version->funcs->set_powergating_state) + continue; + r = adev->ip_blocks[i].version->funcs->set_powergating_state( + (void *)adev, state); + if (r) + DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + return r; +} + +/** + * amdgpu_device_ip_get_clockgating_state - get the CG state + * + * @adev: amdgpu_device pointer + * @flags: clockgating feature flags + * + * Walks the list of IPs on the device and updates the clockgating + * flags for each IP. + * Updates @flags with the feature flags for each hardware IP where + * clockgating is enabled. + */ +void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags) +{ + int i; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->funcs->get_clockgating_state) + adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); + } +} + +/** + * amdgpu_device_ip_wait_for_idle - wait for idle + * + * @adev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Waits for the request hardware IP to be idle. + * Returns 0 for success or a negative error code on failure. + */ +int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type == block_type) { + r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); + if (r) + return r; + break; + } + } + return 0; + +} + +/** + * amdgpu_device_ip_is_idle - is the hardware IP idle + * + * @adev: amdgpu_device pointer + * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Check if the hardware IP is idle or not. + * Returns true if it the IP is idle, false if not. + */ +bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type) +{ + int i; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type == block_type) + return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); + } + return true; + +} + +/** + * amdgpu_device_ip_get_ip_block - get a hw IP pointer + * + * @adev: amdgpu_device pointer + * @type: Type of hardware IP (SMU, GFX, UVD, etc.) + * + * Returns a pointer to the hardware IP block structure + * if it exists for the asic, otherwise NULL. + */ +struct amdgpu_ip_block * +amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, + enum amd_ip_block_type type) +{ + int i; + + for (i = 0; i < adev->num_ip_blocks; i++) + if (adev->ip_blocks[i].version->type == type) + return &adev->ip_blocks[i]; + + return NULL; +} + +/** + * amdgpu_device_ip_block_version_cmp + * + * @adev: amdgpu_device pointer + * @type: enum amd_ip_block_type + * @major: major version + * @minor: minor version + * + * return 0 if equal or greater + * return 1 if smaller or the ip_block doesn't exist + */ +int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, + enum amd_ip_block_type type, + u32 major, u32 minor) +{ + struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); + + if (ip_block && ((ip_block->version->major > major) || + ((ip_block->version->major == major) && + (ip_block->version->minor >= minor)))) + return 0; + + return 1; +} + +/** + * amdgpu_device_ip_block_add + * + * @adev: amdgpu_device pointer + * @ip_block_version: pointer to the IP to add + * + * Adds the IP block driver information to the collection of IPs + * on the asic. + */ +int amdgpu_device_ip_block_add(struct amdgpu_device *adev, + const struct amdgpu_ip_block_version *ip_block_version) +{ + if (!ip_block_version) + return -EINVAL; + + DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, + ip_block_version->funcs->name); + + adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; + + return 0; +} + +/** + * amdgpu_device_enable_virtual_display - enable virtual display feature + * + * @adev: amdgpu_device pointer + * + * Enabled the virtual display feature if the user has enabled it via + * the module parameter virtual_display. This feature provides a virtual + * display hardware on headless boards or in virtualized environments. + * This function parses and validates the configuration string specified by + * the user and configues the virtual display configuration (number of + * virtual connectors, crtcs, etc.) specified. + */ +static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) +{ + adev->enable_virtual_display = false; + + if (amdgpu_virtual_display) { + struct drm_device *ddev = adev->ddev; + const char *pci_address_name = pci_name(ddev->pdev); + char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; + + pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); + pciaddstr_tmp = pciaddstr; + while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { + pciaddname = strsep(&pciaddname_tmp, ","); + if (!strcmp("all", pciaddname) + || !strcmp(pci_address_name, pciaddname)) { + long num_crtc; + int res = -1; + + adev->enable_virtual_display = true; + + if (pciaddname_tmp) + res = kstrtol(pciaddname_tmp, 10, + &num_crtc); + + if (!res) { + if (num_crtc < 1) + num_crtc = 1; + if (num_crtc > 6) + num_crtc = 6; + adev->mode_info.num_crtc = num_crtc; + } else { + adev->mode_info.num_crtc = 1; + } + break; + } + } + + DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", + amdgpu_virtual_display, pci_address_name, + adev->enable_virtual_display, adev->mode_info.num_crtc); + + kfree(pciaddstr); + } +} + +/** + * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware + * + * @adev: amdgpu_device pointer + * + * Parses the asic configuration parameters specified in the gpu info + * firmware and makes them availale to the driver for use in configuring + * the asic. + * Returns 0 on success, -EINVAL on failure. + */ +static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) +{ + const char *chip_name; + char fw_name[30]; + int err; + const struct gpu_info_firmware_header_v1_0 *hdr; + + adev->firmware.gpu_info_fw = NULL; + + switch (adev->asic_type) { + case CHIP_TOPAZ: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + case CHIP_CARRIZO: + case CHIP_STONEY: +#ifdef CONFIG_DRM_AMDGPU_SI + case CHIP_VERDE: + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_OLAND: + case CHIP_HAINAN: +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + case CHIP_HAWAII: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: +#endif + case CHIP_VEGA20: + default: + return 0; + case CHIP_VEGA10: + chip_name = "vega10"; + break; + case CHIP_VEGA12: + chip_name = "vega12"; + break; + case CHIP_RAVEN: + if (adev->rev_id >= 8) + chip_name = "raven2"; + else if (adev->pdev->device == 0x15d8) + chip_name = "picasso"; + else + chip_name = "raven"; + break; + case CHIP_NAVI10: + chip_name = "navi10"; + break; + } + + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); + err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); + if (err) { + dev_err(adev->dev, + "Failed to load gpu_info firmware \"%s\"\n", + fw_name); + goto out; + } + err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); + if (err) { + dev_err(adev->dev, + "Failed to validate gpu_info firmware \"%s\"\n", + fw_name); + goto out; + } + + hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; + amdgpu_ucode_print_gpu_info_hdr(&hdr->header); + + switch (hdr->version_major) { + case 1: + { + const struct gpu_info_firmware_v1_0 *gpu_info_fw = + (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); + adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); + adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); + adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); + adev->gfx.config.max_texture_channel_caches = + le32_to_cpu(gpu_info_fw->gc_num_tccs); + adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); + adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); + adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); + adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); + adev->gfx.config.double_offchip_lds_buf = + le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); + adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); + adev->gfx.cu_info.max_waves_per_simd = + le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); + adev->gfx.cu_info.max_scratch_slots_per_cu = + le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); + adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); + if (hdr->version_minor >= 1) { + const struct gpu_info_firmware_v1_1 *gpu_info_fw = + (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + adev->gfx.config.num_sc_per_sh = + le32_to_cpu(gpu_info_fw->num_sc_per_sh); + adev->gfx.config.num_packer_per_sc = + le32_to_cpu(gpu_info_fw->num_packer_per_sc); + } +#ifdef CONFIG_DRM_AMD_DC_DCN2_0 + if (hdr->version_minor == 2) { + const struct gpu_info_firmware_v1_2 *gpu_info_fw = + (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; + } +#endif + break; + } + default: + dev_err(adev->dev, + "Unsupported gpu_info table %d\n", hdr->header.ucode_version); + err = -EINVAL; + goto out; + } +out: + return err; +} + +/** + * amdgpu_device_ip_early_init - run early init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Early initialization pass for hardware IPs. The hardware IPs that make + * up each asic are discovered each IP's early_init callback is run. This + * is the first stage in initializing the asic. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) +{ + int i, r; + + amdgpu_device_enable_virtual_display(adev); + + switch (adev->asic_type) { + case CHIP_TOPAZ: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + case CHIP_CARRIZO: + case CHIP_STONEY: + if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) + adev->family = AMDGPU_FAMILY_CZ; + else + adev->family = AMDGPU_FAMILY_VI; + + r = vi_set_ip_blocks(adev); + if (r) + return r; + break; +#ifdef CONFIG_DRM_AMDGPU_SI + case CHIP_VERDE: + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_OLAND: + case CHIP_HAINAN: + adev->family = AMDGPU_FAMILY_SI; + r = si_set_ip_blocks(adev); + if (r) + return r; + break; +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + case CHIP_HAWAII: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: + if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII)) + adev->family = AMDGPU_FAMILY_CI; + else + adev->family = AMDGPU_FAMILY_KV; + + r = cik_set_ip_blocks(adev); + if (r) + return r; + break; +#endif + case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: + case CHIP_RAVEN: + if (adev->asic_type == CHIP_RAVEN) + adev->family = AMDGPU_FAMILY_RV; + else + adev->family = AMDGPU_FAMILY_AI; + + r = soc15_set_ip_blocks(adev); + if (r) + return r; + break; + case CHIP_NAVI10: + adev->family = AMDGPU_FAMILY_NV; + + r = nv_set_ip_blocks(adev); + if (r) + return r; + break; + default: + /* FIXME: not supported yet */ + return -EINVAL; + } + + r = amdgpu_device_parse_gpu_info_fw(adev); + if (r) + return r; + + amdgpu_amdkfd_device_probe(adev); + + if (amdgpu_sriov_vf(adev)) { + r = amdgpu_virt_request_full_gpu(adev, true); + if (r) + return -EAGAIN; + + /* query the reg access mode at the very beginning */ + amdgpu_virt_init_reg_access_mode(adev); + } + + adev->pm.pp_feature = amdgpu_pp_feature_mask; + if (amdgpu_sriov_vf(adev)) + adev->pm.pp_feature &= ~PP_GFXOFF_MASK; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if ((amdgpu_ip_block_mask & (1 << i)) == 0) { + DRM_ERROR("disabled ip block: %d <%s>\n", + i, adev->ip_blocks[i].version->funcs->name); + adev->ip_blocks[i].status.valid = false; + } else { + if (adev->ip_blocks[i].version->funcs->early_init) { + r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); + if (r == -ENOENT) { + adev->ip_blocks[i].status.valid = false; + } else if (r) { + DRM_ERROR("early_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } else { + adev->ip_blocks[i].status.valid = true; + } + } else { + adev->ip_blocks[i].status.valid = true; + } + } + /* get the vbios after the asic_funcs are set up */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { + /* Read BIOS */ + if (!amdgpu_get_bios(adev)) + return -EINVAL; + + r = amdgpu_atombios_init(adev); + if (r) { + dev_err(adev->dev, "amdgpu_atombios_init failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); + return r; + } + } + } + + adev->cg_flags &= amdgpu_cg_mask; + adev->pg_flags &= amdgpu_pg_mask; + + return 0; +} + +static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.sw) + continue; + if (adev->ip_blocks[i].status.hw) + continue; + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || + (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { + r = adev->ip_blocks[i].version->funcs->hw_init(adev); + if (r) { + DRM_ERROR("hw_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + adev->ip_blocks[i].status.hw = true; + } + } + + return 0; +} + +static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.sw) + continue; + if (adev->ip_blocks[i].status.hw) + continue; + r = adev->ip_blocks[i].version->funcs->hw_init(adev); + if (r) { + DRM_ERROR("hw_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + adev->ip_blocks[i].status.hw = true; + } + + return 0; +} + +static int amdgpu_device_fw_loading(struct amdgpu_device *adev) +{ + int r = 0; + int i; + uint32_t smu_version; + + if (adev->asic_type >= CHIP_VEGA10) { + for (i = 0; i < adev->num_ip_blocks; i++) { + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { + if (adev->in_gpu_reset || adev->in_suspend) { + if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) + break; /* sriov gpu reset, psp need to do hw_init before IH because of hw limit */ + r = adev->ip_blocks[i].version->funcs->resume(adev); + if (r) { + DRM_ERROR("resume of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } else { + r = adev->ip_blocks[i].version->funcs->hw_init(adev); + if (r) { + DRM_ERROR("hw_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + adev->ip_blocks[i].status.hw = true; + } + } + } + r = amdgpu_pm_load_smu_firmware(adev, &smu_version); + + return r; +} + +/** + * amdgpu_device_ip_init - run init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main initialization pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the sw_init and hw_init callbacks + * are run. sw_init initializes the software state associated with each IP + * and hw_init initializes the hardware associated with each IP. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_init(struct amdgpu_device *adev) +{ + int i, r; + + r = amdgpu_ras_init(adev); + if (r) + return r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); + if (r) { + DRM_ERROR("sw_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + goto init_failed; + } + adev->ip_blocks[i].status.sw = true; + + /* need to do gmc hw init early so we can allocate gpu mem */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { + r = amdgpu_device_vram_scratch_init(adev); + if (r) { + DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); + goto init_failed; + } + r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); + if (r) { + DRM_ERROR("hw_init %d failed %d\n", i, r); + goto init_failed; + } + r = amdgpu_device_wb_init(adev); + if (r) { + DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); + goto init_failed; + } + adev->ip_blocks[i].status.hw = true; + + /* right after GMC hw init, we create CSA */ + if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_CSA_SIZE); + if (r) { + DRM_ERROR("allocate CSA failed %d\n", r); + goto init_failed; + } + } + } + } + + r = amdgpu_ib_pool_init(adev); + if (r) { + dev_err(adev->dev, "IB initialization failed (%d).\n", r); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); + goto init_failed; + } + + r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ + if (r) + goto init_failed; + + r = amdgpu_device_ip_hw_init_phase1(adev); + if (r) + goto init_failed; + + r = amdgpu_device_fw_loading(adev); + if (r) + goto init_failed; + + r = amdgpu_device_ip_hw_init_phase2(adev); + if (r) + goto init_failed; + + if (adev->gmc.xgmi.num_physical_nodes > 1) + amdgpu_xgmi_add_device(adev); + amdgpu_amdkfd_device_init(adev); + +init_failed: + if (amdgpu_sriov_vf(adev)) { + if (!r) + amdgpu_virt_init_data_exchange(adev); + amdgpu_virt_release_full_gpu(adev, true); + } + + return r; +} + +/** + * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer + * + * @adev: amdgpu_device pointer + * + * Writes a reset magic value to the gart pointer in VRAM. The driver calls + * this function before a GPU reset. If the value is retained after a + * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. + */ +static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) +{ + memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); +} + +/** + * amdgpu_device_check_vram_lost - check if vram is valid + * + * @adev: amdgpu_device pointer + * + * Checks the reset magic value written to the gart pointer in VRAM. + * The driver calls this after a GPU reset to see if the contents of + * VRAM is lost or now. + * returns true if vram is lost, false if not. + */ +static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) +{ + return !!memcmp(adev->gart.ptr, adev->reset_magic, + AMDGPU_RESET_MAGIC_NUM); +} + +/** + * amdgpu_device_set_cg_state - set clockgating for amdgpu device + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * set_clockgating_state callbacks are run. + * Late initialization pass enabling clockgating for hardware IPs. + * Fini or suspend, pass disabling clockgating for hardware IPs. + * Returns 0 on success, negative error code on failure. + */ + +static int amdgpu_device_set_cg_state(struct amdgpu_device *adev, + enum amd_clockgating_state state) +{ + int i, j, r; + + if (amdgpu_emu_mode == 1) + return 0; + + for (j = 0; j < adev->num_ip_blocks; j++) { + i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; + if (!adev->ip_blocks[i].status.late_initialized) + continue; + /* skip CG for VCE/UVD, it's handled specially */ + if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && + adev->ip_blocks[i].version->funcs->set_clockgating_state) { + /* enable clockgating to save power */ + r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, + state); + if (r) { + DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + } + + return 0; +} + +static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state) +{ + int i, j, r; + + if (amdgpu_emu_mode == 1) + return 0; + + for (j = 0; j < adev->num_ip_blocks; j++) { + i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; + if (!adev->ip_blocks[i].status.late_initialized) + continue; + /* skip CG for VCE/UVD, it's handled specially */ + if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && + adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && + adev->ip_blocks[i].version->funcs->set_powergating_state) { + /* enable powergating to save power */ + r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, + state); + if (r) { + DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + } + return 0; +} + +static int amdgpu_device_enable_mgpu_fan_boost(void) +{ + struct amdgpu_gpu_instance *gpu_ins; + struct amdgpu_device *adev; + int i, ret = 0; + + mutex_lock(&mgpu_info.mutex); + + /* + * MGPU fan boost feature should be enabled + * only when there are two or more dGPUs in + * the system + */ + if (mgpu_info.num_dgpu < 2) + goto out; + + for (i = 0; i < mgpu_info.num_dgpu; i++) { + gpu_ins = &(mgpu_info.gpu_ins[i]); + adev = gpu_ins->adev; + if (!(adev->flags & AMD_IS_APU) && + !gpu_ins->mgpu_fan_enabled && + adev->powerplay.pp_funcs && + adev->powerplay.pp_funcs->enable_mgpu_fan_boost) { + ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); + if (ret) + break; + + gpu_ins->mgpu_fan_enabled = 1; + } + } + +out: + mutex_unlock(&mgpu_info.mutex); + + return ret; +} + +/** + * amdgpu_device_ip_late_init - run late init for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Late initialization pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the late_init callbacks are run. + * late_init covers any special initialization that an IP requires + * after all of the have been initialized or something that needs to happen + * late in the init process. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) +{ + int i = 0, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.hw) + continue; + if (adev->ip_blocks[i].version->funcs->late_init) { + r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); + if (r) { + DRM_ERROR("late_init of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + adev->ip_blocks[i].status.late_initialized = true; + } + + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); + + amdgpu_device_fill_reset_magic(adev); + + r = amdgpu_device_enable_mgpu_fan_boost(); + if (r) + DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); + + /* set to low pstate by default */ + amdgpu_xgmi_set_pstate(adev, 0); + + return 0; +} + +/** + * amdgpu_device_ip_fini - run fini for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main teardown pass for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks + * are run. hw_fini tears down the hardware associated with each IP + * and sw_fini tears down any software state associated with each IP. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_fini(struct amdgpu_device *adev) +{ + int i, r; + + amdgpu_ras_pre_fini(adev); + + if (adev->gmc.xgmi.num_physical_nodes > 1) + amdgpu_xgmi_remove_device(adev); + + amdgpu_amdkfd_device_fini(adev); + + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + + /* need to disable SMC first */ + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.hw) + continue; + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { + r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); + /* XXX handle errors */ + if (r) { + DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + adev->ip_blocks[i].status.hw = false; + break; + } + } + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.hw) + continue; + + r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); + /* XXX handle errors */ + if (r) { + DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + + adev->ip_blocks[i].status.hw = false; + } + + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.sw) + continue; + + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { + amdgpu_ucode_free_bo(adev); + amdgpu_free_static_csa(&adev->virt.csa_obj); + amdgpu_device_wb_fini(adev); + amdgpu_device_vram_scratch_fini(adev); + amdgpu_ib_pool_fini(adev); + } + + r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); + /* XXX handle errors */ + if (r) { + DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + adev->ip_blocks[i].status.sw = false; + adev->ip_blocks[i].status.valid = false; + } + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.late_initialized) + continue; + if (adev->ip_blocks[i].version->funcs->late_fini) + adev->ip_blocks[i].version->funcs->late_fini((void *)adev); + adev->ip_blocks[i].status.late_initialized = false; + } + + amdgpu_ras_fini(adev); + + if (amdgpu_sriov_vf(adev)) + if (amdgpu_virt_release_full_gpu(adev, false)) + DRM_ERROR("failed to release exclusive mode on fini\n"); + + return 0; +} + +/** + * amdgpu_device_delayed_init_work_handler - work handler for IB tests + * + * @work: work_struct. + */ +static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, delayed_init_work.work); + int r; + + r = amdgpu_ib_ring_tests(adev); + if (r) + DRM_ERROR("ib ring test failed (%d).\n", r); +} + +static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); + + mutex_lock(&adev->gfx.gfx_off_mutex); + if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { + if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) + adev->gfx.gfx_off_state = true; + } + mutex_unlock(&adev->gfx.gfx_off_mutex); +} + +/** + * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) +{ + int i, r; + + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.valid) + continue; + /* displays are handled separately */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { + /* XXX handle errors */ + r = adev->ip_blocks[i].version->funcs->suspend(adev); + /* XXX handle errors */ + if (r) { + DRM_ERROR("suspend of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + } + } + + return 0; +} + +/** + * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) +{ + int i, r; + + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { + if (!adev->ip_blocks[i].status.valid) + continue; + /* displays are handled in phase1 */ + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) + continue; + /* XXX handle errors */ + r = adev->ip_blocks[i].version->funcs->suspend(adev); + /* XXX handle errors */ + if (r) { + DRM_ERROR("suspend of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + } + } + + return 0; +} + +/** + * amdgpu_device_ip_suspend - run suspend for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main suspend function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked, clockgating is disabled and the + * suspend callbacks are run. suspend puts the hardware and software state + * in each IP into a state suitable for suspend. + * Returns 0 on success, negative error code on failure. + */ +int amdgpu_device_ip_suspend(struct amdgpu_device *adev) +{ + int r; + + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_request_full_gpu(adev, false); + + r = amdgpu_device_ip_suspend_phase1(adev); + if (r) + return r; + r = amdgpu_device_ip_suspend_phase2(adev); + + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_release_full_gpu(adev, false); + + return r; +} + +static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) +{ + int i, r; + + static enum amd_ip_block_type ip_order[] = { + AMD_IP_BLOCK_TYPE_GMC, + AMD_IP_BLOCK_TYPE_COMMON, + AMD_IP_BLOCK_TYPE_PSP, + AMD_IP_BLOCK_TYPE_IH, + }; + + for (i = 0; i < ARRAY_SIZE(ip_order); i++) { + int j; + struct amdgpu_ip_block *block; + + for (j = 0; j < adev->num_ip_blocks; j++) { + block = &adev->ip_blocks[j]; + + if (block->version->type != ip_order[i] || + !block->status.valid) + continue; + + r = block->version->funcs->hw_init(adev); + DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + if (r) + return r; + } + } + + return 0; +} + +static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) +{ + int i, r; + + static enum amd_ip_block_type ip_order[] = { + AMD_IP_BLOCK_TYPE_SMC, + AMD_IP_BLOCK_TYPE_DCE, + AMD_IP_BLOCK_TYPE_GFX, + AMD_IP_BLOCK_TYPE_SDMA, + AMD_IP_BLOCK_TYPE_UVD, + AMD_IP_BLOCK_TYPE_VCE + }; + + for (i = 0; i < ARRAY_SIZE(ip_order); i++) { + int j; + struct amdgpu_ip_block *block; + + for (j = 0; j < adev->num_ip_blocks; j++) { + block = &adev->ip_blocks[j]; + + if (block->version->type != ip_order[i] || + !block->status.valid) + continue; + + r = block->version->funcs->hw_init(adev); + DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); + if (r) + return r; + } + } + + return 0; +} + +/** + * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * First resume function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the resume callbacks are run for + * COMMON, GMC, and IH. resume puts the hardware into a functional state + * after a suspend and updates the software state as necessary. This + * function is also used for restoring the GPU after a GPU reset. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { + r = adev->ip_blocks[i].version->funcs->resume(adev); + if (r) { + DRM_ERROR("resume of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + } + + return 0; +} + +/** + * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * First resume function for hardware IPs. The list of all the hardware + * IPs that make up the asic is walked and the resume callbacks are run for + * all blocks except COMMON, GMC, and IH. resume puts the hardware into a + * functional state after a suspend and updates the software state as + * necessary. This function is also used for restoring the GPU after a GPU + * reset. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) + continue; + r = adev->ip_blocks[i].version->funcs->resume(adev); + if (r) { + DRM_ERROR("resume of IP block <%s> failed %d\n", + adev->ip_blocks[i].version->funcs->name, r); + return r; + } + } + + return 0; +} + +/** + * amdgpu_device_ip_resume - run resume for hardware IPs + * + * @adev: amdgpu_device pointer + * + * Main resume function for hardware IPs. The hardware IPs + * are split into two resume functions because they are + * are also used in in recovering from a GPU reset and some additional + * steps need to be take between them. In this case (S3/S4) they are + * run sequentially. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_resume(struct amdgpu_device *adev) +{ + int r; + + r = amdgpu_device_ip_resume_phase1(adev); + if (r) + return r; + + r = amdgpu_device_fw_loading(adev); + if (r) + return r; + + r = amdgpu_device_ip_resume_phase2(adev); + + return r; +} + +/** + * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV + * + * @adev: amdgpu_device pointer + * + * Query the VBIOS data tables to determine if the board supports SR-IOV. + */ +static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) +{ + if (amdgpu_sriov_vf(adev)) { + if (adev->is_atom_fw) { + if (amdgpu_atomfirmware_gpu_supports_virtualization(adev)) + adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; + } else { + if (amdgpu_atombios_has_gpu_virtualization_table(adev)) + adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; + } + + if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); + } +} + +/** + * amdgpu_device_asic_has_dc_support - determine if DC supports the asic + * + * @asic_type: AMD asic type + * + * Check if there is DC (new modesetting infrastructre) support for an asic. + * returns true if DC has support, false if not. + */ +bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) +{ + switch (asic_type) { +#if defined(CONFIG_DRM_AMD_DC) + case CHIP_BONAIRE: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_MULLINS: + /* + * We have systems in the wild with these ASICs that require + * LVDS and VGA support which is not supported with DC. + * + * Fallback to the non-DC driver here by default so as not to + * cause regressions. + */ + return amdgpu_dc > 0; + case CHIP_HAWAII: + case CHIP_CARRIZO: + case CHIP_STONEY: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_VEGA10: + case CHIP_VEGA12: + case CHIP_VEGA20: +#if defined(CONFIG_DRM_AMD_DC_DCN1_0) + case CHIP_RAVEN: +#endif +#if defined(CONFIG_DRM_AMD_DC_DCN2_0) + case CHIP_NAVI10: +#endif + return amdgpu_dc != 0; +#endif + default: + return false; + } +} + +/** + * amdgpu_device_has_dc_support - check if dc is supported + * + * @adev: amdgpu_device_pointer + * + * Returns true for supported, false for not supported + */ +bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) +{ + if (amdgpu_sriov_vf(adev)) + return false; + + return amdgpu_device_asic_has_dc_support(adev->asic_type); +} + + +static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) +{ + struct amdgpu_device *adev = + container_of(__work, struct amdgpu_device, xgmi_reset_work); + + adev->asic_reset_res = amdgpu_asic_reset(adev); + if (adev->asic_reset_res) + DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", + adev->asic_reset_res, adev->ddev->unique); +} + + +/** + * amdgpu_device_init - initialize the driver + * + * @adev: amdgpu_device pointer + * @ddev: drm dev pointer + * @pdev: pci dev pointer + * @flags: driver flags + * + * Initializes the driver info and hw (all asics). + * Returns 0 for success or an error on failure. + * Called at driver startup. + */ +int amdgpu_device_init(struct amdgpu_device *adev, + struct drm_device *ddev, + struct pci_dev *pdev, + uint32_t flags) +{ + int r, i; + bool runtime = false; + u32 max_MBps; + + adev->shutdown = false; + adev->dev = &pdev->dev; + adev->ddev = ddev; + adev->pdev = pdev; + adev->flags = flags; + adev->asic_type = flags & AMD_ASIC_MASK; + adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; + if (amdgpu_emu_mode == 1) + adev->usec_timeout *= 2; + adev->gmc.gart_size = 512 * 1024 * 1024; + adev->accel_working = false; + adev->num_rings = 0; + adev->mman.buffer_funcs = NULL; + adev->mman.buffer_funcs_ring = NULL; + adev->vm_manager.vm_pte_funcs = NULL; + adev->vm_manager.vm_pte_num_rqs = 0; + adev->gmc.gmc_funcs = NULL; + adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); + bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); + + adev->smc_rreg = &amdgpu_invalid_rreg; + adev->smc_wreg = &amdgpu_invalid_wreg; + adev->pcie_rreg = &amdgpu_invalid_rreg; + adev->pcie_wreg = &amdgpu_invalid_wreg; + adev->pciep_rreg = &amdgpu_invalid_rreg; + adev->pciep_wreg = &amdgpu_invalid_wreg; + adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; + adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; + adev->didt_rreg = &amdgpu_invalid_rreg; + adev->didt_wreg = &amdgpu_invalid_wreg; + adev->gc_cac_rreg = &amdgpu_invalid_rreg; + adev->gc_cac_wreg = &amdgpu_invalid_wreg; + adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; + adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; + + DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", + amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, + pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); + + /* mutex initialization are all done here so we + * can recall function without having locking issues */ + atomic_set(&adev->irq.ih.lock, 0); + mutex_init(&adev->firmware.mutex); + mutex_init(&adev->pm.mutex); + mutex_init(&adev->gfx.gpu_clock_mutex); + mutex_init(&adev->srbm_mutex); + mutex_init(&adev->gfx.pipe_reserve_mutex); + mutex_init(&adev->gfx.gfx_off_mutex); + mutex_init(&adev->grbm_idx_mutex); + mutex_init(&adev->mn_lock); + mutex_init(&adev->virt.vf_errors.lock); + hash_init(adev->mn_hash); + mutex_init(&adev->lock_reset); + mutex_init(&adev->virt.dpm_mutex); + mutex_init(&adev->psp.mutex); + + r = amdgpu_device_check_arguments(adev); + if (r) + return r; + + spin_lock_init(&adev->mmio_idx_lock); + spin_lock_init(&adev->smc_idx_lock); + spin_lock_init(&adev->pcie_idx_lock); + spin_lock_init(&adev->uvd_ctx_idx_lock); + spin_lock_init(&adev->didt_idx_lock); + spin_lock_init(&adev->gc_cac_idx_lock); + spin_lock_init(&adev->se_cac_idx_lock); + spin_lock_init(&adev->audio_endpt_idx_lock); + spin_lock_init(&adev->mm_stats.lock); + + INIT_LIST_HEAD(&adev->shadow_list); + mutex_init(&adev->shadow_list_lock); + + INIT_LIST_HEAD(&adev->ring_lru_list); + spin_lock_init(&adev->ring_lru_list_lock); + + INIT_DELAYED_WORK(&adev->delayed_init_work, + amdgpu_device_delayed_init_work_handler); + INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, + amdgpu_device_delay_enable_gfx_off); + + INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); + + adev->gfx.gfx_off_req_count = 1; + adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false; + + /* Registers mapping */ + /* TODO: block userspace mapping of io register */ + if (adev->asic_type >= CHIP_BONAIRE) { + adev->rmmio_base = pci_resource_start(adev->pdev, 5); + adev->rmmio_size = pci_resource_len(adev->pdev, 5); + } else { + adev->rmmio_base = pci_resource_start(adev->pdev, 2); + adev->rmmio_size = pci_resource_len(adev->pdev, 2); + } + + adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); + if (adev->rmmio == NULL) { + return -ENOMEM; + } + DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); + DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); + + /* io port mapping */ + for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { + if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) { + adev->rio_mem_size = pci_resource_len(adev->pdev, i); + adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size); + break; + } + } + if (adev->rio_mem == NULL) + DRM_INFO("PCI I/O BAR is not found.\n"); + + /* enable PCIE atomic ops */ + r = pci_enable_atomic_ops_to_root(adev->pdev, + PCI_EXP_DEVCAP2_ATOMIC_COMP32 | + PCI_EXP_DEVCAP2_ATOMIC_COMP64); + if (r) { + adev->have_atomics_support = false; + DRM_INFO("PCIE atomic ops is not supported\n"); + } else { + adev->have_atomics_support = true; + } + + amdgpu_device_get_pcie_info(adev); + + if (amdgpu_mcbp) + DRM_INFO("MCBP is enabled\n"); + + if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10) + adev->enable_mes = true; + + if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) { + r = amdgpu_discovery_init(adev); + if (r) { + dev_err(adev->dev, "amdgpu_discovery_init failed\n"); + return r; + } + } + + /* early init functions */ + r = amdgpu_device_ip_early_init(adev); + if (r) + return r; + + /* doorbell bar mapping and doorbell index init*/ + amdgpu_device_doorbell_init(adev); + + /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ + /* this will fail for cards that aren't VGA class devices, just + * ignore it */ + vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode); + + if (amdgpu_device_is_px(ddev)) + runtime = true; + if (!pci_is_thunderbolt_attached(adev->pdev)) + vga_switcheroo_register_client(adev->pdev, + &amdgpu_switcheroo_ops, runtime); + if (runtime) + vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); + + if (amdgpu_emu_mode == 1) { + /* post the asic on emulation mode */ + emu_soc_asic_init(adev); + goto fence_driver_init; + } + + /* detect if we are with an SRIOV vbios */ + amdgpu_device_detect_sriov_bios(adev); + + /* check if we need to reset the asic + * E.g., driver was not cleanly unloaded previously, etc. + */ + if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { + r = amdgpu_asic_reset(adev); + if (r) { + dev_err(adev->dev, "asic reset on init failed\n"); + goto failed; + } + } + + /* Post card if necessary */ + if (amdgpu_device_need_post(adev)) { + if (!adev->bios) { + dev_err(adev->dev, "no vBIOS found\n"); + r = -EINVAL; + goto failed; + } + DRM_INFO("GPU posting now...\n"); + r = amdgpu_atom_asic_init(adev->mode_info.atom_context); + if (r) { + dev_err(adev->dev, "gpu post error!\n"); + goto failed; + } + } + + if (adev->is_atom_fw) { + /* Initialize clocks */ + r = amdgpu_atomfirmware_get_clock_info(adev); + if (r) { + dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); + goto failed; + } + } else { + /* Initialize clocks */ + r = amdgpu_atombios_get_clock_info(adev); + if (r) { + dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); + goto failed; + } + /* init i2c buses */ + if (!amdgpu_device_has_dc_support(adev)) + amdgpu_atombios_i2c_init(adev); + } + +fence_driver_init: + /* Fence driver */ + r = amdgpu_fence_driver_init(adev); + if (r) { + dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); + goto failed; + } + + /* init the mode config */ + drm_mode_config_init(adev->ddev); + + r = amdgpu_device_ip_init(adev); + if (r) { + /* failed in exclusive mode due to timeout */ + if (amdgpu_sriov_vf(adev) && + !amdgpu_sriov_runtime(adev) && + amdgpu_virt_mmio_blocked(adev) && + !amdgpu_virt_wait_reset(adev)) { + dev_err(adev->dev, "VF exclusive mode timeout\n"); + /* Don't send request since VF is inactive. */ + adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; + adev->virt.ops = NULL; + r = -EAGAIN; + goto failed; + } + dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); + if (amdgpu_virt_request_full_gpu(adev, false)) + amdgpu_virt_release_full_gpu(adev, false); + goto failed; + } + + adev->accel_working = true; + + amdgpu_vm_check_compute_bug(adev); + + /* Initialize the buffer migration limit. */ + if (amdgpu_moverate >= 0) + max_MBps = amdgpu_moverate; + else + max_MBps = 8; /* Allow 8 MB/s. */ + /* Get a log2 for easy divisions. */ + adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); + + amdgpu_fbdev_init(adev); + + if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)) + amdgpu_pm_virt_sysfs_init(adev); + + r = amdgpu_pm_sysfs_init(adev); + if (r) + DRM_ERROR("registering pm debugfs failed (%d).\n", r); + + r = amdgpu_ucode_sysfs_init(adev); + if (r) + DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); + + r = amdgpu_debugfs_gem_init(adev); + if (r) + DRM_ERROR("registering gem debugfs failed (%d).\n", r); + + r = amdgpu_debugfs_regs_init(adev); + if (r) + DRM_ERROR("registering register debugfs failed (%d).\n", r); + + r = amdgpu_debugfs_firmware_init(adev); + if (r) + DRM_ERROR("registering firmware debugfs failed (%d).\n", r); + + r = amdgpu_debugfs_init(adev); + if (r) + DRM_ERROR("Creating debugfs files failed (%d).\n", r); + + if ((amdgpu_testing & 1)) { + if (adev->accel_working) + amdgpu_test_moves(adev); + else + DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n"); + } + if (amdgpu_benchmarking) { + if (adev->accel_working) + amdgpu_benchmark(adev, amdgpu_benchmarking); + else + DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n"); + } + + /* enable clockgating, etc. after ib tests, etc. since some blocks require + * explicit gating rather than handling it automatically. + */ + r = amdgpu_device_ip_late_init(adev); + if (r) { + dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); + amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); + goto failed; + } + + /* must succeed. */ + amdgpu_ras_resume(adev); + + queue_delayed_work(system_wq, &adev->delayed_init_work, + msecs_to_jiffies(AMDGPU_RESUME_MS)); + + r = device_create_file(adev->dev, &dev_attr_pcie_replay_count); + if (r) { + dev_err(adev->dev, "Could not create pcie_replay_count"); + return r; + } + + if (IS_ENABLED(CONFIG_PERF_EVENTS)) + r = amdgpu_pmu_init(adev); + if (r) + dev_err(adev->dev, "amdgpu_pmu_init failed\n"); + + return 0; + +failed: + amdgpu_vf_error_trans_all(adev); + if (runtime) + vga_switcheroo_fini_domain_pm_ops(adev->dev); + + return r; +} + +/** + * amdgpu_device_fini - tear down the driver + * + * @adev: amdgpu_device pointer + * + * Tear down the driver info (all asics). + * Called at driver shutdown. + */ +void amdgpu_device_fini(struct amdgpu_device *adev) +{ + int r; + + DRM_INFO("amdgpu: finishing device.\n"); + adev->shutdown = true; + /* disable all interrupts */ + amdgpu_irq_disable_all(adev); + if (adev->mode_info.mode_config_initialized){ + if (!amdgpu_device_has_dc_support(adev)) + drm_helper_force_disable_all(adev->ddev); + else + drm_atomic_helper_shutdown(adev->ddev); + } + amdgpu_fence_driver_fini(adev); + amdgpu_pm_sysfs_fini(adev); + amdgpu_fbdev_fini(adev); + r = amdgpu_device_ip_fini(adev); + if (adev->firmware.gpu_info_fw) { + release_firmware(adev->firmware.gpu_info_fw); + adev->firmware.gpu_info_fw = NULL; + } + adev->accel_working = false; + cancel_delayed_work_sync(&adev->delayed_init_work); + /* free i2c buses */ + if (!amdgpu_device_has_dc_support(adev)) + amdgpu_i2c_fini(adev); + + if (amdgpu_emu_mode != 1) + amdgpu_atombios_fini(adev); + + kfree(adev->bios); + adev->bios = NULL; + if (!pci_is_thunderbolt_attached(adev->pdev)) + vga_switcheroo_unregister_client(adev->pdev); + if (adev->flags & AMD_IS_PX) + vga_switcheroo_fini_domain_pm_ops(adev->dev); + vga_client_register(adev->pdev, NULL, NULL, NULL); + if (adev->rio_mem) + pci_iounmap(adev->pdev, adev->rio_mem); + adev->rio_mem = NULL; + iounmap(adev->rmmio); + adev->rmmio = NULL; + amdgpu_device_doorbell_fini(adev); + if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)) + amdgpu_pm_virt_sysfs_fini(adev); + + amdgpu_debugfs_regs_cleanup(adev); + device_remove_file(adev->dev, &dev_attr_pcie_replay_count); + amdgpu_ucode_sysfs_fini(adev); + if (IS_ENABLED(CONFIG_PERF_EVENTS)) + amdgpu_pmu_fini(adev); + amdgpu_debugfs_preempt_cleanup(adev); + if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) + amdgpu_discovery_fini(adev); +} + + +/* + * Suspend & resume. + */ +/** + * amdgpu_device_suspend - initiate device suspend + * + * @dev: drm dev pointer + * @suspend: suspend state + * @fbcon : notify the fbdev of suspend + * + * Puts the hw in the suspend state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver suspend. + */ +int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) +{ + struct amdgpu_device *adev; + struct drm_crtc *crtc; + struct drm_connector *connector; + int r; + + if (dev == NULL || dev->dev_private == NULL) { + return -ENODEV; + } + + adev = dev->dev_private; + + if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; + + adev->in_suspend = true; + drm_kms_helper_poll_disable(dev); + + if (fbcon) + amdgpu_fbdev_set_suspend(adev, 1); + + cancel_delayed_work_sync(&adev->delayed_init_work); + + if (!amdgpu_device_has_dc_support(adev)) { + /* turn off display hw */ + drm_modeset_lock_all(dev); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); + } + drm_modeset_unlock_all(dev); + /* unpin the front buffers and cursors */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_framebuffer *fb = crtc->primary->fb; + struct amdgpu_bo *robj; + + if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { + struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); + r = amdgpu_bo_reserve(aobj, true); + if (r == 0) { + amdgpu_bo_unpin(aobj); + amdgpu_bo_unreserve(aobj); + } + } + + if (fb == NULL || fb->obj[0] == NULL) { + continue; + } + robj = gem_to_amdgpu_bo(fb->obj[0]); + /* don't unpin kernel fb objects */ + if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { + r = amdgpu_bo_reserve(robj, true); + if (r == 0) { + amdgpu_bo_unpin(robj); + amdgpu_bo_unreserve(robj); + } + } + } + } + + amdgpu_amdkfd_suspend(adev); + + amdgpu_ras_suspend(adev); + + r = amdgpu_device_ip_suspend_phase1(adev); + + /* evict vram memory */ + amdgpu_bo_evict_vram(adev); + + amdgpu_fence_driver_suspend(adev); + + r = amdgpu_device_ip_suspend_phase2(adev); + + /* evict remaining vram memory + * This second call to evict vram is to evict the gart page table + * using the CPU. + */ + amdgpu_bo_evict_vram(adev); + + pci_save_state(dev->pdev); + if (suspend) { + /* Shut down the device */ + pci_disable_device(dev->pdev); + pci_set_power_state(dev->pdev, PCI_D3hot); + } else { + r = amdgpu_asic_reset(adev); + if (r) + DRM_ERROR("amdgpu asic reset failed\n"); + } + + return 0; +} + +/** + * amdgpu_device_resume - initiate device resume + * + * @dev: drm dev pointer + * @resume: resume state + * @fbcon : notify the fbdev of resume + * + * Bring the hw back to operating state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver resume. + */ +int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) +{ + struct drm_connector *connector; + struct amdgpu_device *adev = dev->dev_private; + struct drm_crtc *crtc; + int r = 0; + + if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; + + if (resume) { + pci_set_power_state(dev->pdev, PCI_D0); + pci_restore_state(dev->pdev); + r = pci_enable_device(dev->pdev); + if (r) + return r; + } + + /* post card */ + if (amdgpu_device_need_post(adev)) { + r = amdgpu_atom_asic_init(adev->mode_info.atom_context); + if (r) + DRM_ERROR("amdgpu asic init failed\n"); + } + + r = amdgpu_device_ip_resume(adev); + if (r) { + DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r); + return r; + } + amdgpu_fence_driver_resume(adev); + + + r = amdgpu_device_ip_late_init(adev); + if (r) + return r; + + queue_delayed_work(system_wq, &adev->delayed_init_work, + msecs_to_jiffies(AMDGPU_RESUME_MS)); + + if (!amdgpu_device_has_dc_support(adev)) { + /* pin cursors */ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + + if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { + struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); + r = amdgpu_bo_reserve(aobj, true); + if (r == 0) { + r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM); + if (r != 0) + DRM_ERROR("Failed to pin cursor BO (%d)\n", r); + amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj); + amdgpu_bo_unreserve(aobj); + } + } + } + } + r = amdgpu_amdkfd_resume(adev); + if (r) + return r; + + /* Make sure IB tests flushed */ + flush_delayed_work(&adev->delayed_init_work); + + /* blat the mode back in */ + if (fbcon) { + if (!amdgpu_device_has_dc_support(adev)) { + /* pre DCE11 */ + drm_helper_resume_force_mode(dev); + + /* turn on display hw */ + drm_modeset_lock_all(dev); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); + } + drm_modeset_unlock_all(dev); + } + amdgpu_fbdev_set_suspend(adev, 0); + } + + drm_kms_helper_poll_enable(dev); + + amdgpu_ras_resume(adev); + + /* + * Most of the connector probing functions try to acquire runtime pm + * refs to ensure that the GPU is powered on when connector polling is + * performed. Since we're calling this from a runtime PM callback, + * trying to acquire rpm refs will cause us to deadlock. + * + * Since we're guaranteed to be holding the rpm lock, it's safe to + * temporarily disable the rpm helpers so this doesn't deadlock us. + */ +#ifdef CONFIG_PM + dev->dev->power.disable_depth++; +#endif + if (!amdgpu_device_has_dc_support(adev)) + drm_helper_hpd_irq_event(dev); + else + drm_kms_helper_hotplug_event(dev); +#ifdef CONFIG_PM + dev->dev->power.disable_depth--; +#endif + adev->in_suspend = false; + + return 0; +} + +/** + * amdgpu_device_ip_check_soft_reset - did soft reset succeed + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and + * the check_soft_reset callbacks are run. check_soft_reset determines + * if the asic is still hung or not. + * Returns true if any of the IPs are still in a hung state, false if not. + */ +static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) +{ + int i; + bool asic_hang = false; + + if (amdgpu_sriov_vf(adev)) + return true; + + if (amdgpu_asic_need_full_reset(adev)) + return true; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].version->funcs->check_soft_reset) + adev->ip_blocks[i].status.hang = + adev->ip_blocks[i].version->funcs->check_soft_reset(adev); + if (adev->ip_blocks[i].status.hang) { + DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); + asic_hang = true; + } + } + return asic_hang; +} + +/** + * amdgpu_device_ip_pre_soft_reset - prepare for soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset + * handles any IP specific hardware or software state changes that are + * necessary for a soft reset to succeed. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) +{ + int i, r = 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].status.hang && + adev->ip_blocks[i].version->funcs->pre_soft_reset) { + r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); + if (r) + return r; + } + } + + return 0; +} + +/** + * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed + * + * @adev: amdgpu_device pointer + * + * Some hardware IPs cannot be soft reset. If they are hung, a full gpu + * reset is necessary to recover. + * Returns true if a full asic reset is required, false if not. + */ +static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) +{ + int i; + + if (amdgpu_asic_need_full_reset(adev)) + return true; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || + (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || + (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || + (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { + if (adev->ip_blocks[i].status.hang) { + DRM_INFO("Some block need full reset!\n"); + return true; + } + } + } + return false; +} + +/** + * amdgpu_device_ip_soft_reset - do a soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * soft_reset callbacks are run if the block is hung. soft_reset handles any + * IP specific hardware or software state changes that are necessary to soft + * reset the IP. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) +{ + int i, r = 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].status.hang && + adev->ip_blocks[i].version->funcs->soft_reset) { + r = adev->ip_blocks[i].version->funcs->soft_reset(adev); + if (r) + return r; + } + } + + return 0; +} + +/** + * amdgpu_device_ip_post_soft_reset - clean up from soft reset + * + * @adev: amdgpu_device pointer + * + * The list of all the hardware IPs that make up the asic is walked and the + * post_soft_reset callbacks are run if the asic was hung. post_soft_reset + * handles any IP specific hardware or software state changes that are + * necessary after the IP has been soft reset. + * Returns 0 on success, negative error code on failure. + */ +static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) +{ + int i, r = 0; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_blocks[i].status.valid) + continue; + if (adev->ip_blocks[i].status.hang && + adev->ip_blocks[i].version->funcs->post_soft_reset) + r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); + if (r) + return r; + } + + return 0; +} + +/** + * amdgpu_device_recover_vram - Recover some VRAM contents + * + * @adev: amdgpu_device pointer + * + * Restores the contents of VRAM buffers from the shadows in GTT. Used to + * restore things like GPUVM page tables after a GPU reset where + * the contents of VRAM might be lost. + * + * Returns: + * 0 on success, negative error code on failure. + */ +static int amdgpu_device_recover_vram(struct amdgpu_device *adev) +{ + struct dma_fence *fence = NULL, *next = NULL; + struct amdgpu_bo *shadow; + long r = 1, tmo; + + if (amdgpu_sriov_runtime(adev)) + tmo = msecs_to_jiffies(8000); + else + tmo = msecs_to_jiffies(100); + + DRM_INFO("recover vram bo from shadow start\n"); + mutex_lock(&adev->shadow_list_lock); + list_for_each_entry(shadow, &adev->shadow_list, shadow_list) { + + /* No need to recover an evicted BO */ + if (shadow->tbo.mem.mem_type != TTM_PL_TT || + shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET || + shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM) + continue; + + r = amdgpu_bo_restore_shadow(shadow, &next); + if (r) + break; + + if (fence) { + tmo = dma_fence_wait_timeout(fence, false, tmo); + dma_fence_put(fence); + fence = next; + if (tmo == 0) { + r = -ETIMEDOUT; + break; + } else if (tmo < 0) { + r = tmo; + break; + } + } else { + fence = next; + } + } + mutex_unlock(&adev->shadow_list_lock); + + if (fence) + tmo = dma_fence_wait_timeout(fence, false, tmo); + dma_fence_put(fence); + + if (r < 0 || tmo <= 0) { + DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); + return -EIO; + } + + DRM_INFO("recover vram bo from shadow done\n"); + return 0; +} + + +/** + * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf + * + * @adev: amdgpu device pointer + * @from_hypervisor: request from hypervisor + * + * do VF FLR and reinitialize Asic + * return 0 means succeeded otherwise failed + */ +static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, + bool from_hypervisor) +{ + int r; + + if (from_hypervisor) + r = amdgpu_virt_request_full_gpu(adev, true); + else + r = amdgpu_virt_reset_gpu(adev); + if (r) + return r; + + amdgpu_amdkfd_pre_reset(adev); + + /* Resume IP prior to SMC */ + r = amdgpu_device_ip_reinit_early_sriov(adev); + if (r) + goto error; + + /* we need recover gart prior to run SMC/CP/SDMA resume */ + amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]); + + r = amdgpu_device_fw_loading(adev); + if (r) + return r; + + /* now we are okay to resume SMC/CP/SDMA */ + r = amdgpu_device_ip_reinit_late_sriov(adev); + if (r) + goto error; + + amdgpu_irq_gpu_reset_resume_helper(adev); + r = amdgpu_ib_ring_tests(adev); + amdgpu_amdkfd_post_reset(adev); + +error: + amdgpu_virt_init_data_exchange(adev); + amdgpu_virt_release_full_gpu(adev, true); + if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { + atomic_inc(&adev->vram_lost_counter); + r = amdgpu_device_recover_vram(adev); + } + + return r; +} + +/** + * amdgpu_device_should_recover_gpu - check if we should try GPU recovery + * + * @adev: amdgpu device pointer + * + * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover + * a hung GPU. + */ +bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) +{ + if (!amdgpu_device_ip_check_soft_reset(adev)) { + DRM_INFO("Timeout, but no hardware hang detected.\n"); + return false; + } + + if (amdgpu_gpu_recovery == 0) + goto disabled; + + if (amdgpu_sriov_vf(adev)) + return true; + + if (amdgpu_gpu_recovery == -1) { + switch (adev->asic_type) { + case CHIP_BONAIRE: + case CHIP_HAWAII: + case CHIP_TOPAZ: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + case CHIP_VEGA20: + case CHIP_VEGA10: + case CHIP_VEGA12: + break; + default: + goto disabled; + } + } + + return true; + +disabled: + DRM_INFO("GPU recovery disabled.\n"); + return false; +} + + +static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, + struct amdgpu_job *job, + bool *need_full_reset_arg) +{ + int i, r = 0; + bool need_full_reset = *need_full_reset_arg; + + /* block all schedulers and reset given job's ring */ + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + + /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ + amdgpu_fence_driver_force_completion(ring); + } + + if(job) + drm_sched_increase_karma(&job->base); + + /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ + if (!amdgpu_sriov_vf(adev)) { + + if (!need_full_reset) + need_full_reset = amdgpu_device_ip_need_full_reset(adev); + + if (!need_full_reset) { + amdgpu_device_ip_pre_soft_reset(adev); + r = amdgpu_device_ip_soft_reset(adev); + amdgpu_device_ip_post_soft_reset(adev); + if (r || amdgpu_device_ip_check_soft_reset(adev)) { + DRM_INFO("soft reset failed, will fallback to full reset!\n"); + need_full_reset = true; + } + } + + if (need_full_reset) + r = amdgpu_device_ip_suspend(adev); + + *need_full_reset_arg = need_full_reset; + } + + return r; +} + +static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, + struct list_head *device_list_handle, + bool *need_full_reset_arg) +{ + struct amdgpu_device *tmp_adev = NULL; + bool need_full_reset = *need_full_reset_arg, vram_lost = false; + int r = 0; + + /* + * ASIC reset has to be done on all HGMI hive nodes ASAP + * to allow proper links negotiation in FW (within 1 sec) + */ + if (need_full_reset) { + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { + /* For XGMI run all resets in parallel to speed up the process */ + if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { + if (!queue_work(system_highpri_wq, &tmp_adev->xgmi_reset_work)) + r = -EALREADY; + } else + r = amdgpu_asic_reset(tmp_adev); + + if (r) { + DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s", + r, tmp_adev->ddev->unique); + break; + } + } + + /* For XGMI wait for all PSP resets to complete before proceed */ + if (!r) { + list_for_each_entry(tmp_adev, device_list_handle, + gmc.xgmi.head) { + if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { + flush_work(&tmp_adev->xgmi_reset_work); + r = tmp_adev->asic_reset_res; + if (r) + break; + } + } + + list_for_each_entry(tmp_adev, device_list_handle, + gmc.xgmi.head) { + amdgpu_ras_reserve_bad_pages(tmp_adev); + } + } + } + + + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { + if (need_full_reset) { + /* post card */ + if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context)) + DRM_WARN("asic atom init failed!"); + + if (!r) { + dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); + r = amdgpu_device_ip_resume_phase1(tmp_adev); + if (r) + goto out; + + vram_lost = amdgpu_device_check_vram_lost(tmp_adev); + if (vram_lost) { + DRM_INFO("VRAM is lost due to GPU reset!\n"); + atomic_inc(&tmp_adev->vram_lost_counter); + } + + r = amdgpu_gtt_mgr_recover( + &tmp_adev->mman.bdev.man[TTM_PL_TT]); + if (r) + goto out; + + r = amdgpu_device_fw_loading(tmp_adev); + if (r) + return r; + + r = amdgpu_device_ip_resume_phase2(tmp_adev); + if (r) + goto out; + + if (vram_lost) + amdgpu_device_fill_reset_magic(tmp_adev); + + /* + * Add this ASIC as tracked as reset was already + * complete successfully. + */ + amdgpu_register_gpu_instance(tmp_adev); + + r = amdgpu_device_ip_late_init(tmp_adev); + if (r) + goto out; + + /* must succeed. */ + amdgpu_ras_resume(tmp_adev); + + /* Update PSP FW topology after reset */ + if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1) + r = amdgpu_xgmi_update_topology(hive, tmp_adev); + } + } + + +out: + if (!r) { + amdgpu_irq_gpu_reset_resume_helper(tmp_adev); + r = amdgpu_ib_ring_tests(tmp_adev); + if (r) { + dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); + r = amdgpu_device_ip_suspend(tmp_adev); + need_full_reset = true; + r = -EAGAIN; + goto end; + } + } + + if (!r) + r = amdgpu_device_recover_vram(tmp_adev); + else + tmp_adev->asic_reset_res = r; + } + +end: + *need_full_reset_arg = need_full_reset; + return r; +} + +static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock) +{ + if (trylock) { + if (!mutex_trylock(&adev->lock_reset)) + return false; + } else + mutex_lock(&adev->lock_reset); + + atomic_inc(&adev->gpu_reset_counter); + adev->in_gpu_reset = 1; + /* Block kfd: SRIOV would do it separately */ + if (!amdgpu_sriov_vf(adev)) + amdgpu_amdkfd_pre_reset(adev); + + return true; +} + +static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) +{ + /*unlock kfd: SRIOV would do it separately */ + if (!amdgpu_sriov_vf(adev)) + amdgpu_amdkfd_post_reset(adev); + amdgpu_vf_error_trans_all(adev); + adev->in_gpu_reset = 0; + mutex_unlock(&adev->lock_reset); +} + + +/** + * amdgpu_device_gpu_recover - reset the asic and recover scheduler + * + * @adev: amdgpu device pointer + * @job: which job trigger hang + * + * Attempt to reset the GPU if it has hung (all asics). + * Attempt to do soft-reset or full-reset and reinitialize Asic + * Returns 0 for success or an error on failure. + */ + +int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + struct amdgpu_job *job) +{ + struct list_head device_list, *device_list_handle = NULL; + bool need_full_reset, job_signaled; + struct amdgpu_hive_info *hive = NULL; + struct amdgpu_device *tmp_adev = NULL; + int i, r = 0; + + need_full_reset = job_signaled = false; + INIT_LIST_HEAD(&device_list); + + dev_info(adev->dev, "GPU reset begin!\n"); + + cancel_delayed_work_sync(&adev->delayed_init_work); + + hive = amdgpu_get_xgmi_hive(adev, false); + + /* + * Here we trylock to avoid chain of resets executing from + * either trigger by jobs on different adevs in XGMI hive or jobs on + * different schedulers for same device while this TO handler is running. + * We always reset all schedulers for device and all devices for XGMI + * hive so that should take care of them too. + */ + + if (hive && !mutex_trylock(&hive->reset_lock)) { + DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", + job->base.id, hive->hive_id); + return 0; + } + + /* Start with adev pre asic reset first for soft reset check.*/ + if (!amdgpu_device_lock_adev(adev, !hive)) { + DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress", + job->base.id); + return 0; + } + + /* Build list of devices to reset */ + if (adev->gmc.xgmi.num_physical_nodes > 1) { + if (!hive) { + amdgpu_device_unlock_adev(adev); + return -ENODEV; + } + + /* + * In case we are in XGMI hive mode device reset is done for all the + * nodes in the hive to retrain all XGMI links and hence the reset + * sequence is executed in loop on all nodes. + */ + device_list_handle = &hive->device_list; + } else { + list_add_tail(&adev->gmc.xgmi.head, &device_list); + device_list_handle = &device_list; + } + + /* + * Mark these ASICs to be reseted as untracked first + * And add them back after reset completed + */ + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) + amdgpu_unregister_gpu_instance(tmp_adev); + + /* block all schedulers and reset given job's ring */ + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { + /* disable ras on ALL IPs */ + if (amdgpu_device_ip_need_full_reset(tmp_adev)) + amdgpu_ras_suspend(tmp_adev); + + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = tmp_adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + + drm_sched_stop(&ring->sched, &job->base); + } + } + + + /* + * Must check guilty signal here since after this point all old + * HW fences are force signaled. + * + * job->base holds a reference to parent fence + */ + if (job && job->base.s_fence->parent && + dma_fence_is_signaled(job->base.s_fence->parent)) + job_signaled = true; + + if (!amdgpu_device_ip_need_full_reset(adev)) + device_list_handle = &device_list; + + if (job_signaled) { + dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); + goto skip_hw_reset; + } + + + /* Guilty job will be freed after this*/ + r = amdgpu_device_pre_asic_reset(adev, + job, + &need_full_reset); + if (r) { + /*TODO Should we stop ?*/ + DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ", + r, adev->ddev->unique); + adev->asic_reset_res = r; + } + +retry: /* Rest of adevs pre asic reset from XGMI hive. */ + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { + + if (tmp_adev == adev) + continue; + + amdgpu_device_lock_adev(tmp_adev, false); + r = amdgpu_device_pre_asic_reset(tmp_adev, + NULL, + &need_full_reset); + /*TODO Should we stop ?*/ + if (r) { + DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ", + r, tmp_adev->ddev->unique); + tmp_adev->asic_reset_res = r; + } + } + + /* Actual ASIC resets if needed.*/ + /* TODO Implement XGMI hive reset logic for SRIOV */ + if (amdgpu_sriov_vf(adev)) { + r = amdgpu_device_reset_sriov(adev, job ? false : true); + if (r) + adev->asic_reset_res = r; + } else { + r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset); + if (r && r == -EAGAIN) + goto retry; + } + +skip_hw_reset: + + /* Post ASIC reset for all devs .*/ + list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = tmp_adev->rings[i]; + + if (!ring || !ring->sched.thread) + continue; + + /* No point to resubmit jobs if we didn't HW reset*/ + if (!tmp_adev->asic_reset_res && !job_signaled) + drm_sched_resubmit_jobs(&ring->sched); + + drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res); + } + + if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) { + drm_helper_resume_force_mode(tmp_adev->ddev); + } + + tmp_adev->asic_reset_res = 0; + + if (r) { + /* bad news, how to tell it to userspace ? */ + dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter)); + amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); + } else { + dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter)); + } + + amdgpu_device_unlock_adev(tmp_adev); + } + + if (hive) + mutex_unlock(&hive->reset_lock); + + if (r) + dev_info(adev->dev, "GPU reset end with ret = %d\n", r); + return r; +} + +/** + * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot + * + * @adev: amdgpu_device pointer + * + * Fetchs and stores in the driver the PCIE capabilities (gen speed + * and lanes) of the slot the device is in. Handles APUs and + * virtualized environments where PCIE config space may not be available. + */ +static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) +{ + struct pci_dev *pdev; + enum pci_bus_speed speed_cap, platform_speed_cap; + enum pcie_link_width platform_link_width; + + if (amdgpu_pcie_gen_cap) + adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; + + if (amdgpu_pcie_lane_cap) + adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; + + /* covers APUs as well */ + if (pci_is_root_bus(adev->pdev->bus)) { + if (adev->pm.pcie_gen_mask == 0) + adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; + if (adev->pm.pcie_mlw_mask == 0) + adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; + return; + } + + if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) + return; + + pcie_bandwidth_available(adev->pdev, NULL, + &platform_speed_cap, &platform_link_width); + + if (adev->pm.pcie_gen_mask == 0) { + /* asic caps */ + pdev = adev->pdev; + speed_cap = pcie_get_speed_cap(pdev); + if (speed_cap == PCI_SPEED_UNKNOWN) { + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); + } else { + if (speed_cap == PCIE_SPEED_16_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); + else if (speed_cap == PCIE_SPEED_8_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); + else if (speed_cap == PCIE_SPEED_5_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); + else + adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; + } + /* platform caps */ + if (platform_speed_cap == PCI_SPEED_UNKNOWN) { + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); + } else { + if (platform_speed_cap == PCIE_SPEED_16_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); + else if (platform_speed_cap == PCIE_SPEED_8_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); + else if (platform_speed_cap == PCIE_SPEED_5_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); + else + adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; + + } + } + if (adev->pm.pcie_mlw_mask == 0) { + if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { + adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; + } else { + switch (platform_link_width) { + case PCIE_LNK_X32: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X16: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X12: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X8: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X4: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X2: + adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | + CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); + break; + case PCIE_LNK_X1: + adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; + break; + default: + break; + } + } + } +} +
diff --git a/amdgpu/amdgpu_discovery.c b/amdgpu/amdgpu_discovery.c new file mode 100644 index 0000000..1481899 --- /dev/null +++ b/amdgpu/amdgpu_discovery.c
@@ -0,0 +1,415 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu.h" +#include "amdgpu_discovery.h" +#include "soc15_common.h" +#include "soc15_hw_ip.h" +#include "nbio/nbio_2_3_offset.h" +#include "discovery.h" + +#define mmRCC_CONFIG_MEMSIZE 0xde3 +#define mmMM_INDEX 0x0 +#define mmMM_INDEX_HI 0x6 +#define mmMM_DATA 0x1 +#define HW_ID_MAX 300 + +const char *hw_id_names[HW_ID_MAX] = { + [MP1_HWID] = "MP1", + [MP2_HWID] = "MP2", + [THM_HWID] = "THM", + [SMUIO_HWID] = "SMUIO", + [FUSE_HWID] = "FUSE", + [CLKA_HWID] = "CLKA", + [PWR_HWID] = "PWR", + [GC_HWID] = "GC", + [UVD_HWID] = "UVD", + [AUDIO_AZ_HWID] = "AUDIO_AZ", + [ACP_HWID] = "ACP", + [DCI_HWID] = "DCI", + [DMU_HWID] = "DMU", + [DCO_HWID] = "DCO", + [DIO_HWID] = "DIO", + [XDMA_HWID] = "XDMA", + [DCEAZ_HWID] = "DCEAZ", + [DAZ_HWID] = "DAZ", + [SDPMUX_HWID] = "SDPMUX", + [NTB_HWID] = "NTB", + [IOHC_HWID] = "IOHC", + [L2IMU_HWID] = "L2IMU", + [VCE_HWID] = "VCE", + [MMHUB_HWID] = "MMHUB", + [ATHUB_HWID] = "ATHUB", + [DBGU_NBIO_HWID] = "DBGU_NBIO", + [DFX_HWID] = "DFX", + [DBGU0_HWID] = "DBGU0", + [DBGU1_HWID] = "DBGU1", + [OSSSYS_HWID] = "OSSSYS", + [HDP_HWID] = "HDP", + [SDMA0_HWID] = "SDMA0", + [SDMA1_HWID] = "SDMA1", + [ISP_HWID] = "ISP", + [DBGU_IO_HWID] = "DBGU_IO", + [DF_HWID] = "DF", + [CLKB_HWID] = "CLKB", + [FCH_HWID] = "FCH", + [DFX_DAP_HWID] = "DFX_DAP", + [L1IMU_PCIE_HWID] = "L1IMU_PCIE", + [L1IMU_NBIF_HWID] = "L1IMU_NBIF", + [L1IMU_IOAGR_HWID] = "L1IMU_IOAGR", + [L1IMU3_HWID] = "L1IMU3", + [L1IMU4_HWID] = "L1IMU4", + [L1IMU5_HWID] = "L1IMU5", + [L1IMU6_HWID] = "L1IMU6", + [L1IMU7_HWID] = "L1IMU7", + [L1IMU8_HWID] = "L1IMU8", + [L1IMU9_HWID] = "L1IMU9", + [L1IMU10_HWID] = "L1IMU10", + [L1IMU11_HWID] = "L1IMU11", + [L1IMU12_HWID] = "L1IMU12", + [L1IMU13_HWID] = "L1IMU13", + [L1IMU14_HWID] = "L1IMU14", + [L1IMU15_HWID] = "L1IMU15", + [WAFLC_HWID] = "WAFLC", + [FCH_USB_PD_HWID] = "FCH_USB_PD", + [PCIE_HWID] = "PCIE", + [PCS_HWID] = "PCS", + [DDCL_HWID] = "DDCL", + [SST_HWID] = "SST", + [IOAGR_HWID] = "IOAGR", + [NBIF_HWID] = "NBIF", + [IOAPIC_HWID] = "IOAPIC", + [SYSTEMHUB_HWID] = "SYSTEMHUB", + [NTBCCP_HWID] = "NTBCCP", + [UMC_HWID] = "UMC", + [SATA_HWID] = "SATA", + [USB_HWID] = "USB", + [CCXSEC_HWID] = "CCXSEC", + [XGMI_HWID] = "XGMI", + [XGBE_HWID] = "XGBE", + [MP0_HWID] = "MP0", +}; + +static int hw_id_map[MAX_HWIP] = { + [GC_HWIP] = GC_HWID, + [HDP_HWIP] = HDP_HWID, + [SDMA0_HWIP] = SDMA0_HWID, + [SDMA1_HWIP] = SDMA1_HWID, + [MMHUB_HWIP] = MMHUB_HWID, + [ATHUB_HWIP] = ATHUB_HWID, + [NBIO_HWIP] = NBIF_HWID, + [MP0_HWIP] = MP0_HWID, + [MP1_HWIP] = MP1_HWID, + [UVD_HWIP] = UVD_HWID, + [VCE_HWIP] = VCE_HWID, + [DF_HWIP] = DF_HWID, + [DCE_HWIP] = DMU_HWID, + [OSSSYS_HWIP] = OSSSYS_HWID, + [SMUIO_HWIP] = SMUIO_HWID, + [PWR_HWIP] = PWR_HWID, + [NBIF_HWIP] = NBIF_HWID, + [THM_HWIP] = THM_HWID, + [CLK_HWIP] = CLKA_HWID, +}; + +static int amdgpu_discovery_read_binary(struct amdgpu_device *adev, uint8_t *binary) +{ + uint32_t *p = (uint32_t *)binary; + uint64_t vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20; + uint64_t pos = vram_size - BINARY_MAX_SIZE; + unsigned long flags; + + while (pos < vram_size) { + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); + WREG32_NO_KIQ(mmMM_INDEX_HI, pos >> 31); + *p++ = RREG32_NO_KIQ(mmMM_DATA); + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + pos += 4; + } + + return 0; +} + +static uint16_t amdgpu_discovery_calculate_checksum(uint8_t *data, uint32_t size) +{ + uint16_t checksum = 0; + int i; + + for (i = 0; i < size; i++) + checksum += data[i]; + + return checksum; +} + +static inline bool amdgpu_discovery_verify_checksum(uint8_t *data, uint32_t size, + uint16_t expected) +{ + return !!(amdgpu_discovery_calculate_checksum(data, size) == expected); +} + +int amdgpu_discovery_init(struct amdgpu_device *adev) +{ + struct table_info *info; + struct binary_header *bhdr; + struct ip_discovery_header *ihdr; + struct gpu_info_header *ghdr; + uint16_t offset; + uint16_t size; + uint16_t checksum; + int r; + + adev->discovery = kzalloc(BINARY_MAX_SIZE, GFP_KERNEL); + if (!adev->discovery) + return -ENOMEM; + + r = amdgpu_discovery_read_binary(adev, adev->discovery); + if (r) { + DRM_ERROR("failed to read ip discovery binary\n"); + goto out; + } + + bhdr = (struct binary_header *)adev->discovery; + + if (le32_to_cpu(bhdr->binary_signature) != BINARY_SIGNATURE) { + DRM_ERROR("invalid ip discovery binary signature\n"); + r = -EINVAL; + goto out; + } + + offset = offsetof(struct binary_header, binary_checksum) + + sizeof(bhdr->binary_checksum); + size = bhdr->binary_size - offset; + checksum = bhdr->binary_checksum; + + if (!amdgpu_discovery_verify_checksum(adev->discovery + offset, + size, checksum)) { + DRM_ERROR("invalid ip discovery binary checksum\n"); + r = -EINVAL; + goto out; + } + + info = &bhdr->table_list[IP_DISCOVERY]; + offset = le16_to_cpu(info->offset); + checksum = le16_to_cpu(info->checksum); + ihdr = (struct ip_discovery_header *)(adev->discovery + offset); + + if (le32_to_cpu(ihdr->signature) != DISCOVERY_TABLE_SIGNATURE) { + DRM_ERROR("invalid ip discovery data table signature\n"); + r = -EINVAL; + goto out; + } + + if (!amdgpu_discovery_verify_checksum(adev->discovery + offset, + ihdr->size, checksum)) { + DRM_ERROR("invalid ip discovery data table checksum\n"); + r = -EINVAL; + goto out; + } + + info = &bhdr->table_list[GC]; + offset = le16_to_cpu(info->offset); + checksum = le16_to_cpu(info->checksum); + ghdr = (struct gpu_info_header *)(adev->discovery + offset); + + if (!amdgpu_discovery_verify_checksum(adev->discovery + offset, + ghdr->size, checksum)) { + DRM_ERROR("invalid gc data table checksum\n"); + r = -EINVAL; + goto out; + } + + return 0; + +out: + kfree(adev->discovery); + adev->discovery = NULL; + + return r; +} + +void amdgpu_discovery_fini(struct amdgpu_device *adev) +{ + kfree(adev->discovery); + adev->discovery = NULL; +} + +int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) +{ + struct binary_header *bhdr; + struct ip_discovery_header *ihdr; + struct die_header *dhdr; + struct ip *ip; + uint16_t die_offset; + uint16_t ip_offset; + uint16_t num_dies; + uint16_t num_ips; + uint8_t num_base_address; + int hw_ip; + int i, j, k; + + if (!adev->discovery) { + DRM_ERROR("ip discovery uninitialized\n"); + return -EINVAL; + } + + bhdr = (struct binary_header *)adev->discovery; + ihdr = (struct ip_discovery_header *)(adev->discovery + + le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset)); + num_dies = le16_to_cpu(ihdr->num_dies); + + DRM_DEBUG("number of dies: %d\n", num_dies); + + for (i = 0; i < num_dies; i++) { + die_offset = le16_to_cpu(ihdr->die_info[i].die_offset); + dhdr = (struct die_header *)(adev->discovery + die_offset); + num_ips = le16_to_cpu(dhdr->num_ips); + ip_offset = die_offset + sizeof(*dhdr); + + if (le16_to_cpu(dhdr->die_id) != i) { + DRM_ERROR("invalid die id %d, expected %d\n", + le16_to_cpu(dhdr->die_id), i); + return -EINVAL; + } + + DRM_DEBUG("number of hardware IPs on die%d: %d\n", + le16_to_cpu(dhdr->die_id), num_ips); + + for (j = 0; j < num_ips; j++) { + ip = (struct ip *)(adev->discovery + ip_offset); + num_base_address = ip->num_base_address; + + DRM_DEBUG("%s(%d) #%d v%d.%d.%d:\n", + hw_id_names[le16_to_cpu(ip->hw_id)], + le16_to_cpu(ip->hw_id), + ip->number_instance, + ip->major, ip->minor, + ip->revision); + + for (k = 0; k < num_base_address; k++) { + /* + * convert the endianness of base addresses in place, + * so that we don't need to convert them when accessing adev->reg_offset. + */ + ip->base_address[k] = le32_to_cpu(ip->base_address[k]); + DRM_DEBUG("\t0x%08x\n", ip->base_address[k]); + } + + for (hw_ip = 0; hw_ip < MAX_HWIP; hw_ip++) { + if (hw_id_map[hw_ip] == le16_to_cpu(ip->hw_id)) { + DRM_INFO("set register base offset for %s\n", + hw_id_names[le16_to_cpu(ip->hw_id)]); + adev->reg_offset[hw_ip][ip->number_instance] = + ip->base_address; + } + + } + + ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1); + } + } + + return 0; +} + +int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id, + int *major, int *minor) +{ + struct binary_header *bhdr; + struct ip_discovery_header *ihdr; + struct die_header *dhdr; + struct ip *ip; + uint16_t die_offset; + uint16_t ip_offset; + uint16_t num_dies; + uint16_t num_ips; + int i, j; + + if (!adev->discovery) { + DRM_ERROR("ip discovery uninitialized\n"); + return -EINVAL; + } + + bhdr = (struct binary_header *)adev->discovery; + ihdr = (struct ip_discovery_header *)(adev->discovery + + le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset)); + num_dies = le16_to_cpu(ihdr->num_dies); + + for (i = 0; i < num_dies; i++) { + die_offset = le16_to_cpu(ihdr->die_info[i].die_offset); + dhdr = (struct die_header *)(adev->discovery + die_offset); + num_ips = le16_to_cpu(dhdr->num_ips); + ip_offset = die_offset + sizeof(*dhdr); + + for (j = 0; j < num_ips; j++) { + ip = (struct ip *)(adev->discovery + ip_offset); + + if (le16_to_cpu(ip->hw_id) == hw_id) { + if (major) + *major = ip->major; + if (minor) + *minor = ip->minor; + return 0; + } + ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1); + } + } + + return -EINVAL; +} + +int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev) +{ + struct binary_header *bhdr; + struct gc_info_v1_0 *gc_info; + + if (!adev->discovery) { + DRM_ERROR("ip discovery uninitialized\n"); + return -EINVAL; + } + + bhdr = (struct binary_header *)adev->discovery; + gc_info = (struct gc_info_v1_0 *)(adev->discovery + + le16_to_cpu(bhdr->table_list[GC].offset)); + + adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->gc_num_se); + adev->gfx.config.max_cu_per_sh = 2 * (le32_to_cpu(gc_info->gc_num_wgp0_per_sa) + + le32_to_cpu(gc_info->gc_num_wgp1_per_sa)); + adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->gc_num_sa_per_se); + adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->gc_num_rb_per_se); + adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->gc_num_gl2c); + adev->gfx.config.max_gprs = le32_to_cpu(gc_info->gc_num_gprs); + adev->gfx.config.max_gs_threads = le32_to_cpu(gc_info->gc_num_max_gs_thds); + adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gc_info->gc_gs_table_depth); + adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gc_info->gc_gsprim_buff_depth); + adev->gfx.config.double_offchip_lds_buf = le32_to_cpu(gc_info->gc_double_offchip_lds_buffer); + adev->gfx.cu_info.wave_front_size = le32_to_cpu(gc_info->gc_wave_size); + adev->gfx.cu_info.max_waves_per_simd = le32_to_cpu(gc_info->gc_max_waves_per_simd); + adev->gfx.cu_info.max_scratch_slots_per_cu = le32_to_cpu(gc_info->gc_max_scratch_slots_per_cu); + adev->gfx.cu_info.lds_size = le32_to_cpu(gc_info->gc_lds_size); + adev->gfx.config.num_sc_per_sh = le32_to_cpu(gc_info->gc_num_sc_per_se) / + le32_to_cpu(gc_info->gc_num_sa_per_se); + adev->gfx.config.num_packer_per_sc = le32_to_cpu(gc_info->gc_num_packer_per_sc); + + return 0; +}
diff --git a/amdgpu/amdgpu_discovery.h b/amdgpu/amdgpu_discovery.h new file mode 100644 index 0000000..85b8c4d --- /dev/null +++ b/amdgpu/amdgpu_discovery.h
@@ -0,0 +1,34 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_DISCOVERY__ +#define __AMDGPU_DISCOVERY__ + +int amdgpu_discovery_init(struct amdgpu_device *adev); +void amdgpu_discovery_fini(struct amdgpu_device *adev); +int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev); +int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id, + int *major, int *minor); +int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev); + +#endif /* __AMDGPU_DISCOVERY__ */
diff --git a/amdgpu/amdgpu_display.c b/amdgpu/amdgpu_display.c new file mode 100644 index 0000000..5356509 --- /dev/null +++ b/amdgpu/amdgpu_display.c
@@ -0,0 +1,902 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_i2c.h" +#include "atom.h" +#include "amdgpu_connectors.h" +#include "amdgpu_display.h" +#include <asm/div64.h> + +#include <linux/pci.h> +#include <linux/pm_runtime.h> +#include <drm/drm_crtc_helper.h> +#include <drm/drm_edid.h> +#include <drm/drm_gem_framebuffer_helper.h> +#include <drm/drm_fb_helper.h> +#include <drm/drm_vblank.h> + +static void amdgpu_display_flip_callback(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct amdgpu_flip_work *work = + container_of(cb, struct amdgpu_flip_work, cb); + + dma_fence_put(f); + schedule_work(&work->flip_work.work); +} + +static bool amdgpu_display_flip_handle_fence(struct amdgpu_flip_work *work, + struct dma_fence **f) +{ + struct dma_fence *fence= *f; + + if (fence == NULL) + return false; + + *f = NULL; + + if (!dma_fence_add_callback(fence, &work->cb, + amdgpu_display_flip_callback)) + return true; + + dma_fence_put(fence); + return false; +} + +static void amdgpu_display_flip_work_func(struct work_struct *__work) +{ + struct delayed_work *delayed_work = + container_of(__work, struct delayed_work, work); + struct amdgpu_flip_work *work = + container_of(delayed_work, struct amdgpu_flip_work, flip_work); + struct amdgpu_device *adev = work->adev; + struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[work->crtc_id]; + + struct drm_crtc *crtc = &amdgpu_crtc->base; + unsigned long flags; + unsigned i; + int vpos, hpos; + + if (amdgpu_display_flip_handle_fence(work, &work->excl)) + return; + + for (i = 0; i < work->shared_count; ++i) + if (amdgpu_display_flip_handle_fence(work, &work->shared[i])) + return; + + /* Wait until we're out of the vertical blank period before the one + * targeted by the flip + */ + if (amdgpu_crtc->enabled && + (amdgpu_display_get_crtc_scanoutpos(adev->ddev, work->crtc_id, 0, + &vpos, &hpos, NULL, NULL, + &crtc->hwmode) + & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) == + (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) && + (int)(work->target_vblank - + amdgpu_get_vblank_counter_kms(adev->ddev, amdgpu_crtc->crtc_id)) > 0) { + schedule_delayed_work(&work->flip_work, usecs_to_jiffies(1000)); + return; + } + + /* We borrow the event spin lock for protecting flip_status */ + spin_lock_irqsave(&crtc->dev->event_lock, flags); + + /* Do the flip (mmio) */ + adev->mode_info.funcs->page_flip(adev, work->crtc_id, work->base, work->async); + + /* Set the flip status */ + amdgpu_crtc->pflip_status = AMDGPU_FLIP_SUBMITTED; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + + + DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_SUBMITTED, work: %p,\n", + amdgpu_crtc->crtc_id, amdgpu_crtc, work); + +} + +/* + * Handle unpin events outside the interrupt handler proper. + */ +static void amdgpu_display_unpin_work_func(struct work_struct *__work) +{ + struct amdgpu_flip_work *work = + container_of(__work, struct amdgpu_flip_work, unpin_work); + int r; + + /* unpin of the old buffer */ + r = amdgpu_bo_reserve(work->old_abo, true); + if (likely(r == 0)) { + r = amdgpu_bo_unpin(work->old_abo); + if (unlikely(r != 0)) { + DRM_ERROR("failed to unpin buffer after flip\n"); + } + amdgpu_bo_unreserve(work->old_abo); + } else + DRM_ERROR("failed to reserve buffer after flip\n"); + + amdgpu_bo_unref(&work->old_abo); + kfree(work->shared); + kfree(work); +} + +int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc, + struct drm_framebuffer *fb, + struct drm_pending_vblank_event *event, + uint32_t page_flip_flags, uint32_t target, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_gem_object *obj; + struct amdgpu_flip_work *work; + struct amdgpu_bo *new_abo; + unsigned long flags; + u64 tiling_flags; + int i, r; + + work = kzalloc(sizeof *work, GFP_KERNEL); + if (work == NULL) + return -ENOMEM; + + INIT_DELAYED_WORK(&work->flip_work, amdgpu_display_flip_work_func); + INIT_WORK(&work->unpin_work, amdgpu_display_unpin_work_func); + + work->event = event; + work->adev = adev; + work->crtc_id = amdgpu_crtc->crtc_id; + work->async = (page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0; + + /* schedule unpin of the old buffer */ + obj = crtc->primary->fb->obj[0]; + + /* take a reference to the old object */ + work->old_abo = gem_to_amdgpu_bo(obj); + amdgpu_bo_ref(work->old_abo); + + obj = fb->obj[0]; + new_abo = gem_to_amdgpu_bo(obj); + + /* pin the new buffer */ + r = amdgpu_bo_reserve(new_abo, false); + if (unlikely(r != 0)) { + DRM_ERROR("failed to reserve new abo buffer before flip\n"); + goto cleanup; + } + + if (!adev->enable_virtual_display) { + r = amdgpu_bo_pin(new_abo, amdgpu_display_supported_domains(adev)); + if (unlikely(r != 0)) { + DRM_ERROR("failed to pin new abo buffer before flip\n"); + goto unreserve; + } + } + + r = amdgpu_ttm_alloc_gart(&new_abo->tbo); + if (unlikely(r != 0)) { + DRM_ERROR("%p bind failed\n", new_abo); + goto unpin; + } + + r = reservation_object_get_fences_rcu(new_abo->tbo.resv, &work->excl, + &work->shared_count, + &work->shared); + if (unlikely(r != 0)) { + DRM_ERROR("failed to get fences for buffer\n"); + goto unpin; + } + + amdgpu_bo_get_tiling_flags(new_abo, &tiling_flags); + amdgpu_bo_unreserve(new_abo); + + if (!adev->enable_virtual_display) + work->base = amdgpu_bo_gpu_offset(new_abo); + work->target_vblank = target - (uint32_t)drm_crtc_vblank_count(crtc) + + amdgpu_get_vblank_counter_kms(dev, work->crtc_id); + + /* we borrow the event spin lock for protecting flip_wrok */ + spin_lock_irqsave(&crtc->dev->event_lock, flags); + if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_NONE) { + DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + r = -EBUSY; + goto pflip_cleanup; + } + + amdgpu_crtc->pflip_status = AMDGPU_FLIP_PENDING; + amdgpu_crtc->pflip_works = work; + + + DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_PENDING, work: %p,\n", + amdgpu_crtc->crtc_id, amdgpu_crtc, work); + /* update crtc fb */ + crtc->primary->fb = fb; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + amdgpu_display_flip_work_func(&work->flip_work.work); + return 0; + +pflip_cleanup: + if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) { + DRM_ERROR("failed to reserve new abo in error path\n"); + goto cleanup; + } +unpin: + if (!adev->enable_virtual_display) + if (unlikely(amdgpu_bo_unpin(new_abo) != 0)) + DRM_ERROR("failed to unpin new abo in error path\n"); + +unreserve: + amdgpu_bo_unreserve(new_abo); + +cleanup: + amdgpu_bo_unref(&work->old_abo); + dma_fence_put(work->excl); + for (i = 0; i < work->shared_count; ++i) + dma_fence_put(work->shared[i]); + kfree(work->shared); + kfree(work); + + return r; +} + +int amdgpu_display_crtc_set_config(struct drm_mode_set *set, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_device *dev; + struct amdgpu_device *adev; + struct drm_crtc *crtc; + bool active = false; + int ret; + + if (!set || !set->crtc) + return -EINVAL; + + dev = set->crtc->dev; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) + return ret; + + ret = drm_crtc_helper_set_config(set, ctx); + + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + if (crtc->enabled) + active = true; + + pm_runtime_mark_last_busy(dev->dev); + + adev = dev->dev_private; + /* if we have active crtcs and we don't have a power ref, + take the current one */ + if (active && !adev->have_disp_power_ref) { + adev->have_disp_power_ref = true; + return ret; + } + /* if we have no active crtcs, then drop the power ref + we got before */ + if (!active && adev->have_disp_power_ref) { + pm_runtime_put_autosuspend(dev->dev); + adev->have_disp_power_ref = false; + } + + /* drop the power reference we got coming in here */ + pm_runtime_put_autosuspend(dev->dev); + return ret; +} + +static const char *encoder_names[41] = { + "NONE", + "INTERNAL_LVDS", + "INTERNAL_TMDS1", + "INTERNAL_TMDS2", + "INTERNAL_DAC1", + "INTERNAL_DAC2", + "INTERNAL_SDVOA", + "INTERNAL_SDVOB", + "SI170B", + "CH7303", + "CH7301", + "INTERNAL_DVO1", + "EXTERNAL_SDVOA", + "EXTERNAL_SDVOB", + "TITFP513", + "INTERNAL_LVTM1", + "VT1623", + "HDMI_SI1930", + "HDMI_INTERNAL", + "INTERNAL_KLDSCP_TMDS1", + "INTERNAL_KLDSCP_DVO1", + "INTERNAL_KLDSCP_DAC1", + "INTERNAL_KLDSCP_DAC2", + "SI178", + "MVPU_FPGA", + "INTERNAL_DDI", + "VT1625", + "HDMI_SI1932", + "DP_AN9801", + "DP_DP501", + "INTERNAL_UNIPHY", + "INTERNAL_KLDSCP_LVTMA", + "INTERNAL_UNIPHY1", + "INTERNAL_UNIPHY2", + "NUTMEG", + "TRAVIS", + "INTERNAL_VCE", + "INTERNAL_UNIPHY3", + "HDMI_ANX9805", + "INTERNAL_AMCLK", + "VIRTUAL", +}; + +static const char *hpd_names[6] = { + "HPD1", + "HPD2", + "HPD3", + "HPD4", + "HPD5", + "HPD6", +}; + +void amdgpu_display_print_display_setup(struct drm_device *dev) +{ + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + uint32_t devices; + int i = 0; + + DRM_INFO("AMDGPU Display Connectors\n"); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + amdgpu_connector = to_amdgpu_connector(connector); + DRM_INFO("Connector %d:\n", i); + DRM_INFO(" %s\n", connector->name); + if (amdgpu_connector->hpd.hpd != AMDGPU_HPD_NONE) + DRM_INFO(" %s\n", hpd_names[amdgpu_connector->hpd.hpd]); + if (amdgpu_connector->ddc_bus) { + DRM_INFO(" DDC: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + amdgpu_connector->ddc_bus->rec.mask_clk_reg, + amdgpu_connector->ddc_bus->rec.mask_data_reg, + amdgpu_connector->ddc_bus->rec.a_clk_reg, + amdgpu_connector->ddc_bus->rec.a_data_reg, + amdgpu_connector->ddc_bus->rec.en_clk_reg, + amdgpu_connector->ddc_bus->rec.en_data_reg, + amdgpu_connector->ddc_bus->rec.y_clk_reg, + amdgpu_connector->ddc_bus->rec.y_data_reg); + if (amdgpu_connector->router.ddc_valid) + DRM_INFO(" DDC Router 0x%x/0x%x\n", + amdgpu_connector->router.ddc_mux_control_pin, + amdgpu_connector->router.ddc_mux_state); + if (amdgpu_connector->router.cd_valid) + DRM_INFO(" Clock/Data Router 0x%x/0x%x\n", + amdgpu_connector->router.cd_mux_control_pin, + amdgpu_connector->router.cd_mux_state); + } else { + if (connector->connector_type == DRM_MODE_CONNECTOR_VGA || + connector->connector_type == DRM_MODE_CONNECTOR_DVII || + connector->connector_type == DRM_MODE_CONNECTOR_DVID || + connector->connector_type == DRM_MODE_CONNECTOR_DVIA || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) + DRM_INFO(" DDC: no ddc bus - possible BIOS bug - please report to xorg-driver-ati@lists.x.org\n"); + } + DRM_INFO(" Encoders:\n"); + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + amdgpu_encoder = to_amdgpu_encoder(encoder); + devices = amdgpu_encoder->devices & amdgpu_connector->devices; + if (devices) { + if (devices & ATOM_DEVICE_CRT1_SUPPORT) + DRM_INFO(" CRT1: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_CRT2_SUPPORT) + DRM_INFO(" CRT2: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_LCD1_SUPPORT) + DRM_INFO(" LCD1: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP1_SUPPORT) + DRM_INFO(" DFP1: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP2_SUPPORT) + DRM_INFO(" DFP2: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP3_SUPPORT) + DRM_INFO(" DFP3: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP4_SUPPORT) + DRM_INFO(" DFP4: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP5_SUPPORT) + DRM_INFO(" DFP5: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_DFP6_SUPPORT) + DRM_INFO(" DFP6: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_TV1_SUPPORT) + DRM_INFO(" TV1: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + if (devices & ATOM_DEVICE_CV_SUPPORT) + DRM_INFO(" CV: %s\n", encoder_names[amdgpu_encoder->encoder_id]); + } + } + i++; + } +} + +/** + * amdgpu_display_ddc_probe + * + */ +bool amdgpu_display_ddc_probe(struct amdgpu_connector *amdgpu_connector, + bool use_aux) +{ + u8 out = 0x0; + u8 buf[8]; + int ret; + struct i2c_msg msgs[] = { + { + .addr = DDC_ADDR, + .flags = 0, + .len = 1, + .buf = &out, + }, + { + .addr = DDC_ADDR, + .flags = I2C_M_RD, + .len = 8, + .buf = buf, + } + }; + + /* on hw with routers, select right port */ + if (amdgpu_connector->router.ddc_valid) + amdgpu_i2c_router_select_ddc_port(amdgpu_connector); + + if (use_aux) { + ret = i2c_transfer(&amdgpu_connector->ddc_bus->aux.ddc, msgs, 2); + } else { + ret = i2c_transfer(&amdgpu_connector->ddc_bus->adapter, msgs, 2); + } + + if (ret != 2) + /* Couldn't find an accessible DDC on this connector */ + return false; + /* Probe also for valid EDID header + * EDID header starts with: + * 0x00,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0x00. + * Only the first 6 bytes must be valid as + * drm_edid_block_valid() can fix the last 2 bytes */ + if (drm_edid_header_is_valid(buf) < 6) { + /* Couldn't find an accessible EDID on this + * connector */ + return false; + } + return true; +} + +static const struct drm_framebuffer_funcs amdgpu_fb_funcs = { + .destroy = drm_gem_fb_destroy, + .create_handle = drm_gem_fb_create_handle, +}; + +uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev) +{ + uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; + +#if defined(CONFIG_DRM_AMD_DC) + if (adev->asic_type >= CHIP_CARRIZO && adev->asic_type < CHIP_RAVEN && + adev->flags & AMD_IS_APU && + amdgpu_device_asic_has_dc_support(adev->asic_type)) + domain |= AMDGPU_GEM_DOMAIN_GTT; +#endif + + return domain; +} + +int amdgpu_display_framebuffer_init(struct drm_device *dev, + struct amdgpu_framebuffer *rfb, + const struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_gem_object *obj) +{ + int ret; + rfb->base.obj[0] = obj; + drm_helper_mode_fill_fb_struct(dev, &rfb->base, mode_cmd); + ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs); + if (ret) { + rfb->base.obj[0] = NULL; + return ret; + } + return 0; +} + +struct drm_framebuffer * +amdgpu_display_user_framebuffer_create(struct drm_device *dev, + struct drm_file *file_priv, + const struct drm_mode_fb_cmd2 *mode_cmd) +{ + struct drm_gem_object *obj; + struct amdgpu_framebuffer *amdgpu_fb; + int ret; + + obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]); + if (obj == NULL) { + dev_err(&dev->pdev->dev, "No GEM object associated to handle 0x%08X, " + "can't create framebuffer\n", mode_cmd->handles[0]); + return ERR_PTR(-ENOENT); + } + + /* Handle is imported dma-buf, so cannot be migrated to VRAM for scanout */ + if (obj->import_attach) { + DRM_DEBUG_KMS("Cannot create framebuffer from imported dma_buf\n"); + return ERR_PTR(-EINVAL); + } + + amdgpu_fb = kzalloc(sizeof(*amdgpu_fb), GFP_KERNEL); + if (amdgpu_fb == NULL) { + drm_gem_object_put_unlocked(obj); + return ERR_PTR(-ENOMEM); + } + + ret = amdgpu_display_framebuffer_init(dev, amdgpu_fb, mode_cmd, obj); + if (ret) { + kfree(amdgpu_fb); + drm_gem_object_put_unlocked(obj); + return ERR_PTR(ret); + } + + return &amdgpu_fb->base; +} + +const struct drm_mode_config_funcs amdgpu_mode_funcs = { + .fb_create = amdgpu_display_user_framebuffer_create, + .output_poll_changed = drm_fb_helper_output_poll_changed, +}; + +static const struct drm_prop_enum_list amdgpu_underscan_enum_list[] = +{ { UNDERSCAN_OFF, "off" }, + { UNDERSCAN_ON, "on" }, + { UNDERSCAN_AUTO, "auto" }, +}; + +static const struct drm_prop_enum_list amdgpu_audio_enum_list[] = +{ { AMDGPU_AUDIO_DISABLE, "off" }, + { AMDGPU_AUDIO_ENABLE, "on" }, + { AMDGPU_AUDIO_AUTO, "auto" }, +}; + +/* XXX support different dither options? spatial, temporal, both, etc. */ +static const struct drm_prop_enum_list amdgpu_dither_enum_list[] = +{ { AMDGPU_FMT_DITHER_DISABLE, "off" }, + { AMDGPU_FMT_DITHER_ENABLE, "on" }, +}; + +int amdgpu_display_modeset_create_props(struct amdgpu_device *adev) +{ + int sz; + + adev->mode_info.coherent_mode_property = + drm_property_create_range(adev->ddev, 0 , "coherent", 0, 1); + if (!adev->mode_info.coherent_mode_property) + return -ENOMEM; + + adev->mode_info.load_detect_property = + drm_property_create_range(adev->ddev, 0, "load detection", 0, 1); + if (!adev->mode_info.load_detect_property) + return -ENOMEM; + + drm_mode_create_scaling_mode_property(adev->ddev); + + sz = ARRAY_SIZE(amdgpu_underscan_enum_list); + adev->mode_info.underscan_property = + drm_property_create_enum(adev->ddev, 0, + "underscan", + amdgpu_underscan_enum_list, sz); + + adev->mode_info.underscan_hborder_property = + drm_property_create_range(adev->ddev, 0, + "underscan hborder", 0, 128); + if (!adev->mode_info.underscan_hborder_property) + return -ENOMEM; + + adev->mode_info.underscan_vborder_property = + drm_property_create_range(adev->ddev, 0, + "underscan vborder", 0, 128); + if (!adev->mode_info.underscan_vborder_property) + return -ENOMEM; + + sz = ARRAY_SIZE(amdgpu_audio_enum_list); + adev->mode_info.audio_property = + drm_property_create_enum(adev->ddev, 0, + "audio", + amdgpu_audio_enum_list, sz); + + sz = ARRAY_SIZE(amdgpu_dither_enum_list); + adev->mode_info.dither_property = + drm_property_create_enum(adev->ddev, 0, + "dither", + amdgpu_dither_enum_list, sz); + + if (amdgpu_device_has_dc_support(adev)) { + adev->mode_info.abm_level_property = + drm_property_create_range(adev->ddev, 0, + "abm level", 0, 4); + if (!adev->mode_info.abm_level_property) + return -ENOMEM; + } + + return 0; +} + +void amdgpu_display_update_priority(struct amdgpu_device *adev) +{ + /* adjustment options for the display watermarks */ + if ((amdgpu_disp_priority == 0) || (amdgpu_disp_priority > 2)) + adev->mode_info.disp_priority = 0; + else + adev->mode_info.disp_priority = amdgpu_disp_priority; + +} + +static bool amdgpu_display_is_hdtv_mode(const struct drm_display_mode *mode) +{ + /* try and guess if this is a tv or a monitor */ + if ((mode->vdisplay == 480 && mode->hdisplay == 720) || /* 480p */ + (mode->vdisplay == 576) || /* 576p */ + (mode->vdisplay == 720) || /* 720p */ + (mode->vdisplay == 1080)) /* 1080p */ + return true; + else + return false; +} + +bool amdgpu_display_crtc_scaling_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct drm_device *dev = crtc->dev; + struct drm_encoder *encoder; + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct amdgpu_encoder *amdgpu_encoder; + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + u32 src_v = 1, dst_v = 1; + u32 src_h = 1, dst_h = 1; + + amdgpu_crtc->h_border = 0; + amdgpu_crtc->v_border = 0; + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + if (encoder->crtc != crtc) + continue; + amdgpu_encoder = to_amdgpu_encoder(encoder); + connector = amdgpu_get_connector_for_encoder(encoder); + amdgpu_connector = to_amdgpu_connector(connector); + + /* set scaling */ + if (amdgpu_encoder->rmx_type == RMX_OFF) + amdgpu_crtc->rmx_type = RMX_OFF; + else if (mode->hdisplay < amdgpu_encoder->native_mode.hdisplay || + mode->vdisplay < amdgpu_encoder->native_mode.vdisplay) + amdgpu_crtc->rmx_type = amdgpu_encoder->rmx_type; + else + amdgpu_crtc->rmx_type = RMX_OFF; + /* copy native mode */ + memcpy(&amdgpu_crtc->native_mode, + &amdgpu_encoder->native_mode, + sizeof(struct drm_display_mode)); + src_v = crtc->mode.vdisplay; + dst_v = amdgpu_crtc->native_mode.vdisplay; + src_h = crtc->mode.hdisplay; + dst_h = amdgpu_crtc->native_mode.hdisplay; + + /* fix up for overscan on hdmi */ + if ((!(mode->flags & DRM_MODE_FLAG_INTERLACE)) && + ((amdgpu_encoder->underscan_type == UNDERSCAN_ON) || + ((amdgpu_encoder->underscan_type == UNDERSCAN_AUTO) && + drm_detect_hdmi_monitor(amdgpu_connector_edid(connector)) && + amdgpu_display_is_hdtv_mode(mode)))) { + if (amdgpu_encoder->underscan_hborder != 0) + amdgpu_crtc->h_border = amdgpu_encoder->underscan_hborder; + else + amdgpu_crtc->h_border = (mode->hdisplay >> 5) + 16; + if (amdgpu_encoder->underscan_vborder != 0) + amdgpu_crtc->v_border = amdgpu_encoder->underscan_vborder; + else + amdgpu_crtc->v_border = (mode->vdisplay >> 5) + 16; + amdgpu_crtc->rmx_type = RMX_FULL; + src_v = crtc->mode.vdisplay; + dst_v = crtc->mode.vdisplay - (amdgpu_crtc->v_border * 2); + src_h = crtc->mode.hdisplay; + dst_h = crtc->mode.hdisplay - (amdgpu_crtc->h_border * 2); + } + } + if (amdgpu_crtc->rmx_type != RMX_OFF) { + fixed20_12 a, b; + a.full = dfixed_const(src_v); + b.full = dfixed_const(dst_v); + amdgpu_crtc->vsc.full = dfixed_div(a, b); + a.full = dfixed_const(src_h); + b.full = dfixed_const(dst_h); + amdgpu_crtc->hsc.full = dfixed_div(a, b); + } else { + amdgpu_crtc->vsc.full = dfixed_const(1); + amdgpu_crtc->hsc.full = dfixed_const(1); + } + return true; +} + +/* + * Retrieve current video scanout position of crtc on a given gpu, and + * an optional accurate timestamp of when query happened. + * + * \param dev Device to query. + * \param pipe Crtc to query. + * \param flags Flags from caller (DRM_CALLED_FROM_VBLIRQ or 0). + * For driver internal use only also supports these flags: + * + * USE_REAL_VBLANKSTART to use the real start of vblank instead + * of a fudged earlier start of vblank. + * + * GET_DISTANCE_TO_VBLANKSTART to return distance to the + * fudged earlier start of vblank in *vpos and the distance + * to true start of vblank in *hpos. + * + * \param *vpos Location where vertical scanout position should be stored. + * \param *hpos Location where horizontal scanout position should go. + * \param *stime Target location for timestamp taken immediately before + * scanout position query. Can be NULL to skip timestamp. + * \param *etime Target location for timestamp taken immediately after + * scanout position query. Can be NULL to skip timestamp. + * + * Returns vpos as a positive number while in active scanout area. + * Returns vpos as a negative number inside vblank, counting the number + * of scanlines to go until end of vblank, e.g., -1 means "one scanline + * until start of active scanout / end of vblank." + * + * \return Flags, or'ed together as follows: + * + * DRM_SCANOUTPOS_VALID = Query successful. + * DRM_SCANOUTPOS_INVBL = Inside vblank. + * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of + * this flag means that returned position may be offset by a constant but + * unknown small number of scanlines wrt. real scanout position. + * + */ +int amdgpu_display_get_crtc_scanoutpos(struct drm_device *dev, + unsigned int pipe, unsigned int flags, int *vpos, + int *hpos, ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode) +{ + u32 vbl = 0, position = 0; + int vbl_start, vbl_end, vtotal, ret = 0; + bool in_vbl = true; + + struct amdgpu_device *adev = dev->dev_private; + + /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ + + /* Get optional system timestamp before query. */ + if (stime) + *stime = ktime_get(); + + if (amdgpu_display_page_flip_get_scanoutpos(adev, pipe, &vbl, &position) == 0) + ret |= DRM_SCANOUTPOS_VALID; + + /* Get optional system timestamp after query. */ + if (etime) + *etime = ktime_get(); + + /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ + + /* Decode into vertical and horizontal scanout position. */ + *vpos = position & 0x1fff; + *hpos = (position >> 16) & 0x1fff; + + /* Valid vblank area boundaries from gpu retrieved? */ + if (vbl > 0) { + /* Yes: Decode. */ + ret |= DRM_SCANOUTPOS_ACCURATE; + vbl_start = vbl & 0x1fff; + vbl_end = (vbl >> 16) & 0x1fff; + } + else { + /* No: Fake something reasonable which gives at least ok results. */ + vbl_start = mode->crtc_vdisplay; + vbl_end = 0; + } + + /* Called from driver internal vblank counter query code? */ + if (flags & GET_DISTANCE_TO_VBLANKSTART) { + /* Caller wants distance from real vbl_start in *hpos */ + *hpos = *vpos - vbl_start; + } + + /* Fudge vblank to start a few scanlines earlier to handle the + * problem that vblank irqs fire a few scanlines before start + * of vblank. Some driver internal callers need the true vblank + * start to be used and signal this via the USE_REAL_VBLANKSTART flag. + * + * The cause of the "early" vblank irq is that the irq is triggered + * by the line buffer logic when the line buffer read position enters + * the vblank, whereas our crtc scanout position naturally lags the + * line buffer read position. + */ + if (!(flags & USE_REAL_VBLANKSTART)) + vbl_start -= adev->mode_info.crtcs[pipe]->lb_vblank_lead_lines; + + /* Test scanout position against vblank region. */ + if ((*vpos < vbl_start) && (*vpos >= vbl_end)) + in_vbl = false; + + /* In vblank? */ + if (in_vbl) + ret |= DRM_SCANOUTPOS_IN_VBLANK; + + /* Called from driver internal vblank counter query code? */ + if (flags & GET_DISTANCE_TO_VBLANKSTART) { + /* Caller wants distance from fudged earlier vbl_start */ + *vpos -= vbl_start; + return ret; + } + + /* Check if inside vblank area and apply corrective offsets: + * vpos will then be >=0 in video scanout area, but negative + * within vblank area, counting down the number of lines until + * start of scanout. + */ + + /* Inside "upper part" of vblank area? Apply corrective offset if so: */ + if (in_vbl && (*vpos >= vbl_start)) { + vtotal = mode->crtc_vtotal; + + /* With variable refresh rate displays the vpos can exceed + * the vtotal value. Clamp to 0 to return -vbl_end instead + * of guessing the remaining number of lines until scanout. + */ + *vpos = (*vpos < vtotal) ? (*vpos - vtotal) : 0; + } + + /* Correct for shifted end of vbl at vbl_end. */ + *vpos = *vpos - vbl_end; + + return ret; +} + +int amdgpu_display_crtc_idx_to_irq_type(struct amdgpu_device *adev, int crtc) +{ + if (crtc < 0 || crtc >= adev->mode_info.num_crtc) + return AMDGPU_CRTC_IRQ_NONE; + + switch (crtc) { + case 0: + return AMDGPU_CRTC_IRQ_VBLANK1; + case 1: + return AMDGPU_CRTC_IRQ_VBLANK2; + case 2: + return AMDGPU_CRTC_IRQ_VBLANK3; + case 3: + return AMDGPU_CRTC_IRQ_VBLANK4; + case 4: + return AMDGPU_CRTC_IRQ_VBLANK5; + case 5: + return AMDGPU_CRTC_IRQ_VBLANK6; + default: + return AMDGPU_CRTC_IRQ_NONE; + } +}
diff --git a/amdgpu/amdgpu_display.h b/amdgpu/amdgpu_display.h new file mode 100644 index 0000000..06b922f --- /dev/null +++ b/amdgpu/amdgpu_display.h
@@ -0,0 +1,47 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_DISPLAY_H__ +#define __AMDGPU_DISPLAY_H__ + +#define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc)) +#define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l)) +#define amdgpu_display_backlight_get_level(adev, e) (adev)->mode_info.funcs->backlight_get_level((e)) +#define amdgpu_display_hpd_sense(adev, h) (adev)->mode_info.funcs->hpd_sense((adev), (h)) +#define amdgpu_display_hpd_set_polarity(adev, h) (adev)->mode_info.funcs->hpd_set_polarity((adev), (h)) +#define amdgpu_display_hpd_get_gpio_reg(adev) (adev)->mode_info.funcs->hpd_get_gpio_reg((adev)) +#define amdgpu_display_bandwidth_update(adev) (adev)->mode_info.funcs->bandwidth_update((adev)) +#define amdgpu_display_page_flip(adev, crtc, base, async) (adev)->mode_info.funcs->page_flip((adev), (crtc), (base), (async)) +#define amdgpu_display_page_flip_get_scanoutpos(adev, crtc, vbl, pos) (adev)->mode_info.funcs->page_flip_get_scanoutpos((adev), (crtc), (vbl), (pos)) +#define amdgpu_display_add_encoder(adev, e, s, c) (adev)->mode_info.funcs->add_encoder((adev), (e), (s), (c)) +#define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r)) + +int amdgpu_display_freesync_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +void amdgpu_display_update_priority(struct amdgpu_device *adev); +uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev); +struct drm_framebuffer * +amdgpu_display_user_framebuffer_create(struct drm_device *dev, + struct drm_file *file_priv, + const struct drm_mode_fb_cmd2 *mode_cmd); + +#endif
diff --git a/amdgpu/amdgpu_dma_buf.c b/amdgpu/amdgpu_dma_buf.c new file mode 100644 index 0000000..489041d --- /dev/null +++ b/amdgpu/amdgpu_dma_buf.c
@@ -0,0 +1,448 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * based on nouveau_prime.c + * + * Authors: Alex Deucher + */ + +/** + * DOC: PRIME Buffer Sharing + * + * The following callback implementations are used for :ref:`sharing GEM buffer + * objects between different devices via PRIME <prime_buffer_sharing>`. + */ + +#include "amdgpu.h" +#include "amdgpu_display.h" +#include "amdgpu_gem.h" +#include <drm/amdgpu_drm.h> +#include <linux/dma-buf.h> +#include <linux/dma-fence-array.h> + +/** + * amdgpu_gem_prime_get_sg_table - &drm_driver.gem_prime_get_sg_table + * implementation + * @obj: GEM buffer object (BO) + * + * Returns: + * A scatter/gather table for the pinned pages of the BO's memory. + */ +struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + int npages = bo->tbo.num_pages; + + return drm_prime_pages_to_sg(bo->tbo.ttm->pages, npages); +} + +/** + * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation + * @obj: GEM BO + * + * Sets up an in-kernel virtual mapping of the BO's memory. + * + * Returns: + * The virtual address of the mapping or an error pointer. + */ +void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + int ret; + + ret = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, + &bo->dma_buf_vmap); + if (ret) + return ERR_PTR(ret); + + return bo->dma_buf_vmap.virtual; +} + +/** + * amdgpu_gem_prime_vunmap - &dma_buf_ops.vunmap implementation + * @obj: GEM BO + * @vaddr: Virtual address (unused) + * + * Tears down the in-kernel virtual mapping of the BO's memory. + */ +void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + ttm_bo_kunmap(&bo->dma_buf_vmap); +} + +/** + * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation + * @obj: GEM BO + * @vma: Virtual memory area + * + * Sets up a userspace mapping of the BO's memory in the given + * virtual memory area. + * + * Returns: + * 0 on success or a negative error code on failure. + */ +int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, + struct vm_area_struct *vma) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + unsigned asize = amdgpu_bo_size(bo); + int ret; + + if (!vma->vm_file) + return -ENODEV; + + if (adev == NULL) + return -ENODEV; + + /* Check for valid size. */ + if (asize < vma->vm_end - vma->vm_start) + return -EINVAL; + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || + (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { + return -EPERM; + } + vma->vm_pgoff += amdgpu_bo_mmap_offset(bo) >> PAGE_SHIFT; + + /* prime mmap does not need to check access, so allow here */ + ret = drm_vma_node_allow(&obj->vma_node, vma->vm_file->private_data); + if (ret) + return ret; + + ret = ttm_bo_mmap(vma->vm_file, vma, &adev->mman.bdev); + drm_vma_node_revoke(&obj->vma_node, vma->vm_file->private_data); + + return ret; +} + +static int +__reservation_object_make_exclusive(struct reservation_object *obj) +{ + struct dma_fence **fences; + unsigned int count; + int r; + + if (!reservation_object_get_list(obj)) /* no shared fences to convert */ + return 0; + + r = reservation_object_get_fences_rcu(obj, NULL, &count, &fences); + if (r) + return r; + + if (count == 0) { + /* Now that was unexpected. */ + } else if (count == 1) { + reservation_object_add_excl_fence(obj, fences[0]); + dma_fence_put(fences[0]); + kfree(fences); + } else { + struct dma_fence_array *array; + + array = dma_fence_array_create(count, fences, + dma_fence_context_alloc(1), 0, + false); + if (!array) + goto err_fences_put; + + reservation_object_add_excl_fence(obj, &array->base); + dma_fence_put(&array->base); + } + + return 0; + +err_fences_put: + while (count--) + dma_fence_put(fences[count]); + kfree(fences); + return -ENOMEM; +} + +/** + * amdgpu_dma_buf_map_attach - &dma_buf_ops.attach implementation + * @dma_buf: Shared DMA buffer + * @attach: DMA-buf attachment + * + * Makes sure that the shared DMA buffer can be accessed by the target device. + * For now, simply pins it to the GTT domain, where it should be accessible by + * all DMA devices. + * + * Returns: + * 0 on success or a negative error code on failure. + */ +static int amdgpu_dma_buf_map_attach(struct dma_buf *dma_buf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = dma_buf->priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + long r; + + r = drm_gem_map_attach(dma_buf, attach); + if (r) + return r; + + r = amdgpu_bo_reserve(bo, false); + if (unlikely(r != 0)) + goto error_detach; + + + if (attach->dev->driver != adev->dev->driver) { + /* + * We only create shared fences for internal use, but importers + * of the dmabuf rely on exclusive fences for implicitly + * tracking write hazards. As any of the current fences may + * correspond to a write, we need to convert all existing + * fences on the reservation object into a single exclusive + * fence. + */ + r = __reservation_object_make_exclusive(bo->tbo.resv); + if (r) + goto error_unreserve; + } + + /* pin buffer into GTT */ + r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); + if (r) + goto error_unreserve; + + if (attach->dev->driver != adev->dev->driver) + bo->prime_shared_count++; + +error_unreserve: + amdgpu_bo_unreserve(bo); + +error_detach: + if (r) + drm_gem_map_detach(dma_buf, attach); + return r; +} + +/** + * amdgpu_dma_buf_map_detach - &dma_buf_ops.detach implementation + * @dma_buf: Shared DMA buffer + * @attach: DMA-buf attachment + * + * This is called when a shared DMA buffer no longer needs to be accessible by + * another device. For now, simply unpins the buffer from GTT. + */ +static void amdgpu_dma_buf_map_detach(struct dma_buf *dma_buf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = dma_buf->priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + int ret = 0; + + ret = amdgpu_bo_reserve(bo, true); + if (unlikely(ret != 0)) + goto error; + + amdgpu_bo_unpin(bo); + if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) + bo->prime_shared_count--; + amdgpu_bo_unreserve(bo); + +error: + drm_gem_map_detach(dma_buf, attach); +} + +/** + * amdgpu_gem_prime_res_obj - &drm_driver.gem_prime_res_obj implementation + * @obj: GEM BO + * + * Returns: + * The BO's reservation object. + */ +struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *obj) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + return bo->tbo.resv; +} + +/** + * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation + * @dma_buf: Shared DMA buffer + * @direction: Direction of DMA transfer + * + * This is called before CPU access to the shared DMA buffer's memory. If it's + * a read access, the buffer is moved to the GTT domain if possible, for optimal + * CPU read performance. + * + * Returns: + * 0 on success or a negative error code on failure. + */ +static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, + enum dma_data_direction direction) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_operation_ctx ctx = { true, false }; + u32 domain = amdgpu_display_supported_domains(adev); + int ret; + bool reads = (direction == DMA_BIDIRECTIONAL || + direction == DMA_FROM_DEVICE); + + if (!reads || !(domain & AMDGPU_GEM_DOMAIN_GTT)) + return 0; + + /* move to gtt */ + ret = amdgpu_bo_reserve(bo, false); + if (unlikely(ret != 0)) + return ret; + + if (!bo->pin_count && (bo->allowed_domains & AMDGPU_GEM_DOMAIN_GTT)) { + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + } + + amdgpu_bo_unreserve(bo); + return ret; +} + +const struct dma_buf_ops amdgpu_dmabuf_ops = { + .attach = amdgpu_dma_buf_map_attach, + .detach = amdgpu_dma_buf_map_detach, + .map_dma_buf = drm_gem_map_dma_buf, + .unmap_dma_buf = drm_gem_unmap_dma_buf, + .release = drm_gem_dmabuf_release, + .begin_cpu_access = amdgpu_dma_buf_begin_cpu_access, + .mmap = drm_gem_dmabuf_mmap, + .vmap = drm_gem_dmabuf_vmap, + .vunmap = drm_gem_dmabuf_vunmap, +}; + +/** + * amdgpu_gem_prime_export - &drm_driver.gem_prime_export implementation + * @dev: DRM device + * @gobj: GEM BO + * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR. + * + * The main work is done by the &drm_gem_prime_export helper, which in turn + * uses &amdgpu_gem_prime_res_obj. + * + * Returns: + * Shared DMA buffer representing the GEM BO from the given device. + */ +struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev, + struct drm_gem_object *gobj, + int flags) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); + struct dma_buf *buf; + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) || + bo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) + return ERR_PTR(-EPERM); + + buf = drm_gem_prime_export(dev, gobj, flags); + if (!IS_ERR(buf)) { + buf->file->f_mapping = dev->anon_inode->i_mapping; + buf->ops = &amdgpu_dmabuf_ops; + } + + return buf; +} + +/** + * amdgpu_gem_prime_import_sg_table - &drm_driver.gem_prime_import_sg_table + * implementation + * @dev: DRM device + * @attach: DMA-buf attachment + * @sg: Scatter/gather table + * + * Imports shared DMA buffer memory exported by another device. + * + * Returns: + * A new GEM BO of the given DRM device, representing the memory + * described by the given DMA-buf attachment and scatter/gather table. + */ +struct drm_gem_object * +amdgpu_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *attach, + struct sg_table *sg) +{ + struct reservation_object *resv = attach->dmabuf->resv; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_bo *bo; + struct amdgpu_bo_param bp; + int ret; + + memset(&bp, 0, sizeof(bp)); + bp.size = attach->dmabuf->size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_CPU; + bp.flags = 0; + bp.type = ttm_bo_type_sg; + bp.resv = resv; + ww_mutex_lock(&resv->lock, NULL); + ret = amdgpu_bo_create(adev, &bp, &bo); + if (ret) + goto error; + + bo->tbo.sg = sg; + bo->tbo.ttm->sg = sg; + bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; + if (attach->dmabuf->ops != &amdgpu_dmabuf_ops) + bo->prime_shared_count = 1; + + ww_mutex_unlock(&resv->lock); + return &bo->gem_base; + +error: + ww_mutex_unlock(&resv->lock); + return ERR_PTR(ret); +} + +/** + * amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation + * @dev: DRM device + * @dma_buf: Shared DMA buffer + * + * The main work is done by the &drm_gem_prime_import helper, which in turn + * uses &amdgpu_gem_prime_import_sg_table. + * + * Returns: + * GEM BO representing the shared DMA buffer for the given device. + */ +struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf) +{ + struct drm_gem_object *obj; + + if (dma_buf->ops == &amdgpu_dmabuf_ops) { + obj = dma_buf->priv; + if (obj->dev == dev) { + /* + * Importing dmabuf exported from out own gem increases + * refcount on gem itself instead of f_count of dmabuf. + */ + drm_gem_object_get(obj); + return obj; + } + } + + return drm_gem_prime_import(dev, dma_buf); +}
diff --git a/amdgpu/amdgpu_dma_buf.h b/amdgpu/amdgpu_dma_buf.h new file mode 100644 index 0000000..c7056cb --- /dev/null +++ b/amdgpu/amdgpu_dma_buf.h
@@ -0,0 +1,46 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_DMA_BUF_H__ +#define __AMDGPU_DMA_BUF_H__ + +#include <drm/drm_gem.h> + +struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj); +struct drm_gem_object * +amdgpu_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *attach, + struct sg_table *sg); +struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev, + struct drm_gem_object *gobj, + int flags); +struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf); +struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *); +void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj); +void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); +int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, + struct vm_area_struct *vma); + +extern const struct dma_buf_ops amdgpu_dmabuf_ops; + +#endif
diff --git a/amdgpu/amdgpu_doorbell.h b/amdgpu/amdgpu_doorbell.h new file mode 100644 index 0000000..790263d --- /dev/null +++ b/amdgpu/amdgpu_doorbell.h
@@ -0,0 +1,287 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +/* + * GPU doorbell structures, functions & helpers + */ +struct amdgpu_doorbell { + /* doorbell mmio */ + resource_size_t base; + resource_size_t size; + u32 __iomem *ptr; + u32 num_doorbells; /* Number of doorbells actually reserved for amdgpu. */ +}; + +/* Reserved doorbells for amdgpu (including multimedia). + * KFD can use all the rest in the 2M doorbell bar. + * For asic before vega10, doorbell is 32-bit, so the + * index/offset is in dword. For vega10 and after, doorbell + * can be 64-bit, so the index defined is in qword. + */ +struct amdgpu_doorbell_index { + uint32_t kiq; + uint32_t mec_ring0; + uint32_t mec_ring1; + uint32_t mec_ring2; + uint32_t mec_ring3; + uint32_t mec_ring4; + uint32_t mec_ring5; + uint32_t mec_ring6; + uint32_t mec_ring7; + uint32_t userqueue_start; + uint32_t userqueue_end; + uint32_t gfx_ring0; + uint32_t gfx_ring1; + uint32_t sdma_engine[8]; + uint32_t ih; + union { + struct { + uint32_t vcn_ring0_1; + uint32_t vcn_ring2_3; + uint32_t vcn_ring4_5; + uint32_t vcn_ring6_7; + } vcn; + struct { + uint32_t uvd_ring0_1; + uint32_t uvd_ring2_3; + uint32_t uvd_ring4_5; + uint32_t uvd_ring6_7; + uint32_t vce_ring0_1; + uint32_t vce_ring2_3; + uint32_t vce_ring4_5; + uint32_t vce_ring6_7; + } uvd_vce; + }; + uint32_t first_non_cp; + uint32_t last_non_cp; + uint32_t max_assignment; + /* Per engine SDMA doorbell size in dword */ + uint32_t sdma_doorbell_range; +}; + +typedef enum _AMDGPU_DOORBELL_ASSIGNMENT +{ + AMDGPU_DOORBELL_KIQ = 0x000, + AMDGPU_DOORBELL_HIQ = 0x001, + AMDGPU_DOORBELL_DIQ = 0x002, + AMDGPU_DOORBELL_MEC_RING0 = 0x010, + AMDGPU_DOORBELL_MEC_RING1 = 0x011, + AMDGPU_DOORBELL_MEC_RING2 = 0x012, + AMDGPU_DOORBELL_MEC_RING3 = 0x013, + AMDGPU_DOORBELL_MEC_RING4 = 0x014, + AMDGPU_DOORBELL_MEC_RING5 = 0x015, + AMDGPU_DOORBELL_MEC_RING6 = 0x016, + AMDGPU_DOORBELL_MEC_RING7 = 0x017, + AMDGPU_DOORBELL_GFX_RING0 = 0x020, + AMDGPU_DOORBELL_sDMA_ENGINE0 = 0x1E0, + AMDGPU_DOORBELL_sDMA_ENGINE1 = 0x1E1, + AMDGPU_DOORBELL_IH = 0x1E8, + AMDGPU_DOORBELL_MAX_ASSIGNMENT = 0x3FF, + AMDGPU_DOORBELL_INVALID = 0xFFFF +} AMDGPU_DOORBELL_ASSIGNMENT; + +typedef enum _AMDGPU_VEGA20_DOORBELL_ASSIGNMENT +{ + /* Compute + GFX: 0~255 */ + AMDGPU_VEGA20_DOORBELL_KIQ = 0x000, + AMDGPU_VEGA20_DOORBELL_HIQ = 0x001, + AMDGPU_VEGA20_DOORBELL_DIQ = 0x002, + AMDGPU_VEGA20_DOORBELL_MEC_RING0 = 0x003, + AMDGPU_VEGA20_DOORBELL_MEC_RING1 = 0x004, + AMDGPU_VEGA20_DOORBELL_MEC_RING2 = 0x005, + AMDGPU_VEGA20_DOORBELL_MEC_RING3 = 0x006, + AMDGPU_VEGA20_DOORBELL_MEC_RING4 = 0x007, + AMDGPU_VEGA20_DOORBELL_MEC_RING5 = 0x008, + AMDGPU_VEGA20_DOORBELL_MEC_RING6 = 0x009, + AMDGPU_VEGA20_DOORBELL_MEC_RING7 = 0x00A, + AMDGPU_VEGA20_DOORBELL_USERQUEUE_START = 0x00B, + AMDGPU_VEGA20_DOORBELL_USERQUEUE_END = 0x08A, + AMDGPU_VEGA20_DOORBELL_GFX_RING0 = 0x08B, + /* SDMA:256~335*/ + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0 = 0x100, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1 = 0x10A, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2 = 0x114, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3 = 0x11E, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4 = 0x128, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5 = 0x132, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6 = 0x13C, + AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7 = 0x146, + /* IH: 376~391 */ + AMDGPU_VEGA20_DOORBELL_IH = 0x178, + /* MMSCH: 392~407 + * overlap the doorbell assignment with VCN as they are mutually exclusive + * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD + */ + AMDGPU_VEGA20_DOORBELL64_VCN0_1 = 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */ + AMDGPU_VEGA20_DOORBELL64_VCN2_3 = 0x189, + AMDGPU_VEGA20_DOORBELL64_VCN4_5 = 0x18A, + AMDGPU_VEGA20_DOORBELL64_VCN6_7 = 0x18B, + + AMDGPU_VEGA20_DOORBELL64_UVD_RING0_1 = 0x188, + AMDGPU_VEGA20_DOORBELL64_UVD_RING2_3 = 0x189, + AMDGPU_VEGA20_DOORBELL64_UVD_RING4_5 = 0x18A, + AMDGPU_VEGA20_DOORBELL64_UVD_RING6_7 = 0x18B, + + AMDGPU_VEGA20_DOORBELL64_VCE_RING0_1 = 0x18C, + AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3 = 0x18D, + AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5 = 0x18E, + AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7 = 0x18F, + + AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0, + AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7, + + AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT = 0x18F, + AMDGPU_VEGA20_DOORBELL_INVALID = 0xFFFF +} AMDGPU_VEGA20_DOORBELL_ASSIGNMENT; + +typedef enum _AMDGPU_NAVI10_DOORBELL_ASSIGNMENT +{ + /* Compute + GFX: 0~255 */ + AMDGPU_NAVI10_DOORBELL_KIQ = 0x000, + AMDGPU_NAVI10_DOORBELL_HIQ = 0x001, + AMDGPU_NAVI10_DOORBELL_DIQ = 0x002, + AMDGPU_NAVI10_DOORBELL_MEC_RING0 = 0x003, + AMDGPU_NAVI10_DOORBELL_MEC_RING1 = 0x004, + AMDGPU_NAVI10_DOORBELL_MEC_RING2 = 0x005, + AMDGPU_NAVI10_DOORBELL_MEC_RING3 = 0x006, + AMDGPU_NAVI10_DOORBELL_MEC_RING4 = 0x007, + AMDGPU_NAVI10_DOORBELL_MEC_RING5 = 0x008, + AMDGPU_NAVI10_DOORBELL_MEC_RING6 = 0x009, + AMDGPU_NAVI10_DOORBELL_MEC_RING7 = 0x00A, + AMDGPU_NAVI10_DOORBELL_USERQUEUE_START = 0x00B, + AMDGPU_NAVI10_DOORBELL_USERQUEUE_END = 0x08A, + AMDGPU_NAVI10_DOORBELL_GFX_RING0 = 0x08B, + AMDGPU_NAVI10_DOORBELL_GFX_RING1 = 0x08C, + /* SDMA:256~335*/ + AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0 = 0x100, + AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1 = 0x10A, + /* IH: 376~391 */ + AMDGPU_NAVI10_DOORBELL_IH = 0x178, + /* MMSCH: 392~407 + * overlap the doorbell assignment with VCN as they are mutually exclusive + * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD + */ + AMDGPU_NAVI10_DOORBELL64_VCN0_1 = 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */ + AMDGPU_NAVI10_DOORBELL64_VCN2_3 = 0x189, + AMDGPU_NAVI10_DOORBELL64_VCN4_5 = 0x18A, + AMDGPU_NAVI10_DOORBELL64_VCN6_7 = 0x18B, + + AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0, + AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP = AMDGPU_NAVI10_DOORBELL64_VCN6_7, + + AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT = 0x18F, + AMDGPU_NAVI10_DOORBELL_INVALID = 0xFFFF +} AMDGPU_NAVI10_DOORBELL_ASSIGNMENT; + +/* + * 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space + */ +typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT +{ + /* + * All compute related doorbells: kiq, hiq, diq, traditional compute queue, user queue, should locate in + * a continues range so that programming CP_MEC_DOORBELL_RANGE_LOWER/UPPER can cover this range. + * Compute related doorbells are allocated from 0x00 to 0x8a + */ + + + /* kernel scheduling */ + AMDGPU_DOORBELL64_KIQ = 0x00, + + /* HSA interface queue and debug queue */ + AMDGPU_DOORBELL64_HIQ = 0x01, + AMDGPU_DOORBELL64_DIQ = 0x02, + + /* Compute engines */ + AMDGPU_DOORBELL64_MEC_RING0 = 0x03, + AMDGPU_DOORBELL64_MEC_RING1 = 0x04, + AMDGPU_DOORBELL64_MEC_RING2 = 0x05, + AMDGPU_DOORBELL64_MEC_RING3 = 0x06, + AMDGPU_DOORBELL64_MEC_RING4 = 0x07, + AMDGPU_DOORBELL64_MEC_RING5 = 0x08, + AMDGPU_DOORBELL64_MEC_RING6 = 0x09, + AMDGPU_DOORBELL64_MEC_RING7 = 0x0a, + + /* User queue doorbell range (128 doorbells) */ + AMDGPU_DOORBELL64_USERQUEUE_START = 0x0b, + AMDGPU_DOORBELL64_USERQUEUE_END = 0x8a, + + /* Graphics engine */ + AMDGPU_DOORBELL64_GFX_RING0 = 0x8b, + + /* + * Other graphics doorbells can be allocated here: from 0x8c to 0xdf + * Graphics voltage island aperture 1 + * default non-graphics QWORD index is 0xe0 - 0xFF inclusive + */ + + /* For vega10 sriov, the sdma doorbell must be fixed as follow + * to keep the same setting with host driver, or it will + * happen conflicts + */ + AMDGPU_DOORBELL64_sDMA_ENGINE0 = 0xF0, + AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE0 = 0xF1, + AMDGPU_DOORBELL64_sDMA_ENGINE1 = 0xF2, + AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE1 = 0xF3, + + /* Interrupt handler */ + AMDGPU_DOORBELL64_IH = 0xF4, /* For legacy interrupt ring buffer */ + AMDGPU_DOORBELL64_IH_RING1 = 0xF5, /* For page migration request log */ + AMDGPU_DOORBELL64_IH_RING2 = 0xF6, /* For page migration translation/invalidation log */ + + /* VCN engine use 32 bits doorbell */ + AMDGPU_DOORBELL64_VCN0_1 = 0xF8, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */ + AMDGPU_DOORBELL64_VCN2_3 = 0xF9, + AMDGPU_DOORBELL64_VCN4_5 = 0xFA, + AMDGPU_DOORBELL64_VCN6_7 = 0xFB, + + /* overlap the doorbell assignment with VCN as they are mutually exclusive + * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD + */ + AMDGPU_DOORBELL64_UVD_RING0_1 = 0xF8, + AMDGPU_DOORBELL64_UVD_RING2_3 = 0xF9, + AMDGPU_DOORBELL64_UVD_RING4_5 = 0xFA, + AMDGPU_DOORBELL64_UVD_RING6_7 = 0xFB, + + AMDGPU_DOORBELL64_VCE_RING0_1 = 0xFC, + AMDGPU_DOORBELL64_VCE_RING2_3 = 0xFD, + AMDGPU_DOORBELL64_VCE_RING4_5 = 0xFE, + AMDGPU_DOORBELL64_VCE_RING6_7 = 0xFF, + + AMDGPU_DOORBELL64_FIRST_NON_CP = AMDGPU_DOORBELL64_sDMA_ENGINE0, + AMDGPU_DOORBELL64_LAST_NON_CP = AMDGPU_DOORBELL64_VCE_RING6_7, + + AMDGPU_DOORBELL64_MAX_ASSIGNMENT = 0xFF, + AMDGPU_DOORBELL64_INVALID = 0xFFFF +} AMDGPU_DOORBELL64_ASSIGNMENT; + +u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index); +void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v); +u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index); +void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v); + +#define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index)) +#define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v)) +#define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index)) +#define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v)) +
diff --git a/amdgpu/amdgpu_dpm.c b/amdgpu/amdgpu_dpm.c new file mode 100644 index 0000000..61bd103 --- /dev/null +++ b/amdgpu/amdgpu_dpm.c
@@ -0,0 +1,968 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "amdgpu_i2c.h" +#include "amdgpu_dpm.h" +#include "atom.h" +#include "amd_pcie.h" + +void amdgpu_dpm_print_class_info(u32 class, u32 class2) +{ + const char *s; + + switch (class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) { + case ATOM_PPLIB_CLASSIFICATION_UI_NONE: + default: + s = "none"; + break; + case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY: + s = "battery"; + break; + case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED: + s = "balanced"; + break; + case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE: + s = "performance"; + break; + } + printk("\tui class: %s\n", s); + printk("\tinternal class:"); + if (((class & ~ATOM_PPLIB_CLASSIFICATION_UI_MASK) == 0) && + (class2 == 0)) + pr_cont(" none"); + else { + if (class & ATOM_PPLIB_CLASSIFICATION_BOOT) + pr_cont(" boot"); + if (class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + pr_cont(" thermal"); + if (class & ATOM_PPLIB_CLASSIFICATION_LIMITEDPOWERSOURCE) + pr_cont(" limited_pwr"); + if (class & ATOM_PPLIB_CLASSIFICATION_REST) + pr_cont(" rest"); + if (class & ATOM_PPLIB_CLASSIFICATION_FORCED) + pr_cont(" forced"); + if (class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) + pr_cont(" 3d_perf"); + if (class & ATOM_PPLIB_CLASSIFICATION_OVERDRIVETEMPLATE) + pr_cont(" ovrdrv"); + if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE) + pr_cont(" uvd"); + if (class & ATOM_PPLIB_CLASSIFICATION_3DLOW) + pr_cont(" 3d_low"); + if (class & ATOM_PPLIB_CLASSIFICATION_ACPI) + pr_cont(" acpi"); + if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + pr_cont(" uvd_hd2"); + if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + pr_cont(" uvd_hd"); + if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + pr_cont(" uvd_sd"); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2) + pr_cont(" limited_pwr2"); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) + pr_cont(" ulv"); + if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + pr_cont(" uvd_mvc"); + } + pr_cont("\n"); +} + +void amdgpu_dpm_print_cap_info(u32 caps) +{ + printk("\tcaps:"); + if (caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) + pr_cont(" single_disp"); + if (caps & ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK) + pr_cont(" video"); + if (caps & ATOM_PPLIB_DISALLOW_ON_DC) + pr_cont(" no_dc"); + pr_cont("\n"); +} + +void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, + struct amdgpu_ps *rps) +{ + printk("\tstatus:"); + if (rps == adev->pm.dpm.current_ps) + pr_cont(" c"); + if (rps == adev->pm.dpm.requested_ps) + pr_cont(" r"); + if (rps == adev->pm.dpm.boot_ps) + pr_cont(" b"); + pr_cont("\n"); +} + +void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev) +{ + struct drm_device *ddev = adev->ddev; + struct drm_crtc *crtc; + struct amdgpu_crtc *amdgpu_crtc; + + adev->pm.dpm.new_active_crtcs = 0; + adev->pm.dpm.new_active_crtc_count = 0; + if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, + &ddev->mode_config.crtc_list, head) { + amdgpu_crtc = to_amdgpu_crtc(crtc); + if (amdgpu_crtc->enabled) { + adev->pm.dpm.new_active_crtcs |= (1 << amdgpu_crtc->crtc_id); + adev->pm.dpm.new_active_crtc_count++; + } + } + } +} + + +u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev) +{ + struct drm_device *dev = adev->ddev; + struct drm_crtc *crtc; + struct amdgpu_crtc *amdgpu_crtc; + u32 vblank_in_pixels; + u32 vblank_time_us = 0xffffffff; /* if the displays are off, vblank time is max */ + + if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + amdgpu_crtc = to_amdgpu_crtc(crtc); + if (crtc->enabled && amdgpu_crtc->enabled && amdgpu_crtc->hw_mode.clock) { + vblank_in_pixels = + amdgpu_crtc->hw_mode.crtc_htotal * + (amdgpu_crtc->hw_mode.crtc_vblank_end - + amdgpu_crtc->hw_mode.crtc_vdisplay + + (amdgpu_crtc->v_border * 2)); + + vblank_time_us = vblank_in_pixels * 1000 / amdgpu_crtc->hw_mode.clock; + break; + } + } + } + + return vblank_time_us; +} + +u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev) +{ + struct drm_device *dev = adev->ddev; + struct drm_crtc *crtc; + struct amdgpu_crtc *amdgpu_crtc; + u32 vrefresh = 0; + + if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) { + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + amdgpu_crtc = to_amdgpu_crtc(crtc); + if (crtc->enabled && amdgpu_crtc->enabled && amdgpu_crtc->hw_mode.clock) { + vrefresh = drm_mode_vrefresh(&amdgpu_crtc->hw_mode); + break; + } + } + } + + return vrefresh; +} + +bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor) +{ + switch (sensor) { + case THERMAL_TYPE_RV6XX: + case THERMAL_TYPE_RV770: + case THERMAL_TYPE_EVERGREEN: + case THERMAL_TYPE_SUMO: + case THERMAL_TYPE_NI: + case THERMAL_TYPE_SI: + case THERMAL_TYPE_CI: + case THERMAL_TYPE_KV: + return true; + case THERMAL_TYPE_ADT7473_WITH_INTERNAL: + case THERMAL_TYPE_EMC2103_WITH_INTERNAL: + return false; /* need special handling */ + case THERMAL_TYPE_NONE: + case THERMAL_TYPE_EXTERNAL: + case THERMAL_TYPE_EXTERNAL_GPIO: + default: + return false; + } +} + +union power_info { + struct _ATOM_POWERPLAY_INFO info; + struct _ATOM_POWERPLAY_INFO_V2 info_2; + struct _ATOM_POWERPLAY_INFO_V3 info_3; + struct _ATOM_PPLIB_POWERPLAYTABLE pplib; + struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2; + struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3; + struct _ATOM_PPLIB_POWERPLAYTABLE4 pplib4; + struct _ATOM_PPLIB_POWERPLAYTABLE5 pplib5; +}; + +union fan_info { + struct _ATOM_PPLIB_FANTABLE fan; + struct _ATOM_PPLIB_FANTABLE2 fan2; + struct _ATOM_PPLIB_FANTABLE3 fan3; +}; + +static int amdgpu_parse_clk_voltage_dep_table(struct amdgpu_clock_voltage_dependency_table *amdgpu_table, + ATOM_PPLIB_Clock_Voltage_Dependency_Table *atom_table) +{ + u32 size = atom_table->ucNumEntries * + sizeof(struct amdgpu_clock_voltage_dependency_entry); + int i; + ATOM_PPLIB_Clock_Voltage_Dependency_Record *entry; + + amdgpu_table->entries = kzalloc(size, GFP_KERNEL); + if (!amdgpu_table->entries) + return -ENOMEM; + + entry = &atom_table->entries[0]; + for (i = 0; i < atom_table->ucNumEntries; i++) { + amdgpu_table->entries[i].clk = le16_to_cpu(entry->usClockLow) | + (entry->ucClockHigh << 16); + amdgpu_table->entries[i].v = le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_Clock_Voltage_Dependency_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_Clock_Voltage_Dependency_Record)); + } + amdgpu_table->count = atom_table->ucNumEntries; + + return 0; +} + +int amdgpu_get_platform_caps(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + union power_info *power_info; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + + if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + adev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps); + adev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime); + adev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime); + + return 0; +} + +/* sizeof(ATOM_PPLIB_EXTENDEDHEADER) */ +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4 16 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5 18 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6 20 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7 22 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8 24 +#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V9 26 + +int amdgpu_parse_extended_power_table(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + union power_info *power_info; + union fan_info *fan_info; + ATOM_PPLIB_Clock_Voltage_Dependency_Table *dep_table; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + u16 data_offset; + u8 frev, crev; + int ret, i; + + if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return -EINVAL; + power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); + + /* fan table */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) { + if (power_info->pplib3.usFanTableOffset) { + fan_info = (union fan_info *)(mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib3.usFanTableOffset)); + adev->pm.dpm.fan.t_hyst = fan_info->fan.ucTHyst; + adev->pm.dpm.fan.t_min = le16_to_cpu(fan_info->fan.usTMin); + adev->pm.dpm.fan.t_med = le16_to_cpu(fan_info->fan.usTMed); + adev->pm.dpm.fan.t_high = le16_to_cpu(fan_info->fan.usTHigh); + adev->pm.dpm.fan.pwm_min = le16_to_cpu(fan_info->fan.usPWMMin); + adev->pm.dpm.fan.pwm_med = le16_to_cpu(fan_info->fan.usPWMMed); + adev->pm.dpm.fan.pwm_high = le16_to_cpu(fan_info->fan.usPWMHigh); + if (fan_info->fan.ucFanTableFormat >= 2) + adev->pm.dpm.fan.t_max = le16_to_cpu(fan_info->fan2.usTMax); + else + adev->pm.dpm.fan.t_max = 10900; + adev->pm.dpm.fan.cycle_delay = 100000; + if (fan_info->fan.ucFanTableFormat >= 3) { + adev->pm.dpm.fan.control_mode = fan_info->fan3.ucFanControlMode; + adev->pm.dpm.fan.default_max_fan_pwm = + le16_to_cpu(fan_info->fan3.usFanPWMMax); + adev->pm.dpm.fan.default_fan_output_sensitivity = 4836; + adev->pm.dpm.fan.fan_output_sensitivity = + le16_to_cpu(fan_info->fan3.usFanOutputSensitivity); + } + adev->pm.dpm.fan.ucode_fan_control = true; + } + } + + /* clock dependancy tables, shedding tables */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE4)) { + if (power_info->pplib4.usVddcDependencyOnSCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcDependencyOnSCLKOffset)); + ret = amdgpu_parse_clk_voltage_dep_table(&adev->pm.dpm.dyn_state.vddc_dependency_on_sclk, + dep_table); + if (ret) { + amdgpu_free_extended_power_table(adev); + return ret; + } + } + if (power_info->pplib4.usVddciDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddciDependencyOnMCLKOffset)); + ret = amdgpu_parse_clk_voltage_dep_table(&adev->pm.dpm.dyn_state.vddci_dependency_on_mclk, + dep_table); + if (ret) { + amdgpu_free_extended_power_table(adev); + return ret; + } + } + if (power_info->pplib4.usVddcDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcDependencyOnMCLKOffset)); + ret = amdgpu_parse_clk_voltage_dep_table(&adev->pm.dpm.dyn_state.vddc_dependency_on_mclk, + dep_table); + if (ret) { + amdgpu_free_extended_power_table(adev); + return ret; + } + } + if (power_info->pplib4.usMvddDependencyOnMCLKOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usMvddDependencyOnMCLKOffset)); + ret = amdgpu_parse_clk_voltage_dep_table(&adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk, + dep_table); + if (ret) { + amdgpu_free_extended_power_table(adev); + return ret; + } + } + if (power_info->pplib4.usMaxClockVoltageOnDCOffset) { + ATOM_PPLIB_Clock_Voltage_Limit_Table *clk_v = + (ATOM_PPLIB_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usMaxClockVoltageOnDCOffset)); + if (clk_v->ucNumEntries) { + adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk = + le16_to_cpu(clk_v->entries[0].usSclkLow) | + (clk_v->entries[0].ucSclkHigh << 16); + adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk = + le16_to_cpu(clk_v->entries[0].usMclkLow) | + (clk_v->entries[0].ucMclkHigh << 16); + adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc = + le16_to_cpu(clk_v->entries[0].usVddc); + adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddci = + le16_to_cpu(clk_v->entries[0].usVddci); + } + } + if (power_info->pplib4.usVddcPhaseShedLimitsTableOffset) { + ATOM_PPLIB_PhaseSheddingLimits_Table *psl = + (ATOM_PPLIB_PhaseSheddingLimits_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib4.usVddcPhaseShedLimitsTableOffset)); + ATOM_PPLIB_PhaseSheddingLimits_Record *entry; + + adev->pm.dpm.dyn_state.phase_shedding_limits_table.entries = + kcalloc(psl->ucNumEntries, + sizeof(struct amdgpu_phase_shedding_limits_entry), + GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.phase_shedding_limits_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + + entry = &psl->entries[0]; + for (i = 0; i < psl->ucNumEntries; i++) { + adev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].sclk = + le16_to_cpu(entry->usSclkLow) | (entry->ucSclkHigh << 16); + adev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].mclk = + le16_to_cpu(entry->usMclkLow) | (entry->ucMclkHigh << 16); + adev->pm.dpm.dyn_state.phase_shedding_limits_table.entries[i].voltage = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_PhaseSheddingLimits_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_PhaseSheddingLimits_Record)); + } + adev->pm.dpm.dyn_state.phase_shedding_limits_table.count = + psl->ucNumEntries; + } + } + + /* cac data */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE5)) { + adev->pm.dpm.tdp_limit = le32_to_cpu(power_info->pplib5.ulTDPLimit); + adev->pm.dpm.near_tdp_limit = le32_to_cpu(power_info->pplib5.ulNearTDPLimit); + adev->pm.dpm.near_tdp_limit_adjusted = adev->pm.dpm.near_tdp_limit; + adev->pm.dpm.tdp_od_limit = le16_to_cpu(power_info->pplib5.usTDPODLimit); + if (adev->pm.dpm.tdp_od_limit) + adev->pm.dpm.power_control = true; + else + adev->pm.dpm.power_control = false; + adev->pm.dpm.tdp_adjustment = 0; + adev->pm.dpm.sq_ramping_threshold = le32_to_cpu(power_info->pplib5.ulSQRampingThreshold); + adev->pm.dpm.cac_leakage = le32_to_cpu(power_info->pplib5.ulCACLeakage); + adev->pm.dpm.load_line_slope = le16_to_cpu(power_info->pplib5.usLoadLineSlope); + if (power_info->pplib5.usCACLeakageTableOffset) { + ATOM_PPLIB_CAC_Leakage_Table *cac_table = + (ATOM_PPLIB_CAC_Leakage_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib5.usCACLeakageTableOffset)); + ATOM_PPLIB_CAC_Leakage_Record *entry; + u32 size = cac_table->ucNumEntries * sizeof(struct amdgpu_cac_leakage_table); + adev->pm.dpm.dyn_state.cac_leakage_table.entries = kzalloc(size, GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.cac_leakage_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + entry = &cac_table->entries[0]; + for (i = 0; i < cac_table->ucNumEntries; i++) { + if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) { + adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1 = + le16_to_cpu(entry->usVddc1); + adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2 = + le16_to_cpu(entry->usVddc2); + adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3 = + le16_to_cpu(entry->usVddc3); + } else { + adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc = + le16_to_cpu(entry->usVddc); + adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage = + le32_to_cpu(entry->ulLeakageValue); + } + entry = (ATOM_PPLIB_CAC_Leakage_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_CAC_Leakage_Record)); + } + adev->pm.dpm.dyn_state.cac_leakage_table.count = cac_table->ucNumEntries; + } + } + + /* ext tables */ + if (le16_to_cpu(power_info->pplib.usTableSize) >= + sizeof(struct _ATOM_PPLIB_POWERPLAYTABLE3)) { + ATOM_PPLIB_EXTENDEDHEADER *ext_hdr = (ATOM_PPLIB_EXTENDEDHEADER *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(power_info->pplib3.usExtendendedHeaderOffset)); + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2) && + ext_hdr->usVCETableOffset) { + VCEClockInfoArray *array = (VCEClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1); + ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *limits = + (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1 + + 1 + array->ucNumEntries * sizeof(VCEClockInfo)); + ATOM_PPLIB_VCE_State_Table *states = + (ATOM_PPLIB_VCE_State_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usVCETableOffset) + 1 + + 1 + (array->ucNumEntries * sizeof (VCEClockInfo)) + + 1 + (limits->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record))); + ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *entry; + ATOM_PPLIB_VCE_State_Record *state_entry; + VCEClockInfo *vce_clk; + u32 size = limits->numEntries * + sizeof(struct amdgpu_vce_clock_voltage_dependency_entry); + adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + state_entry = &states->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + vce_clk = (VCEClockInfo *) + ((u8 *)&array->entries[0] + + (entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo))); + adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].evclk = + le16_to_cpu(vce_clk->usEVClkLow) | (vce_clk->ucEVClkHigh << 16); + adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].ecclk = + le16_to_cpu(vce_clk->usECClkLow) | (vce_clk->ucECClkHigh << 16); + adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record)); + } + adev->pm.dpm.num_of_vce_states = + states->numEntries > AMD_MAX_VCE_LEVELS ? + AMD_MAX_VCE_LEVELS : states->numEntries; + for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) { + vce_clk = (VCEClockInfo *) + ((u8 *)&array->entries[0] + + (state_entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo))); + adev->pm.dpm.vce_states[i].evclk = + le16_to_cpu(vce_clk->usEVClkLow) | (vce_clk->ucEVClkHigh << 16); + adev->pm.dpm.vce_states[i].ecclk = + le16_to_cpu(vce_clk->usECClkLow) | (vce_clk->ucECClkHigh << 16); + adev->pm.dpm.vce_states[i].clk_idx = + state_entry->ucClockInfoIndex & 0x3f; + adev->pm.dpm.vce_states[i].pstate = + (state_entry->ucClockInfoIndex & 0xc0) >> 6; + state_entry = (ATOM_PPLIB_VCE_State_Record *) + ((u8 *)state_entry + sizeof(ATOM_PPLIB_VCE_State_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3) && + ext_hdr->usUVDTableOffset) { + UVDClockInfoArray *array = (UVDClockInfoArray *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usUVDTableOffset) + 1); + ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *limits = + (ATOM_PPLIB_UVD_Clock_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usUVDTableOffset) + 1 + + 1 + (array->ucNumEntries * sizeof (UVDClockInfo))); + ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct amdgpu_uvd_clock_voltage_dependency_entry); + adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + UVDClockInfo *uvd_clk = (UVDClockInfo *) + ((u8 *)&array->entries[0] + + (entry->ucUVDClockInfoIndex * sizeof(UVDClockInfo))); + adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].vclk = + le16_to_cpu(uvd_clk->usVClkLow) | (uvd_clk->ucVClkHigh << 16); + adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].dclk = + le16_to_cpu(uvd_clk->usDClkLow) | (uvd_clk->ucDClkHigh << 16); + adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_UVD_Clock_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4) && + ext_hdr->usSAMUTableOffset) { + ATOM_PPLIB_SAMClk_Voltage_Limit_Table *limits = + (ATOM_PPLIB_SAMClk_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usSAMUTableOffset) + 1); + ATOM_PPLIB_SAMClk_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct amdgpu_clock_voltage_dependency_entry); + adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].clk = + le16_to_cpu(entry->usSAMClockLow) | (entry->ucSAMClockHigh << 16); + adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_SAMClk_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_SAMClk_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5) && + ext_hdr->usPPMTableOffset) { + ATOM_PPLIB_PPM_Table *ppm = (ATOM_PPLIB_PPM_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPPMTableOffset)); + adev->pm.dpm.dyn_state.ppm_table = + kzalloc(sizeof(struct amdgpu_ppm_table), GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.ppm_table) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + adev->pm.dpm.dyn_state.ppm_table->ppm_design = ppm->ucPpmDesign; + adev->pm.dpm.dyn_state.ppm_table->cpu_core_number = + le16_to_cpu(ppm->usCpuCoreNumber); + adev->pm.dpm.dyn_state.ppm_table->platform_tdp = + le32_to_cpu(ppm->ulPlatformTDP); + adev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdp = + le32_to_cpu(ppm->ulSmallACPlatformTDP); + adev->pm.dpm.dyn_state.ppm_table->platform_tdc = + le32_to_cpu(ppm->ulPlatformTDC); + adev->pm.dpm.dyn_state.ppm_table->small_ac_platform_tdc = + le32_to_cpu(ppm->ulSmallACPlatformTDC); + adev->pm.dpm.dyn_state.ppm_table->apu_tdp = + le32_to_cpu(ppm->ulApuTDP); + adev->pm.dpm.dyn_state.ppm_table->dgpu_tdp = + le32_to_cpu(ppm->ulDGpuTDP); + adev->pm.dpm.dyn_state.ppm_table->dgpu_ulv_power = + le32_to_cpu(ppm->ulDGpuUlvPower); + adev->pm.dpm.dyn_state.ppm_table->tj_max = + le32_to_cpu(ppm->ulTjmax); + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6) && + ext_hdr->usACPTableOffset) { + ATOM_PPLIB_ACPClk_Voltage_Limit_Table *limits = + (ATOM_PPLIB_ACPClk_Voltage_Limit_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usACPTableOffset) + 1); + ATOM_PPLIB_ACPClk_Voltage_Limit_Record *entry; + u32 size = limits->numEntries * + sizeof(struct amdgpu_clock_voltage_dependency_entry); + adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries = + kzalloc(size, GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count = + limits->numEntries; + entry = &limits->entries[0]; + for (i = 0; i < limits->numEntries; i++) { + adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].clk = + le16_to_cpu(entry->usACPClockLow) | (entry->ucACPClockHigh << 16); + adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v = + le16_to_cpu(entry->usVoltage); + entry = (ATOM_PPLIB_ACPClk_Voltage_Limit_Record *) + ((u8 *)entry + sizeof(ATOM_PPLIB_ACPClk_Voltage_Limit_Record)); + } + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7) && + ext_hdr->usPowerTuneTableOffset) { + u8 rev = *(u8 *)(mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + ATOM_PowerTune_Table *pt; + adev->pm.dpm.dyn_state.cac_tdp_table = + kzalloc(sizeof(struct amdgpu_cac_tdp_table), GFP_KERNEL); + if (!adev->pm.dpm.dyn_state.cac_tdp_table) { + amdgpu_free_extended_power_table(adev); + return -ENOMEM; + } + if (rev > 0) { + ATOM_PPLIB_POWERTUNE_Table_V1 *ppt = (ATOM_PPLIB_POWERTUNE_Table_V1 *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + adev->pm.dpm.dyn_state.cac_tdp_table->maximum_power_delivery_limit = + ppt->usMaximumPowerDeliveryLimit; + pt = &ppt->power_tune_table; + } else { + ATOM_PPLIB_POWERTUNE_Table *ppt = (ATOM_PPLIB_POWERTUNE_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usPowerTuneTableOffset)); + adev->pm.dpm.dyn_state.cac_tdp_table->maximum_power_delivery_limit = 255; + pt = &ppt->power_tune_table; + } + adev->pm.dpm.dyn_state.cac_tdp_table->tdp = le16_to_cpu(pt->usTDP); + adev->pm.dpm.dyn_state.cac_tdp_table->configurable_tdp = + le16_to_cpu(pt->usConfigurableTDP); + adev->pm.dpm.dyn_state.cac_tdp_table->tdc = le16_to_cpu(pt->usTDC); + adev->pm.dpm.dyn_state.cac_tdp_table->battery_power_limit = + le16_to_cpu(pt->usBatteryPowerLimit); + adev->pm.dpm.dyn_state.cac_tdp_table->small_power_limit = + le16_to_cpu(pt->usSmallPowerLimit); + adev->pm.dpm.dyn_state.cac_tdp_table->low_cac_leakage = + le16_to_cpu(pt->usLowCACLeakage); + adev->pm.dpm.dyn_state.cac_tdp_table->high_cac_leakage = + le16_to_cpu(pt->usHighCACLeakage); + } + if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8) && + ext_hdr->usSclkVddgfxTableOffset) { + dep_table = (ATOM_PPLIB_Clock_Voltage_Dependency_Table *) + (mode_info->atom_context->bios + data_offset + + le16_to_cpu(ext_hdr->usSclkVddgfxTableOffset)); + ret = amdgpu_parse_clk_voltage_dep_table( + &adev->pm.dpm.dyn_state.vddgfx_dependency_on_sclk, + dep_table); + if (ret) { + kfree(adev->pm.dpm.dyn_state.vddgfx_dependency_on_sclk.entries); + return ret; + } + } + } + + return 0; +} + +void amdgpu_free_extended_power_table(struct amdgpu_device *adev) +{ + struct amdgpu_dpm_dynamic_state *dyn_state = &adev->pm.dpm.dyn_state; + + kfree(dyn_state->vddc_dependency_on_sclk.entries); + kfree(dyn_state->vddci_dependency_on_mclk.entries); + kfree(dyn_state->vddc_dependency_on_mclk.entries); + kfree(dyn_state->mvdd_dependency_on_mclk.entries); + kfree(dyn_state->cac_leakage_table.entries); + kfree(dyn_state->phase_shedding_limits_table.entries); + kfree(dyn_state->ppm_table); + kfree(dyn_state->cac_tdp_table); + kfree(dyn_state->vce_clock_voltage_dependency_table.entries); + kfree(dyn_state->uvd_clock_voltage_dependency_table.entries); + kfree(dyn_state->samu_clock_voltage_dependency_table.entries); + kfree(dyn_state->acp_clock_voltage_dependency_table.entries); + kfree(dyn_state->vddgfx_dependency_on_sclk.entries); +} + +static const char *pp_lib_thermal_controller_names[] = { + "NONE", + "lm63", + "adm1032", + "adm1030", + "max6649", + "lm64", + "f75375", + "RV6xx", + "RV770", + "adt7473", + "NONE", + "External GPIO", + "Evergreen", + "emc2103", + "Sumo", + "Northern Islands", + "Southern Islands", + "lm96163", + "Sea Islands", + "Kaveri/Kabini", +}; + +void amdgpu_add_thermal_controller(struct amdgpu_device *adev) +{ + struct amdgpu_mode_info *mode_info = &adev->mode_info; + ATOM_PPLIB_POWERPLAYTABLE *power_table; + int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); + ATOM_PPLIB_THERMALCONTROLLER *controller; + struct amdgpu_i2c_bus_rec i2c_bus; + u16 data_offset; + u8 frev, crev; + + if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) + return; + power_table = (ATOM_PPLIB_POWERPLAYTABLE *) + (mode_info->atom_context->bios + data_offset); + controller = &power_table->sThermalController; + + /* add the i2c bus for thermal/fan chip */ + if (controller->ucType > 0) { + if (controller->ucFanParameters & ATOM_PP_FANPARAMETERS_NOFAN) + adev->pm.no_fan = true; + adev->pm.fan_pulses_per_revolution = + controller->ucFanParameters & ATOM_PP_FANPARAMETERS_TACHOMETER_PULSES_PER_REVOLUTION_MASK; + if (adev->pm.fan_pulses_per_revolution) { + adev->pm.fan_min_rpm = controller->ucFanMinRPM; + adev->pm.fan_max_rpm = controller->ucFanMaxRPM; + } + if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_RV6XX; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV770) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_RV770; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EVERGREEN) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_EVERGREEN; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_SUMO) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_SUMO; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_NISLANDS) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_NI; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_SISLANDS) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_SI; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_CISLANDS) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_CI; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_KAVERI) { + DRM_INFO("Internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_KV; + } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) { + DRM_INFO("External GPIO thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_EXTERNAL_GPIO; + } else if (controller->ucType == + ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL) { + DRM_INFO("ADT7473 with internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_ADT7473_WITH_INTERNAL; + } else if (controller->ucType == + ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL) { + DRM_INFO("EMC2103 with internal thermal controller %s fan control\n", + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_EMC2103_WITH_INTERNAL; + } else if (controller->ucType < ARRAY_SIZE(pp_lib_thermal_controller_names)) { + DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n", + pp_lib_thermal_controller_names[controller->ucType], + controller->ucI2cAddress >> 1, + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + adev->pm.int_thermal_type = THERMAL_TYPE_EXTERNAL; + i2c_bus = amdgpu_atombios_lookup_i2c_gpio(adev, controller->ucI2cLine); + adev->pm.i2c_bus = amdgpu_i2c_lookup(adev, &i2c_bus); + if (adev->pm.i2c_bus) { + struct i2c_board_info info = { }; + const char *name = pp_lib_thermal_controller_names[controller->ucType]; + info.addr = controller->ucI2cAddress >> 1; + strlcpy(info.type, name, sizeof(info.type)); + i2c_new_device(&adev->pm.i2c_bus->adapter, &info); + } + } else { + DRM_INFO("Unknown thermal controller type %d at 0x%02x %s fan control\n", + controller->ucType, + controller->ucI2cAddress >> 1, + (controller->ucFanParameters & + ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); + } + } +} + +enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, + u32 sys_mask, + enum amdgpu_pcie_gen asic_gen, + enum amdgpu_pcie_gen default_gen) +{ + switch (asic_gen) { + case AMDGPU_PCIE_GEN1: + return AMDGPU_PCIE_GEN1; + case AMDGPU_PCIE_GEN2: + return AMDGPU_PCIE_GEN2; + case AMDGPU_PCIE_GEN3: + return AMDGPU_PCIE_GEN3; + default: + if ((sys_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) && + (default_gen == AMDGPU_PCIE_GEN3)) + return AMDGPU_PCIE_GEN3; + else if ((sys_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) && + (default_gen == AMDGPU_PCIE_GEN2)) + return AMDGPU_PCIE_GEN2; + else + return AMDGPU_PCIE_GEN1; + } + return AMDGPU_PCIE_GEN1; +} + +struct amd_vce_state* +amdgpu_get_vce_clock_state(void *handle, u32 idx) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + if (idx < adev->pm.dpm.num_of_vce_states) + return &adev->pm.dpm.vce_states[idx]; + + return NULL; +} + +int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low) +{ + uint32_t clk_freq; + int ret = 0; + if (is_support_sw_smu(adev)) { + ret = smu_get_dpm_freq_range(&adev->smu, SMU_GFXCLK, + low ? &clk_freq : NULL, + !low ? &clk_freq : NULL); + if (ret) + return 0; + return clk_freq * 100; + + } else { + return (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (low)); + } +} + +int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low) +{ + uint32_t clk_freq; + int ret = 0; + if (is_support_sw_smu(adev)) { + ret = smu_get_dpm_freq_range(&adev->smu, SMU_UCLK, + low ? &clk_freq : NULL, + !low ? &clk_freq : NULL); + if (ret) + return 0; + return clk_freq * 100; + + } else { + return (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (low)); + } +} + +int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, uint32_t block_type, bool gate) +{ + int ret = 0; + bool swsmu = is_support_sw_smu(adev); + + switch (block_type) { + case AMD_IP_BLOCK_TYPE_GFX: + case AMD_IP_BLOCK_TYPE_UVD: + case AMD_IP_BLOCK_TYPE_VCN: + case AMD_IP_BLOCK_TYPE_VCE: + if (swsmu) + ret = smu_dpm_set_power_gate(&adev->smu, block_type, gate); + else + ret = ((adev)->powerplay.pp_funcs->set_powergating_by_smu( + (adev)->powerplay.pp_handle, block_type, gate)); + break; + case AMD_IP_BLOCK_TYPE_GMC: + case AMD_IP_BLOCK_TYPE_ACP: + case AMD_IP_BLOCK_TYPE_SDMA: + ret = ((adev)->powerplay.pp_funcs->set_powergating_by_smu( + (adev)->powerplay.pp_handle, block_type, gate)); + break; + default: + break; + } + + return ret; +}
diff --git a/amdgpu/amdgpu_dpm.h b/amdgpu/amdgpu_dpm.h new file mode 100644 index 0000000..1c5c0fd --- /dev/null +++ b/amdgpu/amdgpu_dpm.h
@@ -0,0 +1,526 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_DPM_H__ +#define __AMDGPU_DPM_H__ + +enum amdgpu_int_thermal_type { + THERMAL_TYPE_NONE, + THERMAL_TYPE_EXTERNAL, + THERMAL_TYPE_EXTERNAL_GPIO, + THERMAL_TYPE_RV6XX, + THERMAL_TYPE_RV770, + THERMAL_TYPE_ADT7473_WITH_INTERNAL, + THERMAL_TYPE_EVERGREEN, + THERMAL_TYPE_SUMO, + THERMAL_TYPE_NI, + THERMAL_TYPE_SI, + THERMAL_TYPE_EMC2103_WITH_INTERNAL, + THERMAL_TYPE_CI, + THERMAL_TYPE_KV, +}; + +enum amdgpu_dpm_auto_throttle_src { + AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, + AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL +}; + +enum amdgpu_dpm_event_src { + AMDGPU_DPM_EVENT_SRC_ANALOG = 0, + AMDGPU_DPM_EVENT_SRC_EXTERNAL = 1, + AMDGPU_DPM_EVENT_SRC_DIGITAL = 2, + AMDGPU_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3, + AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4 +}; + +struct amdgpu_ps { + u32 caps; /* vbios flags */ + u32 class; /* vbios flags */ + u32 class2; /* vbios flags */ + /* UVD clocks */ + u32 vclk; + u32 dclk; + /* VCE clocks */ + u32 evclk; + u32 ecclk; + bool vce_active; + enum amd_vce_level vce_level; + /* asic priv */ + void *ps_priv; +}; + +struct amdgpu_dpm_thermal { + /* thermal interrupt work */ + struct work_struct work; + /* low temperature threshold */ + int min_temp; + /* high temperature threshold */ + int max_temp; + /* edge max emergency(shutdown) temp */ + int max_edge_emergency_temp; + /* hotspot low temperature threshold */ + int min_hotspot_temp; + /* hotspot high temperature critical threshold */ + int max_hotspot_crit_temp; + /* hotspot max emergency(shutdown) temp */ + int max_hotspot_emergency_temp; + /* memory low temperature threshold */ + int min_mem_temp; + /* memory high temperature critical threshold */ + int max_mem_crit_temp; + /* memory max emergency(shutdown) temp */ + int max_mem_emergency_temp; + /* was last interrupt low to high or high to low */ + bool high_to_low; + /* interrupt source */ + struct amdgpu_irq_src irq; +}; + +enum amdgpu_clk_action +{ + AMDGPU_SCLK_UP = 1, + AMDGPU_SCLK_DOWN +}; + +struct amdgpu_blacklist_clocks +{ + u32 sclk; + u32 mclk; + enum amdgpu_clk_action action; +}; + +struct amdgpu_clock_and_voltage_limits { + u32 sclk; + u32 mclk; + u16 vddc; + u16 vddci; +}; + +struct amdgpu_clock_array { + u32 count; + u32 *values; +}; + +struct amdgpu_clock_voltage_dependency_entry { + u32 clk; + u16 v; +}; + +struct amdgpu_clock_voltage_dependency_table { + u32 count; + struct amdgpu_clock_voltage_dependency_entry *entries; +}; + +union amdgpu_cac_leakage_entry { + struct { + u16 vddc; + u32 leakage; + }; + struct { + u16 vddc1; + u16 vddc2; + u16 vddc3; + }; +}; + +struct amdgpu_cac_leakage_table { + u32 count; + union amdgpu_cac_leakage_entry *entries; +}; + +struct amdgpu_phase_shedding_limits_entry { + u16 voltage; + u32 sclk; + u32 mclk; +}; + +struct amdgpu_phase_shedding_limits_table { + u32 count; + struct amdgpu_phase_shedding_limits_entry *entries; +}; + +struct amdgpu_uvd_clock_voltage_dependency_entry { + u32 vclk; + u32 dclk; + u16 v; +}; + +struct amdgpu_uvd_clock_voltage_dependency_table { + u8 count; + struct amdgpu_uvd_clock_voltage_dependency_entry *entries; +}; + +struct amdgpu_vce_clock_voltage_dependency_entry { + u32 ecclk; + u32 evclk; + u16 v; +}; + +struct amdgpu_vce_clock_voltage_dependency_table { + u8 count; + struct amdgpu_vce_clock_voltage_dependency_entry *entries; +}; + +struct amdgpu_ppm_table { + u8 ppm_design; + u16 cpu_core_number; + u32 platform_tdp; + u32 small_ac_platform_tdp; + u32 platform_tdc; + u32 small_ac_platform_tdc; + u32 apu_tdp; + u32 dgpu_tdp; + u32 dgpu_ulv_power; + u32 tj_max; +}; + +struct amdgpu_cac_tdp_table { + u16 tdp; + u16 configurable_tdp; + u16 tdc; + u16 battery_power_limit; + u16 small_power_limit; + u16 low_cac_leakage; + u16 high_cac_leakage; + u16 maximum_power_delivery_limit; +}; + +struct amdgpu_dpm_dynamic_state { + struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_sclk; + struct amdgpu_clock_voltage_dependency_table vddci_dependency_on_mclk; + struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_mclk; + struct amdgpu_clock_voltage_dependency_table mvdd_dependency_on_mclk; + struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_dispclk; + struct amdgpu_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table; + struct amdgpu_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table; + struct amdgpu_clock_voltage_dependency_table samu_clock_voltage_dependency_table; + struct amdgpu_clock_voltage_dependency_table acp_clock_voltage_dependency_table; + struct amdgpu_clock_voltage_dependency_table vddgfx_dependency_on_sclk; + struct amdgpu_clock_array valid_sclk_values; + struct amdgpu_clock_array valid_mclk_values; + struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_dc; + struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_ac; + u32 mclk_sclk_ratio; + u32 sclk_mclk_delta; + u16 vddc_vddci_delta; + u16 min_vddc_for_pcie_gen2; + struct amdgpu_cac_leakage_table cac_leakage_table; + struct amdgpu_phase_shedding_limits_table phase_shedding_limits_table; + struct amdgpu_ppm_table *ppm_table; + struct amdgpu_cac_tdp_table *cac_tdp_table; +}; + +struct amdgpu_dpm_fan { + u16 t_min; + u16 t_med; + u16 t_high; + u16 pwm_min; + u16 pwm_med; + u16 pwm_high; + u8 t_hyst; + u32 cycle_delay; + u16 t_max; + u8 control_mode; + u16 default_max_fan_pwm; + u16 default_fan_output_sensitivity; + u16 fan_output_sensitivity; + bool ucode_fan_control; +}; + +enum amdgpu_pcie_gen { + AMDGPU_PCIE_GEN1 = 0, + AMDGPU_PCIE_GEN2 = 1, + AMDGPU_PCIE_GEN3 = 2, + AMDGPU_PCIE_GEN_INVALID = 0xffff +}; + +#define amdgpu_dpm_pre_set_power_state(adev) \ + ((adev)->powerplay.pp_funcs->pre_set_power_state((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_set_power_state(adev) \ + ((adev)->powerplay.pp_funcs->set_power_state((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_post_set_power_state(adev) \ + ((adev)->powerplay.pp_funcs->post_set_power_state((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_display_configuration_changed(adev) \ + ((adev)->powerplay.pp_funcs->display_configuration_changed((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_print_power_state(adev, ps) \ + ((adev)->powerplay.pp_funcs->print_power_state((adev)->powerplay.pp_handle, (ps))) + +#define amdgpu_dpm_vblank_too_short(adev) \ + ((adev)->powerplay.pp_funcs->vblank_too_short((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_enable_bapm(adev, e) \ + ((adev)->powerplay.pp_funcs->enable_bapm((adev)->powerplay.pp_handle, (e))) + +#define amdgpu_dpm_set_fan_control_mode(adev, m) \ + ((adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m))) + +#define amdgpu_dpm_get_fan_control_mode(adev) \ + ((adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_set_fan_speed_percent(adev, s) \ + ((adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s))) + +#define amdgpu_dpm_get_fan_speed_percent(adev, s) \ + ((adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s))) + +#define amdgpu_dpm_get_fan_speed_rpm(adev, s) \ + ((adev)->powerplay.pp_funcs->get_fan_speed_rpm)((adev)->powerplay.pp_handle, (s)) + +#define amdgpu_dpm_set_fan_speed_rpm(adev, s) \ + ((adev)->powerplay.pp_funcs->set_fan_speed_rpm)((adev)->powerplay.pp_handle, (s)) + +#define amdgpu_dpm_force_performance_level(adev, l) \ + ((adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l))) + +#define amdgpu_dpm_get_current_power_state(adev) \ + ((adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)) + +#define amdgpu_smu_get_current_power_state(adev) \ + ((adev)->smu.ppt_funcs->get_current_power_state(&((adev)->smu))) + +#define amdgpu_smu_set_power_state(adev) \ + ((adev)->smu.ppt_funcs->set_power_state(&((adev)->smu))) + +#define amdgpu_dpm_get_pp_num_states(adev, data) \ + ((adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)) + +#define amdgpu_dpm_get_pp_table(adev, table) \ + ((adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table)) + +#define amdgpu_dpm_set_pp_table(adev, buf, size) \ + ((adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size)) + +#define amdgpu_dpm_print_clock_levels(adev, type, buf) \ + ((adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf)) + +#define amdgpu_dpm_force_clock_level(adev, type, level) \ + ((adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)) + +#define amdgpu_dpm_get_sclk_od(adev) \ + ((adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_set_sclk_od(adev, value) \ + ((adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)) + +#define amdgpu_dpm_get_mclk_od(adev) \ + ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_set_mclk_od(adev, value) \ + ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value)) + +#define amdgpu_dpm_dispatch_task(adev, task_id, user_state) \ + ((adev)->powerplay.pp_funcs->dispatch_tasks)((adev)->powerplay.pp_handle, (task_id), (user_state)) + +#define amdgpu_dpm_check_state_equal(adev, cps, rps, equal) \ + ((adev)->powerplay.pp_funcs->check_state_equal((adev)->powerplay.pp_handle, (cps), (rps), (equal))) + +#define amdgpu_dpm_get_vce_clock_state(adev, i) \ + ((adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i))) + +#define amdgpu_dpm_get_performance_level(adev) \ + ((adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_reset_power_profile_state(adev, request) \ + ((adev)->powerplay.pp_funcs->reset_power_profile_state(\ + (adev)->powerplay.pp_handle, request)) + +#define amdgpu_dpm_switch_power_profile(adev, type, en) \ + ((adev)->powerplay.pp_funcs->switch_power_profile(\ + (adev)->powerplay.pp_handle, type, en)) + +#define amdgpu_dpm_set_clockgating_by_smu(adev, msg_id) \ + ((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\ + (adev)->powerplay.pp_handle, msg_id)) + +#define amdgpu_dpm_get_power_profile_mode(adev, buf) \ + ((adev)->powerplay.pp_funcs->get_power_profile_mode(\ + (adev)->powerplay.pp_handle, buf)) + +#define amdgpu_dpm_set_power_profile_mode(adev, parameter, size) \ + ((adev)->powerplay.pp_funcs->set_power_profile_mode(\ + (adev)->powerplay.pp_handle, parameter, size)) + +#define amdgpu_dpm_odn_edit_dpm_table(adev, type, parameter, size) \ + ((adev)->powerplay.pp_funcs->odn_edit_dpm_table(\ + (adev)->powerplay.pp_handle, type, parameter, size)) + +#define amdgpu_dpm_enable_mgpu_fan_boost(adev) \ + ((adev)->powerplay.pp_funcs->enable_mgpu_fan_boost(\ + (adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_get_ppfeature_status(adev, buf) \ + ((adev)->powerplay.pp_funcs->get_ppfeature_status(\ + (adev)->powerplay.pp_handle, (buf))) + +#define amdgpu_dpm_set_ppfeature_status(adev, ppfeatures) \ + ((adev)->powerplay.pp_funcs->set_ppfeature_status(\ + (adev)->powerplay.pp_handle, (ppfeatures))) + +struct amdgpu_dpm { + struct amdgpu_ps *ps; + /* number of valid power states */ + int num_ps; + /* current power state that is active */ + struct amdgpu_ps *current_ps; + /* requested power state */ + struct amdgpu_ps *requested_ps; + /* boot up power state */ + struct amdgpu_ps *boot_ps; + /* default uvd power state */ + struct amdgpu_ps *uvd_ps; + /* vce requirements */ + u32 num_of_vce_states; + struct amd_vce_state vce_states[AMD_MAX_VCE_LEVELS]; + enum amd_vce_level vce_level; + enum amd_pm_state_type state; + enum amd_pm_state_type user_state; + enum amd_pm_state_type last_state; + enum amd_pm_state_type last_user_state; + u32 platform_caps; + u32 voltage_response_time; + u32 backbias_response_time; + void *priv; + u32 new_active_crtcs; + int new_active_crtc_count; + u32 current_active_crtcs; + int current_active_crtc_count; + struct amdgpu_dpm_dynamic_state dyn_state; + struct amdgpu_dpm_fan fan; + u32 tdp_limit; + u32 near_tdp_limit; + u32 near_tdp_limit_adjusted; + u32 sq_ramping_threshold; + u32 cac_leakage; + u16 tdp_od_limit; + u32 tdp_adjustment; + u16 load_line_slope; + bool power_control; + /* special states active */ + bool thermal_active; + bool uvd_active; + bool vce_active; + /* thermal handling */ + struct amdgpu_dpm_thermal thermal; + /* forced levels */ + enum amd_dpm_forced_level forced_level; +}; + +struct amdgpu_pm { + struct mutex mutex; + u32 current_sclk; + u32 current_mclk; + u32 default_sclk; + u32 default_mclk; + struct amdgpu_i2c_chan *i2c_bus; + /* internal thermal controller on rv6xx+ */ + enum amdgpu_int_thermal_type int_thermal_type; + struct device *int_hwmon_dev; + /* fan control parameters */ + bool no_fan; + u8 fan_pulses_per_revolution; + u8 fan_min_rpm; + u8 fan_max_rpm; + /* dpm */ + bool dpm_enabled; + bool sysfs_initialized; + struct amdgpu_dpm dpm; + const struct firmware *fw; /* SMC firmware */ + uint32_t fw_version; + uint32_t pcie_gen_mask; + uint32_t pcie_mlw_mask; + struct amd_pp_display_configuration pm_display_cfg;/* set by dc */ + uint32_t smu_prv_buffer_size; + struct amdgpu_bo *smu_prv_buffer; + bool ac_power; + /* powerplay feature */ + uint32_t pp_feature; + +}; + +#define R600_SSTU_DFLT 0 +#define R600_SST_DFLT 0x00C8 + +/* XXX are these ok? */ +#define R600_TEMP_RANGE_MIN (90 * 1000) +#define R600_TEMP_RANGE_MAX (120 * 1000) + +#define FDO_PWM_MODE_STATIC 1 +#define FDO_PWM_MODE_STATIC_RPM 5 + +enum amdgpu_td { + AMDGPU_TD_AUTO, + AMDGPU_TD_UP, + AMDGPU_TD_DOWN, +}; + +enum amdgpu_display_watermark { + AMDGPU_DISPLAY_WATERMARK_LOW = 0, + AMDGPU_DISPLAY_WATERMARK_HIGH = 1, +}; + +enum amdgpu_display_gap +{ + AMDGPU_PM_DISPLAY_GAP_VBLANK_OR_WM = 0, + AMDGPU_PM_DISPLAY_GAP_VBLANK = 1, + AMDGPU_PM_DISPLAY_GAP_WATERMARK = 2, + AMDGPU_PM_DISPLAY_GAP_IGNORE = 3, +}; + +void amdgpu_dpm_print_class_info(u32 class, u32 class2); +void amdgpu_dpm_print_cap_info(u32 caps); +void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, + struct amdgpu_ps *rps); +u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev); +u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev); +void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev); +int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor, + void *data, uint32_t *size); + +bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor); + +int amdgpu_get_platform_caps(struct amdgpu_device *adev); + +int amdgpu_parse_extended_power_table(struct amdgpu_device *adev); +void amdgpu_free_extended_power_table(struct amdgpu_device *adev); + +void amdgpu_add_thermal_controller(struct amdgpu_device *adev); + +enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev, + u32 sys_mask, + enum amdgpu_pcie_gen asic_gen, + enum amdgpu_pcie_gen default_gen); + +struct amd_vce_state* +amdgpu_get_vce_clock_state(void *handle, u32 idx); + +int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, + uint32_t block_type, bool gate); + +extern int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low); + +extern int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low); + +#endif
diff --git a/amdgpu/amdgpu_drv.h b/amdgpu/amdgpu_drv.h new file mode 100644 index 0000000..e3a4f70 --- /dev/null +++ b/amdgpu/amdgpu_drv.h
@@ -0,0 +1,48 @@ +/* amdgpu_drv.h -- Private header for amdgpu driver -*- linux-c -*- + * + * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas. + * Copyright 2000 VA Linux Systems, Inc., Fremont, California. + * All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_DRV_H__ +#define __AMDGPU_DRV_H__ + +#include <linux/firmware.h> +#include <linux/platform_device.h> + +#include "amd_shared.h" + +/* General customization: + */ + +#define DRIVER_AUTHOR "AMD linux driver team" + +#define DRIVER_NAME "amdgpu" +#define DRIVER_DESC "AMD GPU" +#define DRIVER_DATE "20150101" + +long amdgpu_drm_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg); + +#endif
diff --git a/amdgpu/amdgpu_encoders.c b/amdgpu/amdgpu_encoders.c new file mode 100644 index 0000000..571a6df --- /dev/null +++ b/amdgpu/amdgpu_encoders.c
@@ -0,0 +1,246 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <drm/drm_crtc_helper.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_connectors.h" +#include "amdgpu_display.h" +#include "atom.h" +#include "atombios_encoders.h" + +void +amdgpu_link_encoder_connector(struct drm_device *dev) +{ + struct amdgpu_device *adev = dev->dev_private; + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + struct drm_encoder *encoder; + struct amdgpu_encoder *amdgpu_encoder; + + /* walk the list and link encoders to connectors */ + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + amdgpu_connector = to_amdgpu_connector(connector); + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + amdgpu_encoder = to_amdgpu_encoder(encoder); + if (amdgpu_encoder->devices & amdgpu_connector->devices) { + drm_connector_attach_encoder(connector, encoder); + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + amdgpu_atombios_encoder_init_backlight(amdgpu_encoder, connector); + adev->mode_info.bl_encoder = amdgpu_encoder; + } + } + } + } +} + +void amdgpu_encoder_set_active_device(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_connector *connector; + + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + if (connector->encoder == encoder) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + amdgpu_encoder->active_device = amdgpu_encoder->devices & amdgpu_connector->devices; + DRM_DEBUG_KMS("setting active device to %08x from %08x %08x for encoder %d\n", + amdgpu_encoder->active_device, amdgpu_encoder->devices, + amdgpu_connector->devices, encoder->encoder_type); + } + } +} + +struct drm_connector * +amdgpu_get_connector_for_encoder(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + amdgpu_connector = to_amdgpu_connector(connector); + if (amdgpu_encoder->active_device & amdgpu_connector->devices) + return connector; + } + return NULL; +} + +struct drm_connector * +amdgpu_get_connector_for_encoder_init(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + amdgpu_connector = to_amdgpu_connector(connector); + if (amdgpu_encoder->devices & amdgpu_connector->devices) + return connector; + } + return NULL; +} + +struct drm_encoder *amdgpu_get_external_encoder(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_encoder *other_encoder; + struct amdgpu_encoder *other_amdgpu_encoder; + + if (amdgpu_encoder->is_ext_encoder) + return NULL; + + list_for_each_entry(other_encoder, &dev->mode_config.encoder_list, head) { + if (other_encoder == encoder) + continue; + other_amdgpu_encoder = to_amdgpu_encoder(other_encoder); + if (other_amdgpu_encoder->is_ext_encoder && + (amdgpu_encoder->devices & other_amdgpu_encoder->devices)) + return other_encoder; + } + return NULL; +} + +u16 amdgpu_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder) +{ + struct drm_encoder *other_encoder = amdgpu_get_external_encoder(encoder); + + if (other_encoder) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(other_encoder); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_TRAVIS: + case ENCODER_OBJECT_ID_NUTMEG: + return amdgpu_encoder->encoder_id; + default: + return ENCODER_OBJECT_ID_NONE; + } + } + return ENCODER_OBJECT_ID_NONE; +} + +void amdgpu_panel_mode_fixup(struct drm_encoder *encoder, + struct drm_display_mode *adjusted_mode) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode; + unsigned hblank = native_mode->htotal - native_mode->hdisplay; + unsigned vblank = native_mode->vtotal - native_mode->vdisplay; + unsigned hover = native_mode->hsync_start - native_mode->hdisplay; + unsigned vover = native_mode->vsync_start - native_mode->vdisplay; + unsigned hsync_width = native_mode->hsync_end - native_mode->hsync_start; + unsigned vsync_width = native_mode->vsync_end - native_mode->vsync_start; + + adjusted_mode->clock = native_mode->clock; + adjusted_mode->flags = native_mode->flags; + + adjusted_mode->hdisplay = native_mode->hdisplay; + adjusted_mode->vdisplay = native_mode->vdisplay; + + adjusted_mode->htotal = native_mode->hdisplay + hblank; + adjusted_mode->hsync_start = native_mode->hdisplay + hover; + adjusted_mode->hsync_end = adjusted_mode->hsync_start + hsync_width; + + adjusted_mode->vtotal = native_mode->vdisplay + vblank; + adjusted_mode->vsync_start = native_mode->vdisplay + vover; + adjusted_mode->vsync_end = adjusted_mode->vsync_start + vsync_width; + + drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V); + + adjusted_mode->crtc_hdisplay = native_mode->hdisplay; + adjusted_mode->crtc_vdisplay = native_mode->vdisplay; + + adjusted_mode->crtc_htotal = adjusted_mode->crtc_hdisplay + hblank; + adjusted_mode->crtc_hsync_start = adjusted_mode->crtc_hdisplay + hover; + adjusted_mode->crtc_hsync_end = adjusted_mode->crtc_hsync_start + hsync_width; + + adjusted_mode->crtc_vtotal = adjusted_mode->crtc_vdisplay + vblank; + adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vdisplay + vover; + adjusted_mode->crtc_vsync_end = adjusted_mode->crtc_vsync_start + vsync_width; + +} + +bool amdgpu_dig_monitor_is_duallink(struct drm_encoder *encoder, + u32 pixel_clock) +{ + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + struct amdgpu_connector_atom_dig *dig_connector; + + connector = amdgpu_get_connector_for_encoder(encoder); + /* if we don't have an active device yet, just use one of + * the connectors tied to the encoder. + */ + if (!connector) + connector = amdgpu_get_connector_for_encoder_init(encoder); + amdgpu_connector = to_amdgpu_connector(connector); + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_HDMIB: + if (amdgpu_connector->use_digital) { + /* HDMI 1.3 supports up to 340 Mhz over single link */ + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + if (pixel_clock > 340000) + return true; + else + return false; + } else { + if (pixel_clock > 165000) + return true; + else + return false; + } + } else + return false; + case DRM_MODE_CONNECTOR_DVID: + case DRM_MODE_CONNECTOR_HDMIA: + case DRM_MODE_CONNECTOR_DisplayPort: + dig_connector = amdgpu_connector->con_priv; + if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || + (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) + return false; + else { + /* HDMI 1.3 supports up to 340 Mhz over single link */ + if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector))) { + if (pixel_clock > 340000) + return true; + else + return false; + } else { + if (pixel_clock > 165000) + return true; + else + return false; + } + } + default: + return false; + } +}
diff --git a/amdgpu/amdgpu_fb.c b/amdgpu/amdgpu_fb.c new file mode 100644 index 0000000..eb3569b --- /dev/null +++ b/amdgpu/amdgpu_fb.c
@@ -0,0 +1,394 @@ +/* + * Copyright © 2007 David Airlie + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * David Airlie + */ + +#include <linux/module.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/vga_switcheroo.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_crtc.h> +#include <drm/drm_crtc_helper.h> +#include <drm/drm_fb_helper.h> +#include <drm/drm_fourcc.h> + +#include "amdgpu.h" +#include "cikd.h" +#include "amdgpu_gem.h" + +#include "amdgpu_display.h" + +/* object hierarchy - + this contains a helper + a amdgpu fb + the helper contains a pointer to amdgpu framebuffer baseclass. +*/ + +static int +amdgpufb_open(struct fb_info *info, int user) +{ + struct drm_fb_helper *fb_helper = info->par; + int ret = pm_runtime_get_sync(fb_helper->dev->dev); + if (ret < 0 && ret != -EACCES) { + pm_runtime_mark_last_busy(fb_helper->dev->dev); + pm_runtime_put_autosuspend(fb_helper->dev->dev); + return ret; + } + return 0; +} + +static int +amdgpufb_release(struct fb_info *info, int user) +{ + struct drm_fb_helper *fb_helper = info->par; + + pm_runtime_mark_last_busy(fb_helper->dev->dev); + pm_runtime_put_autosuspend(fb_helper->dev->dev); + return 0; +} + +static struct fb_ops amdgpufb_ops = { + .owner = THIS_MODULE, + DRM_FB_HELPER_DEFAULT_OPS, + .fb_open = amdgpufb_open, + .fb_release = amdgpufb_release, + .fb_fillrect = drm_fb_helper_cfb_fillrect, + .fb_copyarea = drm_fb_helper_cfb_copyarea, + .fb_imageblit = drm_fb_helper_cfb_imageblit, +}; + + +int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int cpp, bool tiled) +{ + int aligned = width; + int pitch_mask = 0; + + switch (cpp) { + case 1: + pitch_mask = 255; + break; + case 2: + pitch_mask = 127; + break; + case 3: + case 4: + pitch_mask = 63; + break; + } + + aligned += pitch_mask; + aligned &= ~pitch_mask; + return aligned * cpp; +} + +static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj) +{ + struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); + int ret; + + ret = amdgpu_bo_reserve(abo, true); + if (likely(ret == 0)) { + amdgpu_bo_kunmap(abo); + amdgpu_bo_unpin(abo); + amdgpu_bo_unreserve(abo); + } + drm_gem_object_put_unlocked(gobj); +} + +static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, + struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_gem_object **gobj_p) +{ + const struct drm_format_info *info; + struct amdgpu_device *adev = rfbdev->adev; + struct drm_gem_object *gobj = NULL; + struct amdgpu_bo *abo = NULL; + bool fb_tiled = false; /* useful for testing */ + u32 tiling_flags = 0, domain; + int ret; + int aligned_size, size; + int height = mode_cmd->height; + u32 cpp; + + info = drm_get_format_info(adev->ddev, mode_cmd); + cpp = info->cpp[0]; + + /* need to align pitch with crtc limits */ + mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp, + fb_tiled); + domain = amdgpu_display_supported_domains(adev); + + height = ALIGN(mode_cmd->height, 8); + size = mode_cmd->pitches[0] * height; + aligned_size = ALIGN(size, PAGE_SIZE); + ret = amdgpu_gem_object_create(adev, aligned_size, 0, domain, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | + AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | + AMDGPU_GEM_CREATE_VRAM_CLEARED, + ttm_bo_type_kernel, NULL, &gobj); + if (ret) { + pr_err("failed to allocate framebuffer (%d)\n", aligned_size); + return -ENOMEM; + } + abo = gem_to_amdgpu_bo(gobj); + + if (fb_tiled) + tiling_flags = AMDGPU_TILING_SET(ARRAY_MODE, GRPH_ARRAY_2D_TILED_THIN1); + + ret = amdgpu_bo_reserve(abo, false); + if (unlikely(ret != 0)) + goto out_unref; + + if (tiling_flags) { + ret = amdgpu_bo_set_tiling_flags(abo, + tiling_flags); + if (ret) + dev_err(adev->dev, "FB failed to set tiling flags\n"); + } + + + ret = amdgpu_bo_pin(abo, domain); + if (ret) { + amdgpu_bo_unreserve(abo); + goto out_unref; + } + + ret = amdgpu_ttm_alloc_gart(&abo->tbo); + if (ret) { + amdgpu_bo_unreserve(abo); + dev_err(adev->dev, "%p bind failed\n", abo); + goto out_unref; + } + + ret = amdgpu_bo_kmap(abo, NULL); + amdgpu_bo_unreserve(abo); + if (ret) { + goto out_unref; + } + + *gobj_p = gobj; + return 0; +out_unref: + amdgpufb_destroy_pinned_object(gobj); + *gobj_p = NULL; + return ret; +} + +static int amdgpufb_create(struct drm_fb_helper *helper, + struct drm_fb_helper_surface_size *sizes) +{ + struct amdgpu_fbdev *rfbdev = (struct amdgpu_fbdev *)helper; + struct amdgpu_device *adev = rfbdev->adev; + struct fb_info *info; + struct drm_framebuffer *fb = NULL; + struct drm_mode_fb_cmd2 mode_cmd; + struct drm_gem_object *gobj = NULL; + struct amdgpu_bo *abo = NULL; + int ret; + unsigned long tmp; + + mode_cmd.width = sizes->surface_width; + mode_cmd.height = sizes->surface_height; + + if (sizes->surface_bpp == 24) + sizes->surface_bpp = 32; + + mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, + sizes->surface_depth); + + ret = amdgpufb_create_pinned_object(rfbdev, &mode_cmd, &gobj); + if (ret) { + DRM_ERROR("failed to create fbcon object %d\n", ret); + return ret; + } + + abo = gem_to_amdgpu_bo(gobj); + + /* okay we have an object now allocate the framebuffer */ + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) { + ret = PTR_ERR(info); + goto out; + } + + ret = amdgpu_display_framebuffer_init(adev->ddev, &rfbdev->rfb, + &mode_cmd, gobj); + if (ret) { + DRM_ERROR("failed to initialize framebuffer %d\n", ret); + goto out; + } + + fb = &rfbdev->rfb.base; + + /* setup helper */ + rfbdev->helper.fb = fb; + + info->fbops = &amdgpufb_ops; + + tmp = amdgpu_bo_gpu_offset(abo) - adev->gmc.vram_start; + info->fix.smem_start = adev->gmc.aper_base + tmp; + info->fix.smem_len = amdgpu_bo_size(abo); + info->screen_base = amdgpu_bo_kptr(abo); + info->screen_size = amdgpu_bo_size(abo); + + drm_fb_helper_fill_info(info, &rfbdev->helper, sizes); + + /* setup aperture base/size for vesafb takeover */ + info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base; + info->apertures->ranges[0].size = adev->gmc.aper_size; + + /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ + + if (info->screen_base == NULL) { + ret = -ENOSPC; + goto out; + } + + DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start); + DRM_INFO("vram apper at 0x%lX\n", (unsigned long)adev->gmc.aper_base); + DRM_INFO("size %lu\n", (unsigned long)amdgpu_bo_size(abo)); + DRM_INFO("fb depth is %d\n", fb->format->depth); + DRM_INFO(" pitch is %d\n", fb->pitches[0]); + + vga_switcheroo_client_fb_set(adev->ddev->pdev, info); + return 0; + +out: + if (abo) { + + } + if (fb && ret) { + drm_gem_object_put_unlocked(gobj); + drm_framebuffer_unregister_private(fb); + drm_framebuffer_cleanup(fb); + kfree(fb); + } + return ret; +} + +static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev) +{ + struct amdgpu_framebuffer *rfb = &rfbdev->rfb; + + drm_fb_helper_unregister_fbi(&rfbdev->helper); + + if (rfb->base.obj[0]) { + amdgpufb_destroy_pinned_object(rfb->base.obj[0]); + rfb->base.obj[0] = NULL; + drm_framebuffer_unregister_private(&rfb->base); + drm_framebuffer_cleanup(&rfb->base); + } + drm_fb_helper_fini(&rfbdev->helper); + + return 0; +} + +static const struct drm_fb_helper_funcs amdgpu_fb_helper_funcs = { + .fb_probe = amdgpufb_create, +}; + +int amdgpu_fbdev_init(struct amdgpu_device *adev) +{ + struct amdgpu_fbdev *rfbdev; + int bpp_sel = 32; + int ret; + + /* don't init fbdev on hw without DCE */ + if (!adev->mode_info.mode_config_initialized) + return 0; + + /* don't init fbdev if there are no connectors */ + if (list_empty(&adev->ddev->mode_config.connector_list)) + return 0; + + /* select 8 bpp console on low vram cards */ + if (adev->gmc.real_vram_size <= (32*1024*1024)) + bpp_sel = 8; + + rfbdev = kzalloc(sizeof(struct amdgpu_fbdev), GFP_KERNEL); + if (!rfbdev) + return -ENOMEM; + + rfbdev->adev = adev; + adev->mode_info.rfbdev = rfbdev; + + drm_fb_helper_prepare(adev->ddev, &rfbdev->helper, + &amdgpu_fb_helper_funcs); + + ret = drm_fb_helper_init(adev->ddev, &rfbdev->helper, + AMDGPUFB_CONN_LIMIT); + if (ret) { + kfree(rfbdev); + return ret; + } + + drm_fb_helper_single_add_all_connectors(&rfbdev->helper); + + /* disable all the possible outputs/crtcs before entering KMS mode */ + if (!amdgpu_device_has_dc_support(adev)) + drm_helper_disable_unused_functions(adev->ddev); + + drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); + return 0; +} + +void amdgpu_fbdev_fini(struct amdgpu_device *adev) +{ + if (!adev->mode_info.rfbdev) + return; + + amdgpu_fbdev_destroy(adev->ddev, adev->mode_info.rfbdev); + kfree(adev->mode_info.rfbdev); + adev->mode_info.rfbdev = NULL; +} + +void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state) +{ + if (adev->mode_info.rfbdev) + drm_fb_helper_set_suspend_unlocked(&adev->mode_info.rfbdev->helper, + state); +} + +int amdgpu_fbdev_total_size(struct amdgpu_device *adev) +{ + struct amdgpu_bo *robj; + int size = 0; + + if (!adev->mode_info.rfbdev) + return 0; + + robj = gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0]); + size += amdgpu_bo_size(robj); + return size; +} + +bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj) +{ + if (!adev->mode_info.rfbdev) + return false; + if (robj == gem_to_amdgpu_bo(adev->mode_info.rfbdev->rfb.base.obj[0])) + return true; + return false; +}
diff --git a/amdgpu/amdgpu_fence.c b/amdgpu/amdgpu_fence.c new file mode 100644 index 0000000..23085b3 --- /dev/null +++ b/amdgpu/amdgpu_fence.c
@@ -0,0 +1,778 @@ +/* + * Copyright 2009 Jerome Glisse. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Jerome Glisse <glisse@freedesktop.org> + * Dave Airlie + */ +#include <linux/seq_file.h> +#include <linux/atomic.h> +#include <linux/wait.h> +#include <linux/kref.h> +#include <linux/slab.h> +#include <linux/firmware.h> + +#include <drm/drm_debugfs.h> + +#include "amdgpu.h" +#include "amdgpu_trace.h" + +/* + * Fences + * Fences mark an event in the GPUs pipeline and are used + * for GPU/CPU synchronization. When the fence is written, + * it is expected that all buffers associated with that fence + * are no longer in use by the associated ring on the GPU and + * that the the relevant GPU caches have been flushed. + */ + +struct amdgpu_fence { + struct dma_fence base; + + /* RB, DMA, etc. */ + struct amdgpu_ring *ring; +}; + +static struct kmem_cache *amdgpu_fence_slab; + +int amdgpu_fence_slab_init(void) +{ + amdgpu_fence_slab = kmem_cache_create( + "amdgpu_fence", sizeof(struct amdgpu_fence), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!amdgpu_fence_slab) + return -ENOMEM; + return 0; +} + +void amdgpu_fence_slab_fini(void) +{ + rcu_barrier(); + kmem_cache_destroy(amdgpu_fence_slab); +} +/* + * Cast helper + */ +static const struct dma_fence_ops amdgpu_fence_ops; +static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f) +{ + struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base); + + if (__f->base.ops == &amdgpu_fence_ops) + return __f; + + return NULL; +} + +/** + * amdgpu_fence_write - write a fence value + * + * @ring: ring the fence is associated with + * @seq: sequence number to write + * + * Writes a fence value to memory (all asics). + */ +static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) +{ + struct amdgpu_fence_driver *drv = &ring->fence_drv; + + if (drv->cpu_addr) + *drv->cpu_addr = cpu_to_le32(seq); +} + +/** + * amdgpu_fence_read - read a fence value + * + * @ring: ring the fence is associated with + * + * Reads a fence value from memory (all asics). + * Returns the value of the fence read from memory. + */ +static u32 amdgpu_fence_read(struct amdgpu_ring *ring) +{ + struct amdgpu_fence_driver *drv = &ring->fence_drv; + u32 seq = 0; + + if (drv->cpu_addr) + seq = le32_to_cpu(*drv->cpu_addr); + else + seq = atomic_read(&drv->last_seq); + + return seq; +} + +/** + * amdgpu_fence_emit - emit a fence on the requested ring + * + * @ring: ring the fence is associated with + * @f: resulting fence object + * + * Emits a fence command on the requested ring (all asics). + * Returns 0 on success, -ENOMEM on failure. + */ +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, + unsigned flags) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_fence *fence; + struct dma_fence __rcu **ptr; + uint32_t seq; + int r; + + fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); + if (fence == NULL) + return -ENOMEM; + + seq = ++ring->fence_drv.sync_seq; + fence->ring = ring; + dma_fence_init(&fence->base, &amdgpu_fence_ops, + &ring->fence_drv.lock, + adev->fence_context + ring->idx, + seq); + amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, + seq, flags | AMDGPU_FENCE_FLAG_INT); + + ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; + if (unlikely(rcu_dereference_protected(*ptr, 1))) { + struct dma_fence *old; + + rcu_read_lock(); + old = dma_fence_get_rcu_safe(ptr); + rcu_read_unlock(); + + if (old) { + r = dma_fence_wait(old, false); + dma_fence_put(old); + if (r) + return r; + } + } + + /* This function can't be called concurrently anyway, otherwise + * emitting the fence would mess up the hardware ring buffer. + */ + rcu_assign_pointer(*ptr, dma_fence_get(&fence->base)); + + *f = &fence->base; + + return 0; +} + +/** + * amdgpu_fence_emit_polling - emit a fence on the requeste ring + * + * @ring: ring the fence is associated with + * @s: resulting sequence number + * + * Emits a fence command on the requested ring (all asics). + * Used For polling fence. + * Returns 0 on success, -ENOMEM on failure. + */ +int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s) +{ + uint32_t seq; + + if (!s) + return -EINVAL; + + seq = ++ring->fence_drv.sync_seq; + amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, + seq, 0); + + *s = seq; + + return 0; +} + +/** + * amdgpu_fence_schedule_fallback - schedule fallback check + * + * @ring: pointer to struct amdgpu_ring + * + * Start a timer as fallback to our interrupts. + */ +static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring) +{ + mod_timer(&ring->fence_drv.fallback_timer, + jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT); +} + +/** + * amdgpu_fence_process - check for fence activity + * + * @ring: pointer to struct amdgpu_ring + * + * Checks the current fence value and calculates the last + * signalled fence value. Wakes the fence queue if the + * sequence number has increased. + * + * Returns true if fence was processed + */ +bool amdgpu_fence_process(struct amdgpu_ring *ring) +{ + struct amdgpu_fence_driver *drv = &ring->fence_drv; + uint32_t seq, last_seq; + int r; + + do { + last_seq = atomic_read(&ring->fence_drv.last_seq); + seq = amdgpu_fence_read(ring); + + } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq); + + if (del_timer(&ring->fence_drv.fallback_timer) && + seq != ring->fence_drv.sync_seq) + amdgpu_fence_schedule_fallback(ring); + + if (unlikely(seq == last_seq)) + return false; + + last_seq &= drv->num_fences_mask; + seq &= drv->num_fences_mask; + + do { + struct dma_fence *fence, **ptr; + + ++last_seq; + last_seq &= drv->num_fences_mask; + ptr = &drv->fences[last_seq]; + + /* There is always exactly one thread signaling this fence slot */ + fence = rcu_dereference_protected(*ptr, 1); + RCU_INIT_POINTER(*ptr, NULL); + + if (!fence) + continue; + + r = dma_fence_signal(fence); + if (!r) + DMA_FENCE_TRACE(fence, "signaled from irq context\n"); + else + BUG(); + + dma_fence_put(fence); + } while (last_seq != seq); + + return true; +} + +/** + * amdgpu_fence_fallback - fallback for hardware interrupts + * + * @work: delayed work item + * + * Checks for fence activity. + */ +static void amdgpu_fence_fallback(struct timer_list *t) +{ + struct amdgpu_ring *ring = from_timer(ring, t, + fence_drv.fallback_timer); + + if (amdgpu_fence_process(ring)) + DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name); +} + +/** + * amdgpu_fence_wait_empty - wait for all fences to signal + * + * @adev: amdgpu device pointer + * @ring: ring index the fence is associated with + * + * Wait for all fences on the requested ring to signal (all asics). + * Returns 0 if the fences have passed, error for all other cases. + */ +int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) +{ + uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq); + struct dma_fence *fence, **ptr; + int r; + + if (!seq) + return 0; + + ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; + rcu_read_lock(); + fence = rcu_dereference(*ptr); + if (!fence || !dma_fence_get_rcu(fence)) { + rcu_read_unlock(); + return 0; + } + rcu_read_unlock(); + + r = dma_fence_wait(fence, false); + dma_fence_put(fence); + return r; +} + +/** + * amdgpu_fence_wait_polling - busy wait for givn sequence number + * + * @ring: ring index the fence is associated with + * @wait_seq: sequence number to wait + * @timeout: the timeout for waiting in usecs + * + * Wait for all fences on the requested ring to signal (all asics). + * Returns left time if no timeout, 0 or minus if timeout. + */ +signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring, + uint32_t wait_seq, + signed long timeout) +{ + uint32_t seq; + + do { + seq = amdgpu_fence_read(ring); + udelay(5); + timeout -= 5; + } while ((int32_t)(wait_seq - seq) > 0 && timeout > 0); + + return timeout > 0 ? timeout : 0; +} +/** + * amdgpu_fence_count_emitted - get the count of emitted fences + * + * @ring: ring the fence is associated with + * + * Get the number of fences emitted on the requested ring (all asics). + * Returns the number of emitted fences on the ring. Used by the + * dynpm code to ring track activity. + */ +unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring) +{ + uint64_t emitted; + + /* We are not protected by ring lock when reading the last sequence + * but it's ok to report slightly wrong fence count here. + */ + amdgpu_fence_process(ring); + emitted = 0x100000000ull; + emitted -= atomic_read(&ring->fence_drv.last_seq); + emitted += READ_ONCE(ring->fence_drv.sync_seq); + return lower_32_bits(emitted); +} + +/** + * amdgpu_fence_driver_start_ring - make the fence driver + * ready for use on the requested ring. + * + * @ring: ring to start the fence driver on + * @irq_src: interrupt source to use for this ring + * @irq_type: interrupt type to use for this ring + * + * Make the fence driver ready for processing (all asics). + * Not all asics have all rings, so each asic will only + * start the fence driver on the rings it has. + * Returns 0 for success, errors for failure. + */ +int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq_src, + unsigned irq_type) +{ + struct amdgpu_device *adev = ring->adev; + uint64_t index; + + if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) { + ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs]; + ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4); + } else { + /* put fence directly behind firmware */ + index = ALIGN(adev->uvd.fw->size, 8); + ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index; + ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index; + } + amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq)); + amdgpu_irq_get(adev, irq_src, irq_type); + + ring->fence_drv.irq_src = irq_src; + ring->fence_drv.irq_type = irq_type; + ring->fence_drv.initialized = true; + + DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr " + "0x%016llx, cpu addr 0x%p\n", ring->name, + ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr); + return 0; +} + +/** + * amdgpu_fence_driver_init_ring - init the fence driver + * for the requested ring. + * + * @ring: ring to init the fence driver on + * @num_hw_submission: number of entries on the hardware queue + * + * Init the fence driver for the requested ring (all asics). + * Helper function for amdgpu_fence_driver_init(). + */ +int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, + unsigned num_hw_submission) +{ + struct amdgpu_device *adev = ring->adev; + long timeout; + int r; + + if (!adev) + return -EINVAL; + + /* Check that num_hw_submission is a power of two */ + if ((num_hw_submission & (num_hw_submission - 1)) != 0) + return -EINVAL; + + ring->fence_drv.cpu_addr = NULL; + ring->fence_drv.gpu_addr = 0; + ring->fence_drv.sync_seq = 0; + atomic_set(&ring->fence_drv.last_seq, 0); + ring->fence_drv.initialized = false; + + timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0); + + ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1; + spin_lock_init(&ring->fence_drv.lock); + ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *), + GFP_KERNEL); + if (!ring->fence_drv.fences) + return -ENOMEM; + + /* No need to setup the GPU scheduler for KIQ ring */ + if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) { + switch (ring->funcs->type) { + case AMDGPU_RING_TYPE_GFX: + timeout = adev->gfx_timeout; + break; + case AMDGPU_RING_TYPE_COMPUTE: + /* + * For non-sriov case, no timeout enforce + * on compute ring by default. Unless user + * specifies a timeout for compute ring. + * + * For sriov case, always use the timeout + * as gfx ring + */ + if (!amdgpu_sriov_vf(ring->adev)) + timeout = adev->compute_timeout; + else + timeout = adev->gfx_timeout; + break; + case AMDGPU_RING_TYPE_SDMA: + timeout = adev->sdma_timeout; + break; + default: + timeout = adev->video_timeout; + break; + } + + r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, + num_hw_submission, amdgpu_job_hang_limit, + timeout, ring->name); + if (r) { + DRM_ERROR("Failed to create scheduler on ring %s.\n", + ring->name); + return r; + } + } + + return 0; +} + +/** + * amdgpu_fence_driver_init - init the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Init the fence driver for all possible rings (all asics). + * Not all asics have all rings, so each asic will only + * start the fence driver on the rings it has using + * amdgpu_fence_driver_start_ring(). + * Returns 0 for success. + */ +int amdgpu_fence_driver_init(struct amdgpu_device *adev) +{ + if (amdgpu_debugfs_fence_init(adev)) + dev_err(adev->dev, "fence debugfs file creation failed\n"); + + return 0; +} + +/** + * amdgpu_fence_driver_fini - tear down the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Tear down the fence driver for all possible rings (all asics). + */ +void amdgpu_fence_driver_fini(struct amdgpu_device *adev) +{ + unsigned i, j; + int r; + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + + if (!ring || !ring->fence_drv.initialized) + continue; + r = amdgpu_fence_wait_empty(ring); + if (r) { + /* no need to trigger GPU reset as we are unloading */ + amdgpu_fence_driver_force_completion(ring); + } + amdgpu_irq_put(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + drm_sched_fini(&ring->sched); + del_timer_sync(&ring->fence_drv.fallback_timer); + for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) + dma_fence_put(ring->fence_drv.fences[j]); + kfree(ring->fence_drv.fences); + ring->fence_drv.fences = NULL; + ring->fence_drv.initialized = false; + } +} + +/** + * amdgpu_fence_driver_suspend - suspend the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Suspend the fence driver for all possible rings (all asics). + */ +void amdgpu_fence_driver_suspend(struct amdgpu_device *adev) +{ + int i, r; + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (!ring || !ring->fence_drv.initialized) + continue; + + /* wait for gpu to finish processing current batch */ + r = amdgpu_fence_wait_empty(ring); + if (r) { + /* delay GPU reset to resume */ + amdgpu_fence_driver_force_completion(ring); + } + + /* disable the interrupt */ + amdgpu_irq_put(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + } +} + +/** + * amdgpu_fence_driver_resume - resume the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Resume the fence driver for all possible rings (all asics). + * Not all asics have all rings, so each asic will only + * start the fence driver on the rings it has using + * amdgpu_fence_driver_start_ring(). + * Returns 0 for success. + */ +void amdgpu_fence_driver_resume(struct amdgpu_device *adev) +{ + int i; + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (!ring || !ring->fence_drv.initialized) + continue; + + /* enable the interrupt */ + amdgpu_irq_get(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + } +} + +/** + * amdgpu_fence_driver_force_completion - force signal latest fence of ring + * + * @ring: fence of the ring to signal + * + */ +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring) +{ + amdgpu_fence_write(ring, ring->fence_drv.sync_seq); + amdgpu_fence_process(ring); +} + +/* + * Common fence implementation + */ + +static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) +{ + return "amdgpu"; +} + +static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) +{ + struct amdgpu_fence *fence = to_amdgpu_fence(f); + return (const char *)fence->ring->name; +} + +/** + * amdgpu_fence_enable_signaling - enable signalling on fence + * @fence: fence + * + * This function is called with fence_queue lock held, and adds a callback + * to fence_queue that checks if this fence is signaled, and if so it + * signals the fence and removes itself. + */ +static bool amdgpu_fence_enable_signaling(struct dma_fence *f) +{ + struct amdgpu_fence *fence = to_amdgpu_fence(f); + struct amdgpu_ring *ring = fence->ring; + + if (!timer_pending(&ring->fence_drv.fallback_timer)) + amdgpu_fence_schedule_fallback(ring); + + DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); + + return true; +} + +/** + * amdgpu_fence_free - free up the fence memory + * + * @rcu: RCU callback head + * + * Free up the fence memory after the RCU grace period. + */ +static void amdgpu_fence_free(struct rcu_head *rcu) +{ + struct dma_fence *f = container_of(rcu, struct dma_fence, rcu); + struct amdgpu_fence *fence = to_amdgpu_fence(f); + kmem_cache_free(amdgpu_fence_slab, fence); +} + +/** + * amdgpu_fence_release - callback that fence can be freed + * + * @fence: fence + * + * This function is called when the reference count becomes zero. + * It just RCU schedules freeing up the fence. + */ +static void amdgpu_fence_release(struct dma_fence *f) +{ + call_rcu(&f->rcu, amdgpu_fence_free); +} + +static const struct dma_fence_ops amdgpu_fence_ops = { + .get_driver_name = amdgpu_fence_get_driver_name, + .get_timeline_name = amdgpu_fence_get_timeline_name, + .enable_signaling = amdgpu_fence_enable_signaling, + .release = amdgpu_fence_release, +}; + +/* + * Fence debugfs + */ +#if defined(CONFIG_DEBUG_FS) +static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + int i; + + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + if (!ring || !ring->fence_drv.initialized) + continue; + + amdgpu_fence_process(ring); + + seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name); + seq_printf(m, "Last signaled fence 0x%08x\n", + atomic_read(&ring->fence_drv.last_seq)); + seq_printf(m, "Last emitted 0x%08x\n", + ring->fence_drv.sync_seq); + + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX || + ring->funcs->type == AMDGPU_RING_TYPE_SDMA) { + seq_printf(m, "Last signaled trailing fence 0x%08x\n", + le32_to_cpu(*ring->trail_fence_cpu_addr)); + seq_printf(m, "Last emitted 0x%08x\n", + ring->trail_seq); + } + + if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) + continue; + + /* set in CP_VMID_PREEMPT and preemption occurred */ + seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2))); + /* set in CP_VMID_RESET and reset occurred */ + seq_printf(m, "Last reset 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 4))); + /* Both preemption and reset occurred */ + seq_printf(m, "Last both 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 6))); + } + return 0; +} + +/** + * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover + * + * Manually trigger a gpu reset at the next fence wait. + */ +static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + + seq_printf(m, "gpu recover\n"); + amdgpu_device_gpu_recover(adev, NULL); + + return 0; +} + +static const struct drm_info_list amdgpu_debugfs_fence_list[] = { + {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL}, + {"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL} +}; + +static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = { + {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL}, +}; +#endif + +int amdgpu_debugfs_fence_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + if (amdgpu_sriov_vf(adev)) + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list_sriov, 1); + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2); +#else + return 0; +#endif +} +
diff --git a/amdgpu/amdgpu_gart.c b/amdgpu/amdgpu_gart.c new file mode 100644 index 0000000..d79ab1d --- /dev/null +++ b/amdgpu/amdgpu_gart.c
@@ -0,0 +1,397 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ + +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <drm/amdgpu_drm.h> +#ifdef CONFIG_X86 +#include <asm/set_memory.h> +#endif +#include "amdgpu.h" + +/* + * GART + * The GART (Graphics Aperture Remapping Table) is an aperture + * in the GPU's address space. System pages can be mapped into + * the aperture and look like contiguous pages from the GPU's + * perspective. A page table maps the pages in the aperture + * to the actual backing pages in system memory. + * + * Radeon GPUs support both an internal GART, as described above, + * and AGP. AGP works similarly, but the GART table is configured + * and maintained by the northbridge rather than the driver. + * Radeon hw has a separate AGP aperture that is programmed to + * point to the AGP aperture provided by the northbridge and the + * requests are passed through to the northbridge aperture. + * Both AGP and internal GART can be used at the same time, however + * that is not currently supported by the driver. + * + * This file handles the common internal GART management. + */ + +/* + * Common GART table functions. + */ + +/** + * amdgpu_dummy_page_init - init dummy page used by the driver + * + * @adev: amdgpu_device pointer + * + * Allocate the dummy page used by the driver (all asics). + * This dummy page is used by the driver as a filler for gart entries + * when pages are taken out of the GART + * Returns 0 on sucess, -ENOMEM on failure. + */ +static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) +{ + struct page *dummy_page = adev->mman.bdev.glob->dummy_read_page; + + if (adev->dummy_page_addr) + return 0; + adev->dummy_page_addr = pci_map_page(adev->pdev, dummy_page, 0, + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + if (pci_dma_mapping_error(adev->pdev, adev->dummy_page_addr)) { + dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n"); + adev->dummy_page_addr = 0; + return -ENOMEM; + } + return 0; +} + +/** + * amdgpu_dummy_page_fini - free dummy page used by the driver + * + * @adev: amdgpu_device pointer + * + * Frees the dummy page used by the driver (all asics). + */ +static void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) +{ + if (!adev->dummy_page_addr) + return; + pci_unmap_page(adev->pdev, adev->dummy_page_addr, + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + adev->dummy_page_addr = 0; +} + +/** + * amdgpu_gart_table_vram_alloc - allocate vram for gart page table + * + * @adev: amdgpu_device pointer + * + * Allocate video memory for GART page table + * (pcie r4xx, r5xx+). These asics require the + * gart table to be in video memory. + * Returns 0 for success, error for failure. + */ +int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev) +{ + int r; + + if (adev->gart.bo == NULL) { + struct amdgpu_bo_param bp; + + memset(&bp, 0, sizeof(bp)); + bp.size = adev->gart.table_size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; + bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | + AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + r = amdgpu_bo_create(adev, &bp, &adev->gart.bo); + if (r) { + return r; + } + } + return 0; +} + +/** + * amdgpu_gart_table_vram_pin - pin gart page table in vram + * + * @adev: amdgpu_device pointer + * + * Pin the GART page table in vram so it will not be moved + * by the memory manager (pcie r4xx, r5xx+). These asics require the + * gart table to be in video memory. + * Returns 0 for success, error for failure. + */ +int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev) +{ + int r; + + r = amdgpu_bo_reserve(adev->gart.bo, false); + if (unlikely(r != 0)) + return r; + r = amdgpu_bo_pin(adev->gart.bo, AMDGPU_GEM_DOMAIN_VRAM); + if (r) { + amdgpu_bo_unreserve(adev->gart.bo); + return r; + } + r = amdgpu_bo_kmap(adev->gart.bo, &adev->gart.ptr); + if (r) + amdgpu_bo_unpin(adev->gart.bo); + amdgpu_bo_unreserve(adev->gart.bo); + return r; +} + +/** + * amdgpu_gart_table_vram_unpin - unpin gart page table in vram + * + * @adev: amdgpu_device pointer + * + * Unpin the GART page table in vram (pcie r4xx, r5xx+). + * These asics require the gart table to be in video memory. + */ +void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev) +{ + int r; + + if (adev->gart.bo == NULL) { + return; + } + r = amdgpu_bo_reserve(adev->gart.bo, true); + if (likely(r == 0)) { + amdgpu_bo_kunmap(adev->gart.bo); + amdgpu_bo_unpin(adev->gart.bo); + amdgpu_bo_unreserve(adev->gart.bo); + adev->gart.ptr = NULL; + } +} + +/** + * amdgpu_gart_table_vram_free - free gart page table vram + * + * @adev: amdgpu_device pointer + * + * Free the video memory used for the GART page table + * (pcie r4xx, r5xx+). These asics require the gart table to + * be in video memory. + */ +void amdgpu_gart_table_vram_free(struct amdgpu_device *adev) +{ + if (adev->gart.bo == NULL) { + return; + } + amdgpu_bo_unref(&adev->gart.bo); +} + +/* + * Common gart functions. + */ +/** + * amdgpu_gart_unbind - unbind pages from the gart page table + * + * @adev: amdgpu_device pointer + * @offset: offset into the GPU's gart aperture + * @pages: number of pages to unbind + * + * Unbinds the requested pages from the gart page table and + * replaces them with the dummy page (all asics). + * Returns 0 for success, -EINVAL for failure. + */ +int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, + int pages) +{ + unsigned t; + unsigned p; + int i, j; + u64 page_base; + /* Starting from VEGA10, system bit must be 0 to mean invalid. */ + uint64_t flags = 0; + + if (!adev->gart.ready) { + WARN(1, "trying to unbind memory from uninitialized GART !\n"); + return -EINVAL; + } + + t = offset / AMDGPU_GPU_PAGE_SIZE; + p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE; + for (i = 0; i < pages; i++, p++) { +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + adev->gart.pages[p] = NULL; +#endif + page_base = adev->dummy_page_addr; + if (!adev->gart.ptr) + continue; + + for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) { + amdgpu_gmc_set_pte_pde(adev, adev->gart.ptr, + t, page_base, flags); + page_base += AMDGPU_GPU_PAGE_SIZE; + } + } + mb(); + amdgpu_asic_flush_hdp(adev, NULL); + amdgpu_gmc_flush_gpu_tlb(adev, 0, 0); + return 0; +} + +/** + * amdgpu_gart_map - map dma_addresses into GART entries + * + * @adev: amdgpu_device pointer + * @offset: offset into the GPU's gart aperture + * @pages: number of pages to bind + * @dma_addr: DMA addresses of pages + * @flags: page table entry flags + * @dst: CPU address of the gart table + * + * Map the dma_addresses into GART entries (all asics). + * Returns 0 for success, -EINVAL for failure. + */ +int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, + int pages, dma_addr_t *dma_addr, uint64_t flags, + void *dst) +{ + uint64_t page_base; + unsigned i, j, t; + + if (!adev->gart.ready) { + WARN(1, "trying to bind memory to uninitialized GART !\n"); + return -EINVAL; + } + + t = offset / AMDGPU_GPU_PAGE_SIZE; + + for (i = 0; i < pages; i++) { + page_base = dma_addr[i]; + for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) { + amdgpu_gmc_set_pte_pde(adev, dst, t, page_base, flags); + page_base += AMDGPU_GPU_PAGE_SIZE; + } + } + return 0; +} + +/** + * amdgpu_gart_bind - bind pages into the gart page table + * + * @adev: amdgpu_device pointer + * @offset: offset into the GPU's gart aperture + * @pages: number of pages to bind + * @pagelist: pages to bind + * @dma_addr: DMA addresses of pages + * + * Binds the requested pages to the gart page table + * (all asics). + * Returns 0 for success, -EINVAL for failure. + */ +int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, + int pages, struct page **pagelist, dma_addr_t *dma_addr, + uint64_t flags) +{ +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + unsigned i,t,p; +#endif + int r; + + if (!adev->gart.ready) { + WARN(1, "trying to bind memory to uninitialized GART !\n"); + return -EINVAL; + } + +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + t = offset / AMDGPU_GPU_PAGE_SIZE; + p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE; + for (i = 0; i < pages; i++, p++) + adev->gart.pages[p] = pagelist ? pagelist[i] : NULL; +#endif + + if (!adev->gart.ptr) + return 0; + + r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags, + adev->gart.ptr); + if (r) + return r; + + mb(); + amdgpu_asic_flush_hdp(adev, NULL); + amdgpu_gmc_flush_gpu_tlb(adev, 0, 0); + return 0; +} + +/** + * amdgpu_gart_init - init the driver info for managing the gart + * + * @adev: amdgpu_device pointer + * + * Allocate the dummy page and init the gart driver info (all asics). + * Returns 0 for success, error for failure. + */ +int amdgpu_gart_init(struct amdgpu_device *adev) +{ + int r; + + if (adev->dummy_page_addr) + return 0; + + /* We need PAGE_SIZE >= AMDGPU_GPU_PAGE_SIZE */ + if (PAGE_SIZE < AMDGPU_GPU_PAGE_SIZE) { + DRM_ERROR("Page size is smaller than GPU page size!\n"); + return -EINVAL; + } + r = amdgpu_gart_dummy_page_init(adev); + if (r) + return r; + /* Compute table size */ + adev->gart.num_cpu_pages = adev->gmc.gart_size / PAGE_SIZE; + adev->gart.num_gpu_pages = adev->gmc.gart_size / AMDGPU_GPU_PAGE_SIZE; + DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", + adev->gart.num_cpu_pages, adev->gart.num_gpu_pages); + +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + /* Allocate pages table */ + adev->gart.pages = vzalloc(array_size(sizeof(void *), + adev->gart.num_cpu_pages)); + if (adev->gart.pages == NULL) + return -ENOMEM; +#endif + + return 0; +} + +/** + * amdgpu_gart_fini - tear down the driver info for managing the gart + * + * @adev: amdgpu_device pointer + * + * Tear down the gart driver info and free the dummy page (all asics). + */ +void amdgpu_gart_fini(struct amdgpu_device *adev) +{ +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + vfree(adev->gart.pages); + adev->gart.pages = NULL; +#endif + amdgpu_gart_dummy_page_fini(adev); +}
diff --git a/amdgpu/amdgpu_gart.h b/amdgpu/amdgpu_gart.h new file mode 100644 index 0000000..afa2e28 --- /dev/null +++ b/amdgpu/amdgpu_gart.h
@@ -0,0 +1,73 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_GART_H__ +#define __AMDGPU_GART_H__ + +#include <linux/types.h> + +/* + * GART structures, functions & helpers + */ +struct amdgpu_device; +struct amdgpu_bo; + +#define AMDGPU_GPU_PAGE_SIZE 4096 +#define AMDGPU_GPU_PAGE_MASK (AMDGPU_GPU_PAGE_SIZE - 1) +#define AMDGPU_GPU_PAGE_SHIFT 12 +#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK) + +#define AMDGPU_GPU_PAGES_IN_CPU_PAGE (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE) + +struct amdgpu_gart { + struct amdgpu_bo *bo; + /* CPU kmapped address of gart table */ + void *ptr; + unsigned num_gpu_pages; + unsigned num_cpu_pages; + unsigned table_size; +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + struct page **pages; +#endif + bool ready; + + /* Asic default pte flags */ + uint64_t gart_pte_flags; +}; + +int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev); +void amdgpu_gart_table_vram_free(struct amdgpu_device *adev); +int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev); +void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev); +int amdgpu_gart_init(struct amdgpu_device *adev); +void amdgpu_gart_fini(struct amdgpu_device *adev); +int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, + int pages); +int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, + int pages, dma_addr_t *dma_addr, uint64_t flags, + void *dst); +int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, + int pages, struct page **pagelist, + dma_addr_t *dma_addr, uint64_t flags); + +#endif
diff --git a/amdgpu/amdgpu_gds.h b/amdgpu/amdgpu_gds.h new file mode 100644 index 0000000..f6ac1e9 --- /dev/null +++ b/amdgpu/amdgpu_gds.h
@@ -0,0 +1,44 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_GDS_H__ +#define __AMDGPU_GDS_H__ + +struct amdgpu_ring; +struct amdgpu_bo; + +struct amdgpu_gds { + uint32_t gds_size; + uint32_t gws_size; + uint32_t oa_size; + uint32_t gds_compute_max_wave_id; +}; + +struct amdgpu_gds_reg_offset { + uint32_t mem_base; + uint32_t mem_size; + uint32_t gws; + uint32_t oa; +}; + +#endif /* __AMDGPU_GDS_H__ */
diff --git a/amdgpu/amdgpu_gem.c b/amdgpu/amdgpu_gem.c new file mode 100644 index 0000000..939f830 --- /dev/null +++ b/amdgpu/amdgpu_gem.c
@@ -0,0 +1,890 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#include <linux/ktime.h> +#include <linux/module.h> +#include <linux/pagemap.h> +#include <linux/pci.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_debugfs.h> + +#include "amdgpu.h" +#include "amdgpu_display.h" +#include "amdgpu_xgmi.h" + +void amdgpu_gem_object_free(struct drm_gem_object *gobj) +{ + struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj); + + if (robj) { + amdgpu_mn_unregister(robj); + amdgpu_bo_unref(&robj); + } +} + +int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, + int alignment, u32 initial_domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct drm_gem_object **obj) +{ + struct amdgpu_bo *bo; + struct amdgpu_bo_param bp; + int r; + + memset(&bp, 0, sizeof(bp)); + *obj = NULL; + + bp.size = size; + bp.byte_align = alignment; + bp.type = type; + bp.resv = resv; + bp.preferred_domain = initial_domain; +retry: + bp.flags = flags; + bp.domain = initial_domain; + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) { + if (r != -ERESTARTSYS) { + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + goto retry; + } + + if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { + initial_domain |= AMDGPU_GEM_DOMAIN_GTT; + goto retry; + } + DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n", + size, initial_domain, alignment, r); + } + return r; + } + *obj = &bo->gem_base; + + return 0; +} + +void amdgpu_gem_force_release(struct amdgpu_device *adev) +{ + struct drm_device *ddev = adev->ddev; + struct drm_file *file; + + mutex_lock(&ddev->filelist_mutex); + + list_for_each_entry(file, &ddev->filelist, lhead) { + struct drm_gem_object *gobj; + int handle; + + WARN_ONCE(1, "Still active user space clients!\n"); + spin_lock(&file->table_lock); + idr_for_each_entry(&file->object_idr, gobj, handle) { + WARN_ONCE(1, "And also active allocations!\n"); + drm_gem_object_put_unlocked(gobj); + } + idr_destroy(&file->object_idr); + spin_unlock(&file->table_lock); + } + + mutex_unlock(&ddev->filelist_mutex); +} + +/* + * Call from drm_gem_handle_create which appear in both new and open ioctl + * case. + */ +int amdgpu_gem_object_open(struct drm_gem_object *obj, + struct drm_file *file_priv) +{ + struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_va *bo_va; + struct mm_struct *mm; + int r; + + mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm); + if (mm && mm != current->mm) + return -EPERM; + + if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID && + abo->tbo.resv != vm->root.base.bo->tbo.resv) + return -EPERM; + + r = amdgpu_bo_reserve(abo, false); + if (r) + return r; + + bo_va = amdgpu_vm_bo_find(vm, abo); + if (!bo_va) { + bo_va = amdgpu_vm_bo_add(adev, vm, abo); + } else { + ++bo_va->ref_count; + } + amdgpu_bo_unreserve(abo); + return 0; +} + +void amdgpu_gem_object_close(struct drm_gem_object *obj, + struct drm_file *file_priv) +{ + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + + struct amdgpu_bo_list_entry vm_pd; + struct list_head list, duplicates; + struct ttm_validate_buffer tv; + struct ww_acquire_ctx ticket; + struct amdgpu_bo_va *bo_va; + int r; + + INIT_LIST_HEAD(&list); + INIT_LIST_HEAD(&duplicates); + + tv.bo = &bo->tbo; + tv.num_shared = 1; + list_add(&tv.head, &list); + + amdgpu_vm_get_pd_bo(vm, &list, &vm_pd); + + r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates, false); + if (r) { + dev_err(adev->dev, "leaking bo va because " + "we fail to reserve bo (%d)\n", r); + return; + } + bo_va = amdgpu_vm_bo_find(vm, bo); + if (bo_va && --bo_va->ref_count == 0) { + amdgpu_vm_bo_rmv(adev, bo_va); + + if (amdgpu_vm_ready(vm)) { + struct dma_fence *fence = NULL; + + r = amdgpu_vm_clear_freed(adev, vm, &fence); + if (unlikely(r)) { + dev_err(adev->dev, "failed to clear page " + "tables on GEM object close (%d)\n", r); + } + + if (fence) { + amdgpu_bo_fence(bo, fence, true); + dma_fence_put(fence); + } + } + } + ttm_eu_backoff_reservation(&ticket, &list); +} + +/* + * GEM ioctls. + */ +int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + union drm_amdgpu_gem_create *args = data; + uint64_t flags = args->in.domain_flags; + uint64_t size = args->in.bo_size; + struct reservation_object *resv = NULL; + struct drm_gem_object *gobj; + uint32_t handle; + int r; + + /* reject invalid gem flags */ + if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | + AMDGPU_GEM_CREATE_NO_CPU_ACCESS | + AMDGPU_GEM_CREATE_CPU_GTT_USWC | + AMDGPU_GEM_CREATE_VRAM_CLEARED | + AMDGPU_GEM_CREATE_VM_ALWAYS_VALID | + AMDGPU_GEM_CREATE_EXPLICIT_SYNC)) + + return -EINVAL; + + /* reject invalid gem domains */ + if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK) + return -EINVAL; + + /* create a gem object to contain this object in */ + if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS | + AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { + if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { + /* if gds bo is created from user space, it must be + * passed to bo list + */ + DRM_ERROR("GDS bo cannot be per-vm-bo\n"); + return -EINVAL; + } + flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + } + + if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { + r = amdgpu_bo_reserve(vm->root.base.bo, false); + if (r) + return r; + + resv = vm->root.base.bo->tbo.resv; + } + + r = amdgpu_gem_object_create(adev, size, args->in.alignment, + (u32)(0xffffffff & args->in.domains), + flags, ttm_bo_type_device, resv, &gobj); + if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { + if (!r) { + struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); + + abo->parent = amdgpu_bo_ref(vm->root.base.bo); + } + amdgpu_bo_unreserve(vm->root.base.bo); + } + if (r) + return r; + + r = drm_gem_handle_create(filp, gobj, &handle); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_put_unlocked(gobj); + if (r) + return r; + + memset(args, 0, sizeof(*args)); + args->out.handle = handle; + return 0; +} + +int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct ttm_operation_ctx ctx = { true, false }; + struct amdgpu_device *adev = dev->dev_private; + struct drm_amdgpu_gem_userptr *args = data; + struct drm_gem_object *gobj; + struct amdgpu_bo *bo; + uint32_t handle; + int r; + + if (offset_in_page(args->addr | args->size)) + return -EINVAL; + + /* reject unknown flag values */ + if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY | + AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE | + AMDGPU_GEM_USERPTR_REGISTER)) + return -EINVAL; + + if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) && + !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) { + + /* if we want to write to it we must install a MMU notifier */ + return -EACCES; + } + + /* create a gem object to contain this object in */ + r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU, + 0, ttm_bo_type_device, NULL, &gobj); + if (r) + return r; + + bo = gem_to_amdgpu_bo(gobj); + bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags); + if (r) + goto release_object; + + if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) { + r = amdgpu_mn_register(bo, args->addr); + if (r) + goto release_object; + } + + if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) { + r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); + if (r) + goto release_object; + + r = amdgpu_bo_reserve(bo, true); + if (r) + goto user_pages_done; + + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + amdgpu_bo_unreserve(bo); + if (r) + goto user_pages_done; + } + + r = drm_gem_handle_create(filp, gobj, &handle); + if (r) + goto user_pages_done; + + args->handle = handle; + +user_pages_done: + if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) + amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); + +release_object: + drm_gem_object_put_unlocked(gobj); + + return r; +} + +int amdgpu_mode_dumb_mmap(struct drm_file *filp, + struct drm_device *dev, + uint32_t handle, uint64_t *offset_p) +{ + struct drm_gem_object *gobj; + struct amdgpu_bo *robj; + + gobj = drm_gem_object_lookup(filp, handle); + if (gobj == NULL) { + return -ENOENT; + } + robj = gem_to_amdgpu_bo(gobj); + if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) || + (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { + drm_gem_object_put_unlocked(gobj); + return -EPERM; + } + *offset_p = amdgpu_bo_mmap_offset(robj); + drm_gem_object_put_unlocked(gobj); + return 0; +} + +int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + union drm_amdgpu_gem_mmap *args = data; + uint32_t handle = args->in.handle; + memset(args, 0, sizeof(*args)); + return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr); +} + +/** + * amdgpu_gem_timeout - calculate jiffies timeout from absolute value + * + * @timeout_ns: timeout in ns + * + * Calculate the timeout in jiffies from an absolute timeout in ns. + */ +unsigned long amdgpu_gem_timeout(uint64_t timeout_ns) +{ + unsigned long timeout_jiffies; + ktime_t timeout; + + /* clamp timeout if it's to large */ + if (((int64_t)timeout_ns) < 0) + return MAX_SCHEDULE_TIMEOUT; + + timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get()); + if (ktime_to_ns(timeout) < 0) + return 0; + + timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout)); + /* clamp timeout to avoid unsigned-> signed overflow */ + if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT ) + return MAX_SCHEDULE_TIMEOUT - 1; + + return timeout_jiffies; +} + +int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + union drm_amdgpu_gem_wait_idle *args = data; + struct drm_gem_object *gobj; + struct amdgpu_bo *robj; + uint32_t handle = args->in.handle; + unsigned long timeout = amdgpu_gem_timeout(args->in.timeout); + int r = 0; + long ret; + + gobj = drm_gem_object_lookup(filp, handle); + if (gobj == NULL) { + return -ENOENT; + } + robj = gem_to_amdgpu_bo(gobj); + ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, + timeout); + + /* ret == 0 means not signaled, + * ret > 0 means signaled + * ret < 0 means interrupted before timeout + */ + if (ret >= 0) { + memset(args, 0, sizeof(*args)); + args->out.status = (ret == 0); + } else + r = ret; + + drm_gem_object_put_unlocked(gobj); + return r; +} + +int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct drm_amdgpu_gem_metadata *args = data; + struct drm_gem_object *gobj; + struct amdgpu_bo *robj; + int r = -1; + + DRM_DEBUG("%d \n", args->handle); + gobj = drm_gem_object_lookup(filp, args->handle); + if (gobj == NULL) + return -ENOENT; + robj = gem_to_amdgpu_bo(gobj); + + r = amdgpu_bo_reserve(robj, false); + if (unlikely(r != 0)) + goto out; + + if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) { + amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info); + r = amdgpu_bo_get_metadata(robj, args->data.data, + sizeof(args->data.data), + &args->data.data_size_bytes, + &args->data.flags); + } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) { + if (args->data.data_size_bytes > sizeof(args->data.data)) { + r = -EINVAL; + goto unreserve; + } + r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info); + if (!r) + r = amdgpu_bo_set_metadata(robj, args->data.data, + args->data.data_size_bytes, + args->data.flags); + } + +unreserve: + amdgpu_bo_unreserve(robj); +out: + drm_gem_object_put_unlocked(gobj); + return r; +} + +/** + * amdgpu_gem_va_update_vm -update the bo_va in its VM + * + * @adev: amdgpu_device pointer + * @vm: vm to update + * @bo_va: bo_va to update + * @operation: map, unmap or clear + * + * Update the bo_va directly after setting its address. Errors are not + * vital here, so they are not reported back to userspace. + */ +static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo_va *bo_va, + uint32_t operation) +{ + int r; + + if (!amdgpu_vm_ready(vm)) + return; + + r = amdgpu_vm_clear_freed(adev, vm, NULL); + if (r) + goto error; + + if (operation == AMDGPU_VA_OP_MAP || + operation == AMDGPU_VA_OP_REPLACE) { + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + goto error; + } + + r = amdgpu_vm_update_directories(adev, vm); + +error: + if (r && r != -ERESTARTSYS) + DRM_ERROR("Couldn't update BO_VA (%d)\n", r); +} + +int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE | + AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE | + AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK; + const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE | + AMDGPU_VM_PAGE_PRT; + + struct drm_amdgpu_gem_va *args = data; + struct drm_gem_object *gobj; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_bo *abo; + struct amdgpu_bo_va *bo_va; + struct amdgpu_bo_list_entry vm_pd; + struct ttm_validate_buffer tv; + struct ww_acquire_ctx ticket; + struct list_head list, duplicates; + uint64_t va_flags; + int r = 0; + + if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { + dev_dbg(&dev->pdev->dev, + "va_address 0x%LX is in reserved area 0x%LX\n", + args->va_address, AMDGPU_VA_RESERVED_SIZE); + return -EINVAL; + } + + if (args->va_address >= AMDGPU_GMC_HOLE_START && + args->va_address < AMDGPU_GMC_HOLE_END) { + dev_dbg(&dev->pdev->dev, + "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n", + args->va_address, AMDGPU_GMC_HOLE_START, + AMDGPU_GMC_HOLE_END); + return -EINVAL; + } + + args->va_address &= AMDGPU_GMC_HOLE_MASK; + + if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { + dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n", + args->flags); + return -EINVAL; + } + + switch (args->operation) { + case AMDGPU_VA_OP_MAP: + case AMDGPU_VA_OP_UNMAP: + case AMDGPU_VA_OP_CLEAR: + case AMDGPU_VA_OP_REPLACE: + break; + default: + dev_dbg(&dev->pdev->dev, "unsupported operation %d\n", + args->operation); + return -EINVAL; + } + + INIT_LIST_HEAD(&list); + INIT_LIST_HEAD(&duplicates); + if ((args->operation != AMDGPU_VA_OP_CLEAR) && + !(args->flags & AMDGPU_VM_PAGE_PRT)) { + gobj = drm_gem_object_lookup(filp, args->handle); + if (gobj == NULL) + return -ENOENT; + abo = gem_to_amdgpu_bo(gobj); + tv.bo = &abo->tbo; + if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) + tv.num_shared = 1; + else + tv.num_shared = 0; + list_add(&tv.head, &list); + } else { + gobj = NULL; + abo = NULL; + } + + amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd); + + r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates, false); + if (r) + goto error_unref; + + if (abo) { + bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); + if (!bo_va) { + r = -ENOENT; + goto error_backoff; + } + } else if (args->operation != AMDGPU_VA_OP_CLEAR) { + bo_va = fpriv->prt_va; + } else { + bo_va = NULL; + } + + switch (args->operation) { + case AMDGPU_VA_OP_MAP: + va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags); + r = amdgpu_vm_bo_map(adev, bo_va, args->va_address, + args->offset_in_bo, args->map_size, + va_flags); + break; + case AMDGPU_VA_OP_UNMAP: + r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address); + break; + + case AMDGPU_VA_OP_CLEAR: + r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm, + args->va_address, + args->map_size); + break; + case AMDGPU_VA_OP_REPLACE: + va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags); + r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address, + args->offset_in_bo, args->map_size, + va_flags); + break; + default: + break; + } + if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !amdgpu_vm_debug) + amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, + args->operation); + +error_backoff: + ttm_eu_backoff_reservation(&ticket, &list); + +error_unref: + drm_gem_object_put_unlocked(gobj); + return r; +} + +int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + struct drm_amdgpu_gem_op *args = data; + struct drm_gem_object *gobj; + struct amdgpu_vm_bo_base *base; + struct amdgpu_bo *robj; + int r; + + gobj = drm_gem_object_lookup(filp, args->handle); + if (gobj == NULL) { + return -ENOENT; + } + robj = gem_to_amdgpu_bo(gobj); + + r = amdgpu_bo_reserve(robj, false); + if (unlikely(r)) + goto out; + + switch (args->op) { + case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: { + struct drm_amdgpu_gem_create_in info; + void __user *out = u64_to_user_ptr(args->value); + + info.bo_size = robj->gem_base.size; + info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT; + info.domains = robj->preferred_domains; + info.domain_flags = robj->flags; + amdgpu_bo_unreserve(robj); + if (copy_to_user(out, &info, sizeof(info))) + r = -EFAULT; + break; + } + case AMDGPU_GEM_OP_SET_PLACEMENT: + if (robj->prime_shared_count && (args->value & AMDGPU_GEM_DOMAIN_VRAM)) { + r = -EINVAL; + amdgpu_bo_unreserve(robj); + break; + } + if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) { + r = -EPERM; + amdgpu_bo_unreserve(robj); + break; + } + for (base = robj->vm_bo; base; base = base->next) + if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev), + amdgpu_ttm_adev(base->vm->root.base.bo->tbo.bdev))) { + r = -EINVAL; + amdgpu_bo_unreserve(robj); + goto out; + } + + + robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT | + AMDGPU_GEM_DOMAIN_CPU); + robj->allowed_domains = robj->preferred_domains; + if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) + robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; + + if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) + amdgpu_vm_bo_invalidate(adev, robj, true); + + amdgpu_bo_unreserve(robj); + break; + default: + amdgpu_bo_unreserve(robj); + r = -EINVAL; + } + +out: + drm_gem_object_put_unlocked(gobj); + return r; +} + +int amdgpu_mode_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args) +{ + struct amdgpu_device *adev = dev->dev_private; + struct drm_gem_object *gobj; + uint32_t handle; + u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + u32 domain; + int r; + + /* + * The buffer returned from this function should be cleared, but + * it can only be done if the ring is enabled or we'll fail to + * create the buffer. + */ + if (adev->mman.buffer_funcs_enabled) + flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; + + args->pitch = amdgpu_align_pitch(adev, args->width, + DIV_ROUND_UP(args->bpp, 8), 0); + args->size = (u64)args->pitch * args->height; + args->size = ALIGN(args->size, PAGE_SIZE); + domain = amdgpu_bo_get_preferred_pin_domain(adev, + amdgpu_display_supported_domains(adev)); + r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, + ttm_bo_type_device, NULL, &gobj); + if (r) + return -ENOMEM; + + r = drm_gem_handle_create(file_priv, gobj, &handle); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_put_unlocked(gobj); + if (r) { + return r; + } + args->handle = handle; + return 0; +} + +#if defined(CONFIG_DEBUG_FS) + +#define amdgpu_debugfs_gem_bo_print_flag(m, bo, flag) \ + if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ + seq_printf((m), " " #flag); \ + } + +static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data) +{ + struct drm_gem_object *gobj = ptr; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); + struct seq_file *m = data; + + struct dma_buf_attachment *attachment; + struct dma_buf *dma_buf; + unsigned domain; + const char *placement; + unsigned pin_count; + + domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); + switch (domain) { + case AMDGPU_GEM_DOMAIN_VRAM: + placement = "VRAM"; + break; + case AMDGPU_GEM_DOMAIN_GTT: + placement = " GTT"; + break; + case AMDGPU_GEM_DOMAIN_CPU: + default: + placement = " CPU"; + break; + } + seq_printf(m, "\t0x%08x: %12ld byte %s", + id, amdgpu_bo_size(bo), placement); + + pin_count = READ_ONCE(bo->pin_count); + if (pin_count) + seq_printf(m, " pin count %d", pin_count); + + dma_buf = READ_ONCE(bo->gem_base.dma_buf); + attachment = READ_ONCE(bo->gem_base.import_attach); + + if (attachment) + seq_printf(m, " imported from %p", dma_buf); + else if (dma_buf) + seq_printf(m, " exported as %p", dma_buf); + + amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); + amdgpu_debugfs_gem_bo_print_flag(m, bo, NO_CPU_ACCESS); + amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_GTT_USWC); + amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CLEARED); + amdgpu_debugfs_gem_bo_print_flag(m, bo, SHADOW); + amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CONTIGUOUS); + amdgpu_debugfs_gem_bo_print_flag(m, bo, VM_ALWAYS_VALID); + amdgpu_debugfs_gem_bo_print_flag(m, bo, EXPLICIT_SYNC); + + seq_printf(m, "\n"); + + return 0; +} + +static int amdgpu_debugfs_gem_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct drm_file *file; + int r; + + r = mutex_lock_interruptible(&dev->filelist_mutex); + if (r) + return r; + + list_for_each_entry(file, &dev->filelist, lhead) { + struct task_struct *task; + + /* + * Although we have a valid reference on file->pid, that does + * not guarantee that the task_struct who called get_pid() is + * still alive (e.g. get_pid(current) => fork() => exit()). + * Therefore, we need to protect this ->comm access using RCU. + */ + rcu_read_lock(); + task = pid_task(file->pid, PIDTYPE_PID); + seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid), + task ? task->comm : "<unknown>"); + rcu_read_unlock(); + + spin_lock(&file->table_lock); + idr_for_each(&file->object_idr, amdgpu_debugfs_gem_bo_info, m); + spin_unlock(&file->table_lock); + } + + mutex_unlock(&dev->filelist_mutex); + return 0; +} + +static const struct drm_info_list amdgpu_debugfs_gem_list[] = { + {"amdgpu_gem_info", &amdgpu_debugfs_gem_info, 0, NULL}, +}; +#endif + +int amdgpu_debugfs_gem_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1); +#endif + return 0; +}
diff --git a/amdgpu/amdgpu_gem.h b/amdgpu/amdgpu_gem.h new file mode 100644 index 0000000..b8ba6e2 --- /dev/null +++ b/amdgpu/amdgpu_gem.h
@@ -0,0 +1,78 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_GEM_H__ +#define __AMDGPU_GEM_H__ + +#include <drm/amdgpu_drm.h> +#include <drm/drm_gem.h> + +/* + * GEM. + */ + +#define AMDGPU_GEM_DOMAIN_MAX 0x3 +#define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base) + +void amdgpu_gem_object_free(struct drm_gem_object *obj); +int amdgpu_gem_object_open(struct drm_gem_object *obj, + struct drm_file *file_priv); +void amdgpu_gem_object_close(struct drm_gem_object *obj, + struct drm_file *file_priv); +unsigned long amdgpu_gem_timeout(uint64_t timeout_ns); + +/* + * GEM objects. + */ +void amdgpu_gem_force_release(struct amdgpu_device *adev); +int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, + int alignment, u32 initial_domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct drm_gem_object **obj); + +int amdgpu_mode_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args); +int amdgpu_mode_dumb_mmap(struct drm_file *filp, + struct drm_device *dev, + uint32_t handle, uint64_t *offset_p); + +int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_info_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); +int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +#endif
diff --git a/amdgpu/amdgpu_gfx.c b/amdgpu/amdgpu_gfx.c new file mode 100644 index 0000000..74066e1 --- /dev/null +++ b/amdgpu/amdgpu_gfx.c
@@ -0,0 +1,571 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu.h" +#include "amdgpu_gfx.h" +#include "amdgpu_rlc.h" + +/* delay 0.1 second to enable gfx off feature */ +#define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100) + +/* + * GPU GFX IP block helpers function. + */ + +int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, + int pipe, int queue) +{ + int bit = 0; + + bit += mec * adev->gfx.mec.num_pipe_per_mec + * adev->gfx.mec.num_queue_per_pipe; + bit += pipe * adev->gfx.mec.num_queue_per_pipe; + bit += queue; + + return bit; +} + +void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit, + int *mec, int *pipe, int *queue) +{ + *queue = bit % adev->gfx.mec.num_queue_per_pipe; + *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) + % adev->gfx.mec.num_pipe_per_mec; + *mec = (bit / adev->gfx.mec.num_queue_per_pipe) + / adev->gfx.mec.num_pipe_per_mec; + +} + +bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, + int mec, int pipe, int queue) +{ + return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), + adev->gfx.mec.queue_bitmap); +} + +int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, + int me, int pipe, int queue) +{ + int bit = 0; + + bit += me * adev->gfx.me.num_pipe_per_me + * adev->gfx.me.num_queue_per_pipe; + bit += pipe * adev->gfx.me.num_queue_per_pipe; + bit += queue; + + return bit; +} + +void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, + int *me, int *pipe, int *queue) +{ + *queue = bit % adev->gfx.me.num_queue_per_pipe; + *pipe = (bit / adev->gfx.me.num_queue_per_pipe) + % adev->gfx.me.num_pipe_per_me; + *me = (bit / adev->gfx.me.num_queue_per_pipe) + / adev->gfx.me.num_pipe_per_me; +} + +bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, + int me, int pipe, int queue) +{ + return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), + adev->gfx.me.queue_bitmap); +} + +/** + * amdgpu_gfx_scratch_get - Allocate a scratch register + * + * @adev: amdgpu_device pointer + * @reg: scratch register mmio offset + * + * Allocate a CP scratch register for use by the driver (all asics). + * Returns 0 on success or -EINVAL on failure. + */ +int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg) +{ + int i; + + i = ffs(adev->gfx.scratch.free_mask); + if (i != 0 && i <= adev->gfx.scratch.num_reg) { + i--; + adev->gfx.scratch.free_mask &= ~(1u << i); + *reg = adev->gfx.scratch.reg_base + i; + return 0; + } + return -EINVAL; +} + +/** + * amdgpu_gfx_scratch_free - Free a scratch register + * + * @adev: amdgpu_device pointer + * @reg: scratch register mmio offset + * + * Free a CP scratch register allocated for use by the driver (all asics) + */ +void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg) +{ + adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base); +} + +/** + * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter + * + * @mask: array in which the per-shader array disable masks will be stored + * @max_se: number of SEs + * @max_sh: number of SHs + * + * The bitmask of CUs to be disabled in the shader array determined by se and + * sh is stored in mask[se * max_sh + sh]. + */ +void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh) +{ + unsigned se, sh, cu; + const char *p; + + memset(mask, 0, sizeof(*mask) * max_se * max_sh); + + if (!amdgpu_disable_cu || !*amdgpu_disable_cu) + return; + + p = amdgpu_disable_cu; + for (;;) { + char *next; + int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); + if (ret < 3) { + DRM_ERROR("amdgpu: could not parse disable_cu\n"); + return; + } + + if (se < max_se && sh < max_sh && cu < 16) { + DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); + mask[se * max_sh + sh] |= 1u << cu; + } else { + DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", + se, sh, cu); + } + + next = strchr(p, ','); + if (!next) + break; + p = next + 1; + } +} + +static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev) +{ + if (amdgpu_compute_multipipe != -1) { + DRM_INFO("amdgpu: forcing compute pipe policy %d\n", + amdgpu_compute_multipipe); + return amdgpu_compute_multipipe == 1; + } + + /* FIXME: spreading the queues across pipes causes perf regressions + * on POLARIS11 compute workloads */ + if (adev->asic_type == CHIP_POLARIS11) + return false; + + return adev->gfx.mec.num_mec > 1; +} + +void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) +{ + int i, queue, pipe, mec; + bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); + + /* policy for amdgpu compute queue ownership */ + for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { + queue = i % adev->gfx.mec.num_queue_per_pipe; + pipe = (i / adev->gfx.mec.num_queue_per_pipe) + % adev->gfx.mec.num_pipe_per_mec; + mec = (i / adev->gfx.mec.num_queue_per_pipe) + / adev->gfx.mec.num_pipe_per_mec; + + /* we've run out of HW */ + if (mec >= adev->gfx.mec.num_mec) + break; + + if (multipipe_policy) { + /* policy: amdgpu owns the first two queues of the first MEC */ + if (mec == 0 && queue < 2) + set_bit(i, adev->gfx.mec.queue_bitmap); + } else { + /* policy: amdgpu owns all queues in the first pipe */ + if (mec == 0 && pipe == 0) + set_bit(i, adev->gfx.mec.queue_bitmap); + } + } + + /* update the number of active compute rings */ + adev->gfx.num_compute_rings = + bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); + + /* If you hit this case and edited the policy, you probably just + * need to increase AMDGPU_MAX_COMPUTE_RINGS */ + if (WARN_ON(adev->gfx.num_compute_rings > AMDGPU_MAX_COMPUTE_RINGS)) + adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; +} + +void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) +{ + int i, queue, pipe, me; + + for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) { + queue = i % adev->gfx.me.num_queue_per_pipe; + pipe = (i / adev->gfx.me.num_queue_per_pipe) + % adev->gfx.me.num_pipe_per_me; + me = (i / adev->gfx.me.num_queue_per_pipe) + / adev->gfx.me.num_pipe_per_me; + + if (me >= adev->gfx.me.num_me) + break; + /* policy: amdgpu owns the first queue per pipe at this stage + * will extend to mulitple queues per pipe later */ + if (me == 0 && queue < 1) + set_bit(i, adev->gfx.me.queue_bitmap); + } + + /* update the number of active graphics rings */ + adev->gfx.num_gfx_rings = + bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); +} + +static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + int queue_bit; + int mec, pipe, queue; + + queue_bit = adev->gfx.mec.num_mec + * adev->gfx.mec.num_pipe_per_mec + * adev->gfx.mec.num_queue_per_pipe; + + while (queue_bit-- >= 0) { + if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap)) + continue; + + amdgpu_gfx_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); + + /* + * 1. Using pipes 2/3 from MEC 2 seems cause problems. + * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN + * only can be issued on queue 0. + */ + if ((mec == 1 && pipe > 1) || queue != 0) + continue; + + ring->me = mec + 1; + ring->pipe = pipe; + ring->queue = queue; + + return 0; + } + + dev_err(adev->dev, "Failed to find a queue for KIQ\n"); + return -EINVAL; +} + +int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq) +{ + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + int r = 0; + + spin_lock_init(&kiq->ring_lock); + + r = amdgpu_device_wb_get(adev, &adev->virt.reg_val_offs); + if (r) + return r; + + ring->adev = NULL; + ring->ring_obj = NULL; + ring->use_doorbell = true; + ring->doorbell_index = adev->doorbell_index.kiq; + + r = amdgpu_gfx_kiq_acquire(adev, ring); + if (r) + return r; + + ring->eop_gpu_addr = kiq->eop_gpu_addr; + sprintf(ring->name, "kiq_%d.%d.%d", ring->me, ring->pipe, ring->queue); + r = amdgpu_ring_init(adev, ring, 1024, + irq, AMDGPU_CP_KIQ_IRQ_DRIVER0); + if (r) + dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); + + return r; +} + +void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq) +{ + amdgpu_device_wb_free(ring->adev, ring->adev->virt.reg_val_offs); + amdgpu_ring_fini(ring); +} + +void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev) +{ + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + + amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL); +} + +int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, + unsigned hpd_size) +{ + int r; + u32 *hpd; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + + r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj, + &kiq->eop_gpu_addr, (void **)&hpd); + if (r) { + dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); + return r; + } + + memset(hpd, 0, hpd_size); + + r = amdgpu_bo_reserve(kiq->eop_obj, true); + if (unlikely(r != 0)) + dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); + amdgpu_bo_kunmap(kiq->eop_obj); + amdgpu_bo_unreserve(kiq->eop_obj); + + return 0; +} + +/* create MQD for each compute/gfx queue */ +int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, + unsigned mqd_size) +{ + struct amdgpu_ring *ring = NULL; + int r, i; + + /* create MQD for KIQ */ + ring = &adev->gfx.kiq.ring; + if (!ring->mqd_obj) { + /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must + * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD + * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for + * KIQ MQD no matter SRIOV or Bare-metal + */ + r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj, + &ring->mqd_gpu_addr, &ring->mqd_ptr); + if (r) { + dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); + return r; + } + + /* prepare MQD backup */ + adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL); + if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]) + dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); + } + + if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring) { + /* create MQD for each KGQ */ + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + if (!ring->mqd_obj) { + r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, + &ring->mqd_gpu_addr, &ring->mqd_ptr); + if (r) { + dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); + return r; + } + + /* prepare MQD backup */ + adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); + if (!adev->gfx.me.mqd_backup[i]) + dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); + } + } + } + + /* create MQD for each KCQ */ + for (i = 0; i < adev->gfx.num_compute_rings; i++) { + ring = &adev->gfx.compute_ring[i]; + if (!ring->mqd_obj) { + r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj, + &ring->mqd_gpu_addr, &ring->mqd_ptr); + if (r) { + dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); + return r; + } + + /* prepare MQD backup */ + adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL); + if (!adev->gfx.mec.mqd_backup[i]) + dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); + } + } + + return 0; +} + +void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring = NULL; + int i; + + if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring) { + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + kfree(adev->gfx.me.mqd_backup[i]); + amdgpu_bo_free_kernel(&ring->mqd_obj, + &ring->mqd_gpu_addr, + &ring->mqd_ptr); + } + } + + for (i = 0; i < adev->gfx.num_compute_rings; i++) { + ring = &adev->gfx.compute_ring[i]; + kfree(adev->gfx.mec.mqd_backup[i]); + amdgpu_bo_free_kernel(&ring->mqd_obj, + &ring->mqd_gpu_addr, + &ring->mqd_ptr); + } + + ring = &adev->gfx.kiq.ring; + if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring) + kfree(adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS]); + kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]); + amdgpu_bo_free_kernel(&ring->mqd_obj, + &ring->mqd_gpu_addr, + &ring->mqd_ptr); +} + +int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) +{ + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *kiq_ring = &kiq->ring; + int i; + + if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) + return -EINVAL; + + if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * + adev->gfx.num_compute_rings)) + return -ENOMEM; + + for (i = 0; i < adev->gfx.num_compute_rings; i++) + kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i], + RESET_QUEUES, 0, 0); + + return amdgpu_ring_test_ring(kiq_ring); +} + +int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev) +{ + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint64_t queue_mask = 0; + int r, i; + + if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources) + return -EINVAL; + + for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { + if (!test_bit(i, adev->gfx.mec.queue_bitmap)) + continue; + + /* This situation may be hit in the future if a new HW + * generation exposes more than 64 queues. If so, the + * definition of queue_mask needs updating */ + if (WARN_ON(i > (sizeof(queue_mask)*8))) { + DRM_ERROR("Invalid KCQ enabled: %d\n", i); + break; + } + + queue_mask |= (1ull << i); + } + + DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe, + kiq_ring->queue); + + r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * + adev->gfx.num_compute_rings + + kiq->pmf->set_resources_size); + if (r) { + DRM_ERROR("Failed to lock KIQ (%d).\n", r); + return r; + } + + kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); + for (i = 0; i < adev->gfx.num_compute_rings; i++) + kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]); + + r = amdgpu_ring_test_helper(kiq_ring); + if (r) + DRM_ERROR("KCQ enable failed\n"); + + return r; +} + +/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable + * + * @adev: amdgpu_device pointer + * @bool enable true: enable gfx off feature, false: disable gfx off feature + * + * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled. + * 2. other client can send request to disable gfx off feature, the request should be honored. + * 3. other client can cancel their request of disable gfx off feature + * 4. other client should not send request to enable gfx off feature before disable gfx off feature. + */ + +void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) +{ + if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) + return; + + if (!is_support_sw_smu(adev) && + (!adev->powerplay.pp_funcs || + !adev->powerplay.pp_funcs->set_powergating_by_smu)) + return; + + + mutex_lock(&adev->gfx.gfx_off_mutex); + + if (!enable) + adev->gfx.gfx_off_req_count++; + else if (adev->gfx.gfx_off_req_count > 0) + adev->gfx.gfx_off_req_count--; + + if (enable && !adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { + schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE); + } else if (!enable && adev->gfx.gfx_off_state) { + if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) + adev->gfx.gfx_off_state = false; + } + + mutex_unlock(&adev->gfx.gfx_off_mutex); +}
diff --git a/amdgpu/amdgpu_gfx.h b/amdgpu/amdgpu_gfx.h new file mode 100644 index 0000000..1199b58 --- /dev/null +++ b/amdgpu/amdgpu_gfx.h
@@ -0,0 +1,385 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_GFX_H__ +#define __AMDGPU_GFX_H__ + +/* + * GFX stuff + */ +#include "clearstate_defs.h" +#include "amdgpu_ring.h" +#include "amdgpu_rlc.h" + +/* GFX current status */ +#define AMDGPU_GFX_NORMAL_MODE 0x00000000L +#define AMDGPU_GFX_SAFE_MODE 0x00000001L +#define AMDGPU_GFX_PG_DISABLED_MODE 0x00000002L +#define AMDGPU_GFX_CG_DISABLED_MODE 0x00000004L +#define AMDGPU_GFX_LBPW_DISABLED_MODE 0x00000008L + +#define AMDGPU_MAX_GFX_QUEUES KGD_MAX_QUEUES +#define AMDGPU_MAX_COMPUTE_QUEUES KGD_MAX_QUEUES + +struct amdgpu_mec { + struct amdgpu_bo *hpd_eop_obj; + u64 hpd_eop_gpu_addr; + struct amdgpu_bo *mec_fw_obj; + u64 mec_fw_gpu_addr; + u32 num_mec; + u32 num_pipe_per_mec; + u32 num_queue_per_pipe; + void *mqd_backup[AMDGPU_MAX_COMPUTE_RINGS + 1]; + + /* These are the resources for which amdgpu takes ownership */ + DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); +}; + +enum amdgpu_unmap_queues_action { + PREEMPT_QUEUES = 0, + RESET_QUEUES, + DISABLE_PROCESS_QUEUES, + PREEMPT_QUEUES_NO_UNMAP, +}; + +struct kiq_pm4_funcs { + /* Support ASIC-specific kiq pm4 packets*/ + void (*kiq_set_resources)(struct amdgpu_ring *kiq_ring, + uint64_t queue_mask); + void (*kiq_map_queues)(struct amdgpu_ring *kiq_ring, + struct amdgpu_ring *ring); + void (*kiq_unmap_queues)(struct amdgpu_ring *kiq_ring, + struct amdgpu_ring *ring, + enum amdgpu_unmap_queues_action action, + u64 gpu_addr, u64 seq); + void (*kiq_query_status)(struct amdgpu_ring *kiq_ring, + struct amdgpu_ring *ring, + u64 addr, + u64 seq); + /* Packet sizes */ + int set_resources_size; + int map_queues_size; + int unmap_queues_size; + int query_status_size; +}; + +struct amdgpu_kiq { + u64 eop_gpu_addr; + struct amdgpu_bo *eop_obj; + spinlock_t ring_lock; + struct amdgpu_ring ring; + struct amdgpu_irq_src irq; + const struct kiq_pm4_funcs *pmf; +}; + +/* + * GPU scratch registers structures, functions & helpers + */ +struct amdgpu_scratch { + unsigned num_reg; + uint32_t reg_base; + uint32_t free_mask; +}; + +/* + * GFX configurations + */ +#define AMDGPU_GFX_MAX_SE 4 +#define AMDGPU_GFX_MAX_SH_PER_SE 2 + +struct amdgpu_rb_config { + uint32_t rb_backend_disable; + uint32_t user_rb_backend_disable; + uint32_t raster_config; + uint32_t raster_config_1; +}; + +struct gb_addr_config { + uint16_t pipe_interleave_size; + uint8_t num_pipes; + uint8_t max_compress_frags; + uint8_t num_banks; + uint8_t num_se; + uint8_t num_rb_per_se; +}; + +struct amdgpu_gfx_config { + unsigned max_shader_engines; + unsigned max_tile_pipes; + unsigned max_cu_per_sh; + unsigned max_sh_per_se; + unsigned max_backends_per_se; + unsigned max_texture_channel_caches; + unsigned max_gprs; + unsigned max_gs_threads; + unsigned max_hw_contexts; + unsigned sc_prim_fifo_size_frontend; + unsigned sc_prim_fifo_size_backend; + unsigned sc_hiz_tile_fifo_size; + unsigned sc_earlyz_tile_fifo_size; + + unsigned num_tile_pipes; + unsigned backend_enable_mask; + unsigned mem_max_burst_length_bytes; + unsigned mem_row_size_in_kb; + unsigned shader_engine_tile_size; + unsigned num_gpus; + unsigned multi_gpu_tile_size; + unsigned mc_arb_ramcfg; + unsigned gb_addr_config; + unsigned num_rbs; + unsigned gs_vgt_table_depth; + unsigned gs_prim_buffer_depth; + + uint32_t tile_mode_array[32]; + uint32_t macrotile_mode_array[16]; + + struct gb_addr_config gb_addr_config_fields; + struct amdgpu_rb_config rb_config[AMDGPU_GFX_MAX_SE][AMDGPU_GFX_MAX_SH_PER_SE]; + + /* gfx configure feature */ + uint32_t double_offchip_lds_buf; + /* cached value of DB_DEBUG2 */ + uint32_t db_debug2; + /* gfx10 specific config */ + uint32_t num_sc_per_sh; + uint32_t num_packer_per_sc; + uint32_t pa_sc_tile_steering_override; +}; + +struct amdgpu_cu_info { + uint32_t simd_per_cu; + uint32_t max_waves_per_simd; + uint32_t wave_front_size; + uint32_t max_scratch_slots_per_cu; + uint32_t lds_size; + + /* total active CU number */ + uint32_t number; + uint32_t ao_cu_mask; + uint32_t ao_cu_bitmap[4][4]; + uint32_t bitmap[4][4]; +}; + +struct amdgpu_gfx_funcs { + /* get the gpu clock counter */ + uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev); + void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, + u32 sh_num, u32 instance); + void (*read_wave_data)(struct amdgpu_device *adev, uint32_t simd, + uint32_t wave, uint32_t *dst, int *no_fields); + void (*read_wave_vgprs)(struct amdgpu_device *adev, uint32_t simd, + uint32_t wave, uint32_t thread, uint32_t start, + uint32_t size, uint32_t *dst); + void (*read_wave_sgprs)(struct amdgpu_device *adev, uint32_t simd, + uint32_t wave, uint32_t start, uint32_t size, + uint32_t *dst); + void (*select_me_pipe_q)(struct amdgpu_device *adev, u32 me, u32 pipe, + u32 queue, u32 vmid); +}; + +struct amdgpu_ngg_buf { + struct amdgpu_bo *bo; + uint64_t gpu_addr; + uint32_t size; + uint32_t bo_size; +}; + +enum { + NGG_PRIM = 0, + NGG_POS, + NGG_CNTL, + NGG_PARAM, + NGG_BUF_MAX +}; + +struct amdgpu_ngg { + struct amdgpu_ngg_buf buf[NGG_BUF_MAX]; + uint32_t gds_reserve_addr; + uint32_t gds_reserve_size; + bool init; +}; + +struct sq_work { + struct work_struct work; + unsigned ih_data; +}; + +struct amdgpu_pfp { + struct amdgpu_bo *pfp_fw_obj; + uint64_t pfp_fw_gpu_addr; + uint32_t *pfp_fw_ptr; +}; + +struct amdgpu_ce { + struct amdgpu_bo *ce_fw_obj; + uint64_t ce_fw_gpu_addr; + uint32_t *ce_fw_ptr; +}; + +struct amdgpu_me { + struct amdgpu_bo *me_fw_obj; + uint64_t me_fw_gpu_addr; + uint32_t *me_fw_ptr; + uint32_t num_me; + uint32_t num_pipe_per_me; + uint32_t num_queue_per_pipe; + void *mqd_backup[AMDGPU_MAX_GFX_RINGS + 1]; + + /* These are the resources for which amdgpu takes ownership */ + DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_GFX_QUEUES); +}; + +struct amdgpu_gfx { + struct mutex gpu_clock_mutex; + struct amdgpu_gfx_config config; + struct amdgpu_rlc rlc; + struct amdgpu_pfp pfp; + struct amdgpu_ce ce; + struct amdgpu_me me; + struct amdgpu_mec mec; + struct amdgpu_kiq kiq; + struct amdgpu_scratch scratch; + const struct firmware *me_fw; /* ME firmware */ + uint32_t me_fw_version; + const struct firmware *pfp_fw; /* PFP firmware */ + uint32_t pfp_fw_version; + const struct firmware *ce_fw; /* CE firmware */ + uint32_t ce_fw_version; + const struct firmware *rlc_fw; /* RLC firmware */ + uint32_t rlc_fw_version; + const struct firmware *mec_fw; /* MEC firmware */ + uint32_t mec_fw_version; + const struct firmware *mec2_fw; /* MEC2 firmware */ + uint32_t mec2_fw_version; + uint32_t me_feature_version; + uint32_t ce_feature_version; + uint32_t pfp_feature_version; + uint32_t rlc_feature_version; + uint32_t rlc_srlc_fw_version; + uint32_t rlc_srlc_feature_version; + uint32_t rlc_srlg_fw_version; + uint32_t rlc_srlg_feature_version; + uint32_t rlc_srls_fw_version; + uint32_t rlc_srls_feature_version; + uint32_t mec_feature_version; + uint32_t mec2_feature_version; + bool mec_fw_write_wait; + bool me_fw_write_wait; + struct amdgpu_ring gfx_ring[AMDGPU_MAX_GFX_RINGS]; + unsigned num_gfx_rings; + struct amdgpu_ring compute_ring[AMDGPU_MAX_COMPUTE_RINGS]; + unsigned num_compute_rings; + struct amdgpu_irq_src eop_irq; + struct amdgpu_irq_src priv_reg_irq; + struct amdgpu_irq_src priv_inst_irq; + struct amdgpu_irq_src cp_ecc_error_irq; + struct amdgpu_irq_src sq_irq; + struct sq_work sq_work; + + /* gfx status */ + uint32_t gfx_current_status; + /* ce ram size*/ + unsigned ce_ram_size; + struct amdgpu_cu_info cu_info; + const struct amdgpu_gfx_funcs *funcs; + + /* reset mask */ + uint32_t grbm_soft_reset; + uint32_t srbm_soft_reset; + + /* NGG */ + struct amdgpu_ngg ngg; + + /* gfx off */ + bool gfx_off_state; /* true: enabled, false: disabled */ + struct mutex gfx_off_mutex; + uint32_t gfx_off_req_count; /* default 1, enable gfx off: dec 1, disable gfx off: add 1 */ + struct delayed_work gfx_off_delay_work; + + /* pipe reservation */ + struct mutex pipe_reserve_mutex; + DECLARE_BITMAP (pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); + + /*ras */ + struct ras_common_if *ras_if; +}; + +#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev)) +#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance)) +#define amdgpu_gfx_select_me_pipe_q(adev, me, pipe, q, vmid) (adev)->gfx.funcs->select_me_pipe_q((adev), (me), (pipe), (q), (vmid)) + +/** + * amdgpu_gfx_create_bitmask - create a bitmask + * + * @bit_width: length of the mask + * + * create a variable length bit mask. + * Returns the bitmask. + */ +static inline u32 amdgpu_gfx_create_bitmask(u32 bit_width) +{ + return (u32)((1ULL << bit_width) - 1); +} + +int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg); +void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg); + +void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, + unsigned max_sh); + +int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq); + +void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq); + +void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev); +int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, + unsigned hpd_size); + +int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, + unsigned mqd_size); +void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev); +int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev); +int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev); + +void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev); +void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev); + +int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, + int pipe, int queue); +void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit, + int *mec, int *pipe, int *queue); +bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, int mec, + int pipe, int queue); +int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, int me, + int pipe, int queue); +void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, + int *me, int *pipe, int *queue); +bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, int me, + int pipe, int queue); +void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable); + +#endif
diff --git a/amdgpu/amdgpu_gmc.c b/amdgpu/amdgpu_gmc.c new file mode 100644 index 0000000..924d83e --- /dev/null +++ b/amdgpu/amdgpu_gmc.c
@@ -0,0 +1,299 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ + +#include <linux/io-64-nonatomic-lo-hi.h> + +#include "amdgpu.h" + +/** + * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO + * + * @bo: the BO to get the PDE for + * @level: the level in the PD hirarchy + * @addr: resulting addr + * @flags: resulting flags + * + * Get the address and flags to be used for a PDE (Page Directory Entry). + */ +void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, + uint64_t *addr, uint64_t *flags) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_dma_tt *ttm; + + switch (bo->tbo.mem.mem_type) { + case TTM_PL_TT: + ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); + *addr = ttm->dma_address[0]; + break; + case TTM_PL_VRAM: + *addr = amdgpu_bo_gpu_offset(bo); + break; + default: + *addr = 0; + break; + } + *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, &bo->tbo.mem); + amdgpu_gmc_get_vm_pde(adev, level, addr, flags); +} + +/** + * amdgpu_gmc_pd_addr - return the address of the root directory + * + */ +uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + uint64_t pd_addr; + + /* TODO: move that into ASIC specific code */ + if (adev->asic_type >= CHIP_VEGA10) { + uint64_t flags = AMDGPU_PTE_VALID; + + amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); + pd_addr |= flags; + } else { + pd_addr = amdgpu_bo_gpu_offset(bo); + } + return pd_addr; +} + +/** + * amdgpu_gmc_set_pte_pde - update the page tables using CPU + * + * @adev: amdgpu_device pointer + * @cpu_pt_addr: cpu address of the page table + * @gpu_page_idx: entry in the page table to update + * @addr: dst addr to write into pte/pde + * @flags: access flags + * + * Update the page tables using CPU. + */ +int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, + uint32_t gpu_page_idx, uint64_t addr, + uint64_t flags) +{ + void __iomem *ptr = (void *)cpu_pt_addr; + uint64_t value; + + /* + * The following is for PTE only. GART does not have PDEs. + */ + value = addr & 0x0000FFFFFFFFF000ULL; + value |= flags; + writeq(value, ptr + (gpu_page_idx * 8)); + return 0; +} + +/** + * amdgpu_gmc_agp_addr - return the address in the AGP address space + * + * @tbo: TTM BO which needs the address, must be in GTT domain + * + * Tries to figure out how to access the BO through the AGP aperture. Returns + * AMDGPU_BO_INVALID_OFFSET if that is not possible. + */ +uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct ttm_dma_tt *ttm; + + if (bo->num_pages != 1 || bo->ttm->caching_state == tt_cached) + return AMDGPU_BO_INVALID_OFFSET; + + ttm = container_of(bo->ttm, struct ttm_dma_tt, ttm); + if (ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) + return AMDGPU_BO_INVALID_OFFSET; + + return adev->gmc.agp_start + ttm->dma_address[0]; +} + +/** + * amdgpu_gmc_vram_location - try to find VRAM location + * + * @adev: amdgpu device structure holding all necessary informations + * @mc: memory controller structure holding memory informations + * @base: base address at which to put VRAM + * + * Function will try to place VRAM at base address provided + * as parameter. + */ +void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, + u64 base) +{ + uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; + + mc->vram_start = base; + mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; + if (limit && limit < mc->real_vram_size) + mc->real_vram_size = limit; + + if (mc->xgmi.num_physical_nodes == 0) { + mc->fb_start = mc->vram_start; + mc->fb_end = mc->vram_end; + } + dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", + mc->mc_vram_size >> 20, mc->vram_start, + mc->vram_end, mc->real_vram_size >> 20); +} + +/** + * amdgpu_gmc_gart_location - try to find GART location + * + * @adev: amdgpu device structure holding all necessary informations + * @mc: memory controller structure holding memory informations + * + * Function will place try to place GART before or after VRAM. + * + * If GART size is bigger than space left then we ajust GART size. + * Thus function will never fails. + */ +void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) +{ + const uint64_t four_gb = 0x100000000ULL; + u64 size_af, size_bf; + /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/ + u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); + + mc->gart_size += adev->pm.smu_prv_buffer_size; + + /* VCE doesn't like it when BOs cross a 4GB segment, so align + * the GART base on a 4GB boundary as well. + */ + size_bf = mc->fb_start; + size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); + + if (mc->gart_size > max(size_bf, size_af)) { + dev_warn(adev->dev, "limiting GART\n"); + mc->gart_size = max(size_bf, size_af); + } + + if ((size_bf >= mc->gart_size && size_bf < size_af) || + (size_af < mc->gart_size)) + mc->gart_start = 0; + else + mc->gart_start = max_mc_address - mc->gart_size + 1; + + mc->gart_start &= ~(four_gb - 1); + mc->gart_end = mc->gart_start + mc->gart_size - 1; + dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", + mc->gart_size >> 20, mc->gart_start, mc->gart_end); +} + +/** + * amdgpu_gmc_agp_location - try to find AGP location + * @adev: amdgpu device structure holding all necessary informations + * @mc: memory controller structure holding memory informations + * + * Function will place try to find a place for the AGP BAR in the MC address + * space. + * + * AGP BAR will be assigned the largest available hole in the address space. + * Should be called after VRAM and GART locations are setup. + */ +void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) +{ + const uint64_t sixteen_gb = 1ULL << 34; + const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); + u64 size_af, size_bf; + + if (mc->fb_start > mc->gart_start) { + size_bf = (mc->fb_start & sixteen_gb_mask) - + ALIGN(mc->gart_end + 1, sixteen_gb); + size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); + } else { + size_bf = mc->fb_start & sixteen_gb_mask; + size_af = (mc->gart_start & sixteen_gb_mask) - + ALIGN(mc->fb_end + 1, sixteen_gb); + } + + if (size_bf > size_af) { + mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; + mc->agp_size = size_bf; + } else { + mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); + mc->agp_size = size_af; + } + + mc->agp_end = mc->agp_start + mc->agp_size - 1; + dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", + mc->agp_size >> 20, mc->agp_start, mc->agp_end); +} + +/** + * amdgpu_gmc_filter_faults - filter VM faults + * + * @adev: amdgpu device structure + * @addr: address of the VM fault + * @pasid: PASID of the process causing the fault + * @timestamp: timestamp of the fault + * + * Returns: + * True if the fault was filtered and should not be processed further. + * False if the fault is a new one and needs to be handled. + */ +bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, + uint16_t pasid, uint64_t timestamp) +{ + struct amdgpu_gmc *gmc = &adev->gmc; + + uint64_t stamp, key = addr << 4 | pasid; + struct amdgpu_gmc_fault *fault; + uint32_t hash; + + /* If we don't have space left in the ring buffer return immediately */ + stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - + AMDGPU_GMC_FAULT_TIMEOUT; + if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) + return true; + + /* Try to find the fault in the hash */ + hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); + fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; + while (fault->timestamp >= stamp) { + uint64_t tmp; + + if (fault->key == key) + return true; + + tmp = fault->timestamp; + fault = &gmc->fault_ring[fault->next]; + + /* Check if the entry was reused */ + if (fault->timestamp >= tmp) + break; + } + + /* Add the fault to the ring */ + fault = &gmc->fault_ring[gmc->last_fault]; + fault->key = key; + fault->timestamp = timestamp; + + /* And update the hash */ + fault->next = gmc->fault_hash[hash].idx; + gmc->fault_hash[hash].idx = gmc->last_fault++; + return false; +}
diff --git a/amdgpu/amdgpu_gmc.h b/amdgpu/amdgpu_gmc.h new file mode 100644 index 0000000..071145a --- /dev/null +++ b/amdgpu/amdgpu_gmc.h
@@ -0,0 +1,233 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +#ifndef __AMDGPU_GMC_H__ +#define __AMDGPU_GMC_H__ + +#include <linux/types.h> + +#include "amdgpu_irq.h" + +/* VA hole for 48bit addresses on Vega10 */ +#define AMDGPU_GMC_HOLE_START 0x0000800000000000ULL +#define AMDGPU_GMC_HOLE_END 0xffff800000000000ULL + +/* + * Hardware is programmed as if the hole doesn't exists with start and end + * address values. + * + * This mask is used to remove the upper 16bits of the VA and so come up with + * the linear addr value. + */ +#define AMDGPU_GMC_HOLE_MASK 0x0000ffffffffffffULL + +/* + * Ring size as power of two for the log of recent faults. + */ +#define AMDGPU_GMC_FAULT_RING_ORDER 8 +#define AMDGPU_GMC_FAULT_RING_SIZE (1 << AMDGPU_GMC_FAULT_RING_ORDER) + +/* + * Hash size as power of two for the log of recent faults + */ +#define AMDGPU_GMC_FAULT_HASH_ORDER 8 +#define AMDGPU_GMC_FAULT_HASH_SIZE (1 << AMDGPU_GMC_FAULT_HASH_ORDER) + +/* + * Number of IH timestamp ticks until a fault is considered handled + */ +#define AMDGPU_GMC_FAULT_TIMEOUT 5000ULL + +struct firmware; + +/* + * GMC page fault information + */ +struct amdgpu_gmc_fault { + uint64_t timestamp; + uint64_t next:AMDGPU_GMC_FAULT_RING_ORDER; + uint64_t key:52; +}; + +/* + * VMHUB structures, functions & helpers + */ +struct amdgpu_vmhub { + uint32_t ctx0_ptb_addr_lo32; + uint32_t ctx0_ptb_addr_hi32; + uint32_t vm_inv_eng0_req; + uint32_t vm_inv_eng0_ack; + uint32_t vm_context0_cntl; + uint32_t vm_l2_pro_fault_status; + uint32_t vm_l2_pro_fault_cntl; +}; + +/* + * GPU MC structures, functions & helpers + */ +struct amdgpu_gmc_funcs { + /* flush the vm tlb via mmio */ + void (*flush_gpu_tlb)(struct amdgpu_device *adev, + uint32_t vmid, uint32_t flush_type); + /* flush the vm tlb via ring */ + uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid, + uint64_t pd_addr); + /* Change the VMID -> PASID mapping */ + void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid, + unsigned pasid); + /* enable/disable PRT support */ + void (*set_prt)(struct amdgpu_device *adev, bool enable); + /* set pte flags based per asic */ + uint64_t (*get_vm_pte_flags)(struct amdgpu_device *adev, + uint32_t flags); + /* get the pde for a given mc addr */ + void (*get_vm_pde)(struct amdgpu_device *adev, int level, + u64 *dst, u64 *flags); +}; + +struct amdgpu_xgmi { + /* from psp */ + u64 node_id; + u64 hive_id; + /* fixed per family */ + u64 node_segment_size; + /* physical node (0-3) */ + unsigned physical_node_id; + /* number of nodes (0-4) */ + unsigned num_physical_nodes; + /* gpu list in the same hive */ + struct list_head head; + bool supported; +}; + +struct amdgpu_gmc { + resource_size_t aper_size; + resource_size_t aper_base; + /* for some chips with <= 32MB we need to lie + * about vram size near mc fb location */ + u64 mc_vram_size; + u64 visible_vram_size; + u64 agp_size; + u64 agp_start; + u64 agp_end; + u64 gart_size; + u64 gart_start; + u64 gart_end; + u64 vram_start; + u64 vram_end; + /* FB region , it's same as local vram region in single GPU, in XGMI + * configuration, this region covers all GPUs in the same hive , + * each GPU in the hive has the same view of this FB region . + * GPU0's vram starts at offset (0 * segment size) , + * GPU1 starts at offset (1 * segment size), etc. + */ + u64 fb_start; + u64 fb_end; + unsigned vram_width; + u64 real_vram_size; + int vram_mtrr; + u64 mc_mask; + const struct firmware *fw; /* MC firmware */ + uint32_t fw_version; + struct amdgpu_irq_src vm_fault; + uint32_t vram_type; + uint32_t srbm_soft_reset; + bool prt_warning; + uint64_t stolen_size; + /* apertures */ + u64 shared_aperture_start; + u64 shared_aperture_end; + u64 private_aperture_start; + u64 private_aperture_end; + /* protects concurrent invalidation */ + spinlock_t invalidate_lock; + bool translate_further; + struct kfd_vm_fault_info *vm_fault_info; + atomic_t vm_fault_info_updated; + + struct amdgpu_gmc_fault fault_ring[AMDGPU_GMC_FAULT_RING_SIZE]; + struct { + uint64_t idx:AMDGPU_GMC_FAULT_RING_ORDER; + } fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE]; + uint64_t last_fault:AMDGPU_GMC_FAULT_RING_ORDER; + + const struct amdgpu_gmc_funcs *gmc_funcs; + + struct amdgpu_xgmi xgmi; + struct amdgpu_irq_src ecc_irq; + struct ras_common_if *ras_if; +}; + +#define amdgpu_gmc_flush_gpu_tlb(adev, vmid, type) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (type)) +#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr)) +#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid)) +#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags)) +#define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags)) + +/** + * amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR + * + * @adev: amdgpu_device pointer + * + * Returns: + * True if full VRAM is visible through the BAR + */ +static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc) +{ + WARN_ON(gmc->real_vram_size < gmc->visible_vram_size); + + return (gmc->real_vram_size == gmc->visible_vram_size); +} + +/** + * amdgpu_gmc_sign_extend - sign extend the given gmc address + * + * @addr: address to extend + */ +static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr) +{ + if (addr >= AMDGPU_GMC_HOLE_START) + addr |= AMDGPU_GMC_HOLE_END; + + return addr; +} + +void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, + uint64_t *addr, uint64_t *flags); +int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, + uint32_t gpu_page_idx, uint64_t addr, + uint64_t flags); +uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo); +uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo); +void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, + u64 base); +void amdgpu_gmc_gart_location(struct amdgpu_device *adev, + struct amdgpu_gmc *mc); +void amdgpu_gmc_agp_location(struct amdgpu_device *adev, + struct amdgpu_gmc *mc); +bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, + uint16_t pasid, uint64_t timestamp); + +#endif
diff --git a/amdgpu/amdgpu_gtt_mgr.c b/amdgpu/amdgpu_gtt_mgr.c new file mode 100644 index 0000000..6271044 --- /dev/null +++ b/amdgpu/amdgpu_gtt_mgr.c
@@ -0,0 +1,358 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include "amdgpu.h" + +struct amdgpu_gtt_mgr { + struct drm_mm mm; + spinlock_t lock; + atomic64_t available; +}; + +struct amdgpu_gtt_node { + struct drm_mm_node node; + struct ttm_buffer_object *tbo; +}; + +/** + * DOC: mem_info_gtt_total + * + * The amdgpu driver provides a sysfs API for reporting current total size of + * the GTT. + * The file mem_info_gtt_total is used for this, and returns the total size of + * the GTT block, in bytes + */ +static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", + (adev->mman.bdev.man[TTM_PL_TT].size) * PAGE_SIZE); +} + +/** + * DOC: mem_info_gtt_used + * + * The amdgpu driver provides a sysfs API for reporting current total amount of + * used GTT. + * The file mem_info_gtt_used is used for this, and returns the current used + * size of the GTT block, in bytes + */ +static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", + amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT])); +} + +static DEVICE_ATTR(mem_info_gtt_total, S_IRUGO, + amdgpu_mem_info_gtt_total_show, NULL); +static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO, + amdgpu_mem_info_gtt_used_show, NULL); + +/** + * amdgpu_gtt_mgr_init - init GTT manager and DRM MM + * + * @man: TTM memory type manager + * @p_size: maximum size of GTT + * + * Allocate and initialize the GTT manager. + */ +static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man, + unsigned long p_size) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_gtt_mgr *mgr; + uint64_t start, size; + int ret; + + mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); + if (!mgr) + return -ENOMEM; + + start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS; + size = (adev->gmc.gart_size >> PAGE_SHIFT) - start; + drm_mm_init(&mgr->mm, start, size); + spin_lock_init(&mgr->lock); + atomic64_set(&mgr->available, p_size); + man->priv = mgr; + + ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_gtt_total\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_gtt_used\n"); + return ret; + } + + return 0; +} + +/** + * amdgpu_gtt_mgr_fini - free and destroy GTT manager + * + * @man: TTM memory type manager + * + * Destroy and free the GTT manager, returns -EBUSY if ranges are still + * allocated inside it. + */ +static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_gtt_mgr *mgr = man->priv; + spin_lock(&mgr->lock); + drm_mm_takedown(&mgr->mm); + spin_unlock(&mgr->lock); + kfree(mgr); + man->priv = NULL; + + device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total); + device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used); + + return 0; +} + +/** + * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space + * + * @mem: the mem object to check + * + * Check if a mem object has already address space allocated. + */ +bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem) +{ + struct amdgpu_gtt_node *node = mem->mm_node; + + return (node->node.start != AMDGPU_BO_INVALID_OFFSET); +} + +/** + * amdgpu_gtt_mgr_alloc - allocate new ranges + * + * @man: TTM memory type manager + * @tbo: TTM BO we need this range for + * @place: placement flags and restrictions + * @mem: the resulting mem object + * + * Allocate the address space for a node. + */ +static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_mem_reg *mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_gtt_mgr *mgr = man->priv; + struct amdgpu_gtt_node *node = mem->mm_node; + enum drm_mm_insert_mode mode; + unsigned long fpfn, lpfn; + int r; + + if (amdgpu_gtt_mgr_has_gart_addr(mem)) + return 0; + + if (place) + fpfn = place->fpfn; + else + fpfn = 0; + + if (place && place->lpfn) + lpfn = place->lpfn; + else + lpfn = adev->gart.num_cpu_pages; + + mode = DRM_MM_INSERT_BEST; + if (place && place->flags & TTM_PL_FLAG_TOPDOWN) + mode = DRM_MM_INSERT_HIGH; + + spin_lock(&mgr->lock); + r = drm_mm_insert_node_in_range(&mgr->mm, &node->node, mem->num_pages, + mem->page_alignment, 0, fpfn, lpfn, + mode); + spin_unlock(&mgr->lock); + + if (!r) + mem->start = node->node.start; + + return r; +} + +/** + * amdgpu_gtt_mgr_new - allocate a new node + * + * @man: TTM memory type manager + * @tbo: TTM BO we need this range for + * @place: placement flags and restrictions + * @mem: the resulting mem object + * + * Dummy, allocate the node but no space for it yet. + */ +static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_mem_reg *mem) +{ + struct amdgpu_gtt_mgr *mgr = man->priv; + struct amdgpu_gtt_node *node; + int r; + + spin_lock(&mgr->lock); + if ((&tbo->mem == mem || tbo->mem.mem_type != TTM_PL_TT) && + atomic64_read(&mgr->available) < mem->num_pages) { + spin_unlock(&mgr->lock); + return 0; + } + atomic64_sub(mem->num_pages, &mgr->available); + spin_unlock(&mgr->lock); + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) { + r = -ENOMEM; + goto err_out; + } + + node->node.start = AMDGPU_BO_INVALID_OFFSET; + node->node.size = mem->num_pages; + node->tbo = tbo; + mem->mm_node = node; + + if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) { + r = amdgpu_gtt_mgr_alloc(man, tbo, place, mem); + if (unlikely(r)) { + kfree(node); + mem->mm_node = NULL; + r = 0; + goto err_out; + } + } else { + mem->start = node->node.start; + } + + return 0; +err_out: + atomic64_add(mem->num_pages, &mgr->available); + + return r; +} + +/** + * amdgpu_gtt_mgr_del - free ranges + * + * @man: TTM memory type manager + * @tbo: TTM BO we need this range for + * @place: placement flags and restrictions + * @mem: TTM memory object + * + * Free the allocated GTT again. + */ +static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + struct amdgpu_gtt_mgr *mgr = man->priv; + struct amdgpu_gtt_node *node = mem->mm_node; + + if (!node) + return; + + spin_lock(&mgr->lock); + if (node->node.start != AMDGPU_BO_INVALID_OFFSET) + drm_mm_remove_node(&node->node); + spin_unlock(&mgr->lock); + atomic64_add(mem->num_pages, &mgr->available); + + kfree(node); + mem->mm_node = NULL; +} + +/** + * amdgpu_gtt_mgr_usage - return usage of GTT domain + * + * @man: TTM memory type manager + * + * Return how many bytes are used in the GTT domain + */ +uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man) +{ + struct amdgpu_gtt_mgr *mgr = man->priv; + s64 result = man->size - atomic64_read(&mgr->available); + + return (result > 0 ? result : 0) * PAGE_SIZE; +} + +int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man) +{ + struct amdgpu_gtt_mgr *mgr = man->priv; + struct amdgpu_gtt_node *node; + struct drm_mm_node *mm_node; + int r = 0; + + spin_lock(&mgr->lock); + drm_mm_for_each_node(mm_node, &mgr->mm) { + node = container_of(mm_node, struct amdgpu_gtt_node, node); + r = amdgpu_ttm_recover_gart(node->tbo); + if (r) + break; + } + spin_unlock(&mgr->lock); + + return r; +} + +/** + * amdgpu_gtt_mgr_debug - dump VRAM table + * + * @man: TTM memory type manager + * @printer: DRM printer to use + * + * Dump the table content using printk. + */ +static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man, + struct drm_printer *printer) +{ + struct amdgpu_gtt_mgr *mgr = man->priv; + + spin_lock(&mgr->lock); + drm_mm_print(&mgr->mm, printer); + spin_unlock(&mgr->lock); + + drm_printf(printer, "man size:%llu pages, gtt available:%lld pages, usage:%lluMB\n", + man->size, (u64)atomic64_read(&mgr->available), + amdgpu_gtt_mgr_usage(man) >> 20); +} + +const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func = { + .init = amdgpu_gtt_mgr_init, + .takedown = amdgpu_gtt_mgr_fini, + .get_node = amdgpu_gtt_mgr_new, + .put_node = amdgpu_gtt_mgr_del, + .debug = amdgpu_gtt_mgr_debug +};
diff --git a/amdgpu/amdgpu_i2c.c b/amdgpu/amdgpu_i2c.c new file mode 100644 index 0000000..70dbe34 --- /dev/null +++ b/amdgpu/amdgpu_i2c.c
@@ -0,0 +1,394 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <linux/export.h> +#include <linux/pci.h> + +#include <drm/drm_edid.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_i2c.h" +#include "amdgpu_atombios.h" +#include "atom.h" +#include "atombios_dp.h" +#include "atombios_i2c.h" + +/* bit banging i2c */ +static int amdgpu_i2c_pre_xfer(struct i2c_adapter *i2c_adap) +{ + struct amdgpu_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t temp; + + mutex_lock(&i2c->mutex); + + /* switch the pads to ddc mode */ + if (rec->hw_capable) { + temp = RREG32(rec->mask_clk_reg); + temp &= ~(1 << 16); + WREG32(rec->mask_clk_reg, temp); + } + + /* clear the output pin values */ + temp = RREG32(rec->a_clk_reg) & ~rec->a_clk_mask; + WREG32(rec->a_clk_reg, temp); + + temp = RREG32(rec->a_data_reg) & ~rec->a_data_mask; + WREG32(rec->a_data_reg, temp); + + /* set the pins to input */ + temp = RREG32(rec->en_clk_reg) & ~rec->en_clk_mask; + WREG32(rec->en_clk_reg, temp); + + temp = RREG32(rec->en_data_reg) & ~rec->en_data_mask; + WREG32(rec->en_data_reg, temp); + + /* mask the gpio pins for software use */ + temp = RREG32(rec->mask_clk_reg) | rec->mask_clk_mask; + WREG32(rec->mask_clk_reg, temp); + temp = RREG32(rec->mask_clk_reg); + + temp = RREG32(rec->mask_data_reg) | rec->mask_data_mask; + WREG32(rec->mask_data_reg, temp); + temp = RREG32(rec->mask_data_reg); + + return 0; +} + +static void amdgpu_i2c_post_xfer(struct i2c_adapter *i2c_adap) +{ + struct amdgpu_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t temp; + + /* unmask the gpio pins for software use */ + temp = RREG32(rec->mask_clk_reg) & ~rec->mask_clk_mask; + WREG32(rec->mask_clk_reg, temp); + temp = RREG32(rec->mask_clk_reg); + + temp = RREG32(rec->mask_data_reg) & ~rec->mask_data_mask; + WREG32(rec->mask_data_reg, temp); + temp = RREG32(rec->mask_data_reg); + + mutex_unlock(&i2c->mutex); +} + +static int amdgpu_i2c_get_clock(void *i2c_priv) +{ + struct amdgpu_i2c_chan *i2c = i2c_priv; + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t val; + + /* read the value off the pin */ + val = RREG32(rec->y_clk_reg); + val &= rec->y_clk_mask; + + return (val != 0); +} + + +static int amdgpu_i2c_get_data(void *i2c_priv) +{ + struct amdgpu_i2c_chan *i2c = i2c_priv; + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t val; + + /* read the value off the pin */ + val = RREG32(rec->y_data_reg); + val &= rec->y_data_mask; + + return (val != 0); +} + +static void amdgpu_i2c_set_clock(void *i2c_priv, int clock) +{ + struct amdgpu_i2c_chan *i2c = i2c_priv; + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t val; + + /* set pin direction */ + val = RREG32(rec->en_clk_reg) & ~rec->en_clk_mask; + val |= clock ? 0 : rec->en_clk_mask; + WREG32(rec->en_clk_reg, val); +} + +static void amdgpu_i2c_set_data(void *i2c_priv, int data) +{ + struct amdgpu_i2c_chan *i2c = i2c_priv; + struct amdgpu_device *adev = i2c->dev->dev_private; + struct amdgpu_i2c_bus_rec *rec = &i2c->rec; + uint32_t val; + + /* set pin direction */ + val = RREG32(rec->en_data_reg) & ~rec->en_data_mask; + val |= data ? 0 : rec->en_data_mask; + WREG32(rec->en_data_reg, val); +} + +static const struct i2c_algorithm amdgpu_atombios_i2c_algo = { + .master_xfer = amdgpu_atombios_i2c_xfer, + .functionality = amdgpu_atombios_i2c_func, +}; + +struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev, + const struct amdgpu_i2c_bus_rec *rec, + const char *name) +{ + struct amdgpu_i2c_chan *i2c; + int ret; + + /* don't add the mm_i2c bus unless hw_i2c is enabled */ + if (rec->mm_i2c && (amdgpu_hw_i2c == 0)) + return NULL; + + i2c = kzalloc(sizeof(struct amdgpu_i2c_chan), GFP_KERNEL); + if (i2c == NULL) + return NULL; + + i2c->rec = *rec; + i2c->adapter.owner = THIS_MODULE; + i2c->adapter.class = I2C_CLASS_DDC; + i2c->adapter.dev.parent = &dev->pdev->dev; + i2c->dev = dev; + i2c_set_adapdata(&i2c->adapter, i2c); + mutex_init(&i2c->mutex); + if (rec->hw_capable && + amdgpu_hw_i2c) { + /* hw i2c using atom */ + snprintf(i2c->adapter.name, sizeof(i2c->adapter.name), + "AMDGPU i2c hw bus %s", name); + i2c->adapter.algo = &amdgpu_atombios_i2c_algo; + ret = i2c_add_adapter(&i2c->adapter); + if (ret) + goto out_free; + } else { + /* set the amdgpu bit adapter */ + snprintf(i2c->adapter.name, sizeof(i2c->adapter.name), + "AMDGPU i2c bit bus %s", name); + i2c->adapter.algo_data = &i2c->bit; + i2c->bit.pre_xfer = amdgpu_i2c_pre_xfer; + i2c->bit.post_xfer = amdgpu_i2c_post_xfer; + i2c->bit.setsda = amdgpu_i2c_set_data; + i2c->bit.setscl = amdgpu_i2c_set_clock; + i2c->bit.getsda = amdgpu_i2c_get_data; + i2c->bit.getscl = amdgpu_i2c_get_clock; + i2c->bit.udelay = 10; + i2c->bit.timeout = usecs_to_jiffies(2200); /* from VESA */ + i2c->bit.data = i2c; + ret = i2c_bit_add_bus(&i2c->adapter); + if (ret) { + DRM_ERROR("Failed to register bit i2c %s\n", name); + goto out_free; + } + } + + return i2c; +out_free: + kfree(i2c); + return NULL; + +} + +void amdgpu_i2c_destroy(struct amdgpu_i2c_chan *i2c) +{ + if (!i2c) + return; + WARN_ON(i2c->has_aux); + i2c_del_adapter(&i2c->adapter); + kfree(i2c); +} + +/* Add the default buses */ +void amdgpu_i2c_init(struct amdgpu_device *adev) +{ + if (amdgpu_hw_i2c) + DRM_INFO("hw_i2c forced on, you may experience display detection problems!\n"); + + amdgpu_atombios_i2c_init(adev); +} + +/* remove all the buses */ +void amdgpu_i2c_fini(struct amdgpu_device *adev) +{ + int i; + + for (i = 0; i < AMDGPU_MAX_I2C_BUS; i++) { + if (adev->i2c_bus[i]) { + amdgpu_i2c_destroy(adev->i2c_bus[i]); + adev->i2c_bus[i] = NULL; + } + } +} + +/* Add additional buses */ +void amdgpu_i2c_add(struct amdgpu_device *adev, + const struct amdgpu_i2c_bus_rec *rec, + const char *name) +{ + struct drm_device *dev = adev->ddev; + int i; + + for (i = 0; i < AMDGPU_MAX_I2C_BUS; i++) { + if (!adev->i2c_bus[i]) { + adev->i2c_bus[i] = amdgpu_i2c_create(dev, rec, name); + return; + } + } +} + +/* looks up bus based on id */ +struct amdgpu_i2c_chan * +amdgpu_i2c_lookup(struct amdgpu_device *adev, + const struct amdgpu_i2c_bus_rec *i2c_bus) +{ + int i; + + for (i = 0; i < AMDGPU_MAX_I2C_BUS; i++) { + if (adev->i2c_bus[i] && + (adev->i2c_bus[i]->rec.i2c_id == i2c_bus->i2c_id)) { + return adev->i2c_bus[i]; + } + } + return NULL; +} + +static void amdgpu_i2c_get_byte(struct amdgpu_i2c_chan *i2c_bus, + u8 slave_addr, + u8 addr, + u8 *val) +{ + u8 out_buf[2]; + u8 in_buf[2]; + struct i2c_msg msgs[] = { + { + .addr = slave_addr, + .flags = 0, + .len = 1, + .buf = out_buf, + }, + { + .addr = slave_addr, + .flags = I2C_M_RD, + .len = 1, + .buf = in_buf, + } + }; + + out_buf[0] = addr; + out_buf[1] = 0; + + if (i2c_transfer(&i2c_bus->adapter, msgs, 2) == 2) { + *val = in_buf[0]; + DRM_DEBUG("val = 0x%02x\n", *val); + } else { + DRM_DEBUG("i2c 0x%02x 0x%02x read failed\n", + addr, *val); + } +} + +static void amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus, + u8 slave_addr, + u8 addr, + u8 val) +{ + uint8_t out_buf[2]; + struct i2c_msg msg = { + .addr = slave_addr, + .flags = 0, + .len = 2, + .buf = out_buf, + }; + + out_buf[0] = addr; + out_buf[1] = val; + + if (i2c_transfer(&i2c_bus->adapter, &msg, 1) != 1) + DRM_DEBUG("i2c 0x%02x 0x%02x write failed\n", + addr, val); +} + +/* ddc router switching */ +void +amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *amdgpu_connector) +{ + u8 val; + + if (!amdgpu_connector->router.ddc_valid) + return; + + if (!amdgpu_connector->router_bus) + return; + + amdgpu_i2c_get_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x3, &val); + val &= ~amdgpu_connector->router.ddc_mux_control_pin; + amdgpu_i2c_put_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x3, val); + amdgpu_i2c_get_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x1, &val); + val &= ~amdgpu_connector->router.ddc_mux_control_pin; + val |= amdgpu_connector->router.ddc_mux_state; + amdgpu_i2c_put_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x1, val); +} + +/* clock/data router switching */ +void +amdgpu_i2c_router_select_cd_port(const struct amdgpu_connector *amdgpu_connector) +{ + u8 val; + + if (!amdgpu_connector->router.cd_valid) + return; + + if (!amdgpu_connector->router_bus) + return; + + amdgpu_i2c_get_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x3, &val); + val &= ~amdgpu_connector->router.cd_mux_control_pin; + amdgpu_i2c_put_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x3, val); + amdgpu_i2c_get_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x1, &val); + val &= ~amdgpu_connector->router.cd_mux_control_pin; + val |= amdgpu_connector->router.cd_mux_state; + amdgpu_i2c_put_byte(amdgpu_connector->router_bus, + amdgpu_connector->router.i2c_addr, + 0x1, val); +}
diff --git a/amdgpu/amdgpu_i2c.h b/amdgpu/amdgpu_i2c.h new file mode 100644 index 0000000..63c2ff7 --- /dev/null +++ b/amdgpu/amdgpu_i2c.h
@@ -0,0 +1,44 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_I2C_H__ +#define __AMDGPU_I2C_H__ + +struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev, + const struct amdgpu_i2c_bus_rec *rec, + const char *name); +void amdgpu_i2c_destroy(struct amdgpu_i2c_chan *i2c); +void amdgpu_i2c_init(struct amdgpu_device *adev); +void amdgpu_i2c_fini(struct amdgpu_device *adev); +void amdgpu_i2c_add(struct amdgpu_device *adev, + const struct amdgpu_i2c_bus_rec *rec, + const char *name); +struct amdgpu_i2c_chan * +amdgpu_i2c_lookup(struct amdgpu_device *adev, + const struct amdgpu_i2c_bus_rec *i2c_bus); +void +amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *connector); +void +amdgpu_i2c_router_select_cd_port(const struct amdgpu_connector *connector); + +#endif
diff --git a/amdgpu/amdgpu_ib.c b/amdgpu/amdgpu_ib.c new file mode 100644 index 0000000..7850084 --- /dev/null +++ b/amdgpu/amdgpu_ib.c
@@ -0,0 +1,430 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + * Christian König + */ +#include <linux/seq_file.h> +#include <linux/slab.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_debugfs.h> + +#include "amdgpu.h" +#include "atom.h" +#include "amdgpu_trace.h" + +#define AMDGPU_IB_TEST_TIMEOUT msecs_to_jiffies(1000) +#define AMDGPU_IB_TEST_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000) + +/* + * IB + * IBs (Indirect Buffers) and areas of GPU accessible memory where + * commands are stored. You can put a pointer to the IB in the + * command ring and the hw will fetch the commands from the IB + * and execute them. Generally userspace acceleration drivers + * produce command buffers which are send to the kernel and + * put in IBs for execution by the requested ring. + */ +static int amdgpu_debugfs_sa_init(struct amdgpu_device *adev); + +/** + * amdgpu_ib_get - request an IB (Indirect Buffer) + * + * @ring: ring index the IB is associated with + * @size: requested IB size + * @ib: IB object returned + * + * Request an IB (all asics). IBs are allocated using the + * suballocator. + * Returns 0 on success, error on failure. + */ +int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, + unsigned size, struct amdgpu_ib *ib) +{ + int r; + + if (size) { + r = amdgpu_sa_bo_new(&adev->ring_tmp_bo, + &ib->sa_bo, size, 256); + if (r) { + dev_err(adev->dev, "failed to get a new IB (%d)\n", r); + return r; + } + + ib->ptr = amdgpu_sa_bo_cpu_addr(ib->sa_bo); + + if (!vm) + ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); + } + + return 0; +} + +/** + * amdgpu_ib_free - free an IB (Indirect Buffer) + * + * @adev: amdgpu_device pointer + * @ib: IB object to free + * @f: the fence SA bo need wait on for the ib alloation + * + * Free an IB (all asics). + */ +void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib, + struct dma_fence *f) +{ + amdgpu_sa_bo_free(adev, &ib->sa_bo, f); +} + +/** + * amdgpu_ib_schedule - schedule an IB (Indirect Buffer) on the ring + * + * @adev: amdgpu_device pointer + * @num_ibs: number of IBs to schedule + * @ibs: IB objects to schedule + * @f: fence created during this submission + * + * Schedule an IB on the associated ring (all asics). + * Returns 0 on success, error on failure. + * + * On SI, there are two parallel engines fed from the primary ring, + * the CE (Constant Engine) and the DE (Drawing Engine). Since + * resource descriptors have moved to memory, the CE allows you to + * prime the caches while the DE is updating register state so that + * the resource descriptors will be already in cache when the draw is + * processed. To accomplish this, the userspace driver submits two + * IBs, one for the CE and one for the DE. If there is a CE IB (called + * a CONST_IB), it will be put on the ring prior to the DE IB. Prior + * to SI there was just a DE IB. + */ +int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, + struct amdgpu_ib *ibs, struct amdgpu_job *job, + struct dma_fence **f) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_ib *ib = &ibs[0]; + struct dma_fence *tmp = NULL; + bool skip_preamble, need_ctx_switch; + unsigned patch_offset = ~0; + struct amdgpu_vm *vm; + uint64_t fence_ctx; + uint32_t status = 0, alloc_size; + unsigned fence_flags = 0; + + unsigned i; + int r = 0; + bool need_pipe_sync = false; + + if (num_ibs == 0) + return -EINVAL; + + /* ring tests don't use a job */ + if (job) { + vm = job->vm; + fence_ctx = job->base.s_fence->scheduled.context; + } else { + vm = NULL; + fence_ctx = 0; + } + + if (!ring->sched.ready) { + dev_err(adev->dev, "couldn't schedule ib on ring <%s>\n", ring->name); + return -EINVAL; + } + + if (vm && !job->vmid) { + dev_err(adev->dev, "VM IB without ID\n"); + return -EINVAL; + } + + alloc_size = ring->funcs->emit_frame_size + num_ibs * + ring->funcs->emit_ib_size; + + r = amdgpu_ring_alloc(ring, alloc_size); + if (r) { + dev_err(adev->dev, "scheduling IB failed (%d).\n", r); + return r; + } + + need_ctx_switch = ring->current_ctx != fence_ctx; + if (ring->funcs->emit_pipeline_sync && job && + ((tmp = amdgpu_sync_get_fence(&job->sched_sync, NULL)) || + (amdgpu_sriov_vf(adev) && need_ctx_switch) || + amdgpu_vm_need_pipeline_sync(ring, job))) { + need_pipe_sync = true; + + if (tmp) + trace_amdgpu_ib_pipe_sync(job, tmp); + + dma_fence_put(tmp); + } + + if (ring->funcs->insert_start) + ring->funcs->insert_start(ring); + + if (job) { + r = amdgpu_vm_flush(ring, job, need_pipe_sync); + if (r) { + amdgpu_ring_undo(ring); + return r; + } + } + + if (job && ring->funcs->init_cond_exec) + patch_offset = amdgpu_ring_init_cond_exec(ring); + +#ifdef CONFIG_X86_64 + if (!(adev->flags & AMD_IS_APU)) +#endif + { + if (ring->funcs->emit_hdp_flush) + amdgpu_ring_emit_hdp_flush(ring); + else + amdgpu_asic_flush_hdp(adev, ring); + } + + if (need_ctx_switch) + status |= AMDGPU_HAVE_CTX_SWITCH; + + skip_preamble = ring->current_ctx == fence_ctx; + if (job && ring->funcs->emit_cntxcntl) { + status |= job->preamble_status; + status |= job->preemption_status; + amdgpu_ring_emit_cntxcntl(ring, status); + } + + for (i = 0; i < num_ibs; ++i) { + ib = &ibs[i]; + + /* drop preamble IBs if we don't have a context switch */ + if ((ib->flags & AMDGPU_IB_FLAG_PREAMBLE) && + skip_preamble && + !(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST) && + !amdgpu_mcbp && + !amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */ + continue; + + amdgpu_ring_emit_ib(ring, job, ib, status); + status &= ~AMDGPU_HAVE_CTX_SWITCH; + } + + if (ring->funcs->emit_tmz) + amdgpu_ring_emit_tmz(ring, false); + +#ifdef CONFIG_X86_64 + if (!(adev->flags & AMD_IS_APU)) +#endif + amdgpu_asic_invalidate_hdp(adev, ring); + + if (ib->flags & AMDGPU_IB_FLAG_TC_WB_NOT_INVALIDATE) + fence_flags |= AMDGPU_FENCE_FLAG_TC_WB_ONLY; + + /* wrap the last IB with fence */ + if (job && job->uf_addr) { + amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence, + fence_flags | AMDGPU_FENCE_FLAG_64BIT); + } + + r = amdgpu_fence_emit(ring, f, fence_flags); + if (r) { + dev_err(adev->dev, "failed to emit fence (%d)\n", r); + if (job && job->vmid) + amdgpu_vmid_reset(adev, ring->funcs->vmhub, job->vmid); + amdgpu_ring_undo(ring); + return r; + } + + if (ring->funcs->insert_end) + ring->funcs->insert_end(ring); + + if (patch_offset != ~0 && ring->funcs->patch_cond_exec) + amdgpu_ring_patch_cond_exec(ring, patch_offset); + + ring->current_ctx = fence_ctx; + if (vm && ring->funcs->emit_switch_buffer) + amdgpu_ring_emit_switch_buffer(ring); + amdgpu_ring_commit(ring); + return 0; +} + +/** + * amdgpu_ib_pool_init - Init the IB (Indirect Buffer) pool + * + * @adev: amdgpu_device pointer + * + * Initialize the suballocator to manage a pool of memory + * for use as IBs (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_ib_pool_init(struct amdgpu_device *adev) +{ + int r; + + if (adev->ib_pool_ready) { + return 0; + } + r = amdgpu_sa_bo_manager_init(adev, &adev->ring_tmp_bo, + AMDGPU_IB_POOL_SIZE*64*1024, + AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT); + if (r) { + return r; + } + + adev->ib_pool_ready = true; + if (amdgpu_debugfs_sa_init(adev)) { + dev_err(adev->dev, "failed to register debugfs file for SA\n"); + } + return 0; +} + +/** + * amdgpu_ib_pool_fini - Free the IB (Indirect Buffer) pool + * + * @adev: amdgpu_device pointer + * + * Tear down the suballocator managing the pool of memory + * for use as IBs (all asics). + */ +void amdgpu_ib_pool_fini(struct amdgpu_device *adev) +{ + if (adev->ib_pool_ready) { + amdgpu_sa_bo_manager_fini(adev, &adev->ring_tmp_bo); + adev->ib_pool_ready = false; + } +} + +/** + * amdgpu_ib_ring_tests - test IBs on the rings + * + * @adev: amdgpu_device pointer + * + * Test an IB (Indirect Buffer) on each ring. + * If the test fails, disable the ring. + * Returns 0 on success, error if the primary GFX ring + * IB test fails. + */ +int amdgpu_ib_ring_tests(struct amdgpu_device *adev) +{ + unsigned i; + int r, ret = 0; + long tmo_gfx, tmo_mm; + + tmo_mm = tmo_gfx = AMDGPU_IB_TEST_TIMEOUT; + if (amdgpu_sriov_vf(adev)) { + /* for MM engines in hypervisor side they are not scheduled together + * with CP and SDMA engines, so even in exclusive mode MM engine could + * still running on other VF thus the IB TEST TIMEOUT for MM engines + * under SR-IOV should be set to a long time. 8 sec should be enough + * for the MM comes back to this VF. + */ + tmo_mm = 8 * AMDGPU_IB_TEST_TIMEOUT; + } + + if (amdgpu_sriov_runtime(adev)) { + /* for CP & SDMA engines since they are scheduled together so + * need to make the timeout width enough to cover the time + * cost waiting for it coming back under RUNTIME only + */ + tmo_gfx = 8 * AMDGPU_IB_TEST_TIMEOUT; + } else if (adev->gmc.xgmi.hive_id) { + tmo_gfx = AMDGPU_IB_TEST_GFX_XGMI_TIMEOUT; + } + + for (i = 0; i < adev->num_rings; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + long tmo; + + /* KIQ rings don't have an IB test because we never submit IBs + * to them and they have no interrupt support. + */ + if (!ring->sched.ready || !ring->funcs->test_ib) + continue; + + /* MM engine need more time */ + if (ring->funcs->type == AMDGPU_RING_TYPE_UVD || + ring->funcs->type == AMDGPU_RING_TYPE_VCE || + ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC || + ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC || + ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || + ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) + tmo = tmo_mm; + else + tmo = tmo_gfx; + + r = amdgpu_ring_test_ib(ring, tmo); + if (!r) { + DRM_DEV_DEBUG(adev->dev, "ib test on %s succeeded\n", + ring->name); + continue; + } + + ring->sched.ready = false; + DRM_DEV_ERROR(adev->dev, "IB test failed on %s (%d).\n", + ring->name, r); + + if (ring == &adev->gfx.gfx_ring[0]) { + /* oh, oh, that's really bad */ + adev->accel_working = false; + return r; + + } else { + ret = r; + } + } + return ret; +} + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +static int amdgpu_debugfs_sa_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + + amdgpu_sa_bo_dump_debug_info(&adev->ring_tmp_bo, m); + + return 0; + +} + +static const struct drm_info_list amdgpu_debugfs_sa_list[] = { + {"amdgpu_sa_info", &amdgpu_debugfs_sa_info, 0, NULL}, +}; + +#endif + +static int amdgpu_debugfs_sa_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_sa_list, 1); +#else + return 0; +#endif +}
diff --git a/amdgpu/amdgpu_ids.c b/amdgpu/amdgpu_ids.c new file mode 100644 index 0000000..57b3d8a --- /dev/null +++ b/amdgpu/amdgpu_ids.c
@@ -0,0 +1,611 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu_ids.h" + +#include <linux/idr.h> +#include <linux/dma-fence-array.h> + + +#include "amdgpu.h" +#include "amdgpu_trace.h" + +/* + * PASID manager + * + * PASIDs are global address space identifiers that can be shared + * between the GPU, an IOMMU and the driver. VMs on different devices + * may use the same PASID if they share the same address + * space. Therefore PASIDs are allocated using a global IDA. VMs are + * looked up from the PASID per amdgpu_device. + */ +static DEFINE_IDA(amdgpu_pasid_ida); + +/* Helper to free pasid from a fence callback */ +struct amdgpu_pasid_cb { + struct dma_fence_cb cb; + unsigned int pasid; +}; + +/** + * amdgpu_pasid_alloc - Allocate a PASID + * @bits: Maximum width of the PASID in bits, must be at least 1 + * + * Allocates a PASID of the given width while keeping smaller PASIDs + * available if possible. + * + * Returns a positive integer on success. Returns %-EINVAL if bits==0. + * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on + * memory allocation failure. + */ +int amdgpu_pasid_alloc(unsigned int bits) +{ + int pasid = -EINVAL; + + for (bits = min(bits, 31U); bits > 0; bits--) { + pasid = ida_simple_get(&amdgpu_pasid_ida, + 1U << (bits - 1), 1U << bits, + GFP_KERNEL); + if (pasid != -ENOSPC) + break; + } + + if (pasid >= 0) + trace_amdgpu_pasid_allocated(pasid); + + return pasid; +} + +/** + * amdgpu_pasid_free - Free a PASID + * @pasid: PASID to free + */ +void amdgpu_pasid_free(unsigned int pasid) +{ + trace_amdgpu_pasid_freed(pasid); + ida_simple_remove(&amdgpu_pasid_ida, pasid); +} + +static void amdgpu_pasid_free_cb(struct dma_fence *fence, + struct dma_fence_cb *_cb) +{ + struct amdgpu_pasid_cb *cb = + container_of(_cb, struct amdgpu_pasid_cb, cb); + + amdgpu_pasid_free(cb->pasid); + dma_fence_put(fence); + kfree(cb); +} + +/** + * amdgpu_pasid_free_delayed - free pasid when fences signal + * + * @resv: reservation object with the fences to wait for + * @pasid: pasid to free + * + * Free the pasid only after all the fences in resv are signaled. + */ +void amdgpu_pasid_free_delayed(struct reservation_object *resv, + unsigned int pasid) +{ + struct dma_fence *fence, **fences; + struct amdgpu_pasid_cb *cb; + unsigned count; + int r; + + r = reservation_object_get_fences_rcu(resv, NULL, &count, &fences); + if (r) + goto fallback; + + if (count == 0) { + amdgpu_pasid_free(pasid); + return; + } + + if (count == 1) { + fence = fences[0]; + kfree(fences); + } else { + uint64_t context = dma_fence_context_alloc(1); + struct dma_fence_array *array; + + array = dma_fence_array_create(count, fences, context, + 1, false); + if (!array) { + kfree(fences); + goto fallback; + } + fence = &array->base; + } + + cb = kmalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) { + /* Last resort when we are OOM */ + dma_fence_wait(fence, false); + dma_fence_put(fence); + amdgpu_pasid_free(pasid); + } else { + cb->pasid = pasid; + if (dma_fence_add_callback(fence, &cb->cb, + amdgpu_pasid_free_cb)) + amdgpu_pasid_free_cb(fence, &cb->cb); + } + + return; + +fallback: + /* Not enough memory for the delayed delete, as last resort + * block for all the fences to complete. + */ + reservation_object_wait_timeout_rcu(resv, true, false, + MAX_SCHEDULE_TIMEOUT); + amdgpu_pasid_free(pasid); +} + +/* + * VMID manager + * + * VMIDs are a per VMHUB identifier for page tables handling. + */ + +/** + * amdgpu_vmid_had_gpu_reset - check if reset occured since last use + * + * @adev: amdgpu_device pointer + * @id: VMID structure + * + * Check if GPU reset occured since last use of the VMID. + */ +bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, + struct amdgpu_vmid *id) +{ + return id->current_gpu_reset_count != + atomic_read(&adev->gpu_reset_counter); +} + +/** + * amdgpu_vm_grab_idle - grab idle VMID + * + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * @sync: sync object where we add dependencies + * @idle: resulting idle VMID + * + * Try to find an idle VMID, if none is idle add a fence to wait to the sync + * object. Returns -ENOMEM when we are out of memory. + */ +static int amdgpu_vmid_grab_idle(struct amdgpu_vm *vm, + struct amdgpu_ring *ring, + struct amdgpu_sync *sync, + struct amdgpu_vmid **idle) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + struct dma_fence **fences; + unsigned i; + int r; + + if (ring->vmid_wait && !dma_fence_is_signaled(ring->vmid_wait)) + return amdgpu_sync_fence(adev, sync, ring->vmid_wait, false); + + fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL); + if (!fences) + return -ENOMEM; + + /* Check if we have an idle VMID */ + i = 0; + list_for_each_entry((*idle), &id_mgr->ids_lru, list) { + fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, ring); + if (!fences[i]) + break; + ++i; + } + + /* If we can't find a idle VMID to use, wait till one becomes available */ + if (&(*idle)->list == &id_mgr->ids_lru) { + u64 fence_context = adev->vm_manager.fence_context + ring->idx; + unsigned seqno = ++adev->vm_manager.seqno[ring->idx]; + struct dma_fence_array *array; + unsigned j; + + *idle = NULL; + for (j = 0; j < i; ++j) + dma_fence_get(fences[j]); + + array = dma_fence_array_create(i, fences, fence_context, + seqno, true); + if (!array) { + for (j = 0; j < i; ++j) + dma_fence_put(fences[j]); + kfree(fences); + return -ENOMEM; + } + + r = amdgpu_sync_fence(adev, sync, &array->base, false); + dma_fence_put(ring->vmid_wait); + ring->vmid_wait = &array->base; + return r; + } + kfree(fences); + + return 0; +} + +/** + * amdgpu_vm_grab_reserved - try to assign reserved VMID + * + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * @sync: sync object where we add dependencies + * @fence: fence protecting ID from reuse + * @job: job who wants to use the VMID + * + * Try to assign a reserved VMID. + */ +static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, + struct amdgpu_ring *ring, + struct amdgpu_sync *sync, + struct dma_fence *fence, + struct amdgpu_job *job, + struct amdgpu_vmid **id) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + uint64_t fence_context = adev->fence_context + ring->idx; + struct dma_fence *updates = sync->last_vm_update; + bool needs_flush = vm->use_cpu_for_update; + int r = 0; + + *id = vm->reserved_vmid[vmhub]; + if (updates && (*id)->flushed_updates && + updates->context == (*id)->flushed_updates->context && + !dma_fence_is_later(updates, (*id)->flushed_updates)) + updates = NULL; + + if ((*id)->owner != vm->entity.fence_context || + job->vm_pd_addr != (*id)->pd_gpu_addr || + updates || !(*id)->last_flush || + ((*id)->last_flush->context != fence_context && + !dma_fence_is_signaled((*id)->last_flush))) { + struct dma_fence *tmp; + + /* to prevent one context starved by another context */ + (*id)->pd_gpu_addr = 0; + tmp = amdgpu_sync_peek_fence(&(*id)->active, ring); + if (tmp) { + *id = NULL; + r = amdgpu_sync_fence(adev, sync, tmp, false); + return r; + } + needs_flush = true; + } + + /* Good we can use this VMID. Remember this submission as + * user of the VMID. + */ + r = amdgpu_sync_fence(ring->adev, &(*id)->active, fence, false); + if (r) + return r; + + if (updates) { + dma_fence_put((*id)->flushed_updates); + (*id)->flushed_updates = dma_fence_get(updates); + } + job->vm_needs_flush = needs_flush; + return 0; +} + +/** + * amdgpu_vm_grab_used - try to reuse a VMID + * + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * @sync: sync object where we add dependencies + * @fence: fence protecting ID from reuse + * @job: job who wants to use the VMID + * @id: resulting VMID + * + * Try to reuse a VMID for this submission. + */ +static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, + struct amdgpu_ring *ring, + struct amdgpu_sync *sync, + struct dma_fence *fence, + struct amdgpu_job *job, + struct amdgpu_vmid **id) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + uint64_t fence_context = adev->fence_context + ring->idx; + struct dma_fence *updates = sync->last_vm_update; + int r; + + job->vm_needs_flush = vm->use_cpu_for_update; + + /* Check if we can use a VMID already assigned to this VM */ + list_for_each_entry_reverse((*id), &id_mgr->ids_lru, list) { + bool needs_flush = vm->use_cpu_for_update; + struct dma_fence *flushed; + + /* Check all the prerequisites to using this VMID */ + if ((*id)->owner != vm->entity.fence_context) + continue; + + if ((*id)->pd_gpu_addr != job->vm_pd_addr) + continue; + + if (!(*id)->last_flush || + ((*id)->last_flush->context != fence_context && + !dma_fence_is_signaled((*id)->last_flush))) + needs_flush = true; + + flushed = (*id)->flushed_updates; + if (updates && (!flushed || dma_fence_is_later(updates, flushed))) + needs_flush = true; + + /* Concurrent flushes are only possible starting with Vega10 and + * are broken on Navi10 and Navi14. + */ + if (needs_flush && (adev->asic_type < CHIP_VEGA10 || + adev->asic_type == CHIP_NAVI10)) + continue; + + /* Good, we can use this VMID. Remember this submission as + * user of the VMID. + */ + r = amdgpu_sync_fence(ring->adev, &(*id)->active, fence, false); + if (r) + return r; + + if (updates && (!flushed || dma_fence_is_later(updates, flushed))) { + dma_fence_put((*id)->flushed_updates); + (*id)->flushed_updates = dma_fence_get(updates); + } + + job->vm_needs_flush |= needs_flush; + return 0; + } + + *id = NULL; + return 0; +} + +/** + * amdgpu_vm_grab_id - allocate the next free VMID + * + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * @sync: sync object where we add dependencies + * @fence: fence protecting ID from reuse + * @job: job who wants to use the VMID + * + * Allocate an id for the vm, adding fences to the sync obj as necessary. + */ +int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring, + struct amdgpu_sync *sync, struct dma_fence *fence, + struct amdgpu_job *job) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + struct amdgpu_vmid *idle = NULL; + struct amdgpu_vmid *id = NULL; + int r = 0; + + mutex_lock(&id_mgr->lock); + r = amdgpu_vmid_grab_idle(vm, ring, sync, &idle); + if (r || !idle) + goto error; + + if (vm->reserved_vmid[vmhub]) { + r = amdgpu_vmid_grab_reserved(vm, ring, sync, fence, job, &id); + if (r || !id) + goto error; + } else { + r = amdgpu_vmid_grab_used(vm, ring, sync, fence, job, &id); + if (r) + goto error; + + if (!id) { + struct dma_fence *updates = sync->last_vm_update; + + /* Still no ID to use? Then use the idle one found earlier */ + id = idle; + + /* Remember this submission as user of the VMID */ + r = amdgpu_sync_fence(ring->adev, &id->active, + fence, false); + if (r) + goto error; + + dma_fence_put(id->flushed_updates); + id->flushed_updates = dma_fence_get(updates); + job->vm_needs_flush = true; + } + + list_move_tail(&id->list, &id_mgr->ids_lru); + } + + id->pd_gpu_addr = job->vm_pd_addr; + id->owner = vm->entity.fence_context; + + if (job->vm_needs_flush) { + dma_fence_put(id->last_flush); + id->last_flush = NULL; + } + job->vmid = id - id_mgr->ids; + job->pasid = vm->pasid; + trace_amdgpu_vm_grab_id(vm, ring, job); + +error: + mutex_unlock(&id_mgr->lock); + return r; +} + +int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + unsigned vmhub) +{ + struct amdgpu_vmid_mgr *id_mgr; + struct amdgpu_vmid *idle; + int r = 0; + + id_mgr = &adev->vm_manager.id_mgr[vmhub]; + mutex_lock(&id_mgr->lock); + if (vm->reserved_vmid[vmhub]) + goto unlock; + if (atomic_inc_return(&id_mgr->reserved_vmid_num) > + AMDGPU_VM_MAX_RESERVED_VMID) { + DRM_ERROR("Over limitation of reserved vmid\n"); + atomic_dec(&id_mgr->reserved_vmid_num); + r = -EINVAL; + goto unlock; + } + /* Select the first entry VMID */ + idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, list); + list_del_init(&idle->list); + vm->reserved_vmid[vmhub] = idle; + mutex_unlock(&id_mgr->lock); + + return 0; +unlock: + mutex_unlock(&id_mgr->lock); + return r; +} + +void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + unsigned vmhub) +{ + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + + mutex_lock(&id_mgr->lock); + if (vm->reserved_vmid[vmhub]) { + list_add(&vm->reserved_vmid[vmhub]->list, + &id_mgr->ids_lru); + vm->reserved_vmid[vmhub] = NULL; + atomic_dec(&id_mgr->reserved_vmid_num); + } + mutex_unlock(&id_mgr->lock); +} + +/** + * amdgpu_vmid_reset - reset VMID to zero + * + * @adev: amdgpu device structure + * @vmid: vmid number to use + * + * Reset saved GDW, GWS and OA to force switch on next flush. + */ +void amdgpu_vmid_reset(struct amdgpu_device *adev, unsigned vmhub, + unsigned vmid) +{ + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + struct amdgpu_vmid *id = &id_mgr->ids[vmid]; + + mutex_lock(&id_mgr->lock); + id->owner = 0; + id->gds_base = 0; + id->gds_size = 0; + id->gws_base = 0; + id->gws_size = 0; + id->oa_base = 0; + id->oa_size = 0; + mutex_unlock(&id_mgr->lock); +} + +/** + * amdgpu_vmid_reset_all - reset VMID to zero + * + * @adev: amdgpu device structure + * + * Reset VMID to force flush on next use + */ +void amdgpu_vmid_reset_all(struct amdgpu_device *adev) +{ + unsigned i, j; + + for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { + struct amdgpu_vmid_mgr *id_mgr = + &adev->vm_manager.id_mgr[i]; + + for (j = 1; j < id_mgr->num_ids; ++j) + amdgpu_vmid_reset(adev, i, j); + } +} + +/** + * amdgpu_vmid_mgr_init - init the VMID manager + * + * @adev: amdgpu_device pointer + * + * Initialize the VM manager structures + */ +void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) +{ + unsigned i, j; + + for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { + struct amdgpu_vmid_mgr *id_mgr = + &adev->vm_manager.id_mgr[i]; + + mutex_init(&id_mgr->lock); + INIT_LIST_HEAD(&id_mgr->ids_lru); + atomic_set(&id_mgr->reserved_vmid_num, 0); + + /* skip over VMID 0, since it is the system VM */ + for (j = 1; j < id_mgr->num_ids; ++j) { + amdgpu_vmid_reset(adev, i, j); + amdgpu_sync_create(&id_mgr->ids[j].active); + list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru); + } + } +} + +/** + * amdgpu_vmid_mgr_fini - cleanup VM manager + * + * @adev: amdgpu_device pointer + * + * Cleanup the VM manager and free resources. + */ +void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) +{ + unsigned i, j; + + for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { + struct amdgpu_vmid_mgr *id_mgr = + &adev->vm_manager.id_mgr[i]; + + mutex_destroy(&id_mgr->lock); + for (j = 0; j < AMDGPU_NUM_VMID; ++j) { + struct amdgpu_vmid *id = &id_mgr->ids[j]; + + amdgpu_sync_free(&id->active); + dma_fence_put(id->flushed_updates); + dma_fence_put(id->last_flush); + dma_fence_put(id->pasid_mapping); + } + } +}
diff --git a/amdgpu/amdgpu_ids.h b/amdgpu/amdgpu_ids.h new file mode 100644 index 0000000..7625419 --- /dev/null +++ b/amdgpu/amdgpu_ids.h
@@ -0,0 +1,96 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_IDS_H__ +#define __AMDGPU_IDS_H__ + +#include <linux/types.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/dma-fence.h> + +#include "amdgpu_sync.h" + +/* maximum number of VMIDs */ +#define AMDGPU_NUM_VMID 16 + +struct amdgpu_device; +struct amdgpu_vm; +struct amdgpu_ring; +struct amdgpu_sync; +struct amdgpu_job; + +struct amdgpu_vmid { + struct list_head list; + struct amdgpu_sync active; + struct dma_fence *last_flush; + uint64_t owner; + + uint64_t pd_gpu_addr; + /* last flushed PD/PT update */ + struct dma_fence *flushed_updates; + + uint32_t current_gpu_reset_count; + + uint32_t gds_base; + uint32_t gds_size; + uint32_t gws_base; + uint32_t gws_size; + uint32_t oa_base; + uint32_t oa_size; + + unsigned pasid; + struct dma_fence *pasid_mapping; +}; + +struct amdgpu_vmid_mgr { + struct mutex lock; + unsigned num_ids; + struct list_head ids_lru; + struct amdgpu_vmid ids[AMDGPU_NUM_VMID]; + atomic_t reserved_vmid_num; +}; + +int amdgpu_pasid_alloc(unsigned int bits); +void amdgpu_pasid_free(unsigned int pasid); +void amdgpu_pasid_free_delayed(struct reservation_object *resv, + unsigned int pasid); + +bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, + struct amdgpu_vmid *id); +int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + unsigned vmhub); +void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + unsigned vmhub); +int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring, + struct amdgpu_sync *sync, struct dma_fence *fence, + struct amdgpu_job *job); +void amdgpu_vmid_reset(struct amdgpu_device *adev, unsigned vmhub, + unsigned vmid); +void amdgpu_vmid_reset_all(struct amdgpu_device *adev); + +void amdgpu_vmid_mgr_init(struct amdgpu_device *adev); +void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_ih.c b/amdgpu/amdgpu_ih.c new file mode 100644 index 0000000..6d8f055 --- /dev/null +++ b/amdgpu/amdgpu_ih.c
@@ -0,0 +1,179 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/dma-mapping.h> + +#include "amdgpu.h" +#include "amdgpu_ih.h" + +/** + * amdgpu_ih_ring_init - initialize the IH state + * + * @adev: amdgpu_device pointer + * @ih: ih ring to initialize + * @ring_size: ring size to allocate + * @use_bus_addr: true when we can use dma_alloc_coherent + * + * Initializes the IH state and allocates a buffer + * for the IH ring buffer. + * Returns 0 for success, errors for failure. + */ +int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, + unsigned ring_size, bool use_bus_addr) +{ + u32 rb_bufsz; + int r; + + /* Align ring size */ + rb_bufsz = order_base_2(ring_size / 4); + ring_size = (1 << rb_bufsz) * 4; + ih->ring_size = ring_size; + ih->ptr_mask = ih->ring_size - 1; + ih->rptr = 0; + ih->use_bus_addr = use_bus_addr; + + if (use_bus_addr) { + dma_addr_t dma_addr; + + if (ih->ring) + return 0; + + /* add 8 bytes for the rptr/wptr shadows and + * add them to the end of the ring allocation. + */ + ih->ring = dma_alloc_coherent(adev->dev, ih->ring_size + 8, + &dma_addr, GFP_KERNEL); + if (ih->ring == NULL) + return -ENOMEM; + + memset((void *)ih->ring, 0, ih->ring_size + 8); + ih->gpu_addr = dma_addr; + ih->wptr_addr = dma_addr + ih->ring_size; + ih->wptr_cpu = &ih->ring[ih->ring_size / 4]; + ih->rptr_addr = dma_addr + ih->ring_size + 4; + ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1]; + } else { + unsigned wptr_offs, rptr_offs; + + r = amdgpu_device_wb_get(adev, &wptr_offs); + if (r) + return r; + + r = amdgpu_device_wb_get(adev, &rptr_offs); + if (r) { + amdgpu_device_wb_free(adev, wptr_offs); + return r; + } + + r = amdgpu_bo_create_kernel(adev, ih->ring_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, + &ih->ring_obj, &ih->gpu_addr, + (void **)&ih->ring); + if (r) { + amdgpu_device_wb_free(adev, rptr_offs); + amdgpu_device_wb_free(adev, wptr_offs); + return r; + } + + ih->wptr_addr = adev->wb.gpu_addr + wptr_offs * 4; + ih->wptr_cpu = &adev->wb.wb[wptr_offs]; + ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4; + ih->rptr_cpu = &adev->wb.wb[rptr_offs]; + } + return 0; +} + +/** + * amdgpu_ih_ring_fini - tear down the IH state + * + * @adev: amdgpu_device pointer + * @ih: ih ring to tear down + * + * Tears down the IH state and frees buffer + * used for the IH ring buffer. + */ +void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) +{ + if (ih->use_bus_addr) { + if (!ih->ring) + return; + + /* add 8 bytes for the rptr/wptr shadows and + * add them to the end of the ring allocation. + */ + dma_free_coherent(adev->dev, ih->ring_size + 8, + (void *)ih->ring, ih->gpu_addr); + ih->ring = NULL; + } else { + amdgpu_bo_free_kernel(&ih->ring_obj, &ih->gpu_addr, + (void **)&ih->ring); + amdgpu_device_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4); + amdgpu_device_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4); + } +} + +/** + * amdgpu_ih_process - interrupt handler + * + * @adev: amdgpu_device pointer + * @ih: ih ring to process + * + * Interrupt hander (VI), walk the IH ring. + * Returns irq process return code. + */ +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) +{ + unsigned int count = AMDGPU_IH_MAX_NUM_IVS; + u32 wptr; + + if (!ih->enabled || adev->shutdown) + return IRQ_NONE; + + wptr = amdgpu_ih_get_wptr(adev, ih); + +restart_ih: + /* is somebody else already processing irqs? */ + if (atomic_xchg(&ih->lock, 1)) + return IRQ_NONE; + + DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, ih->rptr, wptr); + + /* Order reading of wptr vs. reading of IH ring data */ + rmb(); + + while (ih->rptr != wptr && --count) { + amdgpu_irq_dispatch(adev, ih); + ih->rptr &= ih->ptr_mask; + } + + amdgpu_ih_set_rptr(adev, ih); + atomic_set(&ih->lock, 0); + + /* make sure wptr hasn't changed while processing */ + wptr = amdgpu_ih_get_wptr(adev, ih); + if (wptr != ih->rptr) + goto restart_ih; + + return IRQ_HANDLED; +} +
diff --git a/amdgpu/amdgpu_ih.h b/amdgpu/amdgpu_ih.h new file mode 100644 index 0000000..4e0bb64 --- /dev/null +++ b/amdgpu/amdgpu_ih.h
@@ -0,0 +1,77 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_IH_H__ +#define __AMDGPU_IH_H__ + +/* Maximum number of IVs processed at once */ +#define AMDGPU_IH_MAX_NUM_IVS 32 + +struct amdgpu_device; +struct amdgpu_iv_entry; + +/* + * R6xx+ IH ring + */ +struct amdgpu_ih_ring { + unsigned ring_size; + uint32_t ptr_mask; + u32 doorbell_index; + bool use_doorbell; + bool use_bus_addr; + + struct amdgpu_bo *ring_obj; + volatile uint32_t *ring; + uint64_t gpu_addr; + + uint64_t wptr_addr; + volatile uint32_t *wptr_cpu; + + uint64_t rptr_addr; + volatile uint32_t *rptr_cpu; + + bool enabled; + unsigned rptr; + atomic_t lock; +}; + +/* provided by the ih block */ +struct amdgpu_ih_funcs { + /* ring read/write ptr handling, called from interrupt context */ + u32 (*get_wptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); + void (*decode_iv)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry); + void (*set_rptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); +}; + +#define amdgpu_ih_get_wptr(adev, ih) (adev)->irq.ih_funcs->get_wptr((adev), (ih)) +#define amdgpu_ih_decode_iv(adev, iv) \ + (adev)->irq.ih_funcs->decode_iv((adev), (ih), (iv)) +#define amdgpu_ih_set_rptr(adev, ih) (adev)->irq.ih_funcs->set_rptr((adev), (ih)) + +int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, + unsigned ring_size, bool use_bus_addr); +void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); +int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); + +#endif
diff --git a/amdgpu/amdgpu_ioc32.c b/amdgpu/amdgpu_ioc32.c new file mode 100644 index 0000000..5cf142e --- /dev/null +++ b/amdgpu/amdgpu_ioc32.c
@@ -0,0 +1,48 @@ +/** + * \file amdgpu_ioc32.c + * + * 32-bit ioctl compatibility routines for the AMDGPU DRM. + * + * \author Paul Mackerras <paulus@samba.org> + * + * Copyright (C) Paul Mackerras 2005 + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include <linux/compat.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_ioctl.h> + +#include "amdgpu_drv.h" + +long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + unsigned int nr = DRM_IOCTL_NR(cmd); + int ret; + + if (nr < DRM_COMMAND_BASE) + return drm_compat_ioctl(filp, cmd, arg); + + ret = amdgpu_drm_ioctl(filp, cmd, arg); + + return ret; +}
diff --git a/amdgpu/amdgpu_irq.c b/amdgpu/amdgpu_irq.c new file mode 100644 index 0000000..2a3f5ec --- /dev/null +++ b/amdgpu/amdgpu_irq.c
@@ -0,0 +1,672 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ + +/** + * DOC: Interrupt Handling + * + * Interrupts generated within GPU hardware raise interrupt requests that are + * passed to amdgpu IRQ handler which is responsible for detecting source and + * type of the interrupt and dispatching matching handlers. If handling an + * interrupt requires calling kernel functions that may sleep processing is + * dispatched to work handlers. + * + * If MSI functionality is not disabled by module parameter then MSI + * support will be enabled. + * + * For GPU interrupt sources that may be driven by another driver, IRQ domain + * support is used (with mapping between virtual and hardware IRQs). + */ + +#include <linux/irq.h> +#include <linux/pci.h> + +#include <drm/drm_crtc_helper.h> +#include <drm/drm_irq.h> +#include <drm/drm_vblank.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_ih.h" +#include "atom.h" +#include "amdgpu_connectors.h" +#include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" + +#include <linux/pm_runtime.h> + +#ifdef CONFIG_DRM_AMD_DC +#include "amdgpu_dm_irq.h" +#endif + +#define AMDGPU_WAIT_IDLE_TIMEOUT 200 + +/** + * amdgpu_hotplug_work_func - work handler for display hotplug event + * + * @work: work struct pointer + * + * This is the hotplug event work handler (all ASICs). + * The work gets scheduled from the IRQ handler if there + * was a hotplug interrupt. It walks through the connector table + * and calls hotplug handler for each connector. After this, it sends + * a DRM hotplug event to alert userspace. + * + * This design approach is required in order to defer hotplug event handling + * from the IRQ handler to a work handler because hotplug handler has to use + * mutexes which cannot be locked in an IRQ handler (since &mutex_lock may + * sleep). + */ +static void amdgpu_hotplug_work_func(struct work_struct *work) +{ + struct amdgpu_device *adev = container_of(work, struct amdgpu_device, + hotplug_work); + struct drm_device *dev = adev->ddev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct drm_connector *connector; + + mutex_lock(&mode_config->mutex); + list_for_each_entry(connector, &mode_config->connector_list, head) + amdgpu_connector_hotplug(connector); + mutex_unlock(&mode_config->mutex); + /* Just fire off a uevent and let userspace tell us what to do */ + drm_helper_hpd_irq_event(dev); +} + +/** + * amdgpu_irq_disable_all - disable *all* interrupts + * + * @adev: amdgpu device pointer + * + * Disable all types of interrupts from all sources. + */ +void amdgpu_irq_disable_all(struct amdgpu_device *adev) +{ + unsigned long irqflags; + unsigned i, j, k; + int r; + + spin_lock_irqsave(&adev->irq.lock, irqflags); + for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) { + if (!adev->irq.client[i].sources) + continue; + + for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) { + struct amdgpu_irq_src *src = adev->irq.client[i].sources[j]; + + if (!src || !src->funcs->set || !src->num_types) + continue; + + for (k = 0; k < src->num_types; ++k) { + atomic_set(&src->enabled_types[k], 0); + r = src->funcs->set(adev, src, k, + AMDGPU_IRQ_STATE_DISABLE); + if (r) + DRM_ERROR("error disabling interrupt (%d)\n", + r); + } + } + } + spin_unlock_irqrestore(&adev->irq.lock, irqflags); +} + +/** + * amdgpu_irq_handler - IRQ handler + * + * @irq: IRQ number (unused) + * @arg: pointer to DRM device + * + * IRQ handler for amdgpu driver (all ASICs). + * + * Returns: + * result of handling the IRQ, as defined by &irqreturn_t + */ +irqreturn_t amdgpu_irq_handler(int irq, void *arg) +{ + struct drm_device *dev = (struct drm_device *) arg; + struct amdgpu_device *adev = dev->dev_private; + irqreturn_t ret; + + ret = amdgpu_ih_process(adev, &adev->irq.ih); + if (ret == IRQ_HANDLED) + pm_runtime_mark_last_busy(dev->dev); + return ret; +} + +/** + * amdgpu_irq_handle_ih1 - kick of processing for IH1 + * + * @work: work structure in struct amdgpu_irq + * + * Kick of processing IH ring 1. + */ +static void amdgpu_irq_handle_ih1(struct work_struct *work) +{ + struct amdgpu_device *adev = container_of(work, struct amdgpu_device, + irq.ih1_work); + + amdgpu_ih_process(adev, &adev->irq.ih1); +} + +/** + * amdgpu_irq_handle_ih2 - kick of processing for IH2 + * + * @work: work structure in struct amdgpu_irq + * + * Kick of processing IH ring 2. + */ +static void amdgpu_irq_handle_ih2(struct work_struct *work) +{ + struct amdgpu_device *adev = container_of(work, struct amdgpu_device, + irq.ih2_work); + + amdgpu_ih_process(adev, &adev->irq.ih2); +} + +/** + * amdgpu_msi_ok - check whether MSI functionality is enabled + * + * @adev: amdgpu device pointer (unused) + * + * Checks whether MSI functionality has been disabled via module parameter + * (all ASICs). + * + * Returns: + * *true* if MSIs are allowed to be enabled or *false* otherwise + */ +static bool amdgpu_msi_ok(struct amdgpu_device *adev) +{ + if (amdgpu_msi == 1) + return true; + else if (amdgpu_msi == 0) + return false; + + return true; +} + +/** + * amdgpu_irq_init - initialize interrupt handling + * + * @adev: amdgpu device pointer + * + * Sets up work functions for hotplug and reset interrupts, enables MSI + * functionality, initializes vblank, hotplug and reset interrupt handling. + * + * Returns: + * 0 on success or error code on failure + */ +int amdgpu_irq_init(struct amdgpu_device *adev) +{ + int r = 0; + + spin_lock_init(&adev->irq.lock); + + /* Enable MSI if not disabled by module parameter */ + adev->irq.msi_enabled = false; + + if (amdgpu_msi_ok(adev)) { + int ret = pci_enable_msi(adev->pdev); + if (!ret) { + adev->irq.msi_enabled = true; + dev_dbg(adev->dev, "amdgpu: using MSI.\n"); + } + } + + if (!amdgpu_device_has_dc_support(adev)) { + if (!adev->enable_virtual_display) + /* Disable vblank IRQs aggressively for power-saving */ + /* XXX: can this be enabled for DC? */ + adev->ddev->vblank_disable_immediate = true; + + r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc); + if (r) + return r; + + /* Pre-DCE11 */ + INIT_WORK(&adev->hotplug_work, + amdgpu_hotplug_work_func); + } + + INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1); + INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2); + + adev->irq.installed = true; + r = drm_irq_install(adev->ddev, adev->ddev->pdev->irq); + if (r) { + adev->irq.installed = false; + if (!amdgpu_device_has_dc_support(adev)) + flush_work(&adev->hotplug_work); + return r; + } + adev->ddev->max_vblank_count = 0x00ffffff; + + DRM_DEBUG("amdgpu: irq initialized.\n"); + return 0; +} + +/** + * amdgpu_irq_fini - shut down interrupt handling + * + * @adev: amdgpu device pointer + * + * Tears down work functions for hotplug and reset interrupts, disables MSI + * functionality, shuts down vblank, hotplug and reset interrupt handling, + * turns off interrupts from all sources (all ASICs). + */ +void amdgpu_irq_fini(struct amdgpu_device *adev) +{ + unsigned i, j; + + if (adev->irq.installed) { + drm_irq_uninstall(adev->ddev); + adev->irq.installed = false; + if (adev->irq.msi_enabled) + pci_disable_msi(adev->pdev); + if (!amdgpu_device_has_dc_support(adev)) + flush_work(&adev->hotplug_work); + } + + for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) { + if (!adev->irq.client[i].sources) + continue; + + for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) { + struct amdgpu_irq_src *src = adev->irq.client[i].sources[j]; + + if (!src) + continue; + + kfree(src->enabled_types); + src->enabled_types = NULL; + if (src->data) { + kfree(src->data); + kfree(src); + adev->irq.client[i].sources[j] = NULL; + } + } + kfree(adev->irq.client[i].sources); + adev->irq.client[i].sources = NULL; + } +} + +/** + * amdgpu_irq_add_id - register IRQ source + * + * @adev: amdgpu device pointer + * @client_id: client id + * @src_id: source id + * @source: IRQ source pointer + * + * Registers IRQ source on a client. + * + * Returns: + * 0 on success or error code otherwise + */ +int amdgpu_irq_add_id(struct amdgpu_device *adev, + unsigned client_id, unsigned src_id, + struct amdgpu_irq_src *source) +{ + if (client_id >= AMDGPU_IRQ_CLIENTID_MAX) + return -EINVAL; + + if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) + return -EINVAL; + + if (!source->funcs) + return -EINVAL; + + if (!adev->irq.client[client_id].sources) { + adev->irq.client[client_id].sources = + kcalloc(AMDGPU_MAX_IRQ_SRC_ID, + sizeof(struct amdgpu_irq_src *), + GFP_KERNEL); + if (!adev->irq.client[client_id].sources) + return -ENOMEM; + } + + if (adev->irq.client[client_id].sources[src_id] != NULL) + return -EINVAL; + + if (source->num_types && !source->enabled_types) { + atomic_t *types; + + types = kcalloc(source->num_types, sizeof(atomic_t), + GFP_KERNEL); + if (!types) + return -ENOMEM; + + source->enabled_types = types; + } + + adev->irq.client[client_id].sources[src_id] = source; + return 0; +} + +/** + * amdgpu_irq_dispatch - dispatch IRQ to IP blocks + * + * @adev: amdgpu device pointer + * @entry: interrupt vector pointer + * + * Dispatches IRQ to IP blocks. + */ +void amdgpu_irq_dispatch(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + u32 ring_index = ih->rptr >> 2; + struct amdgpu_iv_entry entry; + unsigned client_id, src_id; + struct amdgpu_irq_src *src; + bool handled = false; + int r; + + entry.iv_entry = (const uint32_t *)&ih->ring[ring_index]; + amdgpu_ih_decode_iv(adev, &entry); + + trace_amdgpu_iv(ih - &adev->irq.ih, &entry); + + client_id = entry.client_id; + src_id = entry.src_id; + + if (client_id >= AMDGPU_IRQ_CLIENTID_MAX) { + DRM_DEBUG("Invalid client_id in IV: %d\n", client_id); + + } else if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) { + DRM_DEBUG("Invalid src_id in IV: %d\n", src_id); + + } else if (adev->irq.virq[src_id]) { + generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id)); + + } else if (!adev->irq.client[client_id].sources) { + DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n", + client_id, src_id); + + } else if ((src = adev->irq.client[client_id].sources[src_id])) { + r = src->funcs->process(adev, src, &entry); + if (r < 0) + DRM_ERROR("error processing interrupt (%d)\n", r); + else if (r) + handled = true; + + } else { + DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id); + } + + /* Send it to amdkfd as well if it isn't already handled */ + if (!handled) + amdgpu_amdkfd_interrupt(adev, entry.iv_entry); +} + +/** + * amdgpu_irq_update - update hardware interrupt state + * + * @adev: amdgpu device pointer + * @src: interrupt source pointer + * @type: type of interrupt + * + * Updates interrupt state for the specific source (all ASICs). + */ +int amdgpu_irq_update(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, unsigned type) +{ + unsigned long irqflags; + enum amdgpu_interrupt_state state; + int r; + + spin_lock_irqsave(&adev->irq.lock, irqflags); + + /* We need to determine after taking the lock, otherwise + we might disable just enabled interrupts again */ + if (amdgpu_irq_enabled(adev, src, type)) + state = AMDGPU_IRQ_STATE_ENABLE; + else + state = AMDGPU_IRQ_STATE_DISABLE; + + r = src->funcs->set(adev, src, type, state); + spin_unlock_irqrestore(&adev->irq.lock, irqflags); + return r; +} + +/** + * amdgpu_irq_gpu_reset_resume_helper - update interrupt states on all sources + * + * @adev: amdgpu device pointer + * + * Updates state of all types of interrupts on all sources on resume after + * reset. + */ +void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev) +{ + int i, j, k; + + for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) { + if (!adev->irq.client[i].sources) + continue; + + for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) { + struct amdgpu_irq_src *src = adev->irq.client[i].sources[j]; + + if (!src) + continue; + for (k = 0; k < src->num_types; k++) + amdgpu_irq_update(adev, src, k); + } + } +} + +/** + * amdgpu_irq_get - enable interrupt + * + * @adev: amdgpu device pointer + * @src: interrupt source pointer + * @type: type of interrupt + * + * Enables specified type of interrupt on the specified source (all ASICs). + * + * Returns: + * 0 on success or error code otherwise + */ +int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type) +{ + if (!adev->ddev->irq_enabled) + return -ENOENT; + + if (type >= src->num_types) + return -EINVAL; + + if (!src->enabled_types || !src->funcs->set) + return -EINVAL; + + if (atomic_inc_return(&src->enabled_types[type]) == 1) + return amdgpu_irq_update(adev, src, type); + + return 0; +} + +/** + * amdgpu_irq_put - disable interrupt + * + * @adev: amdgpu device pointer + * @src: interrupt source pointer + * @type: type of interrupt + * + * Enables specified type of interrupt on the specified source (all ASICs). + * + * Returns: + * 0 on success or error code otherwise + */ +int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type) +{ + if (!adev->ddev->irq_enabled) + return -ENOENT; + + if (type >= src->num_types) + return -EINVAL; + + if (!src->enabled_types || !src->funcs->set) + return -EINVAL; + + if (atomic_dec_and_test(&src->enabled_types[type])) + return amdgpu_irq_update(adev, src, type); + + return 0; +} + +/** + * amdgpu_irq_enabled - check whether interrupt is enabled or not + * + * @adev: amdgpu device pointer + * @src: interrupt source pointer + * @type: type of interrupt + * + * Checks whether the given type of interrupt is enabled on the given source. + * + * Returns: + * *true* if interrupt is enabled, *false* if interrupt is disabled or on + * invalid parameters + */ +bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type) +{ + if (!adev->ddev->irq_enabled) + return false; + + if (type >= src->num_types) + return false; + + if (!src->enabled_types || !src->funcs->set) + return false; + + return !!atomic_read(&src->enabled_types[type]); +} + +/* XXX: Generic IRQ handling */ +static void amdgpu_irq_mask(struct irq_data *irqd) +{ + /* XXX */ +} + +static void amdgpu_irq_unmask(struct irq_data *irqd) +{ + /* XXX */ +} + +/* amdgpu hardware interrupt chip descriptor */ +static struct irq_chip amdgpu_irq_chip = { + .name = "amdgpu-ih", + .irq_mask = amdgpu_irq_mask, + .irq_unmask = amdgpu_irq_unmask, +}; + +/** + * amdgpu_irqdomain_map - create mapping between virtual and hardware IRQ numbers + * + * @d: amdgpu IRQ domain pointer (unused) + * @irq: virtual IRQ number + * @hwirq: hardware irq number + * + * Current implementation assigns simple interrupt handler to the given virtual + * IRQ. + * + * Returns: + * 0 on success or error code otherwise + */ +static int amdgpu_irqdomain_map(struct irq_domain *d, + unsigned int irq, irq_hw_number_t hwirq) +{ + if (hwirq >= AMDGPU_MAX_IRQ_SRC_ID) + return -EPERM; + + irq_set_chip_and_handler(irq, + &amdgpu_irq_chip, handle_simple_irq); + return 0; +} + +/* Implementation of methods for amdgpu IRQ domain */ +static const struct irq_domain_ops amdgpu_hw_irqdomain_ops = { + .map = amdgpu_irqdomain_map, +}; + +/** + * amdgpu_irq_add_domain - create a linear IRQ domain + * + * @adev: amdgpu device pointer + * + * Creates an IRQ domain for GPU interrupt sources + * that may be driven by another driver (e.g., ACP). + * + * Returns: + * 0 on success or error code otherwise + */ +int amdgpu_irq_add_domain(struct amdgpu_device *adev) +{ + adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID, + &amdgpu_hw_irqdomain_ops, adev); + if (!adev->irq.domain) { + DRM_ERROR("GPU irq add domain failed\n"); + return -ENODEV; + } + + return 0; +} + +/** + * amdgpu_irq_remove_domain - remove the IRQ domain + * + * @adev: amdgpu device pointer + * + * Removes the IRQ domain for GPU interrupt sources + * that may be driven by another driver (e.g., ACP). + */ +void amdgpu_irq_remove_domain(struct amdgpu_device *adev) +{ + if (adev->irq.domain) { + irq_domain_remove(adev->irq.domain); + adev->irq.domain = NULL; + } +} + +/** + * amdgpu_irq_create_mapping - create mapping between domain Linux IRQs + * + * @adev: amdgpu device pointer + * @src_id: IH source id + * + * Creates mapping between a domain IRQ (GPU IH src id) and a Linux IRQ + * Use this for components that generate a GPU interrupt, but are driven + * by a different driver (e.g., ACP). + * + * Returns: + * Linux IRQ + */ +unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id) +{ + adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id); + + return adev->irq.virq[src_id]; +}
diff --git a/amdgpu/amdgpu_irq.h b/amdgpu/amdgpu_irq.h new file mode 100644 index 0000000..c718e94 --- /dev/null +++ b/amdgpu/amdgpu_irq.h
@@ -0,0 +1,126 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_IRQ_H__ +#define __AMDGPU_IRQ_H__ + +#include <linux/irqdomain.h> +#include "soc15_ih_clientid.h" +#include "amdgpu_ih.h" + +#define AMDGPU_MAX_IRQ_SRC_ID 0x100 +#define AMDGPU_MAX_IRQ_CLIENT_ID 0x100 + +#define AMDGPU_IRQ_CLIENTID_LEGACY 0 +#define AMDGPU_IRQ_CLIENTID_MAX SOC15_IH_CLIENTID_MAX + +#define AMDGPU_IRQ_SRC_DATA_MAX_SIZE_DW 4 + +struct amdgpu_device; + +enum amdgpu_interrupt_state { + AMDGPU_IRQ_STATE_DISABLE, + AMDGPU_IRQ_STATE_ENABLE, +}; + +struct amdgpu_iv_entry { + unsigned client_id; + unsigned src_id; + unsigned ring_id; + unsigned vmid; + unsigned vmid_src; + uint64_t timestamp; + unsigned timestamp_src; + unsigned pasid; + unsigned pasid_src; + unsigned src_data[AMDGPU_IRQ_SRC_DATA_MAX_SIZE_DW]; + const uint32_t *iv_entry; +}; + +struct amdgpu_irq_src { + unsigned num_types; + atomic_t *enabled_types; + const struct amdgpu_irq_src_funcs *funcs; + void *data; +}; + +struct amdgpu_irq_client { + struct amdgpu_irq_src **sources; +}; + +/* provided by interrupt generating IP blocks */ +struct amdgpu_irq_src_funcs { + int (*set)(struct amdgpu_device *adev, struct amdgpu_irq_src *source, + unsigned type, enum amdgpu_interrupt_state state); + + int (*process)(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry); +}; + +struct amdgpu_irq { + bool installed; + spinlock_t lock; + /* interrupt sources */ + struct amdgpu_irq_client client[AMDGPU_IRQ_CLIENTID_MAX]; + + /* status, etc. */ + bool msi_enabled; /* msi enabled */ + + /* interrupt rings */ + struct amdgpu_ih_ring ih, ih1, ih2; + const struct amdgpu_ih_funcs *ih_funcs; + struct work_struct ih1_work, ih2_work; + struct amdgpu_irq_src self_irq; + + /* gen irq stuff */ + struct irq_domain *domain; /* GPU irq controller domain */ + unsigned virq[AMDGPU_MAX_IRQ_SRC_ID]; + uint32_t srbm_soft_reset; +}; + +void amdgpu_irq_disable_all(struct amdgpu_device *adev); +irqreturn_t amdgpu_irq_handler(int irq, void *arg); + +int amdgpu_irq_init(struct amdgpu_device *adev); +void amdgpu_irq_fini(struct amdgpu_device *adev); +int amdgpu_irq_add_id(struct amdgpu_device *adev, + unsigned client_id, unsigned src_id, + struct amdgpu_irq_src *source); +void amdgpu_irq_dispatch(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih); +int amdgpu_irq_update(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type); +int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type); +int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type); +bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src, + unsigned type); +void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev); + +int amdgpu_irq_add_domain(struct amdgpu_device *adev); +void amdgpu_irq_remove_domain(struct amdgpu_device *adev); +unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id); + +#endif
diff --git a/amdgpu/amdgpu_job.c b/amdgpu/amdgpu_job.c new file mode 100644 index 0000000..9d76e09 --- /dev/null +++ b/amdgpu/amdgpu_job.c
@@ -0,0 +1,254 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include <linux/kthread.h> +#include <linux/wait.h> +#include <linux/sched.h> + +#include "amdgpu.h" +#include "amdgpu_trace.h" + +static void amdgpu_job_timedout(struct drm_sched_job *s_job) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched); + struct amdgpu_job *job = to_amdgpu_job(s_job); + struct amdgpu_task_info ti; + + memset(&ti, 0, sizeof(struct amdgpu_task_info)); + + if (amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) { + DRM_ERROR("ring %s timeout, but soft recovered\n", + s_job->sched->name); + return; + } + + amdgpu_vm_get_task_info(ring->adev, job->pasid, &ti); + DRM_ERROR("ring %s timeout, signaled seq=%u, emitted seq=%u\n", + job->base.sched->name, atomic_read(&ring->fence_drv.last_seq), + ring->fence_drv.sync_seq); + DRM_ERROR("Process information: process %s pid %d thread %s pid %d\n", + ti.process_name, ti.tgid, ti.task_name, ti.pid); + + if (amdgpu_device_should_recover_gpu(ring->adev)) + amdgpu_device_gpu_recover(ring->adev, job); + else + drm_sched_suspend_timeout(&ring->sched); +} + +int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, + struct amdgpu_job **job, struct amdgpu_vm *vm) +{ + size_t size = sizeof(struct amdgpu_job); + + if (num_ibs == 0) + return -EINVAL; + + size += sizeof(struct amdgpu_ib) * num_ibs; + + *job = kzalloc(size, GFP_KERNEL); + if (!*job) + return -ENOMEM; + + /* + * Initialize the scheduler to at least some ring so that we always + * have a pointer to adev. + */ + (*job)->base.sched = &adev->rings[0]->sched; + (*job)->vm = vm; + (*job)->ibs = (void *)&(*job)[1]; + (*job)->num_ibs = num_ibs; + + amdgpu_sync_create(&(*job)->sync); + amdgpu_sync_create(&(*job)->sched_sync); + (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter); + (*job)->vm_pd_addr = AMDGPU_BO_INVALID_OFFSET; + + return 0; +} + +int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, + struct amdgpu_job **job) +{ + int r; + + r = amdgpu_job_alloc(adev, 1, job, NULL); + if (r) + return r; + + r = amdgpu_ib_get(adev, NULL, size, &(*job)->ibs[0]); + if (r) + kfree(*job); + + return r; +} + +void amdgpu_job_free_resources(struct amdgpu_job *job) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched); + struct dma_fence *f; + unsigned i; + + /* use sched fence if available */ + f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; + + for (i = 0; i < job->num_ibs; ++i) + amdgpu_ib_free(ring->adev, &job->ibs[i], f); +} + +static void amdgpu_job_free_cb(struct drm_sched_job *s_job) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched); + struct amdgpu_job *job = to_amdgpu_job(s_job); + + drm_sched_job_cleanup(s_job); + + amdgpu_ring_priority_put(ring, s_job->s_priority); + dma_fence_put(job->fence); + amdgpu_sync_free(&job->sync); + amdgpu_sync_free(&job->sched_sync); + kfree(job); +} + +void amdgpu_job_free(struct amdgpu_job *job) +{ + amdgpu_job_free_resources(job); + + dma_fence_put(job->fence); + amdgpu_sync_free(&job->sync); + amdgpu_sync_free(&job->sched_sync); + kfree(job); +} + +int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, + void *owner, struct dma_fence **f) +{ + enum drm_sched_priority priority; + struct amdgpu_ring *ring; + int r; + + if (!f) + return -EINVAL; + + r = drm_sched_job_init(&job->base, entity, owner); + if (r) + return r; + + job->owner = owner; + *f = dma_fence_get(&job->base.s_fence->finished); + amdgpu_job_free_resources(job); + priority = job->base.s_priority; + drm_sched_entity_push_job(&job->base, entity); + + ring = to_amdgpu_ring(entity->rq->sched); + amdgpu_ring_priority_get(ring, priority); + + return 0; +} + +int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, + struct dma_fence **fence) +{ + int r; + + job->base.sched = &ring->sched; + r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence); + job->fence = dma_fence_get(*fence); + if (r) + return r; + + amdgpu_job_free(job); + return 0; +} + +static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job, + struct drm_sched_entity *s_entity) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched); + struct amdgpu_job *job = to_amdgpu_job(sched_job); + struct amdgpu_vm *vm = job->vm; + struct dma_fence *fence; + bool explicit = false; + int r; + + fence = amdgpu_sync_get_fence(&job->sync, &explicit); + if (fence && explicit) { + if (drm_sched_dependency_optimized(fence, s_entity)) { + r = amdgpu_sync_fence(ring->adev, &job->sched_sync, + fence, false); + if (r) + DRM_ERROR("Error adding fence (%d)\n", r); + } + } + + while (fence == NULL && vm && !job->vmid) { + r = amdgpu_vmid_grab(vm, ring, &job->sync, + &job->base.s_fence->finished, + job); + if (r) + DRM_ERROR("Error getting VM ID (%d)\n", r); + + fence = amdgpu_sync_get_fence(&job->sync, NULL); + } + + return fence; +} + +static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) +{ + struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched); + struct dma_fence *fence = NULL, *finished; + struct amdgpu_job *job; + int r; + + job = to_amdgpu_job(sched_job); + finished = &job->base.s_fence->finished; + + BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL)); + + trace_amdgpu_sched_run_job(job); + + if (job->vram_lost_counter != atomic_read(&ring->adev->vram_lost_counter)) + dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */ + + if (finished->error < 0) { + DRM_INFO("Skip scheduling IBs!\n"); + } else { + r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job, + &fence); + if (r) + DRM_ERROR("Error scheduling IBs (%d)\n", r); + } + /* if gpu reset, hw fence will be replaced here */ + dma_fence_put(job->fence); + job->fence = dma_fence_get(fence); + + amdgpu_job_free_resources(job); + return fence; +} + +const struct drm_sched_backend_ops amdgpu_sched_ops = { + .dependency = amdgpu_job_dependency, + .run_job = amdgpu_job_run, + .timedout_job = amdgpu_job_timedout, + .free_job = amdgpu_job_free_cb +};
diff --git a/amdgpu/amdgpu_job.h b/amdgpu/amdgpu_job.h new file mode 100644 index 0000000..51e6250 --- /dev/null +++ b/amdgpu/amdgpu_job.h
@@ -0,0 +1,79 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_JOB_H__ +#define __AMDGPU_JOB_H__ + +/* bit set means command submit involves a preamble IB */ +#define AMDGPU_PREAMBLE_IB_PRESENT (1 << 0) +/* bit set means preamble IB is first presented in belonging context */ +#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST (1 << 1) +/* bit set means context switch occured */ +#define AMDGPU_HAVE_CTX_SWITCH (1 << 2) +/* bit set means IB is preempted */ +#define AMDGPU_IB_PREEMPTED (1 << 3) + +#define to_amdgpu_job(sched_job) \ + container_of((sched_job), struct amdgpu_job, base) + +#define AMDGPU_JOB_GET_VMID(job) ((job) ? (job)->vmid : 0) + +struct amdgpu_fence; + +struct amdgpu_job { + struct drm_sched_job base; + struct amdgpu_vm *vm; + struct amdgpu_sync sync; + struct amdgpu_sync sched_sync; + struct amdgpu_ib *ibs; + struct dma_fence *fence; /* the hw fence */ + uint32_t preamble_status; + uint32_t preemption_status; + uint32_t num_ibs; + void *owner; + bool vm_needs_flush; + uint64_t vm_pd_addr; + unsigned vmid; + unsigned pasid; + uint32_t gds_base, gds_size; + uint32_t gws_base, gws_size; + uint32_t oa_base, oa_size; + uint32_t vram_lost_counter; + + /* user fence handling */ + uint64_t uf_addr; + uint64_t uf_sequence; + +}; + +int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, + struct amdgpu_job **job, struct amdgpu_vm *vm); +int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, + struct amdgpu_job **job); + +void amdgpu_job_free_resources(struct amdgpu_job *job); +void amdgpu_job_free(struct amdgpu_job *job); +int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, + void *owner, struct dma_fence **f); +int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, + struct dma_fence **fence); +#endif
diff --git a/amdgpu/amdgpu_kms.c b/amdgpu/amdgpu_kms.c new file mode 100644 index 0000000..0cf7e86 --- /dev/null +++ b/amdgpu/amdgpu_kms.c
@@ -0,0 +1,1427 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ + +#include "amdgpu.h" +#include <drm/drm_debugfs.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu_sched.h" +#include "amdgpu_uvd.h" +#include "amdgpu_vce.h" +#include "atom.h" + +#include <linux/vga_switcheroo.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/pci.h> +#include <linux/pm_runtime.h> +#include "amdgpu_amdkfd.h" +#include "amdgpu_gem.h" +#include "amdgpu_display.h" +#include "amdgpu_ras.h" + +void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev) +{ + struct amdgpu_gpu_instance *gpu_instance; + int i; + + mutex_lock(&mgpu_info.mutex); + + for (i = 0; i < mgpu_info.num_gpu; i++) { + gpu_instance = &(mgpu_info.gpu_ins[i]); + if (gpu_instance->adev == adev) { + mgpu_info.gpu_ins[i] = + mgpu_info.gpu_ins[mgpu_info.num_gpu - 1]; + mgpu_info.num_gpu--; + if (adev->flags & AMD_IS_APU) + mgpu_info.num_apu--; + else + mgpu_info.num_dgpu--; + break; + } + } + + mutex_unlock(&mgpu_info.mutex); +} + +/** + * amdgpu_driver_unload_kms - Main unload function for KMS. + * + * @dev: drm dev pointer + * + * This is the main unload function for KMS (all asics). + * Returns 0 on success. + */ +void amdgpu_driver_unload_kms(struct drm_device *dev) +{ + struct amdgpu_device *adev = dev->dev_private; + + if (adev == NULL) + return; + + amdgpu_unregister_gpu_instance(adev); + + if (adev->rmmio == NULL) + goto done_free; + + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_request_full_gpu(adev, false); + + if (amdgpu_device_is_px(dev)) { + pm_runtime_get_sync(dev->dev); + pm_runtime_forbid(dev->dev); + } + + amdgpu_acpi_fini(adev); + + amdgpu_device_fini(adev); + +done_free: + kfree(adev); + dev->dev_private = NULL; +} + +void amdgpu_register_gpu_instance(struct amdgpu_device *adev) +{ + struct amdgpu_gpu_instance *gpu_instance; + + mutex_lock(&mgpu_info.mutex); + + if (mgpu_info.num_gpu >= MAX_GPU_INSTANCE) { + DRM_ERROR("Cannot register more gpu instance\n"); + mutex_unlock(&mgpu_info.mutex); + return; + } + + gpu_instance = &(mgpu_info.gpu_ins[mgpu_info.num_gpu]); + gpu_instance->adev = adev; + gpu_instance->mgpu_fan_enabled = 0; + + mgpu_info.num_gpu++; + if (adev->flags & AMD_IS_APU) + mgpu_info.num_apu++; + else + mgpu_info.num_dgpu++; + + mutex_unlock(&mgpu_info.mutex); +} + +/** + * amdgpu_driver_load_kms - Main load function for KMS. + * + * @dev: drm dev pointer + * @flags: device flags + * + * This is the main load function for KMS (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) +{ + struct amdgpu_device *adev; + int r, acpi_status; + +#ifdef CONFIG_DRM_AMDGPU_SI + if (!amdgpu_si_support) { + switch (flags & AMD_ASIC_MASK) { + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + case CHIP_HAINAN: + dev_info(dev->dev, + "SI support provided by radeon.\n"); + dev_info(dev->dev, + "Use radeon.si_support=0 amdgpu.si_support=1 to override.\n" + ); + return -ENODEV; + } + } +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK + if (!amdgpu_cik_support) { + switch (flags & AMD_ASIC_MASK) { + case CHIP_KAVERI: + case CHIP_BONAIRE: + case CHIP_HAWAII: + case CHIP_KABINI: + case CHIP_MULLINS: + dev_info(dev->dev, + "CIK support provided by radeon.\n"); + dev_info(dev->dev, + "Use radeon.cik_support=0 amdgpu.cik_support=1 to override.\n" + ); + return -ENODEV; + } + } +#endif + + adev = kzalloc(sizeof(struct amdgpu_device), GFP_KERNEL); + if (adev == NULL) { + return -ENOMEM; + } + dev->dev_private = (void *)adev; + + if ((amdgpu_runtime_pm != 0) && + amdgpu_has_atpx() && + (amdgpu_is_atpx_hybrid() || + amdgpu_has_atpx_dgpu_power_cntl()) && + ((flags & AMD_IS_APU) == 0) && + !pci_is_thunderbolt_attached(dev->pdev)) + flags |= AMD_IS_PX; + + /* amdgpu_device_init should report only fatal error + * like memory allocation failure or iomapping failure, + * or memory manager initialization failure, it must + * properly initialize the GPU MC controller and permit + * VRAM allocation + */ + r = amdgpu_device_init(adev, dev, dev->pdev, flags); + if (r) { + dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); + goto out; + } + + /* Call ACPI methods: require modeset init + * but failure is not fatal + */ + if (!r) { + acpi_status = amdgpu_acpi_init(adev); + if (acpi_status) + dev_dbg(&dev->pdev->dev, + "Error during ACPI methods call\n"); + } + + if (amdgpu_device_is_px(dev)) { + dev_pm_set_driver_flags(dev->dev, DPM_FLAG_NEVER_SKIP); + pm_runtime_use_autosuspend(dev->dev); + pm_runtime_set_autosuspend_delay(dev->dev, 5000); + pm_runtime_set_active(dev->dev); + pm_runtime_allow(dev->dev); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + } + + amdgpu_register_gpu_instance(adev); +out: + if (r) { + /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */ + if (adev->rmmio && amdgpu_device_is_px(dev)) + pm_runtime_put_noidle(dev->dev); + amdgpu_driver_unload_kms(dev); + } + + return r; +} + +static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, + struct drm_amdgpu_query_fw *query_fw, + struct amdgpu_device *adev) +{ + switch (query_fw->fw_type) { + case AMDGPU_INFO_FW_VCE: + fw_info->ver = adev->vce.fw_version; + fw_info->feature = adev->vce.fb_version; + break; + case AMDGPU_INFO_FW_UVD: + fw_info->ver = adev->uvd.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_VCN: + fw_info->ver = adev->vcn.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_GMC: + fw_info->ver = adev->gmc.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_GFX_ME: + fw_info->ver = adev->gfx.me_fw_version; + fw_info->feature = adev->gfx.me_feature_version; + break; + case AMDGPU_INFO_FW_GFX_PFP: + fw_info->ver = adev->gfx.pfp_fw_version; + fw_info->feature = adev->gfx.pfp_feature_version; + break; + case AMDGPU_INFO_FW_GFX_CE: + fw_info->ver = adev->gfx.ce_fw_version; + fw_info->feature = adev->gfx.ce_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLC: + fw_info->ver = adev->gfx.rlc_fw_version; + fw_info->feature = adev->gfx.rlc_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL: + fw_info->ver = adev->gfx.rlc_srlc_fw_version; + fw_info->feature = adev->gfx.rlc_srlc_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM: + fw_info->ver = adev->gfx.rlc_srlg_fw_version; + fw_info->feature = adev->gfx.rlc_srlg_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM: + fw_info->ver = adev->gfx.rlc_srls_fw_version; + fw_info->feature = adev->gfx.rlc_srls_feature_version; + break; + case AMDGPU_INFO_FW_GFX_MEC: + if (query_fw->index == 0) { + fw_info->ver = adev->gfx.mec_fw_version; + fw_info->feature = adev->gfx.mec_feature_version; + } else if (query_fw->index == 1) { + fw_info->ver = adev->gfx.mec2_fw_version; + fw_info->feature = adev->gfx.mec2_feature_version; + } else + return -EINVAL; + break; + case AMDGPU_INFO_FW_SMC: + fw_info->ver = adev->pm.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_TA: + if (query_fw->index > 1) + return -EINVAL; + if (query_fw->index == 0) { + fw_info->ver = adev->psp.ta_fw_version; + fw_info->feature = adev->psp.ta_xgmi_ucode_version; + } else { + fw_info->ver = adev->psp.ta_fw_version; + fw_info->feature = adev->psp.ta_ras_ucode_version; + } + break; + case AMDGPU_INFO_FW_SDMA: + if (query_fw->index >= adev->sdma.num_instances) + return -EINVAL; + fw_info->ver = adev->sdma.instance[query_fw->index].fw_version; + fw_info->feature = adev->sdma.instance[query_fw->index].feature_version; + break; + case AMDGPU_INFO_FW_SOS: + fw_info->ver = adev->psp.sos_fw_version; + fw_info->feature = adev->psp.sos_feature_version; + break; + case AMDGPU_INFO_FW_ASD: + fw_info->ver = adev->psp.asd_fw_version; + fw_info->feature = adev->psp.asd_feature_version; + break; + case AMDGPU_INFO_FW_DMCU: + fw_info->ver = adev->dm.dmcu_fw_version; + fw_info->feature = 0; + break; + default: + return -EINVAL; + } + return 0; +} + +static int amdgpu_hw_ip_info(struct amdgpu_device *adev, + struct drm_amdgpu_info *info, + struct drm_amdgpu_info_hw_ip *result) +{ + uint32_t ib_start_alignment = 0; + uint32_t ib_size_alignment = 0; + enum amd_ip_block_type type; + unsigned int num_rings = 0; + unsigned int i, j; + + if (info->query_hw_ip.ip_instance >= AMDGPU_HW_IP_INSTANCE_MAX_COUNT) + return -EINVAL; + + switch (info->query_hw_ip.type) { + case AMDGPU_HW_IP_GFX: + type = AMD_IP_BLOCK_TYPE_GFX; + for (i = 0; i < adev->gfx.num_gfx_rings; i++) + if (adev->gfx.gfx_ring[i].sched.ready) + ++num_rings; + ib_start_alignment = 32; + ib_size_alignment = 32; + break; + case AMDGPU_HW_IP_COMPUTE: + type = AMD_IP_BLOCK_TYPE_GFX; + for (i = 0; i < adev->gfx.num_compute_rings; i++) + if (adev->gfx.compute_ring[i].sched.ready) + ++num_rings; + ib_start_alignment = 32; + ib_size_alignment = 32; + break; + case AMDGPU_HW_IP_DMA: + type = AMD_IP_BLOCK_TYPE_SDMA; + for (i = 0; i < adev->sdma.num_instances; i++) + if (adev->sdma.instance[i].ring.sched.ready) + ++num_rings; + ib_start_alignment = 256; + ib_size_alignment = 4; + break; + case AMDGPU_HW_IP_UVD: + type = AMD_IP_BLOCK_TYPE_UVD; + for (i = 0; i < adev->uvd.num_uvd_inst; i++) { + if (adev->uvd.harvest_config & (1 << i)) + continue; + + if (adev->uvd.inst[i].ring.sched.ready) + ++num_rings; + } + ib_start_alignment = 64; + ib_size_alignment = 64; + break; + case AMDGPU_HW_IP_VCE: + type = AMD_IP_BLOCK_TYPE_VCE; + for (i = 0; i < adev->vce.num_rings; i++) + if (adev->vce.ring[i].sched.ready) + ++num_rings; + ib_start_alignment = 4; + ib_size_alignment = 1; + break; + case AMDGPU_HW_IP_UVD_ENC: + type = AMD_IP_BLOCK_TYPE_UVD; + for (i = 0; i < adev->uvd.num_uvd_inst; i++) { + if (adev->uvd.harvest_config & (1 << i)) + continue; + + for (j = 0; j < adev->uvd.num_enc_rings; j++) + if (adev->uvd.inst[i].ring_enc[j].sched.ready) + ++num_rings; + } + ib_start_alignment = 64; + ib_size_alignment = 64; + break; + case AMDGPU_HW_IP_VCN_DEC: + type = AMD_IP_BLOCK_TYPE_VCN; + if (adev->vcn.ring_dec.sched.ready) + ++num_rings; + ib_start_alignment = 16; + ib_size_alignment = 16; + break; + case AMDGPU_HW_IP_VCN_ENC: + type = AMD_IP_BLOCK_TYPE_VCN; + for (i = 0; i < adev->vcn.num_enc_rings; i++) + if (adev->vcn.ring_enc[i].sched.ready) + ++num_rings; + ib_start_alignment = 64; + ib_size_alignment = 1; + break; + case AMDGPU_HW_IP_VCN_JPEG: + type = AMD_IP_BLOCK_TYPE_VCN; + if (adev->vcn.ring_jpeg.sched.ready) + ++num_rings; + ib_start_alignment = 16; + ib_size_alignment = 16; + break; + default: + return -EINVAL; + } + + for (i = 0; i < adev->num_ip_blocks; i++) + if (adev->ip_blocks[i].version->type == type && + adev->ip_blocks[i].status.valid) + break; + + if (i == adev->num_ip_blocks) + return 0; + + num_rings = min(amdgpu_ctx_num_entities[info->query_hw_ip.type], + num_rings); + + result->hw_ip_version_major = adev->ip_blocks[i].version->major; + result->hw_ip_version_minor = adev->ip_blocks[i].version->minor; + result->capabilities_flags = 0; + result->available_rings = (1 << num_rings) - 1; + result->ib_start_alignment = ib_start_alignment; + result->ib_size_alignment = ib_size_alignment; + return 0; +} + +/* + * Userspace get information ioctl + */ +/** + * amdgpu_info_ioctl - answer a device specific request. + * + * @adev: amdgpu device pointer + * @data: request object + * @filp: drm filp + * + * This function is used to pass device specific parameters to the userspace + * drivers. Examples include: pci device id, pipeline parms, tiling params, + * etc. (all asics). + * Returns 0 on success, -EINVAL on failure. + */ +static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + struct amdgpu_device *adev = dev->dev_private; + struct drm_amdgpu_info *info = data; + struct amdgpu_mode_info *minfo = &adev->mode_info; + void __user *out = (void __user *)(uintptr_t)info->return_pointer; + uint32_t size = info->return_size; + struct drm_crtc *crtc; + uint32_t ui32 = 0; + uint64_t ui64 = 0; + int i, found; + int ui32_size = sizeof(ui32); + + if (!info->return_size || !info->return_pointer) + return -EINVAL; + + switch (info->query) { + case AMDGPU_INFO_ACCEL_WORKING: + ui32 = adev->accel_working; + return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0; + case AMDGPU_INFO_CRTC_FROM_ID: + for (i = 0, found = 0; i < adev->mode_info.num_crtc; i++) { + crtc = (struct drm_crtc *)minfo->crtcs[i]; + if (crtc && crtc->base.id == info->mode_crtc.id) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + ui32 = amdgpu_crtc->crtc_id; + found = 1; + break; + } + } + if (!found) { + DRM_DEBUG_KMS("unknown crtc id %d\n", info->mode_crtc.id); + return -EINVAL; + } + return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0; + case AMDGPU_INFO_HW_IP_INFO: { + struct drm_amdgpu_info_hw_ip ip = {}; + int ret; + + ret = amdgpu_hw_ip_info(adev, info, &ip); + if (ret) + return ret; + + ret = copy_to_user(out, &ip, min((size_t)size, sizeof(ip))); + return ret ? -EFAULT : 0; + } + case AMDGPU_INFO_HW_IP_COUNT: { + enum amd_ip_block_type type; + uint32_t count = 0; + + switch (info->query_hw_ip.type) { + case AMDGPU_HW_IP_GFX: + type = AMD_IP_BLOCK_TYPE_GFX; + break; + case AMDGPU_HW_IP_COMPUTE: + type = AMD_IP_BLOCK_TYPE_GFX; + break; + case AMDGPU_HW_IP_DMA: + type = AMD_IP_BLOCK_TYPE_SDMA; + break; + case AMDGPU_HW_IP_UVD: + type = AMD_IP_BLOCK_TYPE_UVD; + break; + case AMDGPU_HW_IP_VCE: + type = AMD_IP_BLOCK_TYPE_VCE; + break; + case AMDGPU_HW_IP_UVD_ENC: + type = AMD_IP_BLOCK_TYPE_UVD; + break; + case AMDGPU_HW_IP_VCN_DEC: + case AMDGPU_HW_IP_VCN_ENC: + case AMDGPU_HW_IP_VCN_JPEG: + type = AMD_IP_BLOCK_TYPE_VCN; + break; + default: + return -EINVAL; + } + + for (i = 0; i < adev->num_ip_blocks; i++) + if (adev->ip_blocks[i].version->type == type && + adev->ip_blocks[i].status.valid && + count < AMDGPU_HW_IP_INSTANCE_MAX_COUNT) + count++; + + return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0; + } + case AMDGPU_INFO_TIMESTAMP: + ui64 = amdgpu_gfx_get_gpu_clock_counter(adev); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_FW_VERSION: { + struct drm_amdgpu_info_firmware fw_info; + int ret; + + /* We only support one instance of each IP block right now. */ + if (info->query_fw.ip_instance != 0) + return -EINVAL; + + ret = amdgpu_firmware_info(&fw_info, &info->query_fw, adev); + if (ret) + return ret; + + return copy_to_user(out, &fw_info, + min((size_t)size, sizeof(fw_info))) ? -EFAULT : 0; + } + case AMDGPU_INFO_NUM_BYTES_MOVED: + ui64 = atomic64_read(&adev->num_bytes_moved); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_NUM_EVICTIONS: + ui64 = atomic64_read(&adev->num_evictions); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_NUM_VRAM_CPU_PAGE_FAULTS: + ui64 = atomic64_read(&adev->num_vram_cpu_page_faults); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_VRAM_USAGE: + ui64 = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_VIS_VRAM_USAGE: + ui64 = amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_GTT_USAGE: + ui64 = amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]); + return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + case AMDGPU_INFO_GDS_CONFIG: { + struct drm_amdgpu_info_gds gds_info; + + memset(&gds_info, 0, sizeof(gds_info)); + gds_info.compute_partition_size = adev->gds.gds_size; + gds_info.gds_total_size = adev->gds.gds_size; + gds_info.gws_per_compute_partition = adev->gds.gws_size; + gds_info.oa_per_compute_partition = adev->gds.oa_size; + return copy_to_user(out, &gds_info, + min((size_t)size, sizeof(gds_info))) ? -EFAULT : 0; + } + case AMDGPU_INFO_VRAM_GTT: { + struct drm_amdgpu_info_vram_gtt vram_gtt; + + vram_gtt.vram_size = adev->gmc.real_vram_size - + atomic64_read(&adev->vram_pin_size); + vram_gtt.vram_cpu_accessible_size = adev->gmc.visible_vram_size - + atomic64_read(&adev->visible_pin_size); + vram_gtt.gtt_size = adev->mman.bdev.man[TTM_PL_TT].size; + vram_gtt.gtt_size *= PAGE_SIZE; + vram_gtt.gtt_size -= atomic64_read(&adev->gart_pin_size); + return copy_to_user(out, &vram_gtt, + min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0; + } + case AMDGPU_INFO_MEMORY: { + struct drm_amdgpu_memory_info mem; + + memset(&mem, 0, sizeof(mem)); + mem.vram.total_heap_size = adev->gmc.real_vram_size; + mem.vram.usable_heap_size = adev->gmc.real_vram_size - + atomic64_read(&adev->vram_pin_size); + mem.vram.heap_usage = + amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4; + + mem.cpu_accessible_vram.total_heap_size = + adev->gmc.visible_vram_size; + mem.cpu_accessible_vram.usable_heap_size = adev->gmc.visible_vram_size - + atomic64_read(&adev->visible_pin_size); + mem.cpu_accessible_vram.heap_usage = + amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); + mem.cpu_accessible_vram.max_allocation = + mem.cpu_accessible_vram.usable_heap_size * 3 / 4; + + mem.gtt.total_heap_size = adev->mman.bdev.man[TTM_PL_TT].size; + mem.gtt.total_heap_size *= PAGE_SIZE; + mem.gtt.usable_heap_size = mem.gtt.total_heap_size - + atomic64_read(&adev->gart_pin_size); + mem.gtt.heap_usage = + amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]); + mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4; + + return copy_to_user(out, &mem, + min((size_t)size, sizeof(mem))) + ? -EFAULT : 0; + } + case AMDGPU_INFO_READ_MMR_REG: { + unsigned n, alloc_size; + uint32_t *regs; + unsigned se_num = (info->read_mmr_reg.instance >> + AMDGPU_INFO_MMR_SE_INDEX_SHIFT) & + AMDGPU_INFO_MMR_SE_INDEX_MASK; + unsigned sh_num = (info->read_mmr_reg.instance >> + AMDGPU_INFO_MMR_SH_INDEX_SHIFT) & + AMDGPU_INFO_MMR_SH_INDEX_MASK; + + /* set full masks if the userspace set all bits + * in the bitfields */ + if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) + se_num = 0xffffffff; + if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) + sh_num = 0xffffffff; + + regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL); + if (!regs) + return -ENOMEM; + alloc_size = info->read_mmr_reg.count * sizeof(*regs); + + for (i = 0; i < info->read_mmr_reg.count; i++) + if (amdgpu_asic_read_register(adev, se_num, sh_num, + info->read_mmr_reg.dword_offset + i, + ®s[i])) { + DRM_DEBUG_KMS("unallowed offset %#x\n", + info->read_mmr_reg.dword_offset + i); + kfree(regs); + return -EFAULT; + } + n = copy_to_user(out, regs, min(size, alloc_size)); + kfree(regs); + return n ? -EFAULT : 0; + } + case AMDGPU_INFO_DEV_INFO: { + struct drm_amdgpu_info_device dev_info = {}; + uint64_t vm_size; + + dev_info.device_id = dev->pdev->device; + dev_info.chip_rev = adev->rev_id; + dev_info.external_rev = adev->external_rev_id; + dev_info.pci_rev = dev->pdev->revision; + dev_info.family = adev->family; + dev_info.num_shader_engines = adev->gfx.config.max_shader_engines; + dev_info.num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; + /* return all clocks in KHz */ + dev_info.gpu_counter_freq = amdgpu_asic_get_xclk(adev) * 10; + if (adev->pm.dpm_enabled) { + dev_info.max_engine_clock = amdgpu_dpm_get_sclk(adev, false) * 10; + dev_info.max_memory_clock = amdgpu_dpm_get_mclk(adev, false) * 10; + } else if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) && + adev->virt.ops->get_pp_clk) { + dev_info.max_engine_clock = amdgpu_virt_get_sclk(adev, false) * 10; + dev_info.max_memory_clock = amdgpu_virt_get_mclk(adev, false) * 10; + } else { + dev_info.max_engine_clock = adev->clock.default_sclk * 10; + dev_info.max_memory_clock = adev->clock.default_mclk * 10; + } + dev_info.enabled_rb_pipes_mask = adev->gfx.config.backend_enable_mask; + dev_info.num_rb_pipes = adev->gfx.config.max_backends_per_se * + adev->gfx.config.max_shader_engines; + dev_info.num_hw_gfx_contexts = adev->gfx.config.max_hw_contexts; + dev_info._pad = 0; + dev_info.ids_flags = 0; + if (adev->flags & AMD_IS_APU) + dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION; + if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) + dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION; + + vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; + vm_size -= AMDGPU_VA_RESERVED_SIZE; + + /* Older VCE FW versions are buggy and can handle only 40bits */ + if (adev->vce.fw_version && + adev->vce.fw_version < AMDGPU_VCE_FW_53_45) + vm_size = min(vm_size, 1ULL << 40); + + dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE; + dev_info.virtual_address_max = + min(vm_size, AMDGPU_GMC_HOLE_START); + + if (vm_size > AMDGPU_GMC_HOLE_START) { + dev_info.high_va_offset = AMDGPU_GMC_HOLE_END; + dev_info.high_va_max = AMDGPU_GMC_HOLE_END | vm_size; + } + dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE); + dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE; + dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE; + dev_info.cu_active_number = adev->gfx.cu_info.number; + dev_info.cu_ao_mask = adev->gfx.cu_info.ao_cu_mask; + dev_info.ce_ram_size = adev->gfx.ce_ram_size; + memcpy(&dev_info.cu_ao_bitmap[0], &adev->gfx.cu_info.ao_cu_bitmap[0], + sizeof(adev->gfx.cu_info.ao_cu_bitmap)); + memcpy(&dev_info.cu_bitmap[0], &adev->gfx.cu_info.bitmap[0], + sizeof(adev->gfx.cu_info.bitmap)); + dev_info.vram_type = adev->gmc.vram_type; + dev_info.vram_bit_width = adev->gmc.vram_width; + dev_info.vce_harvest_config = adev->vce.harvest_config; + dev_info.gc_double_offchip_lds_buf = + adev->gfx.config.double_offchip_lds_buf; + + if (amdgpu_ngg) { + dev_info.prim_buf_gpu_addr = adev->gfx.ngg.buf[NGG_PRIM].gpu_addr; + dev_info.prim_buf_size = adev->gfx.ngg.buf[NGG_PRIM].size; + dev_info.pos_buf_gpu_addr = adev->gfx.ngg.buf[NGG_POS].gpu_addr; + dev_info.pos_buf_size = adev->gfx.ngg.buf[NGG_POS].size; + dev_info.cntl_sb_buf_gpu_addr = adev->gfx.ngg.buf[NGG_CNTL].gpu_addr; + dev_info.cntl_sb_buf_size = adev->gfx.ngg.buf[NGG_CNTL].size; + dev_info.param_buf_gpu_addr = adev->gfx.ngg.buf[NGG_PARAM].gpu_addr; + dev_info.param_buf_size = adev->gfx.ngg.buf[NGG_PARAM].size; + } + dev_info.wave_front_size = adev->gfx.cu_info.wave_front_size; + dev_info.num_shader_visible_vgprs = adev->gfx.config.max_gprs; + dev_info.num_cu_per_sh = adev->gfx.config.max_cu_per_sh; + dev_info.num_tcc_blocks = adev->gfx.config.max_texture_channel_caches; + dev_info.gs_vgt_table_depth = adev->gfx.config.gs_vgt_table_depth; + dev_info.gs_prim_buffer_depth = adev->gfx.config.gs_prim_buffer_depth; + dev_info.max_gs_waves_per_vgt = adev->gfx.config.max_gs_threads; + + if (adev->family >= AMDGPU_FAMILY_NV) + dev_info.pa_sc_tile_steering_override = + adev->gfx.config.pa_sc_tile_steering_override; + + return copy_to_user(out, &dev_info, + min((size_t)size, sizeof(dev_info))) ? -EFAULT : 0; + } + case AMDGPU_INFO_VCE_CLOCK_TABLE: { + unsigned i; + struct drm_amdgpu_info_vce_clock_table vce_clk_table = {}; + struct amd_vce_state *vce_state; + + for (i = 0; i < AMDGPU_VCE_CLOCK_TABLE_ENTRIES; i++) { + vce_state = amdgpu_dpm_get_vce_clock_state(adev, i); + if (vce_state) { + vce_clk_table.entries[i].sclk = vce_state->sclk; + vce_clk_table.entries[i].mclk = vce_state->mclk; + vce_clk_table.entries[i].eclk = vce_state->evclk; + vce_clk_table.num_valid_entries++; + } + } + + return copy_to_user(out, &vce_clk_table, + min((size_t)size, sizeof(vce_clk_table))) ? -EFAULT : 0; + } + case AMDGPU_INFO_VBIOS: { + uint32_t bios_size = adev->bios_size; + + switch (info->vbios_info.type) { + case AMDGPU_INFO_VBIOS_SIZE: + return copy_to_user(out, &bios_size, + min((size_t)size, sizeof(bios_size))) + ? -EFAULT : 0; + case AMDGPU_INFO_VBIOS_IMAGE: { + uint8_t *bios; + uint32_t bios_offset = info->vbios_info.offset; + + if (bios_offset >= bios_size) + return -EINVAL; + + bios = adev->bios + bios_offset; + return copy_to_user(out, bios, + min((size_t)size, (size_t)(bios_size - bios_offset))) + ? -EFAULT : 0; + } + default: + DRM_DEBUG_KMS("Invalid request %d\n", + info->vbios_info.type); + return -EINVAL; + } + } + case AMDGPU_INFO_NUM_HANDLES: { + struct drm_amdgpu_info_num_handles handle; + + switch (info->query_hw_ip.type) { + case AMDGPU_HW_IP_UVD: + /* Starting Polaris, we support unlimited UVD handles */ + if (adev->asic_type < CHIP_POLARIS10) { + handle.uvd_max_handles = adev->uvd.max_handles; + handle.uvd_used_handles = amdgpu_uvd_used_handles(adev); + + return copy_to_user(out, &handle, + min((size_t)size, sizeof(handle))) ? -EFAULT : 0; + } else { + return -ENODATA; + } + + break; + default: + return -EINVAL; + } + } + case AMDGPU_INFO_SENSOR: { + if (!adev->pm.dpm_enabled) + return -ENOENT; + + switch (info->sensor_info.type) { + case AMDGPU_INFO_SENSOR_GFX_SCLK: + /* get sclk in Mhz */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_GFX_SCLK, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + ui32 /= 100; + break; + case AMDGPU_INFO_SENSOR_GFX_MCLK: + /* get mclk in Mhz */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_GFX_MCLK, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + ui32 /= 100; + break; + case AMDGPU_INFO_SENSOR_GPU_TEMP: + /* get temperature in millidegrees C */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_GPU_TEMP, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + break; + case AMDGPU_INFO_SENSOR_GPU_LOAD: + /* get GPU load */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_GPU_LOAD, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + break; + case AMDGPU_INFO_SENSOR_GPU_AVG_POWER: + /* get average GPU power */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_GPU_POWER, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + ui32 >>= 8; + break; + case AMDGPU_INFO_SENSOR_VDDNB: + /* get VDDNB in millivolts */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_VDDNB, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + break; + case AMDGPU_INFO_SENSOR_VDDGFX: + /* get VDDGFX in millivolts */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_VDDGFX, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + break; + case AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_SCLK: + /* get stable pstate sclk in Mhz */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_STABLE_PSTATE_SCLK, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + ui32 /= 100; + break; + case AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_MCLK: + /* get stable pstate mclk in Mhz */ + if (amdgpu_dpm_read_sensor(adev, + AMDGPU_PP_SENSOR_STABLE_PSTATE_MCLK, + (void *)&ui32, &ui32_size)) { + return -EINVAL; + } + ui32 /= 100; + break; + default: + DRM_DEBUG_KMS("Invalid request %d\n", + info->sensor_info.type); + return -EINVAL; + } + return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0; + } + case AMDGPU_INFO_VRAM_LOST_COUNTER: + ui32 = atomic_read(&adev->vram_lost_counter); + return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0; + case AMDGPU_INFO_RAS_ENABLED_FEATURES: { + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); + uint64_t ras_mask; + + if (!ras) + return -EINVAL; + ras_mask = (uint64_t)ras->supported << 32 | ras->features; + + return copy_to_user(out, &ras_mask, + min_t(u64, size, sizeof(ras_mask))) ? + -EFAULT : 0; + } + default: + DRM_DEBUG_KMS("Invalid request %d\n", info->query); + return -EINVAL; + } + return 0; +} + + +/* + * Outdated mess for old drm with Xorg being in charge (void function now). + */ +/** + * amdgpu_driver_lastclose_kms - drm callback for last close + * + * @dev: drm dev pointer + * + * Switch vga_switcheroo state after last close (all asics). + */ +void amdgpu_driver_lastclose_kms(struct drm_device *dev) +{ + drm_fb_helper_lastclose(dev); + vga_switcheroo_process_delayed_switch(); +} + +/** + * amdgpu_driver_open_kms - drm callback for open + * + * @dev: drm dev pointer + * @file_priv: drm file + * + * On device open, init vm on cayman+ (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) +{ + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv; + int r, pasid; + + /* Ensure IB tests are run on ring */ + flush_delayed_work(&adev->delayed_init_work); + + file_priv->driver_priv = NULL; + + r = pm_runtime_get_sync(dev->dev); + if (r < 0) + return r; + + fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL); + if (unlikely(!fpriv)) { + r = -ENOMEM; + goto out_suspend; + } + + pasid = amdgpu_pasid_alloc(16); + if (pasid < 0) { + dev_warn(adev->dev, "No more PASIDs available!"); + pasid = 0; + } + r = amdgpu_vm_init(adev, &fpriv->vm, AMDGPU_VM_CONTEXT_GFX, pasid); + if (r) + goto error_pasid; + + fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL); + if (!fpriv->prt_va) { + r = -ENOMEM; + goto error_vm; + } + + if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK; + + r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj, + &fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE); + if (r) + goto error_vm; + } + + mutex_init(&fpriv->bo_list_lock); + idr_init(&fpriv->bo_list_handles); + + amdgpu_ctx_mgr_init(&fpriv->ctx_mgr); + + file_priv->driver_priv = fpriv; + goto out_suspend; + +error_vm: + amdgpu_vm_fini(adev, &fpriv->vm); + +error_pasid: + if (pasid) + amdgpu_pasid_free(pasid); + + kfree(fpriv); + +out_suspend: + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + return r; +} + +/** + * amdgpu_driver_postclose_kms - drm callback for post close + * + * @dev: drm dev pointer + * @file_priv: drm file + * + * On device post close, tear down vm on cayman+ (all asics). + */ +void amdgpu_driver_postclose_kms(struct drm_device *dev, + struct drm_file *file_priv) +{ + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct amdgpu_bo_list *list; + struct amdgpu_bo *pd; + unsigned int pasid; + int handle; + + if (!fpriv) + return; + + pm_runtime_get_sync(dev->dev); + + if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_UVD) != NULL) + amdgpu_uvd_free_handles(adev, file_priv); + if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_VCE) != NULL) + amdgpu_vce_free_handles(adev, file_priv); + + amdgpu_vm_bo_rmv(adev, fpriv->prt_va); + + if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + /* TODO: how to handle reserve failure */ + BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true)); + amdgpu_vm_bo_rmv(adev, fpriv->csa_va); + fpriv->csa_va = NULL; + amdgpu_bo_unreserve(adev->virt.csa_obj); + } + + pasid = fpriv->vm.pasid; + pd = amdgpu_bo_ref(fpriv->vm.root.base.bo); + + amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); + amdgpu_vm_fini(adev, &fpriv->vm); + + if (pasid) + amdgpu_pasid_free_delayed(pd->tbo.resv, pasid); + amdgpu_bo_unref(&pd); + + idr_for_each_entry(&fpriv->bo_list_handles, list, handle) + amdgpu_bo_list_put(list); + + idr_destroy(&fpriv->bo_list_handles); + mutex_destroy(&fpriv->bo_list_lock); + + kfree(fpriv); + file_priv->driver_priv = NULL; + + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); +} + +/* + * VBlank related functions. + */ +/** + * amdgpu_get_vblank_counter_kms - get frame count + * + * @dev: drm dev pointer + * @pipe: crtc to get the frame count from + * + * Gets the frame count on the requested crtc (all asics). + * Returns frame count on success, -EINVAL on failure. + */ +u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe) +{ + struct amdgpu_device *adev = dev->dev_private; + int vpos, hpos, stat; + u32 count; + + if (pipe >= adev->mode_info.num_crtc) { + DRM_ERROR("Invalid crtc %u\n", pipe); + return -EINVAL; + } + + /* The hw increments its frame counter at start of vsync, not at start + * of vblank, as is required by DRM core vblank counter handling. + * Cook the hw count here to make it appear to the caller as if it + * incremented at start of vblank. We measure distance to start of + * vblank in vpos. vpos therefore will be >= 0 between start of vblank + * and start of vsync, so vpos >= 0 means to bump the hw frame counter + * result by 1 to give the proper appearance to caller. + */ + if (adev->mode_info.crtcs[pipe]) { + /* Repeat readout if needed to provide stable result if + * we cross start of vsync during the queries. + */ + do { + count = amdgpu_display_vblank_get_counter(adev, pipe); + /* Ask amdgpu_display_get_crtc_scanoutpos to return + * vpos as distance to start of vblank, instead of + * regular vertical scanout pos. + */ + stat = amdgpu_display_get_crtc_scanoutpos( + dev, pipe, GET_DISTANCE_TO_VBLANKSTART, + &vpos, &hpos, NULL, NULL, + &adev->mode_info.crtcs[pipe]->base.hwmode); + } while (count != amdgpu_display_vblank_get_counter(adev, pipe)); + + if (((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) != + (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE))) { + DRM_DEBUG_VBL("Query failed! stat %d\n", stat); + } else { + DRM_DEBUG_VBL("crtc %d: dist from vblank start %d\n", + pipe, vpos); + + /* Bump counter if we are at >= leading edge of vblank, + * but before vsync where vpos would turn negative and + * the hw counter really increments. + */ + if (vpos >= 0) + count++; + } + } else { + /* Fallback to use value as is. */ + count = amdgpu_display_vblank_get_counter(adev, pipe); + DRM_DEBUG_VBL("NULL mode info! Returned count may be wrong.\n"); + } + + return count; +} + +/** + * amdgpu_enable_vblank_kms - enable vblank interrupt + * + * @dev: drm dev pointer + * @pipe: crtc to enable vblank interrupt for + * + * Enable the interrupt on the requested crtc (all asics). + * Returns 0 on success, -EINVAL on failure. + */ +int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe) +{ + struct amdgpu_device *adev = dev->dev_private; + int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe); + + return amdgpu_irq_get(adev, &adev->crtc_irq, idx); +} + +/** + * amdgpu_disable_vblank_kms - disable vblank interrupt + * + * @dev: drm dev pointer + * @pipe: crtc to disable vblank interrupt for + * + * Disable the interrupt on the requested crtc (all asics). + */ +void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe) +{ + struct amdgpu_device *adev = dev->dev_private; + int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe); + + amdgpu_irq_put(adev, &adev->crtc_irq, idx); +} + +const struct drm_ioctl_desc amdgpu_ioctls_kms[] = { + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_CREATE, amdgpu_gem_create_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_CTX, amdgpu_ctx_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_VM, amdgpu_vm_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_SCHED, amdgpu_sched_ioctl, DRM_MASTER), + DRM_IOCTL_DEF_DRV(AMDGPU_BO_LIST, amdgpu_bo_list_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_FENCE_TO_HANDLE, amdgpu_cs_fence_to_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + /* KMS */ + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_MMAP, amdgpu_gem_mmap_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_WAIT_IDLE, amdgpu_gem_wait_idle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_CS, amdgpu_cs_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_INFO, amdgpu_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_CS, amdgpu_cs_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_FENCES, amdgpu_cs_wait_fences_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW) +}; +const int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms); + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + struct drm_amdgpu_info_firmware fw_info; + struct drm_amdgpu_query_fw query_fw; + struct atom_context *ctx = adev->mode_info.atom_context; + int ret, i; + + /* VCE */ + query_fw.fw_type = AMDGPU_INFO_FW_VCE; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "VCE feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* UVD */ + query_fw.fw_type = AMDGPU_INFO_FW_UVD; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "UVD feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* GMC */ + query_fw.fw_type = AMDGPU_INFO_FW_GMC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* ME */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_ME; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "ME feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* PFP */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_PFP; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "PFP feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* CE */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_CE; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "CE feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLC */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLC SAVE RESTORE LIST CNTL */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLC SRLC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLC SAVE RESTORE LIST GPM MEM */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLC SRLG feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLC SAVE RESTORE LIST SRM MEM */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLC SRLS feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* MEC */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC; + query_fw.index = 0; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MEC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* MEC2 */ + if (adev->asic_type == CHIP_KAVERI || + (adev->asic_type > CHIP_TOPAZ && adev->asic_type != CHIP_STONEY)) { + query_fw.index = 1; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MEC2 feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + } + + /* PSP SOS */ + query_fw.fw_type = AMDGPU_INFO_FW_SOS; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "SOS feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + + /* PSP ASD */ + query_fw.fw_type = AMDGPU_INFO_FW_ASD; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "ASD feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + query_fw.fw_type = AMDGPU_INFO_FW_TA; + for (i = 0; i < 2; i++) { + query_fw.index = i; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + continue; + seq_printf(m, "TA %s feature version: %u, firmware version: 0x%08x\n", + i ? "RAS" : "XGMI", fw_info.feature, fw_info.ver); + } + + /* SMC */ + query_fw.fw_type = AMDGPU_INFO_FW_SMC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "SMC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* SDMA */ + query_fw.fw_type = AMDGPU_INFO_FW_SDMA; + for (i = 0; i < adev->sdma.num_instances; i++) { + query_fw.index = i; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "SDMA%d feature version: %u, firmware version: 0x%08x\n", + i, fw_info.feature, fw_info.ver); + } + + /* VCN */ + query_fw.fw_type = AMDGPU_INFO_FW_VCN; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "VCN feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* DMCU */ + query_fw.fw_type = AMDGPU_INFO_FW_DMCU; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "DMCU feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + + seq_printf(m, "VBIOS version: %s\n", ctx->vbios_version); + + return 0; +} + +static const struct drm_info_list amdgpu_firmware_info_list[] = { + {"amdgpu_firmware_info", amdgpu_debugfs_firmware_info, 0, NULL}, +}; +#endif + +int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + return amdgpu_debugfs_add_files(adev, amdgpu_firmware_info_list, + ARRAY_SIZE(amdgpu_firmware_info_list)); +#else + return 0; +#endif +}
diff --git a/amdgpu/amdgpu_mes.h b/amdgpu/amdgpu_mes.h new file mode 100644 index 0000000..78fe490 --- /dev/null +++ b/amdgpu/amdgpu_mes.h
@@ -0,0 +1,101 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_MES_H__ +#define __AMDGPU_MES_H__ + +struct amdgpu_mes_funcs; + +struct amdgpu_mes { + struct amdgpu_adev *adev; + + const struct firmware *fw; + + /* mes ucode */ + struct amdgpu_bo *ucode_fw_obj; + uint64_t ucode_fw_gpu_addr; + uint32_t *ucode_fw_ptr; + uint32_t ucode_fw_version; + uint64_t uc_start_addr; + + /* mes ucode data */ + struct amdgpu_bo *data_fw_obj; + uint64_t data_fw_gpu_addr; + uint32_t *data_fw_ptr; + uint32_t data_fw_version; + uint64_t data_start_addr; + + /* ip specific functions */ + struct amdgpu_mes_funcs *funcs; +}; + +struct mes_add_queue_input { + uint32_t process_id; + uint64_t page_table_base_addr; + uint64_t process_va_start; + uint64_t process_va_end; + uint64_t process_quantum; + uint64_t process_context_addr; + uint64_t gang_quantum; + uint64_t gang_context_addr; + uint32_t inprocess_gang_priority; + uint32_t gang_global_priority_level; + uint32_t doorbell_offset; + uint64_t mqd_addr; + uint64_t wptr_addr; + uint32_t queue_type; + uint32_t paging; +}; + +struct mes_remove_queue_input { + uint32_t doorbell_offset; + uint64_t gang_context_addr; +}; + +struct mes_suspend_gang_input { + bool suspend_all_gangs; + uint64_t gang_context_addr; + uint64_t suspend_fence_addr; + uint32_t suspend_fence_value; +}; + +struct mes_resume_gang_input { + bool resume_all_gangs; + uint64_t gang_context_addr; +}; + +struct amdgpu_mes_funcs { + int (*add_hw_queue)(struct amdgpu_mes *mes, + struct mes_add_queue_input *input); + + int (*remove_hw_queue)(struct amdgpu_mes *mes, + struct mes_remove_queue_input *input); + + int (*suspend_gang)(struct amdgpu_mes *mes, + struct mes_suspend_gang_input *input); + + int (*resume_gang)(struct amdgpu_mes *mes, + struct mes_resume_gang_input *input); +}; + +#endif /* __AMDGPU_MES_H__ */
diff --git a/amdgpu/amdgpu_mn.c b/amdgpu/amdgpu_mn.c new file mode 100644 index 0000000..3971c20 --- /dev/null +++ b/amdgpu/amdgpu_mn.c
@@ -0,0 +1,487 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König <christian.koenig@amd.com> + */ + +/** + * DOC: MMU Notifier + * + * For coherent userptr handling registers an MMU notifier to inform the driver + * about updates on the page tables of a process. + * + * When somebody tries to invalidate the page tables we block the update until + * all operations on the pages in question are completed, then those pages are + * marked as accessed and also dirty if it wasn't a read only access. + * + * New command submissions using the userptrs in question are delayed until all + * page table invalidation are completed and we once more see a coherent process + * address space. + */ + +#include <linux/firmware.h> +#include <linux/module.h> +#include <drm/drm.h> + +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" + +/** + * struct amdgpu_mn_node + * + * @it: interval node defining start-last of the affected address range + * @bos: list of all BOs in the affected address range + * + * Manages all BOs which are affected of a certain range of address space. + */ +struct amdgpu_mn_node { + struct interval_tree_node it; + struct list_head bos; +}; + +/** + * amdgpu_mn_destroy - destroy the HMM mirror + * + * @work: previously sheduled work item + * + * Lazy destroys the notifier from a work item + */ +static void amdgpu_mn_destroy(struct work_struct *work) +{ + struct amdgpu_mn *amn = container_of(work, struct amdgpu_mn, work); + struct amdgpu_device *adev = amn->adev; + struct amdgpu_mn_node *node, *next_node; + struct amdgpu_bo *bo, *next_bo; + + mutex_lock(&adev->mn_lock); + down_write(&amn->lock); + hash_del(&amn->node); + rbtree_postorder_for_each_entry_safe(node, next_node, + &amn->objects.rb_root, it.rb) { + list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) { + bo->mn = NULL; + list_del_init(&bo->mn_list); + } + kfree(node); + } + up_write(&amn->lock); + mutex_unlock(&adev->mn_lock); + + hmm_mirror_unregister(&amn->mirror); + kfree(amn); +} + +/** + * amdgpu_hmm_mirror_release - callback to notify about mm destruction + * + * @mirror: the HMM mirror (mm) this callback is about + * + * Shedule a work item to lazy destroy HMM mirror. + */ +static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror) +{ + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); + + INIT_WORK(&amn->work, amdgpu_mn_destroy); + schedule_work(&amn->work); +} + +/** + * amdgpu_mn_lock - take the write side lock for this notifier + * + * @mn: our notifier + */ +void amdgpu_mn_lock(struct amdgpu_mn *mn) +{ + if (mn) + down_write(&mn->lock); +} + +/** + * amdgpu_mn_unlock - drop the write side lock for this notifier + * + * @mn: our notifier + */ +void amdgpu_mn_unlock(struct amdgpu_mn *mn) +{ + if (mn) + up_write(&mn->lock); +} + +/** + * amdgpu_mn_read_lock - take the read side lock for this notifier + * + * @amn: our notifier + */ +static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable) +{ + if (blockable) + down_read(&amn->lock); + else if (!down_read_trylock(&amn->lock)) + return -EAGAIN; + + return 0; +} + +/** + * amdgpu_mn_read_unlock - drop the read side lock for this notifier + * + * @amn: our notifier + */ +static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn) +{ + up_read(&amn->lock); +} + +/** + * amdgpu_mn_invalidate_node - unmap all BOs of a node + * + * @node: the node with the BOs to unmap + * @start: start of address range affected + * @end: end of address range affected + * + * Block for operations on BOs to finish and mark pages as accessed and + * potentially dirty. + */ +static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, + unsigned long start, + unsigned long end) +{ + struct amdgpu_bo *bo; + long r; + + list_for_each_entry(bo, &node->bos, mn_list) { + + if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end)) + continue; + + r = reservation_object_wait_timeout_rcu(bo->tbo.resv, + true, false, MAX_SCHEDULE_TIMEOUT); + if (r <= 0) + DRM_ERROR("(%ld) failed to wait for user bo\n", r); + } +} + +/** + * amdgpu_mn_sync_pagetables_gfx - callback to notify about mm change + * + * @mirror: the hmm_mirror (mm) is about to update + * @update: the update start, end address + * + * Block for operations on BOs to finish and mark pages as accessed and + * potentially dirty. + */ +static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror, + const struct hmm_update *update) +{ + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); + unsigned long start = update->start; + unsigned long end = update->end; + bool blockable = update->blockable; + struct interval_tree_node *it; + + /* notification is exclusive, but interval is inclusive */ + end -= 1; + + /* TODO we should be able to split locking for interval tree and + * amdgpu_mn_invalidate_node + */ + if (amdgpu_mn_read_lock(amn, blockable)) + return -EAGAIN; + + it = interval_tree_iter_first(&amn->objects, start, end); + while (it) { + struct amdgpu_mn_node *node; + + if (!blockable) { + amdgpu_mn_read_unlock(amn); + return -EAGAIN; + } + + node = container_of(it, struct amdgpu_mn_node, it); + it = interval_tree_iter_next(it, start, end); + + amdgpu_mn_invalidate_node(node, start, end); + } + + amdgpu_mn_read_unlock(amn); + + return 0; +} + +/** + * amdgpu_mn_sync_pagetables_hsa - callback to notify about mm change + * + * @mirror: the hmm_mirror (mm) is about to update + * @update: the update start, end address + * + * We temporarily evict all BOs between start and end. This + * necessitates evicting all user-mode queues of the process. The BOs + * are restorted in amdgpu_mn_invalidate_range_end_hsa. + */ +static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror, + const struct hmm_update *update) +{ + struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror); + unsigned long start = update->start; + unsigned long end = update->end; + bool blockable = update->blockable; + struct interval_tree_node *it; + + /* notification is exclusive, but interval is inclusive */ + end -= 1; + + if (amdgpu_mn_read_lock(amn, blockable)) + return -EAGAIN; + + it = interval_tree_iter_first(&amn->objects, start, end); + while (it) { + struct amdgpu_mn_node *node; + struct amdgpu_bo *bo; + + if (!blockable) { + amdgpu_mn_read_unlock(amn); + return -EAGAIN; + } + + node = container_of(it, struct amdgpu_mn_node, it); + it = interval_tree_iter_next(it, start, end); + + list_for_each_entry(bo, &node->bos, mn_list) { + struct kgd_mem *mem = bo->kfd_bo; + + if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, + start, end)) + amdgpu_amdkfd_evict_userptr(mem, amn->mm); + } + } + + amdgpu_mn_read_unlock(amn); + + return 0; +} + +/* Low bits of any reasonable mm pointer will be unused due to struct + * alignment. Use these bits to make a unique key from the mm pointer + * and notifier type. + */ +#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type)) + +static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = { + [AMDGPU_MN_TYPE_GFX] = { + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx, + .release = amdgpu_hmm_mirror_release + }, + [AMDGPU_MN_TYPE_HSA] = { + .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa, + .release = amdgpu_hmm_mirror_release + }, +}; + +/** + * amdgpu_mn_get - create HMM mirror context + * + * @adev: amdgpu device pointer + * @type: type of MMU notifier context + * + * Creates a HMM mirror context for current->mm. + */ +struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, + enum amdgpu_mn_type type) +{ + struct mm_struct *mm = current->mm; + struct amdgpu_mn *amn; + unsigned long key = AMDGPU_MN_KEY(mm, type); + int r; + + mutex_lock(&adev->mn_lock); + if (down_write_killable(&mm->mmap_sem)) { + mutex_unlock(&adev->mn_lock); + return ERR_PTR(-EINTR); + } + + hash_for_each_possible(adev->mn_hash, amn, node, key) + if (AMDGPU_MN_KEY(amn->mm, amn->type) == key) + goto release_locks; + + amn = kzalloc(sizeof(*amn), GFP_KERNEL); + if (!amn) { + amn = ERR_PTR(-ENOMEM); + goto release_locks; + } + + amn->adev = adev; + amn->mm = mm; + init_rwsem(&amn->lock); + amn->type = type; + amn->objects = RB_ROOT_CACHED; + + amn->mirror.ops = &amdgpu_hmm_mirror_ops[type]; + r = hmm_mirror_register(&amn->mirror, mm); + if (r) + goto free_amn; + + hash_add(adev->mn_hash, &amn->node, AMDGPU_MN_KEY(mm, type)); + +release_locks: + up_write(&mm->mmap_sem); + mutex_unlock(&adev->mn_lock); + + return amn; + +free_amn: + up_write(&mm->mmap_sem); + mutex_unlock(&adev->mn_lock); + kfree(amn); + + return ERR_PTR(r); +} + +/** + * amdgpu_mn_register - register a BO for notifier updates + * + * @bo: amdgpu buffer object + * @addr: userptr addr we should monitor + * + * Registers an HMM mirror for the given BO at the specified address. + * Returns 0 on success, -ERRNO if anything goes wrong. + */ +int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) +{ + unsigned long end = addr + amdgpu_bo_size(bo) - 1; + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + enum amdgpu_mn_type type = + bo->kfd_bo ? AMDGPU_MN_TYPE_HSA : AMDGPU_MN_TYPE_GFX; + struct amdgpu_mn *amn; + struct amdgpu_mn_node *node = NULL, *new_node; + struct list_head bos; + struct interval_tree_node *it; + + amn = amdgpu_mn_get(adev, type); + if (IS_ERR(amn)) + return PTR_ERR(amn); + + new_node = kmalloc(sizeof(*new_node), GFP_KERNEL); + if (!new_node) + return -ENOMEM; + + INIT_LIST_HEAD(&bos); + + down_write(&amn->lock); + + while ((it = interval_tree_iter_first(&amn->objects, addr, end))) { + kfree(node); + node = container_of(it, struct amdgpu_mn_node, it); + interval_tree_remove(&node->it, &amn->objects); + addr = min(it->start, addr); + end = max(it->last, end); + list_splice(&node->bos, &bos); + } + + if (!node) + node = new_node; + else + kfree(new_node); + + bo->mn = amn; + + node->it.start = addr; + node->it.last = end; + INIT_LIST_HEAD(&node->bos); + list_splice(&bos, &node->bos); + list_add(&bo->mn_list, &node->bos); + + interval_tree_insert(&node->it, &amn->objects); + + up_write(&amn->lock); + + return 0; +} + +/** + * amdgpu_mn_unregister - unregister a BO for HMM mirror updates + * + * @bo: amdgpu buffer object + * + * Remove any registration of HMM mirror updates from the buffer object. + */ +void amdgpu_mn_unregister(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_mn *amn; + struct list_head *head; + + mutex_lock(&adev->mn_lock); + + amn = bo->mn; + if (amn == NULL) { + mutex_unlock(&adev->mn_lock); + return; + } + + down_write(&amn->lock); + + /* save the next list entry for later */ + head = bo->mn_list.next; + + bo->mn = NULL; + list_del_init(&bo->mn_list); + + if (list_empty(head)) { + struct amdgpu_mn_node *node; + + node = container_of(head, struct amdgpu_mn_node, bos); + interval_tree_remove(&node->it, &amn->objects); + kfree(node); + } + + up_write(&amn->lock); + mutex_unlock(&adev->mn_lock); +} + +/* flags used by HMM internal, not related to CPU/GPU PTE flags */ +static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = { + (1 << 0), /* HMM_PFN_VALID */ + (1 << 1), /* HMM_PFN_WRITE */ + 0 /* HMM_PFN_DEVICE_PRIVATE */ +}; + +static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = { + 0xfffffffffffffffeUL, /* HMM_PFN_ERROR */ + 0, /* HMM_PFN_NONE */ + 0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */ +}; + +void amdgpu_hmm_init_range(struct hmm_range *range) +{ + if (range) { + range->flags = hmm_range_flags; + range->values = hmm_range_values; + range->pfn_shift = PAGE_SHIFT; + INIT_LIST_HEAD(&range->list); + } +}
diff --git a/amdgpu/amdgpu_mn.h b/amdgpu/amdgpu_mn.h new file mode 100644 index 0000000..b8ed689 --- /dev/null +++ b/amdgpu/amdgpu_mn.h
@@ -0,0 +1,99 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ +#ifndef __AMDGPU_MN_H__ +#define __AMDGPU_MN_H__ + +#include <linux/types.h> +#include <linux/hmm.h> +#include <linux/rwsem.h> +#include <linux/workqueue.h> +#include <linux/interval_tree.h> + +enum amdgpu_mn_type { + AMDGPU_MN_TYPE_GFX, + AMDGPU_MN_TYPE_HSA, +}; + +/** + * struct amdgpu_mn + * + * @adev: amdgpu device pointer + * @mm: process address space + * @type: type of MMU notifier + * @work: destruction work item + * @node: hash table node to find structure by adev and mn + * @lock: rw semaphore protecting the notifier nodes + * @objects: interval tree containing amdgpu_mn_nodes + * @mirror: HMM mirror function support + * + * Data for each amdgpu device and process address space. + */ +struct amdgpu_mn { + /* constant after initialisation */ + struct amdgpu_device *adev; + struct mm_struct *mm; + enum amdgpu_mn_type type; + + /* only used on destruction */ + struct work_struct work; + + /* protected by adev->mn_lock */ + struct hlist_node node; + + /* objects protected by lock */ + struct rw_semaphore lock; + struct rb_root_cached objects; + +#ifdef CONFIG_HMM_MIRROR + /* HMM mirror */ + struct hmm_mirror mirror; +#endif +}; + +#if defined(CONFIG_HMM_MIRROR) +void amdgpu_mn_lock(struct amdgpu_mn *mn); +void amdgpu_mn_unlock(struct amdgpu_mn *mn); +struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, + enum amdgpu_mn_type type); +int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr); +void amdgpu_mn_unregister(struct amdgpu_bo *bo); +void amdgpu_hmm_init_range(struct hmm_range *range); +#else +static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {} +static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {} +static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev, + enum amdgpu_mn_type type) +{ + return NULL; +} +static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr) +{ + DRM_WARN_ONCE("HMM_MIRROR kernel config option is not enabled, " + "add CONFIG_ZONE_DEVICE=y in config file to fix this\n"); + return -ENODEV; +} +static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {} +#endif + +#endif
diff --git a/amdgpu/amdgpu_mode.h b/amdgpu/amdgpu_mode.h new file mode 100644 index 0000000..eb9975f --- /dev/null +++ b/amdgpu/amdgpu_mode.h
@@ -0,0 +1,636 @@ +/* + * Copyright 2000 ATI Technologies Inc., Markham, Ontario, and + * VA Linux Systems Inc., Fremont, California. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Original Authors: + * Kevin E. Martin, Rickard E. Faith, Alan Hourihane + * + * Kernel port Author: Dave Airlie + */ + +#ifndef AMDGPU_MODE_H +#define AMDGPU_MODE_H + +#include <drm/drm_crtc.h> +#include <drm/drm_edid.h> +#include <drm/drm_encoder.h> +#include <drm/drm_dp_helper.h> +#include <drm/drm_fixed.h> +#include <drm/drm_crtc_helper.h> +#include <drm/drm_fb_helper.h> +#include <drm/drm_plane_helper.h> +#include <drm/drm_probe_helper.h> +#include <linux/i2c.h> +#include <linux/i2c-algo-bit.h> +#include <linux/hrtimer.h> +#include "amdgpu_irq.h" + +#include <drm/drm_dp_mst_helper.h> +#include "modules/inc/mod_freesync.h" + +struct amdgpu_bo; +struct amdgpu_device; +struct amdgpu_encoder; +struct amdgpu_router; +struct amdgpu_hpd; + +#define to_amdgpu_crtc(x) container_of(x, struct amdgpu_crtc, base) +#define to_amdgpu_connector(x) container_of(x, struct amdgpu_connector, base) +#define to_amdgpu_encoder(x) container_of(x, struct amdgpu_encoder, base) +#define to_amdgpu_framebuffer(x) container_of(x, struct amdgpu_framebuffer, base) + +#define to_dm_plane_state(x) container_of(x, struct dm_plane_state, base) + +#define AMDGPU_MAX_HPD_PINS 6 +#define AMDGPU_MAX_CRTCS 6 +#define AMDGPU_MAX_PLANES 6 +#define AMDGPU_MAX_AFMT_BLOCKS 9 + +enum amdgpu_rmx_type { + RMX_OFF, + RMX_FULL, + RMX_CENTER, + RMX_ASPECT +}; + +enum amdgpu_underscan_type { + UNDERSCAN_OFF, + UNDERSCAN_ON, + UNDERSCAN_AUTO, +}; + +#define AMDGPU_HPD_CONNECT_INT_DELAY_IN_MS 50 +#define AMDGPU_HPD_DISCONNECT_INT_DELAY_IN_MS 10 + +enum amdgpu_hpd_id { + AMDGPU_HPD_1 = 0, + AMDGPU_HPD_2, + AMDGPU_HPD_3, + AMDGPU_HPD_4, + AMDGPU_HPD_5, + AMDGPU_HPD_6, + AMDGPU_HPD_NONE = 0xff, +}; + +enum amdgpu_crtc_irq { + AMDGPU_CRTC_IRQ_VBLANK1 = 0, + AMDGPU_CRTC_IRQ_VBLANK2, + AMDGPU_CRTC_IRQ_VBLANK3, + AMDGPU_CRTC_IRQ_VBLANK4, + AMDGPU_CRTC_IRQ_VBLANK5, + AMDGPU_CRTC_IRQ_VBLANK6, + AMDGPU_CRTC_IRQ_VLINE1, + AMDGPU_CRTC_IRQ_VLINE2, + AMDGPU_CRTC_IRQ_VLINE3, + AMDGPU_CRTC_IRQ_VLINE4, + AMDGPU_CRTC_IRQ_VLINE5, + AMDGPU_CRTC_IRQ_VLINE6, + AMDGPU_CRTC_IRQ_NONE = 0xff +}; + +enum amdgpu_pageflip_irq { + AMDGPU_PAGEFLIP_IRQ_D1 = 0, + AMDGPU_PAGEFLIP_IRQ_D2, + AMDGPU_PAGEFLIP_IRQ_D3, + AMDGPU_PAGEFLIP_IRQ_D4, + AMDGPU_PAGEFLIP_IRQ_D5, + AMDGPU_PAGEFLIP_IRQ_D6, + AMDGPU_PAGEFLIP_IRQ_NONE = 0xff +}; + +enum amdgpu_flip_status { + AMDGPU_FLIP_NONE, + AMDGPU_FLIP_PENDING, + AMDGPU_FLIP_SUBMITTED +}; + +#define AMDGPU_MAX_I2C_BUS 16 + +/* amdgpu gpio-based i2c + * 1. "mask" reg and bits + * grabs the gpio pins for software use + * 0=not held 1=held + * 2. "a" reg and bits + * output pin value + * 0=low 1=high + * 3. "en" reg and bits + * sets the pin direction + * 0=input 1=output + * 4. "y" reg and bits + * input pin value + * 0=low 1=high + */ +struct amdgpu_i2c_bus_rec { + bool valid; + /* id used by atom */ + uint8_t i2c_id; + /* id used by atom */ + enum amdgpu_hpd_id hpd; + /* can be used with hw i2c engine */ + bool hw_capable; + /* uses multi-media i2c engine */ + bool mm_i2c; + /* regs and bits */ + uint32_t mask_clk_reg; + uint32_t mask_data_reg; + uint32_t a_clk_reg; + uint32_t a_data_reg; + uint32_t en_clk_reg; + uint32_t en_data_reg; + uint32_t y_clk_reg; + uint32_t y_data_reg; + uint32_t mask_clk_mask; + uint32_t mask_data_mask; + uint32_t a_clk_mask; + uint32_t a_data_mask; + uint32_t en_clk_mask; + uint32_t en_data_mask; + uint32_t y_clk_mask; + uint32_t y_data_mask; +}; + +#define AMDGPU_MAX_BIOS_CONNECTOR 16 + +/* pll flags */ +#define AMDGPU_PLL_USE_BIOS_DIVS (1 << 0) +#define AMDGPU_PLL_NO_ODD_POST_DIV (1 << 1) +#define AMDGPU_PLL_USE_REF_DIV (1 << 2) +#define AMDGPU_PLL_LEGACY (1 << 3) +#define AMDGPU_PLL_PREFER_LOW_REF_DIV (1 << 4) +#define AMDGPU_PLL_PREFER_HIGH_REF_DIV (1 << 5) +#define AMDGPU_PLL_PREFER_LOW_FB_DIV (1 << 6) +#define AMDGPU_PLL_PREFER_HIGH_FB_DIV (1 << 7) +#define AMDGPU_PLL_PREFER_LOW_POST_DIV (1 << 8) +#define AMDGPU_PLL_PREFER_HIGH_POST_DIV (1 << 9) +#define AMDGPU_PLL_USE_FRAC_FB_DIV (1 << 10) +#define AMDGPU_PLL_PREFER_CLOSEST_LOWER (1 << 11) +#define AMDGPU_PLL_USE_POST_DIV (1 << 12) +#define AMDGPU_PLL_IS_LCD (1 << 13) +#define AMDGPU_PLL_PREFER_MINM_OVER_MAXP (1 << 14) + +struct amdgpu_pll { + /* reference frequency */ + uint32_t reference_freq; + + /* fixed dividers */ + uint32_t reference_div; + uint32_t post_div; + + /* pll in/out limits */ + uint32_t pll_in_min; + uint32_t pll_in_max; + uint32_t pll_out_min; + uint32_t pll_out_max; + uint32_t lcd_pll_out_min; + uint32_t lcd_pll_out_max; + uint32_t best_vco; + + /* divider limits */ + uint32_t min_ref_div; + uint32_t max_ref_div; + uint32_t min_post_div; + uint32_t max_post_div; + uint32_t min_feedback_div; + uint32_t max_feedback_div; + uint32_t min_frac_feedback_div; + uint32_t max_frac_feedback_div; + + /* flags for the current clock */ + uint32_t flags; + + /* pll id */ + uint32_t id; +}; + +struct amdgpu_i2c_chan { + struct i2c_adapter adapter; + struct drm_device *dev; + struct i2c_algo_bit_data bit; + struct amdgpu_i2c_bus_rec rec; + struct drm_dp_aux aux; + bool has_aux; + struct mutex mutex; +}; + +struct amdgpu_fbdev; + +struct amdgpu_afmt { + bool enabled; + int offset; + bool last_buffer_filled_status; + int id; + struct amdgpu_audio_pin *pin; +}; + +/* + * Audio + */ +struct amdgpu_audio_pin { + int channels; + int rate; + int bits_per_sample; + u8 status_bits; + u8 category_code; + u32 offset; + bool connected; + u32 id; +}; + +struct amdgpu_audio { + bool enabled; + struct amdgpu_audio_pin pin[AMDGPU_MAX_AFMT_BLOCKS]; + int num_pins; +}; + +struct amdgpu_display_funcs { + /* display watermarks */ + void (*bandwidth_update)(struct amdgpu_device *adev); + /* get frame count */ + u32 (*vblank_get_counter)(struct amdgpu_device *adev, int crtc); + /* set backlight level */ + void (*backlight_set_level)(struct amdgpu_encoder *amdgpu_encoder, + u8 level); + /* get backlight level */ + u8 (*backlight_get_level)(struct amdgpu_encoder *amdgpu_encoder); + /* hotplug detect */ + bool (*hpd_sense)(struct amdgpu_device *adev, enum amdgpu_hpd_id hpd); + void (*hpd_set_polarity)(struct amdgpu_device *adev, + enum amdgpu_hpd_id hpd); + u32 (*hpd_get_gpio_reg)(struct amdgpu_device *adev); + /* pageflipping */ + void (*page_flip)(struct amdgpu_device *adev, + int crtc_id, u64 crtc_base, bool async); + int (*page_flip_get_scanoutpos)(struct amdgpu_device *adev, int crtc, + u32 *vbl, u32 *position); + /* display topology setup */ + void (*add_encoder)(struct amdgpu_device *adev, + uint32_t encoder_enum, + uint32_t supported_device, + u16 caps); + void (*add_connector)(struct amdgpu_device *adev, + uint32_t connector_id, + uint32_t supported_device, + int connector_type, + struct amdgpu_i2c_bus_rec *i2c_bus, + uint16_t connector_object_id, + struct amdgpu_hpd *hpd, + struct amdgpu_router *router); + + +}; + +struct amdgpu_framebuffer { + struct drm_framebuffer base; + + /* caching for later use */ + uint64_t address; +}; + +struct amdgpu_fbdev { + struct drm_fb_helper helper; + struct amdgpu_framebuffer rfb; + struct list_head fbdev_list; + struct amdgpu_device *adev; +}; + +struct amdgpu_mode_info { + struct atom_context *atom_context; + struct card_info *atom_card_info; + bool mode_config_initialized; + struct amdgpu_crtc *crtcs[AMDGPU_MAX_CRTCS]; + struct drm_plane *planes[AMDGPU_MAX_PLANES]; + struct amdgpu_afmt *afmt[AMDGPU_MAX_AFMT_BLOCKS]; + /* DVI-I properties */ + struct drm_property *coherent_mode_property; + /* DAC enable load detect */ + struct drm_property *load_detect_property; + /* underscan */ + struct drm_property *underscan_property; + struct drm_property *underscan_hborder_property; + struct drm_property *underscan_vborder_property; + /* audio */ + struct drm_property *audio_property; + /* FMT dithering */ + struct drm_property *dither_property; + /* Adaptive Backlight Modulation (power feature) */ + struct drm_property *abm_level_property; + /* hardcoded DFP edid from BIOS */ + struct edid *bios_hardcoded_edid; + int bios_hardcoded_edid_size; + + /* pointer to fbdev info structure */ + struct amdgpu_fbdev *rfbdev; + /* firmware flags */ + u16 firmware_flags; + /* pointer to backlight encoder */ + struct amdgpu_encoder *bl_encoder; + u8 bl_level; /* saved backlight level */ + struct amdgpu_audio audio; /* audio stuff */ + int num_crtc; /* number of crtcs */ + int num_hpd; /* number of hpd pins */ + int num_dig; /* number of dig blocks */ + int disp_priority; + const struct amdgpu_display_funcs *funcs; + const enum drm_plane_type *plane_type; +}; + +#define AMDGPU_MAX_BL_LEVEL 0xFF + +#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) + +struct amdgpu_backlight_privdata { + struct amdgpu_encoder *encoder; + uint8_t negative; +}; + +#endif + +struct amdgpu_atom_ss { + uint16_t percentage; + uint16_t percentage_divider; + uint8_t type; + uint16_t step; + uint8_t delay; + uint8_t range; + uint8_t refdiv; + /* asic_ss */ + uint16_t rate; + uint16_t amount; +}; + +struct amdgpu_crtc { + struct drm_crtc base; + int crtc_id; + bool enabled; + bool can_tile; + uint32_t crtc_offset; + struct drm_gem_object *cursor_bo; + uint64_t cursor_addr; + int cursor_x; + int cursor_y; + int cursor_hot_x; + int cursor_hot_y; + int cursor_width; + int cursor_height; + int max_cursor_width; + int max_cursor_height; + enum amdgpu_rmx_type rmx_type; + u8 h_border; + u8 v_border; + fixed20_12 vsc; + fixed20_12 hsc; + struct drm_display_mode native_mode; + u32 pll_id; + /* page flipping */ + struct amdgpu_flip_work *pflip_works; + enum amdgpu_flip_status pflip_status; + int deferred_flip_completion; + u32 last_flip_vblank; + /* pll sharing */ + struct amdgpu_atom_ss ss; + bool ss_enabled; + u32 adjusted_clock; + int bpc; + u32 pll_reference_div; + u32 pll_post_div; + u32 pll_flags; + struct drm_encoder *encoder; + struct drm_connector *connector; + /* for dpm */ + u32 line_time; + u32 wm_low; + u32 wm_high; + u32 lb_vblank_lead_lines; + struct drm_display_mode hw_mode; + /* for virtual dce */ + struct hrtimer vblank_timer; + enum amdgpu_interrupt_state vsync_timer_enabled; + + int otg_inst; + struct drm_pending_vblank_event *event; +}; + +struct amdgpu_encoder_atom_dig { + bool linkb; + /* atom dig */ + bool coherent_mode; + int dig_encoder; /* -1 disabled, 0 DIGA, 1 DIGB, etc. */ + /* atom lvds/edp */ + uint32_t lcd_misc; + uint16_t panel_pwr_delay; + uint32_t lcd_ss_id; + /* panel mode */ + struct drm_display_mode native_mode; + struct backlight_device *bl_dev; + int dpms_mode; + uint8_t backlight_level; + int panel_mode; + struct amdgpu_afmt *afmt; +}; + +struct amdgpu_encoder { + struct drm_encoder base; + uint32_t encoder_enum; + uint32_t encoder_id; + uint32_t devices; + uint32_t active_device; + uint32_t flags; + uint32_t pixel_clock; + enum amdgpu_rmx_type rmx_type; + enum amdgpu_underscan_type underscan_type; + uint32_t underscan_hborder; + uint32_t underscan_vborder; + struct drm_display_mode native_mode; + void *enc_priv; + int audio_polling_active; + bool is_ext_encoder; + u16 caps; +}; + +struct amdgpu_connector_atom_dig { + /* displayport */ + u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 dp_sink_type; + int dp_clock; + int dp_lane_count; + bool edp_on; +}; + +struct amdgpu_gpio_rec { + bool valid; + u8 id; + u32 reg; + u32 mask; + u32 shift; +}; + +struct amdgpu_hpd { + enum amdgpu_hpd_id hpd; + u8 plugged_state; + struct amdgpu_gpio_rec gpio; +}; + +struct amdgpu_router { + u32 router_id; + struct amdgpu_i2c_bus_rec i2c_info; + u8 i2c_addr; + /* i2c mux */ + bool ddc_valid; + u8 ddc_mux_type; + u8 ddc_mux_control_pin; + u8 ddc_mux_state; + /* clock/data mux */ + bool cd_valid; + u8 cd_mux_type; + u8 cd_mux_control_pin; + u8 cd_mux_state; +}; + +enum amdgpu_connector_audio { + AMDGPU_AUDIO_DISABLE = 0, + AMDGPU_AUDIO_ENABLE = 1, + AMDGPU_AUDIO_AUTO = 2 +}; + +enum amdgpu_connector_dither { + AMDGPU_FMT_DITHER_DISABLE = 0, + AMDGPU_FMT_DITHER_ENABLE = 1, +}; + +struct amdgpu_dm_dp_aux { + struct drm_dp_aux aux; + struct ddc_service *ddc_service; +}; + +struct amdgpu_i2c_adapter { + struct i2c_adapter base; + + struct ddc_service *ddc_service; +}; + +#define TO_DM_AUX(x) container_of((x), struct amdgpu_dm_dp_aux, aux) + +struct amdgpu_connector { + struct drm_connector base; + uint32_t connector_id; + uint32_t devices; + struct amdgpu_i2c_chan *ddc_bus; + /* some systems have an hdmi and vga port with a shared ddc line */ + bool shared_ddc; + bool use_digital; + /* we need to mind the EDID between detect + and get modes due to analog/digital/tvencoder */ + struct edid *edid; + void *con_priv; + bool dac_load_detect; + bool detected_by_load; /* if the connection status was determined by load */ + uint16_t connector_object_id; + struct amdgpu_hpd hpd; + struct amdgpu_router router; + struct amdgpu_i2c_chan *router_bus; + enum amdgpu_connector_audio audio; + enum amdgpu_connector_dither dither; + unsigned pixelclock_for_modeset; +}; + +/* TODO: start to use this struct and remove same field from base one */ +struct amdgpu_mst_connector { + struct amdgpu_connector base; + + struct drm_dp_mst_topology_mgr mst_mgr; + struct amdgpu_dm_dp_aux dm_dp_aux; + struct drm_dp_mst_port *port; + struct amdgpu_connector *mst_port; + bool is_mst_connector; + struct amdgpu_encoder *mst_encoder; +}; + +#define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \ + ((em) == ATOM_ENCODER_MODE_DP_MST)) + +/* Driver internal use only flags of amdgpu_display_get_crtc_scanoutpos() */ +#define DRM_SCANOUTPOS_VALID (1 << 0) +#define DRM_SCANOUTPOS_IN_VBLANK (1 << 1) +#define DRM_SCANOUTPOS_ACCURATE (1 << 2) +#define USE_REAL_VBLANKSTART (1 << 30) +#define GET_DISTANCE_TO_VBLANKSTART (1 << 31) + +void amdgpu_link_encoder_connector(struct drm_device *dev); + +struct drm_connector * +amdgpu_get_connector_for_encoder(struct drm_encoder *encoder); +struct drm_connector * +amdgpu_get_connector_for_encoder_init(struct drm_encoder *encoder); +bool amdgpu_dig_monitor_is_duallink(struct drm_encoder *encoder, + u32 pixel_clock); + +u16 amdgpu_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder); +struct drm_encoder *amdgpu_get_external_encoder(struct drm_encoder *encoder); + +bool amdgpu_display_ddc_probe(struct amdgpu_connector *amdgpu_connector, + bool use_aux); + +void amdgpu_encoder_set_active_device(struct drm_encoder *encoder); + +int amdgpu_display_get_crtc_scanoutpos(struct drm_device *dev, + unsigned int pipe, unsigned int flags, int *vpos, + int *hpos, ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode); + +int amdgpu_display_framebuffer_init(struct drm_device *dev, + struct amdgpu_framebuffer *rfb, + const struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_gem_object *obj); + +int amdgpufb_remove(struct drm_device *dev, struct drm_framebuffer *fb); + +void amdgpu_enc_destroy(struct drm_encoder *encoder); +void amdgpu_copy_fb(struct drm_device *dev, struct drm_gem_object *dst_obj); +bool amdgpu_display_crtc_scaling_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); +void amdgpu_panel_mode_fixup(struct drm_encoder *encoder, + struct drm_display_mode *adjusted_mode); +int amdgpu_display_crtc_idx_to_irq_type(struct amdgpu_device *adev, int crtc); + +/* fbdev layer */ +int amdgpu_fbdev_init(struct amdgpu_device *adev); +void amdgpu_fbdev_fini(struct amdgpu_device *adev); +void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state); +int amdgpu_fbdev_total_size(struct amdgpu_device *adev); +bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj); + +int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled); + +/* amdgpu_display.c */ +void amdgpu_display_print_display_setup(struct drm_device *dev); +int amdgpu_display_modeset_create_props(struct amdgpu_device *adev); +int amdgpu_display_crtc_set_config(struct drm_mode_set *set, + struct drm_modeset_acquire_ctx *ctx); +int amdgpu_display_crtc_page_flip_target(struct drm_crtc *crtc, + struct drm_framebuffer *fb, + struct drm_pending_vblank_event *event, + uint32_t page_flip_flags, uint32_t target, + struct drm_modeset_acquire_ctx *ctx); +extern const struct drm_mode_config_funcs amdgpu_mode_funcs; + +#endif
diff --git a/amdgpu/amdgpu_object.c b/amdgpu/amdgpu_object.c new file mode 100644 index 0000000..bea6f29 --- /dev/null +++ b/amdgpu/amdgpu_object.c
@@ -0,0 +1,1357 @@ +/* + * Copyright 2009 Jerome Glisse. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Jerome Glisse <glisse@freedesktop.org> + * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> + * Dave Airlie + */ +#include <linux/list.h> +#include <linux/slab.h> + +#include <drm/amdgpu_drm.h> +#include <drm/drm_cache.h> +#include "amdgpu.h" +#include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" + +/** + * DOC: amdgpu_object + * + * This defines the interfaces to operate on an &amdgpu_bo buffer object which + * represents memory used by driver (VRAM, system memory, etc.). The driver + * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces + * to create/destroy/set buffer object which are then managed by the kernel TTM + * memory manager. + * The interfaces are also used internally by kernel clients, including gfx, + * uvd, etc. for kernel managed allocations used by the GPU. + * + */ + +/** + * amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting + * + * @bo: &amdgpu_bo buffer object + * + * This function is called when a BO stops being pinned, and updates the + * &amdgpu_device pin_size values accordingly. + */ +static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + + if (bo->tbo.mem.mem_type == TTM_PL_VRAM) { + atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); + atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), + &adev->visible_pin_size); + } else if (bo->tbo.mem.mem_type == TTM_PL_TT) { + atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); + } +} + +static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); + + if (bo->pin_count > 0) + amdgpu_bo_subtract_pin_size(bo); + + if (bo->kfd_bo) + amdgpu_amdkfd_unreserve_memory_limit(bo); + + amdgpu_bo_kunmap(bo); + + if (bo->gem_base.import_attach) + drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg); + drm_gem_object_release(&bo->gem_base); + /* in case amdgpu_device_recover_vram got NULL of bo->parent */ + if (!list_empty(&bo->shadow_list)) { + mutex_lock(&adev->shadow_list_lock); + list_del_init(&bo->shadow_list); + mutex_unlock(&adev->shadow_list_lock); + } + amdgpu_bo_unref(&bo->parent); + + kfree(bo->metadata); + kfree(bo); +} + +/** + * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo + * @bo: buffer object to be checked + * + * Uses destroy function associated with the object to determine if this is + * an &amdgpu_bo. + * + * Returns: + * true if the object belongs to &amdgpu_bo, false if not. + */ +bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) +{ + if (bo->destroy == &amdgpu_bo_destroy) + return true; + return false; +} + +/** + * amdgpu_bo_placement_from_domain - set buffer's placement + * @abo: &amdgpu_bo buffer object whose placement is to be set + * @domain: requested domain + * + * Sets buffer's placement according to requested domain and the buffer's + * flags. + */ +void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); + struct ttm_placement *placement = &abo->placement; + struct ttm_place *places = abo->placements; + u64 flags = abo->flags; + u32 c = 0; + + if (domain & AMDGPU_GEM_DOMAIN_VRAM) { + unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; + + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_VRAM; + + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) + places[c].lpfn = visible_pfn; + else + places[c].flags |= TTM_PL_FLAG_TOPDOWN; + + if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) + places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; + c++; + } + + if (domain & AMDGPU_GEM_DOMAIN_GTT) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_TT; + if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + places[c].flags |= TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED; + else + places[c].flags |= TTM_PL_FLAG_CACHED; + c++; + } + + if (domain & AMDGPU_GEM_DOMAIN_CPU) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_SYSTEM; + if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + places[c].flags |= TTM_PL_FLAG_WC | + TTM_PL_FLAG_UNCACHED; + else + places[c].flags |= TTM_PL_FLAG_CACHED; + c++; + } + + if (domain & AMDGPU_GEM_DOMAIN_GDS) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; + c++; + } + + if (domain & AMDGPU_GEM_DOMAIN_GWS) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; + c++; + } + + if (domain & AMDGPU_GEM_DOMAIN_OA) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; + c++; + } + + if (!c) { + places[c].fpfn = 0; + places[c].lpfn = 0; + places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; + c++; + } + + BUG_ON(c >= AMDGPU_BO_MAX_PLACEMENTS); + + placement->num_placement = c; + placement->placement = places; + + placement->num_busy_placement = c; + placement->busy_placement = places; +} + +/** + * amdgpu_bo_create_reserved - create reserved BO for kernel use + * + * @adev: amdgpu device object + * @size: size for the new BO + * @align: alignment for the new BO + * @domain: where to place it + * @bo_ptr: used to initialize BOs in structures + * @gpu_addr: GPU addr of the pinned BO + * @cpu_addr: optional CPU address mapping + * + * Allocates and pins a BO for kernel internal use, and returns it still + * reserved. + * + * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. + * + * Returns: + * 0 on success, negative error code otherwise. + */ +int amdgpu_bo_create_reserved(struct amdgpu_device *adev, + unsigned long size, int align, + u32 domain, struct amdgpu_bo **bo_ptr, + u64 *gpu_addr, void **cpu_addr) +{ + struct amdgpu_bo_param bp; + bool free = false; + int r; + + if (!size) { + amdgpu_bo_unref(bo_ptr); + return 0; + } + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = align; + bp.domain = domain; + bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | + AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + + if (!*bo_ptr) { + r = amdgpu_bo_create(adev, &bp, bo_ptr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", + r); + return r; + } + free = true; + } + + r = amdgpu_bo_reserve(*bo_ptr, false); + if (r) { + dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); + goto error_free; + } + + r = amdgpu_bo_pin(*bo_ptr, domain); + if (r) { + dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); + goto error_unreserve; + } + + r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); + if (r) { + dev_err(adev->dev, "%p bind failed\n", *bo_ptr); + goto error_unpin; + } + + if (gpu_addr) + *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); + + if (cpu_addr) { + r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); + if (r) { + dev_err(adev->dev, "(%d) kernel bo map failed\n", r); + goto error_unpin; + } + } + + return 0; + +error_unpin: + amdgpu_bo_unpin(*bo_ptr); +error_unreserve: + amdgpu_bo_unreserve(*bo_ptr); + +error_free: + if (free) + amdgpu_bo_unref(bo_ptr); + + return r; +} + +/** + * amdgpu_bo_create_kernel - create BO for kernel use + * + * @adev: amdgpu device object + * @size: size for the new BO + * @align: alignment for the new BO + * @domain: where to place it + * @bo_ptr: used to initialize BOs in structures + * @gpu_addr: GPU addr of the pinned BO + * @cpu_addr: optional CPU address mapping + * + * Allocates and pins a BO for kernel internal use. + * + * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. + * + * Returns: + * 0 on success, negative error code otherwise. + */ +int amdgpu_bo_create_kernel(struct amdgpu_device *adev, + unsigned long size, int align, + u32 domain, struct amdgpu_bo **bo_ptr, + u64 *gpu_addr, void **cpu_addr) +{ + int r; + + r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, + gpu_addr, cpu_addr); + + if (r) + return r; + + if (*bo_ptr) + amdgpu_bo_unreserve(*bo_ptr); + + return 0; +} + +/** + * amdgpu_bo_free_kernel - free BO for kernel use + * + * @bo: amdgpu BO to free + * @gpu_addr: pointer to where the BO's GPU memory space address was stored + * @cpu_addr: pointer to where the BO's CPU memory space address was stored + * + * unmaps and unpin a BO for kernel internal use. + */ +void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, + void **cpu_addr) +{ + if (*bo == NULL) + return; + + if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { + if (cpu_addr) + amdgpu_bo_kunmap(*bo); + + amdgpu_bo_unpin(*bo); + amdgpu_bo_unreserve(*bo); + } + amdgpu_bo_unref(bo); + + if (gpu_addr) + *gpu_addr = 0; + + if (cpu_addr) + *cpu_addr = NULL; +} + +/* Validate bo size is bit bigger then the request domain */ +static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, + unsigned long size, u32 domain) +{ + struct ttm_mem_type_manager *man = NULL; + + /* + * If GTT is part of requested domains the check must succeed to + * allow fall back to GTT + */ + if (domain & AMDGPU_GEM_DOMAIN_GTT) { + man = &adev->mman.bdev.man[TTM_PL_TT]; + + if (size < (man->size << PAGE_SHIFT)) + return true; + else + goto fail; + } + + if (domain & AMDGPU_GEM_DOMAIN_VRAM) { + man = &adev->mman.bdev.man[TTM_PL_VRAM]; + + if (size < (man->size << PAGE_SHIFT)) + return true; + else + goto fail; + } + + + /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */ + return true; + +fail: + DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, + man->size << PAGE_SHIFT); + return false; +} + +static int amdgpu_bo_do_create(struct amdgpu_device *adev, + struct amdgpu_bo_param *bp, + struct amdgpu_bo **bo_ptr) +{ + struct ttm_operation_ctx ctx = { + .interruptible = (bp->type != ttm_bo_type_kernel), + .no_wait_gpu = false, + .resv = bp->resv, + .flags = TTM_OPT_FLAG_ALLOW_RES_EVICT + }; + struct amdgpu_bo *bo; + unsigned long page_align, size = bp->size; + size_t acc_size; + int r; + + /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ + if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { + /* GWS and OA don't need any alignment. */ + page_align = bp->byte_align; + size <<= PAGE_SHIFT; + } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { + /* Both size and alignment must be a multiple of 4. */ + page_align = ALIGN(bp->byte_align, 4); + size = ALIGN(size, 4) << PAGE_SHIFT; + } else { + /* Memory should be aligned at least to a page size. */ + page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; + size = ALIGN(size, PAGE_SIZE); + } + + if (!amdgpu_bo_validate_size(adev, size, bp->domain)) + return -ENOMEM; + + *bo_ptr = NULL; + + acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, + sizeof(struct amdgpu_bo)); + + bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); + if (bo == NULL) + return -ENOMEM; + drm_gem_private_object_init(adev->ddev, &bo->gem_base, size); + INIT_LIST_HEAD(&bo->shadow_list); + bo->vm_bo = NULL; + bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : + bp->domain; + bo->allowed_domains = bo->preferred_domains; + if (bp->type != ttm_bo_type_kernel && + bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) + bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; + + bo->flags = bp->flags; + +#ifdef CONFIG_X86_32 + /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit + * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 + */ + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) + /* Don't try to enable write-combining when it can't work, or things + * may be slow + * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 + */ + +#ifndef CONFIG_COMPILE_TEST +#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ + thanks to write-combining +#endif + + if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " + "better performance thanks to write-combining\n"); + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#else + /* For architectures that don't support WC memory, + * mask out the WC flag from the BO + */ + if (!drm_arch_can_wc_memory()) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#endif + + bo->tbo.bdev = &adev->mman.bdev; + if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | + AMDGPU_GEM_DOMAIN_GDS)) + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); + else + amdgpu_bo_placement_from_domain(bo, bp->domain); + if (bp->type == ttm_bo_type_kernel) + bo->tbo.priority = 1; + + r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type, + &bo->placement, page_align, &ctx, acc_size, + NULL, bp->resv, &amdgpu_bo_destroy); + if (unlikely(r != 0)) + return r; + + if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + bo->tbo.mem.mem_type == TTM_PL_VRAM && + bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT) + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, + ctx.bytes_moved); + else + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); + + if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && + bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { + struct dma_fence *fence; + + r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); + if (unlikely(r)) + goto fail_unreserve; + + amdgpu_bo_fence(bo, fence, false); + dma_fence_put(bo->tbo.moving); + bo->tbo.moving = dma_fence_get(fence); + dma_fence_put(fence); + } + if (!bp->resv) + amdgpu_bo_unreserve(bo); + *bo_ptr = bo; + + trace_amdgpu_bo_create(bo); + + /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ + if (bp->type == ttm_bo_type_device) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + + return 0; + +fail_unreserve: + if (!bp->resv) + ww_mutex_unlock(&bo->tbo.resv->lock); + amdgpu_bo_unref(&bo); + return r; +} + +static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, + unsigned long size, + struct amdgpu_bo *bo) +{ + struct amdgpu_bo_param bp; + int r; + + if (bo->shadow) + return 0; + + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.domain = AMDGPU_GEM_DOMAIN_GTT; + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC | + AMDGPU_GEM_CREATE_SHADOW; + bp.type = ttm_bo_type_kernel; + bp.resv = bo->tbo.resv; + + r = amdgpu_bo_do_create(adev, &bp, &bo->shadow); + if (!r) { + bo->shadow->parent = amdgpu_bo_ref(bo); + mutex_lock(&adev->shadow_list_lock); + list_add_tail(&bo->shadow->shadow_list, &adev->shadow_list); + mutex_unlock(&adev->shadow_list_lock); + } + + return r; +} + +/** + * amdgpu_bo_create - create an &amdgpu_bo buffer object + * @adev: amdgpu device object + * @bp: parameters to be used for the buffer object + * @bo_ptr: pointer to the buffer object pointer + * + * Creates an &amdgpu_bo buffer object; and if requested, also creates a + * shadow object. + * Shadow object is used to backup the original buffer object, and is always + * in GTT. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_create(struct amdgpu_device *adev, + struct amdgpu_bo_param *bp, + struct amdgpu_bo **bo_ptr) +{ + u64 flags = bp->flags; + int r; + + bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW; + r = amdgpu_bo_do_create(adev, bp, bo_ptr); + if (r) + return r; + + if ((flags & AMDGPU_GEM_CREATE_SHADOW) && !(adev->flags & AMD_IS_APU)) { + if (!bp->resv) + WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv, + NULL)); + + r = amdgpu_bo_create_shadow(adev, bp->size, *bo_ptr); + + if (!bp->resv) + reservation_object_unlock((*bo_ptr)->tbo.resv); + + if (r) + amdgpu_bo_unref(bo_ptr); + } + + return r; +} + +/** + * amdgpu_bo_validate - validate an &amdgpu_bo buffer object + * @bo: pointer to the buffer object + * + * Sets placement according to domain; and changes placement and caching + * policy of the buffer object according to the placement. + * This is used for validating shadow bos. It calls ttm_bo_validate() to + * make sure the buffer is resident where it needs to be. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_validate(struct amdgpu_bo *bo) +{ + struct ttm_operation_ctx ctx = { false, false }; + uint32_t domain; + int r; + + if (bo->pin_count) + return 0; + + domain = bo->preferred_domains; + +retry: + amdgpu_bo_placement_from_domain(bo, domain); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { + domain = bo->allowed_domains; + goto retry; + } + + return r; +} + +/** + * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow + * + * @shadow: &amdgpu_bo shadow to be restored + * @fence: dma_fence associated with the operation + * + * Copies a buffer object's shadow content back to the object. + * This is used for recovering a buffer from its shadow in case of a gpu + * reset where vram context may be lost. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence) + +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev); + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + uint64_t shadow_addr, parent_addr; + + shadow_addr = amdgpu_bo_gpu_offset(shadow); + parent_addr = amdgpu_bo_gpu_offset(shadow->parent); + + return amdgpu_copy_buffer(ring, shadow_addr, parent_addr, + amdgpu_bo_size(shadow), NULL, fence, + true, false); +} + +/** + * amdgpu_bo_kmap - map an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object to be mapped + * @ptr: kernel virtual address to be returned + * + * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls + * amdgpu_bo_kptr() to get the kernel virtual address. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) +{ + void *kptr; + long r; + + if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) + return -EPERM; + + kptr = amdgpu_bo_kptr(bo); + if (kptr) { + if (ptr) + *ptr = kptr; + return 0; + } + + r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false, + MAX_SCHEDULE_TIMEOUT); + if (r < 0) + return r; + + r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); + if (r) + return r; + + if (ptr) + *ptr = amdgpu_bo_kptr(bo); + + return 0; +} + +/** + * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object + * @bo: &amdgpu_bo buffer object + * + * Calls ttm_kmap_obj_virtual() to get the kernel virtual address + * + * Returns: + * the virtual address of a buffer object area. + */ +void *amdgpu_bo_kptr(struct amdgpu_bo *bo) +{ + bool is_iomem; + + return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); +} + +/** + * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object to be unmapped + * + * Unmaps a kernel map set up by amdgpu_bo_kmap(). + */ +void amdgpu_bo_kunmap(struct amdgpu_bo *bo) +{ + if (bo->kmap.bo) + ttm_bo_kunmap(&bo->kmap); +} + +/** + * amdgpu_bo_ref - reference an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object + * + * References the contained &ttm_buffer_object. + * + * Returns: + * a refcounted pointer to the &amdgpu_bo buffer object. + */ +struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) +{ + if (bo == NULL) + return NULL; + + ttm_bo_get(&bo->tbo); + return bo; +} + +/** + * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object + * + * Unreferences the contained &ttm_buffer_object and clear the pointer + */ +void amdgpu_bo_unref(struct amdgpu_bo **bo) +{ + struct ttm_buffer_object *tbo; + + if ((*bo) == NULL) + return; + + tbo = &((*bo)->tbo); + ttm_bo_put(tbo); + *bo = NULL; +} + +/** + * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object to be pinned + * @domain: domain to be pinned to + * @min_offset: the start of requested address range + * @max_offset: the end of requested address range + * + * Pins the buffer object according to requested domain and address range. If + * the memory is unbound gart memory, binds the pages into gart table. Adjusts + * pin_count and pin_size accordingly. + * + * Pinning means to lock pages in memory along with keeping them at a fixed + * offset. It is required when a buffer can not be moved, for example, when + * a display buffer is being scanned out. + * + * Compared with amdgpu_bo_pin(), this function gives more flexibility on + * where to pin a buffer if there are specific restrictions on where a buffer + * must be located. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, + u64 min_offset, u64 max_offset) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_operation_ctx ctx = { false, false }; + int r, i; + + if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) + return -EPERM; + + if (WARN_ON_ONCE(min_offset > max_offset)) + return -EINVAL; + + /* A shared bo cannot be migrated to VRAM */ + if (bo->prime_shared_count) { + if (domain & AMDGPU_GEM_DOMAIN_GTT) + domain = AMDGPU_GEM_DOMAIN_GTT; + else + return -EINVAL; + } + + /* This assumes only APU display buffers are pinned with (VRAM|GTT). + * See function amdgpu_display_supported_domains() + */ + domain = amdgpu_bo_get_preferred_pin_domain(adev, domain); + + if (bo->pin_count) { + uint32_t mem_type = bo->tbo.mem.mem_type; + + if (!(domain & amdgpu_mem_type_to_domain(mem_type))) + return -EINVAL; + + bo->pin_count++; + + if (max_offset != 0) { + u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset; + WARN_ON_ONCE(max_offset < + (amdgpu_bo_gpu_offset(bo) - domain_start)); + } + + return 0; + } + + bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; + /* force to pin into visible video ram */ + if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) + bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + amdgpu_bo_placement_from_domain(bo, domain); + for (i = 0; i < bo->placement.num_placement; i++) { + unsigned fpfn, lpfn; + + fpfn = min_offset >> PAGE_SHIFT; + lpfn = max_offset >> PAGE_SHIFT; + + if (fpfn > bo->placements[i].fpfn) + bo->placements[i].fpfn = fpfn; + if (!bo->placements[i].lpfn || + (lpfn && lpfn < bo->placements[i].lpfn)) + bo->placements[i].lpfn = lpfn; + bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; + } + + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (unlikely(r)) { + dev_err(adev->dev, "%p pin failed\n", bo); + goto error; + } + + bo->pin_count = 1; + + domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); + if (domain == AMDGPU_GEM_DOMAIN_VRAM) { + atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); + atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), + &adev->visible_pin_size); + } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { + atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); + } + +error: + return r; +} + +/** + * amdgpu_bo_pin - pin an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object to be pinned + * @domain: domain to be pinned to + * + * A simple wrapper to amdgpu_bo_pin_restricted(). + * Provides a simpler API for buffers that do not have any strict restrictions + * on where a buffer must be located. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) +{ + return amdgpu_bo_pin_restricted(bo, domain, 0, 0); +} + +/** + * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object + * @bo: &amdgpu_bo buffer object to be unpinned + * + * Decreases the pin_count, and clears the flags if pin_count reaches 0. + * Changes placement and pin size accordingly. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_unpin(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_operation_ctx ctx = { false, false }; + int r, i; + + if (WARN_ON_ONCE(!bo->pin_count)) { + dev_warn(adev->dev, "%p unpin not necessary\n", bo); + return 0; + } + bo->pin_count--; + if (bo->pin_count) + return 0; + + amdgpu_bo_subtract_pin_size(bo); + + for (i = 0; i < bo->placement.num_placement; i++) { + bo->placements[i].lpfn = 0; + bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; + } + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (unlikely(r)) + dev_err(adev->dev, "%p validate failed for unpin\n", bo); + + return r; +} + +/** + * amdgpu_bo_evict_vram - evict VRAM buffers + * @adev: amdgpu device object + * + * Evicts all VRAM buffers on the lru list of the memory type. + * Mainly used for evicting vram at suspend time. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_evict_vram(struct amdgpu_device *adev) +{ + /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ +#ifndef CONFIG_HIBERNATION + if (adev->flags & AMD_IS_APU) { + /* Useless to evict on IGP chips */ + return 0; + } +#endif + return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM); +} + +static const char *amdgpu_vram_names[] = { + "UNKNOWN", + "GDDR1", + "DDR2", + "GDDR3", + "GDDR4", + "GDDR5", + "HBM", + "DDR3", + "DDR4", + "GDDR6", +}; + +/** + * amdgpu_bo_init - initialize memory manager + * @adev: amdgpu device object + * + * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_init(struct amdgpu_device *adev) +{ + /* reserve PAT memory space to WC for VRAM */ + arch_io_reserve_memtype_wc(adev->gmc.aper_base, + adev->gmc.aper_size); + + /* Add an MTRR for the VRAM */ + adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, + adev->gmc.aper_size); + DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", + adev->gmc.mc_vram_size >> 20, + (unsigned long long)adev->gmc.aper_size >> 20); + DRM_INFO("RAM width %dbits %s\n", + adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); + return amdgpu_ttm_init(adev); +} + +/** + * amdgpu_bo_late_init - late init + * @adev: amdgpu device object + * + * Calls amdgpu_ttm_late_init() to free resources used earlier during + * initialization. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_late_init(struct amdgpu_device *adev) +{ + amdgpu_ttm_late_init(adev); + + return 0; +} + +/** + * amdgpu_bo_fini - tear down memory manager + * @adev: amdgpu device object + * + * Reverses amdgpu_bo_init() to tear down memory manager. + */ +void amdgpu_bo_fini(struct amdgpu_device *adev) +{ + amdgpu_ttm_fini(adev); + arch_phys_wc_del(adev->gmc.vram_mtrr); + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); +} + +/** + * amdgpu_bo_fbdev_mmap - mmap fbdev memory + * @bo: &amdgpu_bo buffer object + * @vma: vma as input from the fbdev mmap method + * + * Calls ttm_fbdev_mmap() to mmap fbdev memory if it is backed by a bo. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, + struct vm_area_struct *vma) +{ + return ttm_fbdev_mmap(vma, &bo->tbo); +} + +/** + * amdgpu_bo_set_tiling_flags - set tiling flags + * @bo: &amdgpu_bo buffer object + * @tiling_flags: new flags + * + * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or + * kernel driver to set the tiling flags on a buffer. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + + if (adev->family <= AMDGPU_FAMILY_CZ && + AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) + return -EINVAL; + + bo->tiling_flags = tiling_flags; + return 0; +} + +/** + * amdgpu_bo_get_tiling_flags - get tiling flags + * @bo: &amdgpu_bo buffer object + * @tiling_flags: returned flags + * + * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to + * set the tiling flags on a buffer. + */ +void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) +{ + lockdep_assert_held(&bo->tbo.resv->lock.base); + + if (tiling_flags) + *tiling_flags = bo->tiling_flags; +} + +/** + * amdgpu_bo_set_metadata - set metadata + * @bo: &amdgpu_bo buffer object + * @metadata: new metadata + * @metadata_size: size of the new metadata + * @flags: flags of the new metadata + * + * Sets buffer object's metadata, its size and flags. + * Used via GEM ioctl. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, + uint32_t metadata_size, uint64_t flags) +{ + void *buffer; + + if (!metadata_size) { + if (bo->metadata_size) { + kfree(bo->metadata); + bo->metadata = NULL; + bo->metadata_size = 0; + } + return 0; + } + + if (metadata == NULL) + return -EINVAL; + + buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); + if (buffer == NULL) + return -ENOMEM; + + kfree(bo->metadata); + bo->metadata_flags = flags; + bo->metadata = buffer; + bo->metadata_size = metadata_size; + + return 0; +} + +/** + * amdgpu_bo_get_metadata - get metadata + * @bo: &amdgpu_bo buffer object + * @buffer: returned metadata + * @buffer_size: size of the buffer + * @metadata_size: size of the returned metadata + * @flags: flags of the returned metadata + * + * Gets buffer object's metadata, its size and flags. buffer_size shall not be + * less than metadata_size. + * Used via GEM ioctl. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, + size_t buffer_size, uint32_t *metadata_size, + uint64_t *flags) +{ + if (!buffer && !metadata_size) + return -EINVAL; + + if (buffer) { + if (buffer_size < bo->metadata_size) + return -EINVAL; + + if (bo->metadata_size) + memcpy(buffer, bo->metadata, bo->metadata_size); + } + + if (metadata_size) + *metadata_size = bo->metadata_size; + if (flags) + *flags = bo->metadata_flags; + + return 0; +} + +/** + * amdgpu_bo_move_notify - notification about a memory move + * @bo: pointer to a buffer object + * @evict: if this move is evicting the buffer from the graphics address space + * @new_mem: new information of the bufer object + * + * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs + * bookkeeping. + * TTM driver callback which is called when ttm moves a buffer. + */ +void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, + bool evict, + struct ttm_mem_reg *new_mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct amdgpu_bo *abo; + struct ttm_mem_reg *old_mem = &bo->mem; + + if (!amdgpu_bo_is_amdgpu_bo(bo)) + return; + + abo = ttm_to_amdgpu_bo(bo); + amdgpu_vm_bo_invalidate(adev, abo, evict); + + amdgpu_bo_kunmap(abo); + + /* remember the eviction */ + if (evict) + atomic64_inc(&adev->num_evictions); + + /* update statistics */ + if (!new_mem) + return; + + /* move_notify is called before move happens */ + trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type); +} + +/** + * amdgpu_bo_fault_reserve_notify - notification about a memory fault + * @bo: pointer to a buffer object + * + * Notifies the driver we are taking a fault on this BO and have reserved it, + * also performs bookkeeping. + * TTM driver callback for dealing with vm faults. + * + * Returns: + * 0 for success or a negative error code on failure. + */ +int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_bo *abo; + unsigned long offset, size; + int r; + + if (!amdgpu_bo_is_amdgpu_bo(bo)) + return 0; + + abo = ttm_to_amdgpu_bo(bo); + + /* Remember that this BO was accessed by the CPU */ + abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + + if (bo->mem.mem_type != TTM_PL_VRAM) + return 0; + + size = bo->mem.num_pages << PAGE_SHIFT; + offset = bo->mem.start << PAGE_SHIFT; + if ((offset + size) <= adev->gmc.visible_vram_size) + return 0; + + /* Can't move a pinned BO to visible VRAM */ + if (abo->pin_count > 0) + return -EINVAL; + + /* hurrah the memory is not visible ! */ + atomic64_inc(&adev->num_vram_cpu_page_faults); + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT); + + /* Avoid costly evictions; only set GTT as a busy placement */ + abo->placement.num_busy_placement = 1; + abo->placement.busy_placement = &abo->placements[1]; + + r = ttm_bo_validate(bo, &abo->placement, &ctx); + if (unlikely(r != 0)) + return r; + + offset = bo->mem.start << PAGE_SHIFT; + /* this should never happen */ + if (bo->mem.mem_type == TTM_PL_VRAM && + (offset + size) > adev->gmc.visible_vram_size) + return -EINVAL; + + return 0; +} + +/** + * amdgpu_bo_fence - add fence to buffer object + * + * @bo: buffer object in question + * @fence: fence to add + * @shared: true if fence should be added shared + * + */ +void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, + bool shared) +{ + struct reservation_object *resv = bo->tbo.resv; + + if (shared) + reservation_object_add_shared_fence(resv, fence); + else + reservation_object_add_excl_fence(resv, fence); +} + +/** + * amdgpu_sync_wait_resv - Wait for BO reservation fences + * + * @bo: buffer object + * @owner: fence owner + * @intr: Whether the wait is interruptible + * + * Returns: + * 0 on success, errno otherwise. + */ +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_sync sync; + int r; + + amdgpu_sync_create(&sync); + amdgpu_sync_resv(adev, &sync, bo->tbo.resv, owner, false); + r = amdgpu_sync_wait(&sync, intr); + amdgpu_sync_free(&sync); + + return r; +} + +/** + * amdgpu_bo_gpu_offset - return GPU offset of bo + * @bo: amdgpu object for which we query the offset + * + * Note: object should either be pinned or reserved when calling this + * function, it might be useful to add check for this for debugging. + * + * Returns: + * current GPU offset of the object. + */ +u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) +{ + WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); + WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && + !bo->pin_count && bo->tbo.type != ttm_bo_type_kernel); + WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); + WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM && + !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); + + return amdgpu_gmc_sign_extend(bo->tbo.offset); +} + +/** + * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout + * @adev: amdgpu device object + * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` + * + * Returns: + * Which of the allowed domains is preferred for pinning the BO for scanout. + */ +uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, + uint32_t domain) +{ + if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { + domain = AMDGPU_GEM_DOMAIN_VRAM; + if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) + domain = AMDGPU_GEM_DOMAIN_GTT; + } + return domain; +}
diff --git a/amdgpu/amdgpu_object.h b/amdgpu/amdgpu_object.h new file mode 100644 index 0000000..d60593c --- /dev/null +++ b/amdgpu/amdgpu_object.h
@@ -0,0 +1,312 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#ifndef __AMDGPU_OBJECT_H__ +#define __AMDGPU_OBJECT_H__ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" + +#define AMDGPU_BO_INVALID_OFFSET LONG_MAX +#define AMDGPU_BO_MAX_PLACEMENTS 3 + +struct amdgpu_bo_param { + unsigned long size; + int byte_align; + u32 domain; + u32 preferred_domain; + u64 flags; + enum ttm_bo_type type; + struct reservation_object *resv; +}; + +/* bo virtual addresses in a vm */ +struct amdgpu_bo_va_mapping { + struct amdgpu_bo_va *bo_va; + struct list_head list; + struct rb_node rb; + uint64_t start; + uint64_t last; + uint64_t __subtree_last; + uint64_t offset; + uint64_t flags; +}; + +/* User space allocated BO in a VM */ +struct amdgpu_bo_va { + struct amdgpu_vm_bo_base base; + + /* protected by bo being reserved */ + unsigned ref_count; + + /* all other members protected by the VM PD being reserved */ + struct dma_fence *last_pt_update; + + /* mappings for this bo_va */ + struct list_head invalids; + struct list_head valids; + + /* If the mappings are cleared or filled */ + bool cleared; + + bool is_xgmi; +}; + +struct amdgpu_bo { + /* Protected by tbo.reserved */ + u32 preferred_domains; + u32 allowed_domains; + struct ttm_place placements[AMDGPU_BO_MAX_PLACEMENTS]; + struct ttm_placement placement; + struct ttm_buffer_object tbo; + struct ttm_bo_kmap_obj kmap; + u64 flags; + unsigned pin_count; + u64 tiling_flags; + u64 metadata_flags; + void *metadata; + u32 metadata_size; + unsigned prime_shared_count; + /* per VM structure for page tables and with virtual addresses */ + struct amdgpu_vm_bo_base *vm_bo; + /* Constant after initialization */ + struct drm_gem_object gem_base; + struct amdgpu_bo *parent; + struct amdgpu_bo *shadow; + + struct ttm_bo_kmap_obj dma_buf_vmap; + struct amdgpu_mn *mn; + + union { + struct list_head mn_list; + struct list_head shadow_list; + }; + + struct kgd_mem *kfd_bo; +}; + +static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo) +{ + return container_of(tbo, struct amdgpu_bo, tbo); +} + +/** + * amdgpu_mem_type_to_domain - return domain corresponding to mem_type + * @mem_type: ttm memory type + * + * Returns corresponding domain of the ttm mem_type + */ +static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type) +{ + switch (mem_type) { + case TTM_PL_VRAM: + return AMDGPU_GEM_DOMAIN_VRAM; + case TTM_PL_TT: + return AMDGPU_GEM_DOMAIN_GTT; + case TTM_PL_SYSTEM: + return AMDGPU_GEM_DOMAIN_CPU; + case AMDGPU_PL_GDS: + return AMDGPU_GEM_DOMAIN_GDS; + case AMDGPU_PL_GWS: + return AMDGPU_GEM_DOMAIN_GWS; + case AMDGPU_PL_OA: + return AMDGPU_GEM_DOMAIN_OA; + default: + break; + } + return 0; +} + +/** + * amdgpu_bo_reserve - reserve bo + * @bo: bo structure + * @no_intr: don't return -ERESTARTSYS on pending signal + * + * Returns: + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + */ +static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + int r; + + r = __ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL); + if (unlikely(r != 0)) { + if (r != -ERESTARTSYS) + dev_err(adev->dev, "%p reserve failed\n", bo); + return r; + } + return 0; +} + +static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo) +{ + ttm_bo_unreserve(&bo->tbo); +} + +static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo) +{ + return bo->tbo.num_pages << PAGE_SHIFT; +} + +static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo) +{ + return (bo->tbo.num_pages << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; +} + +static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo) +{ + return (bo->tbo.mem.page_alignment << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; +} + +/** + * amdgpu_bo_mmap_offset - return mmap offset of bo + * @bo: amdgpu object for which we query the offset + * + * Returns mmap offset of the object. + */ +static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo) +{ + return drm_vma_node_offset_addr(&bo->tbo.vma_node); +} + +/** + * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM + */ +static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + unsigned fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; + struct drm_mm_node *node = bo->tbo.mem.mm_node; + unsigned long pages_left; + + if (bo->tbo.mem.mem_type != TTM_PL_VRAM) + return false; + + for (pages_left = bo->tbo.mem.num_pages; pages_left; + pages_left -= node->size, node++) + if (node->start < fpfn) + return true; + + return false; +} + +/** + * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced + */ +static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo) +{ + return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC; +} + +bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); +void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain); + +int amdgpu_bo_create(struct amdgpu_device *adev, + struct amdgpu_bo_param *bp, + struct amdgpu_bo **bo_ptr); +int amdgpu_bo_create_reserved(struct amdgpu_device *adev, + unsigned long size, int align, + u32 domain, struct amdgpu_bo **bo_ptr, + u64 *gpu_addr, void **cpu_addr); +int amdgpu_bo_create_kernel(struct amdgpu_device *adev, + unsigned long size, int align, + u32 domain, struct amdgpu_bo **bo_ptr, + u64 *gpu_addr, void **cpu_addr); +void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, + void **cpu_addr); +int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr); +void *amdgpu_bo_kptr(struct amdgpu_bo *bo); +void amdgpu_bo_kunmap(struct amdgpu_bo *bo); +struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo); +void amdgpu_bo_unref(struct amdgpu_bo **bo); +int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain); +int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, + u64 min_offset, u64 max_offset); +int amdgpu_bo_unpin(struct amdgpu_bo *bo); +int amdgpu_bo_evict_vram(struct amdgpu_device *adev); +int amdgpu_bo_init(struct amdgpu_device *adev); +int amdgpu_bo_late_init(struct amdgpu_device *adev); +void amdgpu_bo_fini(struct amdgpu_device *adev); +int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, + struct vm_area_struct *vma); +int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags); +void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags); +int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, + uint32_t metadata_size, uint64_t flags); +int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, + size_t buffer_size, uint32_t *metadata_size, + uint64_t *flags); +void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, + bool evict, + struct ttm_mem_reg *new_mem); +int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); +void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, + bool shared); +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr); +u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); +int amdgpu_bo_validate(struct amdgpu_bo *bo); +int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, + struct dma_fence **fence); +uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, + uint32_t domain); + +/* + * sub allocation + */ + +static inline uint64_t amdgpu_sa_bo_gpu_addr(struct amdgpu_sa_bo *sa_bo) +{ + return sa_bo->manager->gpu_addr + sa_bo->soffset; +} + +static inline void * amdgpu_sa_bo_cpu_addr(struct amdgpu_sa_bo *sa_bo) +{ + return sa_bo->manager->cpu_ptr + sa_bo->soffset; +} + +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev, + struct amdgpu_sa_manager *sa_manager, + unsigned size, u32 align, u32 domain); +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev, + struct amdgpu_sa_manager *sa_manager); +int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev, + struct amdgpu_sa_manager *sa_manager); +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager, + struct amdgpu_sa_bo **sa_bo, + unsigned size, unsigned align); +void amdgpu_sa_bo_free(struct amdgpu_device *adev, + struct amdgpu_sa_bo **sa_bo, + struct dma_fence *fence); +#if defined(CONFIG_DEBUG_FS) +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, + struct seq_file *m); +#endif + + +#endif
diff --git a/amdgpu/amdgpu_pll.c b/amdgpu/amdgpu_pll.c new file mode 100644 index 0000000..1f2305b --- /dev/null +++ b/amdgpu/amdgpu_pll.c
@@ -0,0 +1,351 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atom.h" +#include "atombios_encoders.h" +#include "amdgpu_pll.h" +#include <asm/div64.h> +#include <linux/gcd.h> + +/** + * amdgpu_pll_reduce_ratio - fractional number reduction + * + * @nom: nominator + * @den: denominator + * @nom_min: minimum value for nominator + * @den_min: minimum value for denominator + * + * Find the greatest common divisor and apply it on both nominator and + * denominator, but make nominator and denominator are at least as large + * as their minimum values. + */ +static void amdgpu_pll_reduce_ratio(unsigned *nom, unsigned *den, + unsigned nom_min, unsigned den_min) +{ + unsigned tmp; + + /* reduce the numbers to a simpler ratio */ + tmp = gcd(*nom, *den); + *nom /= tmp; + *den /= tmp; + + /* make sure nominator is large enough */ + if (*nom < nom_min) { + tmp = DIV_ROUND_UP(nom_min, *nom); + *nom *= tmp; + *den *= tmp; + } + + /* make sure the denominator is large enough */ + if (*den < den_min) { + tmp = DIV_ROUND_UP(den_min, *den); + *nom *= tmp; + *den *= tmp; + } +} + +/** + * amdgpu_pll_get_fb_ref_div - feedback and ref divider calculation + * + * @nom: nominator + * @den: denominator + * @post_div: post divider + * @fb_div_max: feedback divider maximum + * @ref_div_max: reference divider maximum + * @fb_div: resulting feedback divider + * @ref_div: resulting reference divider + * + * Calculate feedback and reference divider for a given post divider. Makes + * sure we stay within the limits. + */ +static void amdgpu_pll_get_fb_ref_div(unsigned nom, unsigned den, unsigned post_div, + unsigned fb_div_max, unsigned ref_div_max, + unsigned *fb_div, unsigned *ref_div) +{ + /* limit reference * post divider to a maximum */ + ref_div_max = min(128 / post_div, ref_div_max); + + /* get matching reference and feedback divider */ + *ref_div = min(max(DIV_ROUND_CLOSEST(den, post_div), 1u), ref_div_max); + *fb_div = DIV_ROUND_CLOSEST(nom * *ref_div * post_div, den); + + /* limit fb divider to its maximum */ + if (*fb_div > fb_div_max) { + *ref_div = DIV_ROUND_CLOSEST(*ref_div * fb_div_max, *fb_div); + *fb_div = fb_div_max; + } +} + +/** + * amdgpu_pll_compute - compute PLL paramaters + * + * @pll: information about the PLL + * @dot_clock_p: resulting pixel clock + * fb_div_p: resulting feedback divider + * frac_fb_div_p: fractional part of the feedback divider + * ref_div_p: resulting reference divider + * post_div_p: resulting reference divider + * + * Try to calculate the PLL parameters to generate the given frequency: + * dot_clock = (ref_freq * feedback_div) / (ref_div * post_div) + */ +void amdgpu_pll_compute(struct amdgpu_pll *pll, + u32 freq, + u32 *dot_clock_p, + u32 *fb_div_p, + u32 *frac_fb_div_p, + u32 *ref_div_p, + u32 *post_div_p) +{ + unsigned target_clock = pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV ? + freq : freq / 10; + + unsigned fb_div_min, fb_div_max, fb_div; + unsigned post_div_min, post_div_max, post_div; + unsigned ref_div_min, ref_div_max, ref_div; + unsigned post_div_best, diff_best; + unsigned nom, den; + + /* determine allowed feedback divider range */ + fb_div_min = pll->min_feedback_div; + fb_div_max = pll->max_feedback_div; + + if (pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV) { + fb_div_min *= 10; + fb_div_max *= 10; + } + + /* determine allowed ref divider range */ + if (pll->flags & AMDGPU_PLL_USE_REF_DIV) + ref_div_min = pll->reference_div; + else + ref_div_min = pll->min_ref_div; + + if (pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV && + pll->flags & AMDGPU_PLL_USE_REF_DIV) + ref_div_max = pll->reference_div; + else + ref_div_max = pll->max_ref_div; + + /* determine allowed post divider range */ + if (pll->flags & AMDGPU_PLL_USE_POST_DIV) { + post_div_min = pll->post_div; + post_div_max = pll->post_div; + } else { + unsigned vco_min, vco_max; + + if (pll->flags & AMDGPU_PLL_IS_LCD) { + vco_min = pll->lcd_pll_out_min; + vco_max = pll->lcd_pll_out_max; + } else { + vco_min = pll->pll_out_min; + vco_max = pll->pll_out_max; + } + + if (pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV) { + vco_min *= 10; + vco_max *= 10; + } + + post_div_min = vco_min / target_clock; + if ((target_clock * post_div_min) < vco_min) + ++post_div_min; + if (post_div_min < pll->min_post_div) + post_div_min = pll->min_post_div; + + post_div_max = vco_max / target_clock; + if ((target_clock * post_div_max) > vco_max) + --post_div_max; + if (post_div_max > pll->max_post_div) + post_div_max = pll->max_post_div; + } + + /* represent the searched ratio as fractional number */ + nom = target_clock; + den = pll->reference_freq; + + /* reduce the numbers to a simpler ratio */ + amdgpu_pll_reduce_ratio(&nom, &den, fb_div_min, post_div_min); + + /* now search for a post divider */ + if (pll->flags & AMDGPU_PLL_PREFER_MINM_OVER_MAXP) + post_div_best = post_div_min; + else + post_div_best = post_div_max; + diff_best = ~0; + + for (post_div = post_div_min; post_div <= post_div_max; ++post_div) { + unsigned diff; + amdgpu_pll_get_fb_ref_div(nom, den, post_div, fb_div_max, + ref_div_max, &fb_div, &ref_div); + diff = abs(target_clock - (pll->reference_freq * fb_div) / + (ref_div * post_div)); + + if (diff < diff_best || (diff == diff_best && + !(pll->flags & AMDGPU_PLL_PREFER_MINM_OVER_MAXP))) { + + post_div_best = post_div; + diff_best = diff; + } + } + post_div = post_div_best; + + /* get the feedback and reference divider for the optimal value */ + amdgpu_pll_get_fb_ref_div(nom, den, post_div, fb_div_max, ref_div_max, + &fb_div, &ref_div); + + /* reduce the numbers to a simpler ratio once more */ + /* this also makes sure that the reference divider is large enough */ + amdgpu_pll_reduce_ratio(&fb_div, &ref_div, fb_div_min, ref_div_min); + + /* avoid high jitter with small fractional dividers */ + if (pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV && (fb_div % 10)) { + fb_div_min = max(fb_div_min, (9 - (fb_div % 10)) * 20 + 60); + if (fb_div < fb_div_min) { + unsigned tmp = DIV_ROUND_UP(fb_div_min, fb_div); + fb_div *= tmp; + ref_div *= tmp; + } + } + + /* and finally save the result */ + if (pll->flags & AMDGPU_PLL_USE_FRAC_FB_DIV) { + *fb_div_p = fb_div / 10; + *frac_fb_div_p = fb_div % 10; + } else { + *fb_div_p = fb_div; + *frac_fb_div_p = 0; + } + + *dot_clock_p = ((pll->reference_freq * *fb_div_p * 10) + + (pll->reference_freq * *frac_fb_div_p)) / + (ref_div * post_div * 10); + *ref_div_p = ref_div; + *post_div_p = post_div; + + DRM_DEBUG_KMS("%d - %d, pll dividers - fb: %d.%d ref: %d, post %d\n", + freq, *dot_clock_p * 10, *fb_div_p, *frac_fb_div_p, + ref_div, post_div); +} + +/** + * amdgpu_pll_get_use_mask - look up a mask of which pplls are in use + * + * @crtc: drm crtc + * + * Returns the mask of which PPLLs (Pixel PLLs) are in use. + */ +u32 amdgpu_pll_get_use_mask(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct drm_crtc *test_crtc; + struct amdgpu_crtc *test_amdgpu_crtc; + u32 pll_in_use = 0; + + list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) { + if (crtc == test_crtc) + continue; + + test_amdgpu_crtc = to_amdgpu_crtc(test_crtc); + if (test_amdgpu_crtc->pll_id != ATOM_PPLL_INVALID) + pll_in_use |= (1 << test_amdgpu_crtc->pll_id); + } + return pll_in_use; +} + +/** + * amdgpu_pll_get_shared_dp_ppll - return the PPLL used by another crtc for DP + * + * @crtc: drm crtc + * + * Returns the PPLL (Pixel PLL) used by another crtc/encoder which is + * also in DP mode. For DP, a single PPLL can be used for all DP + * crtcs/encoders. + */ +int amdgpu_pll_get_shared_dp_ppll(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct drm_crtc *test_crtc; + struct amdgpu_crtc *test_amdgpu_crtc; + + list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) { + if (crtc == test_crtc) + continue; + test_amdgpu_crtc = to_amdgpu_crtc(test_crtc); + if (test_amdgpu_crtc->encoder && + ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(test_amdgpu_crtc->encoder))) { + /* for DP use the same PLL for all */ + if (test_amdgpu_crtc->pll_id != ATOM_PPLL_INVALID) + return test_amdgpu_crtc->pll_id; + } + } + return ATOM_PPLL_INVALID; +} + +/** + * amdgpu_pll_get_shared_nondp_ppll - return the PPLL used by another non-DP crtc + * + * @crtc: drm crtc + * @encoder: drm encoder + * + * Returns the PPLL (Pixel PLL) used by another non-DP crtc/encoder which can + * be shared (i.e., same clock). + */ +int amdgpu_pll_get_shared_nondp_ppll(struct drm_crtc *crtc) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct drm_crtc *test_crtc; + struct amdgpu_crtc *test_amdgpu_crtc; + u32 adjusted_clock, test_adjusted_clock; + + adjusted_clock = amdgpu_crtc->adjusted_clock; + + if (adjusted_clock == 0) + return ATOM_PPLL_INVALID; + + list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) { + if (crtc == test_crtc) + continue; + test_amdgpu_crtc = to_amdgpu_crtc(test_crtc); + if (test_amdgpu_crtc->encoder && + !ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(test_amdgpu_crtc->encoder))) { + /* check if we are already driving this connector with another crtc */ + if (test_amdgpu_crtc->connector == amdgpu_crtc->connector) { + /* if we are, return that pll */ + if (test_amdgpu_crtc->pll_id != ATOM_PPLL_INVALID) + return test_amdgpu_crtc->pll_id; + } + /* for non-DP check the clock */ + test_adjusted_clock = test_amdgpu_crtc->adjusted_clock; + if ((crtc->mode.clock == test_crtc->mode.clock) && + (adjusted_clock == test_adjusted_clock) && + (amdgpu_crtc->ss_enabled == test_amdgpu_crtc->ss_enabled) && + (test_amdgpu_crtc->pll_id != ATOM_PPLL_INVALID)) + return test_amdgpu_crtc->pll_id; + } + } + return ATOM_PPLL_INVALID; +}
diff --git a/amdgpu/amdgpu_pll.h b/amdgpu/amdgpu_pll.h new file mode 100644 index 0000000..db6136f --- /dev/null +++ b/amdgpu/amdgpu_pll.h
@@ -0,0 +1,38 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_PLL_H__ +#define __AMDGPU_PLL_H__ + +void amdgpu_pll_compute(struct amdgpu_pll *pll, + u32 freq, + u32 *dot_clock_p, + u32 *fb_div_p, + u32 *frac_fb_div_p, + u32 *ref_div_p, + u32 *post_div_p); +u32 amdgpu_pll_get_use_mask(struct drm_crtc *crtc); +int amdgpu_pll_get_shared_dp_ppll(struct drm_crtc *crtc); +int amdgpu_pll_get_shared_nondp_ppll(struct drm_crtc *crtc); + +#endif
diff --git a/amdgpu/amdgpu_pm.c b/amdgpu/amdgpu_pm.c new file mode 100644 index 0000000..2b54656 --- /dev/null +++ b/amdgpu/amdgpu_pm.c
@@ -0,0 +1,3177 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: RafaÅ‚ MiÅ‚ecki <zajec5@gmail.com> + * Alex Deucher <alexdeucher@gmail.com> + */ + +#include <drm/drm_debugfs.h> + +#include "amdgpu.h" +#include "amdgpu_drv.h" +#include "amdgpu_pm.h" +#include "amdgpu_dpm.h" +#include "amdgpu_display.h" +#include "amdgpu_smu.h" +#include "atom.h" +#include <linux/power_supply.h> +#include <linux/pci.h> +#include <linux/hwmon.h> +#include <linux/hwmon-sysfs.h> +#include <linux/nospec.h> +#include "hwmgr.h" +#define WIDTH_4K 3840 + +static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev); + +static const struct cg_flag_name clocks[] = { + {AMD_CG_SUPPORT_GFX_MGCG, "Graphics Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_GFX_MGLS, "Graphics Medium Grain memory Light Sleep"}, + {AMD_CG_SUPPORT_GFX_CGCG, "Graphics Coarse Grain Clock Gating"}, + {AMD_CG_SUPPORT_GFX_CGLS, "Graphics Coarse Grain memory Light Sleep"}, + {AMD_CG_SUPPORT_GFX_CGTS, "Graphics Coarse Grain Tree Shader Clock Gating"}, + {AMD_CG_SUPPORT_GFX_CGTS_LS, "Graphics Coarse Grain Tree Shader Light Sleep"}, + {AMD_CG_SUPPORT_GFX_CP_LS, "Graphics Command Processor Light Sleep"}, + {AMD_CG_SUPPORT_GFX_RLC_LS, "Graphics Run List Controller Light Sleep"}, + {AMD_CG_SUPPORT_GFX_3D_CGCG, "Graphics 3D Coarse Grain Clock Gating"}, + {AMD_CG_SUPPORT_GFX_3D_CGLS, "Graphics 3D Coarse Grain memory Light Sleep"}, + {AMD_CG_SUPPORT_MC_LS, "Memory Controller Light Sleep"}, + {AMD_CG_SUPPORT_MC_MGCG, "Memory Controller Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_SDMA_LS, "System Direct Memory Access Light Sleep"}, + {AMD_CG_SUPPORT_SDMA_MGCG, "System Direct Memory Access Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_BIF_MGCG, "Bus Interface Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_BIF_LS, "Bus Interface Light Sleep"}, + {AMD_CG_SUPPORT_UVD_MGCG, "Unified Video Decoder Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_VCE_MGCG, "Video Compression Engine Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_HDP_LS, "Host Data Path Light Sleep"}, + {AMD_CG_SUPPORT_HDP_MGCG, "Host Data Path Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_DRM_MGCG, "Digital Right Management Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_DRM_LS, "Digital Right Management Light Sleep"}, + {AMD_CG_SUPPORT_ROM_MGCG, "Rom Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_DF_MGCG, "Data Fabric Medium Grain Clock Gating"}, + + {AMD_CG_SUPPORT_ATHUB_MGCG, "Address Translation Hub Medium Grain Clock Gating"}, + {AMD_CG_SUPPORT_ATHUB_LS, "Address Translation Hub Light Sleep"}, + {0, NULL}, +}; + +static const struct hwmon_temp_label { + enum PP_HWMON_TEMP channel; + const char *label; +} temp_label[] = { + {PP_TEMP_EDGE, "edge"}, + {PP_TEMP_JUNCTION, "junction"}, + {PP_TEMP_MEM, "mem"}, +}; + +void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev) +{ + if (adev->pm.dpm_enabled) { + mutex_lock(&adev->pm.mutex); + if (power_supply_is_system_supplied() > 0) + adev->pm.ac_power = true; + else + adev->pm.ac_power = false; + if (adev->powerplay.pp_funcs->enable_bapm) + amdgpu_dpm_enable_bapm(adev, adev->pm.ac_power); + mutex_unlock(&adev->pm.mutex); + } +} + +int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor, + void *data, uint32_t *size) +{ + int ret = 0; + + if (!data || !size) + return -EINVAL; + + if (is_support_sw_smu(adev)) + ret = smu_read_sensor(&adev->smu, sensor, data, size); + else { + if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor) + ret = adev->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, + sensor, data, size); + else + ret = -EINVAL; + } + + return ret; +} + +/** + * DOC: power_dpm_state + * + * The power_dpm_state file is a legacy interface and is only provided for + * backwards compatibility. The amdgpu driver provides a sysfs API for adjusting + * certain power related parameters. The file power_dpm_state is used for this. + * It accepts the following arguments: + * + * - battery + * + * - balanced + * + * - performance + * + * battery + * + * On older GPUs, the vbios provided a special power state for battery + * operation. Selecting battery switched to this state. This is no + * longer provided on newer GPUs so the option does nothing in that case. + * + * balanced + * + * On older GPUs, the vbios provided a special power state for balanced + * operation. Selecting balanced switched to this state. This is no + * longer provided on newer GPUs so the option does nothing in that case. + * + * performance + * + * On older GPUs, the vbios provided a special power state for performance + * operation. Selecting performance switched to this state. This is no + * longer provided on newer GPUs so the option does nothing in that case. + * + */ + +static ssize_t amdgpu_get_dpm_state(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + enum amd_pm_state_type pm; + + if (is_support_sw_smu(adev)) { + if (adev->smu.ppt_funcs->get_current_power_state) + pm = amdgpu_smu_get_current_power_state(adev); + else + pm = adev->pm.dpm.user_state; + } else if (adev->powerplay.pp_funcs->get_current_power_state) { + pm = amdgpu_dpm_get_current_power_state(adev); + } else { + pm = adev->pm.dpm.user_state; + } + + return snprintf(buf, PAGE_SIZE, "%s\n", + (pm == POWER_STATE_TYPE_BATTERY) ? "battery" : + (pm == POWER_STATE_TYPE_BALANCED) ? "balanced" : "performance"); +} + +static ssize_t amdgpu_set_dpm_state(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + enum amd_pm_state_type state; + + if (strncmp("battery", buf, strlen("battery")) == 0) + state = POWER_STATE_TYPE_BATTERY; + else if (strncmp("balanced", buf, strlen("balanced")) == 0) + state = POWER_STATE_TYPE_BALANCED; + else if (strncmp("performance", buf, strlen("performance")) == 0) + state = POWER_STATE_TYPE_PERFORMANCE; + else { + count = -EINVAL; + goto fail; + } + + if (is_support_sw_smu(adev)) { + mutex_lock(&adev->pm.mutex); + adev->pm.dpm.user_state = state; + mutex_unlock(&adev->pm.mutex); + } else if (adev->powerplay.pp_funcs->dispatch_tasks) { + amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state); + } else { + mutex_lock(&adev->pm.mutex); + adev->pm.dpm.user_state = state; + mutex_unlock(&adev->pm.mutex); + + /* Can't set dpm state when the card is off */ + if (!(adev->flags & AMD_IS_PX) || + (ddev->switch_power_state == DRM_SWITCH_POWER_ON)) + amdgpu_pm_compute_clocks(adev); + } +fail: + return count; +} + + +/** + * DOC: power_dpm_force_performance_level + * + * The amdgpu driver provides a sysfs API for adjusting certain power + * related parameters. The file power_dpm_force_performance_level is + * used for this. It accepts the following arguments: + * + * - auto + * + * - low + * + * - high + * + * - manual + * + * - profile_standard + * + * - profile_min_sclk + * + * - profile_min_mclk + * + * - profile_peak + * + * auto + * + * When auto is selected, the driver will attempt to dynamically select + * the optimal power profile for current conditions in the driver. + * + * low + * + * When low is selected, the clocks are forced to the lowest power state. + * + * high + * + * When high is selected, the clocks are forced to the highest power state. + * + * manual + * + * When manual is selected, the user can manually adjust which power states + * are enabled for each clock domain via the sysfs pp_dpm_mclk, pp_dpm_sclk, + * and pp_dpm_pcie files and adjust the power state transition heuristics + * via the pp_power_profile_mode sysfs file. + * + * profile_standard + * profile_min_sclk + * profile_min_mclk + * profile_peak + * + * When the profiling modes are selected, clock and power gating are + * disabled and the clocks are set for different profiling cases. This + * mode is recommended for profiling specific work loads where you do + * not want clock or power gating for clock fluctuation to interfere + * with your results. profile_standard sets the clocks to a fixed clock + * level which varies from asic to asic. profile_min_sclk forces the sclk + * to the lowest level. profile_min_mclk forces the mclk to the lowest level. + * profile_peak sets all clocks (mclk, sclk, pcie) to the highest levels. + * + */ + +static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + enum amd_dpm_forced_level level = 0xff; + + if (amdgpu_sriov_vf(adev)) + return 0; + + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return snprintf(buf, PAGE_SIZE, "off\n"); + + if (is_support_sw_smu(adev)) + level = smu_get_performance_level(&adev->smu); + else if (adev->powerplay.pp_funcs->get_performance_level) + level = amdgpu_dpm_get_performance_level(adev); + else + level = adev->pm.dpm.forced_level; + + return snprintf(buf, PAGE_SIZE, "%s\n", + (level == AMD_DPM_FORCED_LEVEL_AUTO) ? "auto" : + (level == AMD_DPM_FORCED_LEVEL_LOW) ? "low" : + (level == AMD_DPM_FORCED_LEVEL_HIGH) ? "high" : + (level == AMD_DPM_FORCED_LEVEL_MANUAL) ? "manual" : + (level == AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD) ? "profile_standard" : + (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) ? "profile_min_sclk" : + (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) ? "profile_min_mclk" : + (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) ? "profile_peak" : + "unknown"); +} + +static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + enum amd_dpm_forced_level level; + enum amd_dpm_forced_level current_level = 0xff; + int ret = 0; + + /* Can't force performance level when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (!amdgpu_sriov_vf(adev)) { + if (is_support_sw_smu(adev)) + current_level = smu_get_performance_level(&adev->smu); + else if (adev->powerplay.pp_funcs->get_performance_level) + current_level = amdgpu_dpm_get_performance_level(adev); + } + + if (strncmp("low", buf, strlen("low")) == 0) { + level = AMD_DPM_FORCED_LEVEL_LOW; + } else if (strncmp("high", buf, strlen("high")) == 0) { + level = AMD_DPM_FORCED_LEVEL_HIGH; + } else if (strncmp("auto", buf, strlen("auto")) == 0) { + level = AMD_DPM_FORCED_LEVEL_AUTO; + } else if (strncmp("manual", buf, strlen("manual")) == 0) { + level = AMD_DPM_FORCED_LEVEL_MANUAL; + } else if (strncmp("profile_exit", buf, strlen("profile_exit")) == 0) { + level = AMD_DPM_FORCED_LEVEL_PROFILE_EXIT; + } else if (strncmp("profile_standard", buf, strlen("profile_standard")) == 0) { + level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD; + } else if (strncmp("profile_min_sclk", buf, strlen("profile_min_sclk")) == 0) { + level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK; + } else if (strncmp("profile_min_mclk", buf, strlen("profile_min_mclk")) == 0) { + level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK; + } else if (strncmp("profile_peak", buf, strlen("profile_peak")) == 0) { + level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; + } else { + count = -EINVAL; + goto fail; + } + + if (amdgpu_sriov_vf(adev)) { + if (amdgim_is_hwperf(adev) && + adev->virt.ops->force_dpm_level) { + mutex_lock(&adev->pm.mutex); + adev->virt.ops->force_dpm_level(adev, level); + mutex_unlock(&adev->pm.mutex); + return count; + } else { + return -EINVAL; + } + } + + if (current_level == level) + return count; + + /* profile_exit setting is valid only when current mode is in profile mode */ + if (!(current_level & (AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD | + AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK | + AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK | + AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)) && + (level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)) { + pr_err("Currently not in any profile mode!\n"); + return -EINVAL; + } + + if (is_support_sw_smu(adev)) { + ret = smu_force_performance_level(&adev->smu, level); + if (ret) + count = -EINVAL; + } else if (adev->powerplay.pp_funcs->force_performance_level) { + mutex_lock(&adev->pm.mutex); + if (adev->pm.dpm.thermal_active) { + count = -EINVAL; + mutex_unlock(&adev->pm.mutex); + goto fail; + } + ret = amdgpu_dpm_force_performance_level(adev, level); + if (ret) + count = -EINVAL; + else + adev->pm.dpm.forced_level = level; + mutex_unlock(&adev->pm.mutex); + } + +fail: + return count; +} + +static ssize_t amdgpu_get_pp_num_states(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + struct pp_states_info data; + int i, buf_len, ret; + + if (is_support_sw_smu(adev)) { + ret = smu_get_power_num_states(&adev->smu, &data); + if (ret) + return ret; + } else if (adev->powerplay.pp_funcs->get_pp_num_states) + amdgpu_dpm_get_pp_num_states(adev, &data); + + buf_len = snprintf(buf, PAGE_SIZE, "states: %d\n", data.nums); + for (i = 0; i < data.nums; i++) + buf_len += snprintf(buf + buf_len, PAGE_SIZE, "%d %s\n", i, + (data.states[i] == POWER_STATE_TYPE_INTERNAL_BOOT) ? "boot" : + (data.states[i] == POWER_STATE_TYPE_BATTERY) ? "battery" : + (data.states[i] == POWER_STATE_TYPE_BALANCED) ? "balanced" : + (data.states[i] == POWER_STATE_TYPE_PERFORMANCE) ? "performance" : "default"); + + return buf_len; +} + +static ssize_t amdgpu_get_pp_cur_state(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + struct pp_states_info data; + struct smu_context *smu = &adev->smu; + enum amd_pm_state_type pm = 0; + int i = 0, ret = 0; + + if (is_support_sw_smu(adev)) { + pm = smu_get_current_power_state(smu); + ret = smu_get_power_num_states(smu, &data); + if (ret) + return ret; + } else if (adev->powerplay.pp_funcs->get_current_power_state + && adev->powerplay.pp_funcs->get_pp_num_states) { + pm = amdgpu_dpm_get_current_power_state(adev); + amdgpu_dpm_get_pp_num_states(adev, &data); + } + + for (i = 0; i < data.nums; i++) { + if (pm == data.states[i]) + break; + } + + if (i == data.nums) + i = -EINVAL; + + return snprintf(buf, PAGE_SIZE, "%d\n", i); +} + +static ssize_t amdgpu_get_pp_force_state(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->pp_force_state_enabled) + return amdgpu_get_pp_cur_state(dev, attr, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_force_state(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + enum amd_pm_state_type state = 0; + unsigned long idx; + int ret; + + if (strlen(buf) == 1) + adev->pp_force_state_enabled = false; + else if (is_support_sw_smu(adev)) + adev->pp_force_state_enabled = false; + else if (adev->powerplay.pp_funcs->dispatch_tasks && + adev->powerplay.pp_funcs->get_pp_num_states) { + struct pp_states_info data; + + ret = kstrtoul(buf, 0, &idx); + if (ret || idx >= ARRAY_SIZE(data.states)) { + count = -EINVAL; + goto fail; + } + idx = array_index_nospec(idx, ARRAY_SIZE(data.states)); + + amdgpu_dpm_get_pp_num_states(adev, &data); + state = data.states[idx]; + /* only set user selected power states */ + if (state != POWER_STATE_TYPE_INTERNAL_BOOT && + state != POWER_STATE_TYPE_DEFAULT) { + amdgpu_dpm_dispatch_task(adev, + AMD_PP_TASK_ENABLE_USER_STATE, &state); + adev->pp_force_state_enabled = true; + } + } +fail: + return count; +} + +/** + * DOC: pp_table + * + * The amdgpu driver provides a sysfs API for uploading new powerplay + * tables. The file pp_table is used for this. Reading the file + * will dump the current power play table. Writing to the file + * will attempt to upload a new powerplay table and re-initialize + * powerplay using that new table. + * + */ + +static ssize_t amdgpu_get_pp_table(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + char *table = NULL; + int size; + + if (is_support_sw_smu(adev)) { + size = smu_sys_get_pp_table(&adev->smu, (void **)&table); + if (size < 0) + return size; + } + else if (adev->powerplay.pp_funcs->get_pp_table) + size = amdgpu_dpm_get_pp_table(adev, &table); + else + return 0; + + if (size >= PAGE_SIZE) + size = PAGE_SIZE - 1; + + memcpy(buf, table, size); + + return size; +} + +static ssize_t amdgpu_set_pp_table(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret = 0; + + if (is_support_sw_smu(adev)) { + ret = smu_sys_set_pp_table(&adev->smu, (void *)buf, count); + if (ret) + return ret; + } else if (adev->powerplay.pp_funcs->set_pp_table) + amdgpu_dpm_set_pp_table(adev, buf, count); + + return count; +} + +/** + * DOC: pp_od_clk_voltage + * + * The amdgpu driver provides a sysfs API for adjusting the clocks and voltages + * in each power level within a power state. The pp_od_clk_voltage is used for + * this. + * + * < For Vega10 and previous ASICs > + * + * Reading the file will display: + * + * - a list of engine clock levels and voltages labeled OD_SCLK + * + * - a list of memory clock levels and voltages labeled OD_MCLK + * + * - a list of valid ranges for sclk, mclk, and voltage labeled OD_RANGE + * + * To manually adjust these settings, first select manual using + * power_dpm_force_performance_level. Enter a new value for each + * level by writing a string that contains "s/m level clock voltage" to + * the file. E.g., "s 1 500 820" will update sclk level 1 to be 500 MHz + * at 820 mV; "m 0 350 810" will update mclk level 0 to be 350 MHz at + * 810 mV. When you have edited all of the states as needed, write + * "c" (commit) to the file to commit your changes. If you want to reset to the + * default power levels, write "r" (reset) to the file to reset them. + * + * + * < For Vega20 > + * + * Reading the file will display: + * + * - minimum and maximum engine clock labeled OD_SCLK + * + * - maximum memory clock labeled OD_MCLK + * + * - three <frequency, voltage> points labeled OD_VDDC_CURVE. + * They can be used to calibrate the sclk voltage curve. + * + * - a list of valid ranges for sclk, mclk, and voltage curve points + * labeled OD_RANGE + * + * To manually adjust these settings: + * + * - First select manual using power_dpm_force_performance_level + * + * - For clock frequency setting, enter a new value by writing a + * string that contains "s/m index clock" to the file. The index + * should be 0 if to set minimum clock. And 1 if to set maximum + * clock. E.g., "s 0 500" will update minimum sclk to be 500 MHz. + * "m 1 800" will update maximum mclk to be 800Mhz. + * + * For sclk voltage curve, enter the new values by writing a + * string that contains "vc point clock voltage" to the file. The + * points are indexed by 0, 1 and 2. E.g., "vc 0 300 600" will + * update point1 with clock set as 300Mhz and voltage as + * 600mV. "vc 2 1000 1000" will update point3 with clock set + * as 1000Mhz and voltage 1000mV. + * + * - When you have edited all of the states as needed, write "c" (commit) + * to the file to commit your changes + * + * - If you want to reset to the default power levels, write "r" (reset) + * to the file to reset them + * + */ + +static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t parameter_size = 0; + long parameter[64]; + char buf_cpy[128]; + char *tmp_str; + char *sub_str; + const char delimiter[3] = {' ', '\n', '\0'}; + uint32_t type; + + if (count > 127) + return -EINVAL; + + if (*buf == 's') + type = PP_OD_EDIT_SCLK_VDDC_TABLE; + else if (*buf == 'm') + type = PP_OD_EDIT_MCLK_VDDC_TABLE; + else if(*buf == 'r') + type = PP_OD_RESTORE_DEFAULT_TABLE; + else if (*buf == 'c') + type = PP_OD_COMMIT_DPM_TABLE; + else if (!strncmp(buf, "vc", 2)) + type = PP_OD_EDIT_VDDC_CURVE; + else + return -EINVAL; + + memcpy(buf_cpy, buf, count+1); + + tmp_str = buf_cpy; + + if (type == PP_OD_EDIT_VDDC_CURVE) + tmp_str++; + while (isspace(*++tmp_str)); + + while (tmp_str[0]) { + sub_str = strsep(&tmp_str, delimiter); + ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); + if (ret) + return -EINVAL; + parameter_size++; + + while (isspace(*tmp_str)) + tmp_str++; + } + + if (is_support_sw_smu(adev)) { + ret = smu_od_edit_dpm_table(&adev->smu, type, + parameter, parameter_size); + + if (ret) + return -EINVAL; + } else { + if (adev->powerplay.pp_funcs->odn_edit_dpm_table) { + ret = amdgpu_dpm_odn_edit_dpm_table(adev, type, + parameter, parameter_size); + if (ret) + return -EINVAL; + } + + if (type == PP_OD_COMMIT_DPM_TABLE) { + if (adev->powerplay.pp_funcs->dispatch_tasks) { + amdgpu_dpm_dispatch_task(adev, + AMD_PP_TASK_READJUST_POWER_STATE, + NULL); + return count; + } else { + return -EINVAL; + } + } + } + + return count; +} + +static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t size = 0; + + if (is_support_sw_smu(adev)) { + size = smu_print_clk_levels(&adev->smu, SMU_OD_SCLK, buf); + size += smu_print_clk_levels(&adev->smu, SMU_OD_MCLK, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDC_CURVE, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_RANGE, buf+size); + return size; + } else if (adev->powerplay.pp_funcs->print_clock_levels) { + size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf); + size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf+size); + size += amdgpu_dpm_print_clock_levels(adev, OD_VDDC_CURVE, buf+size); + size += amdgpu_dpm_print_clock_levels(adev, OD_RANGE, buf+size); + return size; + } else { + return snprintf(buf, PAGE_SIZE, "\n"); + } + +} + +/** + * DOC: ppfeatures + * + * The amdgpu driver provides a sysfs API for adjusting what powerplay + * features to be enabled. The file ppfeatures is used for this. And + * this is only available for Vega10 and later dGPUs. + * + * Reading back the file will show you the followings: + * - Current ppfeature masks + * - List of the all supported powerplay features with their naming, + * bitmasks and enablement status('Y'/'N' means "enabled"/"disabled"). + * + * To manually enable or disable a specific feature, just set or clear + * the corresponding bit from original ppfeature masks and input the + * new ppfeature masks. + */ +static ssize_t amdgpu_set_ppfeature_status(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint64_t featuremask; + int ret; + + ret = kstrtou64(buf, 0, &featuremask); + if (ret) + return -EINVAL; + + pr_debug("featuremask = 0x%llx\n", featuremask); + + if (is_support_sw_smu(adev)) { + ret = smu_set_ppfeature_status(&adev->smu, featuremask); + if (ret) + return -EINVAL; + } else if (adev->powerplay.pp_funcs->set_ppfeature_status) { + ret = amdgpu_dpm_set_ppfeature_status(adev, featuremask); + if (ret) + return -EINVAL; + } + + return count; +} + +static ssize_t amdgpu_get_ppfeature_status(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) { + return smu_get_ppfeature_status(&adev->smu, buf); + } else if (adev->powerplay.pp_funcs->get_ppfeature_status) + return amdgpu_dpm_get_ppfeature_status(adev, buf); + + return snprintf(buf, PAGE_SIZE, "\n"); +} + +/** + * DOC: pp_dpm_sclk pp_dpm_mclk pp_dpm_socclk pp_dpm_fclk pp_dpm_dcefclk + * pp_dpm_pcie + * + * The amdgpu driver provides a sysfs API for adjusting what power levels + * are enabled for a given power state. The files pp_dpm_sclk, pp_dpm_mclk, + * pp_dpm_socclk, pp_dpm_fclk, pp_dpm_dcefclk and pp_dpm_pcie are used for + * this. + * + * pp_dpm_socclk and pp_dpm_dcefclk interfaces are only available for + * Vega10 and later ASICs. + * pp_dpm_fclk interface is only available for Vega20 and later ASICs. + * + * Reading back the files will show you the available power levels within + * the power state and the clock information for those levels. + * + * To manually adjust these states, first select manual using + * power_dpm_force_performance_level. + * Secondly,Enter a new value for each level by inputing a string that + * contains " echo xx xx xx > pp_dpm_sclk/mclk/pcie" + * E.g., echo 4 5 6 to > pp_dpm_sclk will enable sclk levels 4, 5, and 6. + * + * NOTE: change to the dcefclk max dpm level is not supported now + */ + +static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) && + adev->virt.ops->get_pp_clk) + return adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf); + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_SCLK, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +/* + * Worst case: 32 bits individually specified, in octal at 12 characters + * per line (+1 for \n). + */ +#define AMDGPU_MASK_BUF_MAX (32 * 13) + +static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) +{ + int ret; + long level; + char *sub_str = NULL; + char *tmp; + char buf_cpy[AMDGPU_MASK_BUF_MAX + 1]; + const char delimiter[3] = {' ', '\n', '\0'}; + size_t bytes; + + *mask = 0; + + bytes = min(count, sizeof(buf_cpy) - 1); + memcpy(buf_cpy, buf, bytes); + buf_cpy[bytes] = '\0'; + tmp = buf_cpy; + while (tmp[0]) { + sub_str = strsep(&tmp, delimiter); + if (strlen(sub_str)) { + ret = kstrtol(sub_str, 0, &level); + if (ret) + return -EINVAL; + *mask |= 1 << level; + } else + break; + } + + return 0; +} + +static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + if (amdgpu_sriov_vf(adev)) + return 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_SCLK, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) && + adev->virt.ops->get_pp_clk) + return adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf); + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_MCLK, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + if (amdgpu_sriov_vf(adev)) + return 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_MCLK, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_SOCCLK, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_SOCCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_SOCCLK, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_SOCCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_FCLK, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_FCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_FCLK, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_FCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_DCEFCLK, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_DCEFCLK, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_DCEFCLK, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_DCEFCLK, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) + return smu_print_clk_levels(&adev->smu, SMU_PCIE, buf); + else if (adev->powerplay.pp_funcs->print_clock_levels) + return amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf); + else + return snprintf(buf, PAGE_SIZE, "\n"); +} + +static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + uint32_t mask = 0; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_PCIE, mask); + else if (adev->powerplay.pp_funcs->force_clock_level) + ret = amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_sclk_od(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t value = 0; + + if (is_support_sw_smu(adev)) + value = smu_get_od_percentage(&(adev->smu), SMU_OD_SCLK); + else if (adev->powerplay.pp_funcs->get_sclk_od) + value = amdgpu_dpm_get_sclk_od(adev); + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +static ssize_t amdgpu_set_pp_sclk_od(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + long int value; + + ret = kstrtol(buf, 0, &value); + + if (ret) { + count = -EINVAL; + goto fail; + } + + if (is_support_sw_smu(adev)) { + value = smu_set_od_percentage(&(adev->smu), SMU_OD_SCLK, (uint32_t)value); + } else { + if (adev->powerplay.pp_funcs->set_sclk_od) + amdgpu_dpm_set_sclk_od(adev, (uint32_t)value); + + if (adev->powerplay.pp_funcs->dispatch_tasks) { + amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL); + } else { + adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; + amdgpu_pm_compute_clocks(adev); + } + } + +fail: + return count; +} + +static ssize_t amdgpu_get_pp_mclk_od(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t value = 0; + + if (is_support_sw_smu(adev)) + value = smu_get_od_percentage(&(adev->smu), SMU_OD_MCLK); + else if (adev->powerplay.pp_funcs->get_mclk_od) + value = amdgpu_dpm_get_mclk_od(adev); + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +static ssize_t amdgpu_set_pp_mclk_od(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + long int value; + + ret = kstrtol(buf, 0, &value); + + if (ret) { + count = -EINVAL; + goto fail; + } + + if (is_support_sw_smu(adev)) { + value = smu_set_od_percentage(&(adev->smu), SMU_OD_MCLK, (uint32_t)value); + } else { + if (adev->powerplay.pp_funcs->set_mclk_od) + amdgpu_dpm_set_mclk_od(adev, (uint32_t)value); + + if (adev->powerplay.pp_funcs->dispatch_tasks) { + amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL); + } else { + adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; + amdgpu_pm_compute_clocks(adev); + } + } + +fail: + return count; +} + +/** + * DOC: pp_power_profile_mode + * + * The amdgpu driver provides a sysfs API for adjusting the heuristics + * related to switching between power levels in a power state. The file + * pp_power_profile_mode is used for this. + * + * Reading this file outputs a list of all of the predefined power profiles + * and the relevant heuristics settings for that profile. + * + * To select a profile or create a custom profile, first select manual using + * power_dpm_force_performance_level. Writing the number of a predefined + * profile to pp_power_profile_mode will enable those heuristics. To + * create a custom set of heuristics, write a string of numbers to the file + * starting with the number of the custom profile along with a setting + * for each heuristic parameter. Due to differences across asic families + * the heuristic parameters vary from family to family. + * + */ + +static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (is_support_sw_smu(adev)) + return smu_get_power_profile_mode(&adev->smu, buf); + else if (adev->powerplay.pp_funcs->get_power_profile_mode) + return amdgpu_dpm_get_power_profile_mode(adev, buf); + + return snprintf(buf, PAGE_SIZE, "\n"); +} + + +static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + int ret = 0xff; + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t parameter_size = 0; + long parameter[64]; + char *sub_str, buf_cpy[128]; + char *tmp_str; + uint32_t i = 0; + char tmp[2]; + long int profile_mode = 0; + const char delimiter[3] = {' ', '\n', '\0'}; + + tmp[0] = *(buf); + tmp[1] = '\0'; + ret = kstrtol(tmp, 0, &profile_mode); + if (ret) + goto fail; + + if (profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) { + if (count < 2 || count > 127) + return -EINVAL; + while (isspace(*++buf)) + i++; + memcpy(buf_cpy, buf, count-i); + tmp_str = buf_cpy; + while (tmp_str[0]) { + sub_str = strsep(&tmp_str, delimiter); + ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); + if (ret) { + count = -EINVAL; + goto fail; + } + parameter_size++; + while (isspace(*tmp_str)) + tmp_str++; + } + } + parameter[parameter_size] = profile_mode; + if (is_support_sw_smu(adev)) + ret = smu_set_power_profile_mode(&adev->smu, parameter, parameter_size); + else if (adev->powerplay.pp_funcs->set_power_profile_mode) + ret = amdgpu_dpm_set_power_profile_mode(adev, parameter, parameter_size); + if (!ret) + return count; +fail: + return -EINVAL; +} + +/** + * DOC: busy_percent + * + * The amdgpu driver provides a sysfs API for reading how busy the GPU + * is as a percentage. The file gpu_busy_percent is used for this. + * The SMU firmware computes a percentage of load based on the + * aggregate activity level in the IP cores. + */ +static ssize_t amdgpu_get_busy_percent(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int r, value, size = sizeof(value); + + /* read the IP busy sensor */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD, + (void *)&value, &size); + + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +/** + * DOC: mem_busy_percent + * + * The amdgpu driver provides a sysfs API for reading how busy the VRAM + * is as a percentage. The file mem_busy_percent is used for this. + * The SMU firmware computes a percentage of load based on the + * aggregate activity level in the IP cores. + */ +static ssize_t amdgpu_get_memory_busy_percent(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int r, value, size = sizeof(value); + + /* read the IP busy sensor */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_LOAD, + (void *)&value, &size); + + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +/** + * DOC: pcie_bw + * + * The amdgpu driver provides a sysfs API for estimating how much data + * has been received and sent by the GPU in the last second through PCIe. + * The file pcie_bw is used for this. + * The Perf counters count the number of received and sent messages and return + * those values, as well as the maximum payload size of a PCIe packet (mps). + * Note that it is not possible to easily and quickly obtain the size of each + * packet transmitted, so we output the max payload size (mps) to allow for + * quick estimation of the PCIe bandwidth usage + */ +static ssize_t amdgpu_get_pcie_bw(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint64_t count0, count1; + + amdgpu_asic_get_pcie_usage(adev, &count0, &count1); + return snprintf(buf, PAGE_SIZE, "%llu %llu %i\n", + count0, count1, pcie_get_mps(adev->pdev)); +} + +/** + * DOC: unique_id + * + * The amdgpu driver provides a sysfs API for providing a unique ID for the GPU + * The file unique_id is used for this. + * This will provide a Unique ID that will persist from machine to machine + * + * NOTE: This will only work for GFX9 and newer. This file will be absent + * on unsupported ASICs (GFX8 and older) + */ +static ssize_t amdgpu_get_unique_id(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + if (adev->unique_id) + return snprintf(buf, PAGE_SIZE, "%016llx\n", adev->unique_id); + + return 0; +} + +static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, amdgpu_get_dpm_state, amdgpu_set_dpm_state); +static DEVICE_ATTR(power_dpm_force_performance_level, S_IRUGO | S_IWUSR, + amdgpu_get_dpm_forced_performance_level, + amdgpu_set_dpm_forced_performance_level); +static DEVICE_ATTR(pp_num_states, S_IRUGO, amdgpu_get_pp_num_states, NULL); +static DEVICE_ATTR(pp_cur_state, S_IRUGO, amdgpu_get_pp_cur_state, NULL); +static DEVICE_ATTR(pp_force_state, S_IRUGO | S_IWUSR, + amdgpu_get_pp_force_state, + amdgpu_set_pp_force_state); +static DEVICE_ATTR(pp_table, S_IRUGO | S_IWUSR, + amdgpu_get_pp_table, + amdgpu_set_pp_table); +static DEVICE_ATTR(pp_dpm_sclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_sclk, + amdgpu_set_pp_dpm_sclk); +static DEVICE_ATTR(pp_dpm_mclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_mclk, + amdgpu_set_pp_dpm_mclk); +static DEVICE_ATTR(pp_dpm_socclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_socclk, + amdgpu_set_pp_dpm_socclk); +static DEVICE_ATTR(pp_dpm_fclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_fclk, + amdgpu_set_pp_dpm_fclk); +static DEVICE_ATTR(pp_dpm_dcefclk, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_dcefclk, + amdgpu_set_pp_dpm_dcefclk); +static DEVICE_ATTR(pp_dpm_pcie, S_IRUGO | S_IWUSR, + amdgpu_get_pp_dpm_pcie, + amdgpu_set_pp_dpm_pcie); +static DEVICE_ATTR(pp_sclk_od, S_IRUGO | S_IWUSR, + amdgpu_get_pp_sclk_od, + amdgpu_set_pp_sclk_od); +static DEVICE_ATTR(pp_mclk_od, S_IRUGO | S_IWUSR, + amdgpu_get_pp_mclk_od, + amdgpu_set_pp_mclk_od); +static DEVICE_ATTR(pp_power_profile_mode, S_IRUGO | S_IWUSR, + amdgpu_get_pp_power_profile_mode, + amdgpu_set_pp_power_profile_mode); +static DEVICE_ATTR(pp_od_clk_voltage, S_IRUGO | S_IWUSR, + amdgpu_get_pp_od_clk_voltage, + amdgpu_set_pp_od_clk_voltage); +static DEVICE_ATTR(gpu_busy_percent, S_IRUGO, + amdgpu_get_busy_percent, NULL); +static DEVICE_ATTR(mem_busy_percent, S_IRUGO, + amdgpu_get_memory_busy_percent, NULL); +static DEVICE_ATTR(pcie_bw, S_IRUGO, amdgpu_get_pcie_bw, NULL); +static DEVICE_ATTR(ppfeatures, S_IRUGO | S_IWUSR, + amdgpu_get_ppfeature_status, + amdgpu_set_ppfeature_status); +static DEVICE_ATTR(unique_id, S_IRUGO, amdgpu_get_unique_id, NULL); + +static ssize_t amdgpu_hwmon_show_temp(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + int channel = to_sensor_dev_attr(attr)->index; + int r, temp = 0, size = sizeof(temp); + + /* Can't get temperature when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (channel >= PP_TEMP_MAX) + return -EINVAL; + + switch (channel) { + case PP_TEMP_JUNCTION: + /* get current junction temperature */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_HOTSPOT_TEMP, + (void *)&temp, &size); + if (r) + return r; + break; + case PP_TEMP_EDGE: + /* get current edge temperature */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_EDGE_TEMP, + (void *)&temp, &size); + if (r) + return r; + break; + case PP_TEMP_MEM: + /* get current memory temperature */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_TEMP, + (void *)&temp, &size); + if (r) + return r; + break; + } + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); +} + +static ssize_t amdgpu_hwmon_show_temp_thresh(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int hyst = to_sensor_dev_attr(attr)->index; + int temp; + + if (hyst) + temp = adev->pm.dpm.thermal.min_temp; + else + temp = adev->pm.dpm.thermal.max_temp; + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); +} + +static ssize_t amdgpu_hwmon_show_hotspot_temp_thresh(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int hyst = to_sensor_dev_attr(attr)->index; + int temp; + + if (hyst) + temp = adev->pm.dpm.thermal.min_hotspot_temp; + else + temp = adev->pm.dpm.thermal.max_hotspot_crit_temp; + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); +} + +static ssize_t amdgpu_hwmon_show_mem_temp_thresh(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int hyst = to_sensor_dev_attr(attr)->index; + int temp; + + if (hyst) + temp = adev->pm.dpm.thermal.min_mem_temp; + else + temp = adev->pm.dpm.thermal.max_mem_crit_temp; + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); +} + +static ssize_t amdgpu_hwmon_show_temp_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int channel = to_sensor_dev_attr(attr)->index; + + if (channel >= PP_TEMP_MAX) + return -EINVAL; + + return snprintf(buf, PAGE_SIZE, "%s\n", temp_label[channel].label); +} + +static ssize_t amdgpu_hwmon_show_temp_emergency(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int channel = to_sensor_dev_attr(attr)->index; + int temp = 0; + + if (channel >= PP_TEMP_MAX) + return -EINVAL; + + switch (channel) { + case PP_TEMP_JUNCTION: + temp = adev->pm.dpm.thermal.max_hotspot_emergency_temp; + break; + case PP_TEMP_EDGE: + temp = adev->pm.dpm.thermal.max_edge_emergency_temp; + break; + case PP_TEMP_MEM: + temp = adev->pm.dpm.thermal.max_mem_emergency_temp; + break; + } + + return snprintf(buf, PAGE_SIZE, "%d\n", temp); +} + +static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + u32 pwm_mode = 0; + if (is_support_sw_smu(adev)) { + pwm_mode = smu_get_fan_control_mode(&adev->smu); + } else { + if (!adev->powerplay.pp_funcs->get_fan_control_mode) + return -EINVAL; + + pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); + } + + return sprintf(buf, "%i\n", pwm_mode); +} + +static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + int value; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (is_support_sw_smu(adev)) { + err = kstrtoint(buf, 10, &value); + if (err) + return err; + + smu_set_fan_control_mode(&adev->smu, value); + } else { + if (!adev->powerplay.pp_funcs->set_fan_control_mode) + return -EINVAL; + + err = kstrtoint(buf, 10, &value); + if (err) + return err; + + amdgpu_dpm_set_fan_control_mode(adev, value); + } + + return count; +} + +static ssize_t amdgpu_hwmon_get_pwm1_min(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%i\n", 0); +} + +static ssize_t amdgpu_hwmon_get_pwm1_max(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%i\n", 255); +} + +static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 value; + u32 pwm_mode; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + if (is_support_sw_smu(adev)) + pwm_mode = smu_get_fan_control_mode(&adev->smu); + else + pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); + if (pwm_mode != AMD_FAN_CTRL_MANUAL) { + pr_info("manual fan speed control should be enabled first\n"); + return -EINVAL; + } + + err = kstrtou32(buf, 10, &value); + if (err) + return err; + + value = (value * 100) / 255; + + if (is_support_sw_smu(adev)) { + err = smu_set_fan_speed_percent(&adev->smu, value); + if (err) + return err; + } else if (adev->powerplay.pp_funcs->set_fan_speed_percent) { + err = amdgpu_dpm_set_fan_speed_percent(adev, value); + if (err) + return err; + } + + return count; +} + +static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 speed = 0; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (is_support_sw_smu(adev)) { + err = smu_get_fan_speed_percent(&adev->smu, &speed); + if (err) + return err; + } else if (adev->powerplay.pp_funcs->get_fan_speed_percent) { + err = amdgpu_dpm_get_fan_speed_percent(adev, &speed); + if (err) + return err; + } + + speed = (speed * 255) / 100; + + return sprintf(buf, "%i\n", speed); +} + +static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 speed = 0; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (is_support_sw_smu(adev)) { + err = smu_get_fan_speed_rpm(&adev->smu, &speed); + if (err) + return err; + } else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) { + err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed); + if (err) + return err; + } + + return sprintf(buf, "%i\n", speed); +} + +static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + u32 min_rpm = 0; + u32 size = sizeof(min_rpm); + int r; + + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MIN_FAN_RPM, + (void *)&min_rpm, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", min_rpm); +} + +static ssize_t amdgpu_hwmon_get_fan1_max(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + u32 max_rpm = 0; + u32 size = sizeof(max_rpm); + int r; + + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MAX_FAN_RPM, + (void *)&max_rpm, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", max_rpm); +} + +static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 rpm = 0; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + if (is_support_sw_smu(adev)) { + err = smu_get_fan_speed_rpm(&adev->smu, &rpm); + if (err) + return err; + } else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) { + err = amdgpu_dpm_get_fan_speed_rpm(adev, &rpm); + if (err) + return err; + } + + return sprintf(buf, "%i\n", rpm); +} + +static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 value; + u32 pwm_mode; + + if (is_support_sw_smu(adev)) + pwm_mode = smu_get_fan_control_mode(&adev->smu); + else + pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); + + if (pwm_mode != AMD_FAN_CTRL_MANUAL) + return -ENODATA; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + err = kstrtou32(buf, 10, &value); + if (err) + return err; + + if (is_support_sw_smu(adev)) { + err = smu_set_fan_speed_rpm(&adev->smu, value); + if (err) + return err; + } else if (adev->powerplay.pp_funcs->set_fan_speed_rpm) { + err = amdgpu_dpm_set_fan_speed_rpm(adev, value); + if (err) + return err; + } + + return count; +} + +static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + u32 pwm_mode = 0; + + if (is_support_sw_smu(adev)) { + pwm_mode = smu_get_fan_control_mode(&adev->smu); + } else { + if (!adev->powerplay.pp_funcs->get_fan_control_mode) + return -EINVAL; + + pwm_mode = amdgpu_dpm_get_fan_control_mode(adev); + } + return sprintf(buf, "%i\n", pwm_mode == AMD_FAN_CTRL_AUTO ? 0 : 1); +} + +static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + int value; + u32 pwm_mode; + + /* Can't adjust fan when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + + err = kstrtoint(buf, 10, &value); + if (err) + return err; + + if (value == 0) + pwm_mode = AMD_FAN_CTRL_AUTO; + else if (value == 1) + pwm_mode = AMD_FAN_CTRL_MANUAL; + else + return -EINVAL; + + if (is_support_sw_smu(adev)) { + smu_set_fan_control_mode(&adev->smu, pwm_mode); + } else { + if (!adev->powerplay.pp_funcs->set_fan_control_mode) + return -EINVAL; + amdgpu_dpm_set_fan_control_mode(adev, pwm_mode); + } + + return count; +} + +static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + u32 vddgfx; + int r, size = sizeof(vddgfx); + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* get the voltage */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDGFX, + (void *)&vddgfx, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", vddgfx); +} + +static ssize_t amdgpu_hwmon_show_vddgfx_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "vddgfx\n"); +} + +static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + u32 vddnb; + int r, size = sizeof(vddnb); + + /* only APUs have vddnb */ + if (!(adev->flags & AMD_IS_APU)) + return -EINVAL; + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* get the voltage */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB, + (void *)&vddnb, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", vddnb); +} + +static ssize_t amdgpu_hwmon_show_vddnb_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "vddnb\n"); +} + +static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + u32 query = 0; + int r, size = sizeof(u32); + unsigned uw; + + /* Can't get power when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* get the voltage */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER, + (void *)&query, &size); + if (r) + return r; + + /* convert to microwatts */ + uw = (query >> 8) * 1000000 + (query & 0xff) * 1000; + + return snprintf(buf, PAGE_SIZE, "%u\n", uw); +} + +static ssize_t amdgpu_hwmon_show_power_cap_min(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%i\n", 0); +} + +static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + uint32_t limit = 0; + + if (is_support_sw_smu(adev)) { + smu_get_power_limit(&adev->smu, &limit, true); + return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); + } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { + adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, true); + return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); + } else { + return snprintf(buf, PAGE_SIZE, "\n"); + } +} + +static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + uint32_t limit = 0; + + if (is_support_sw_smu(adev)) { + smu_get_power_limit(&adev->smu, &limit, false); + return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); + } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { + adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, false); + return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); + } else { + return snprintf(buf, PAGE_SIZE, "\n"); + } +} + + +static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + int err; + u32 value; + + err = kstrtou32(buf, 10, &value); + if (err) + return err; + + value = value / 1000000; /* convert to Watt */ + if (is_support_sw_smu(adev)) { + adev->smu.funcs->set_power_limit(&adev->smu, value); + } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->set_power_limit) { + err = adev->powerplay.pp_funcs->set_power_limit(adev->powerplay.pp_handle, value); + if (err) + return err; + } else { + return -EINVAL; + } + + return count; +} + +static ssize_t amdgpu_hwmon_show_sclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + uint32_t sclk; + int r, size = sizeof(sclk); + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* get the sclk */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, + (void *)&sclk, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", sclk * 10 * 1000); +} + +static ssize_t amdgpu_hwmon_show_sclk_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "sclk\n"); +} + +static ssize_t amdgpu_hwmon_show_mclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amdgpu_device *adev = dev_get_drvdata(dev); + struct drm_device *ddev = adev->ddev; + uint32_t mclk; + int r, size = sizeof(mclk); + + /* Can't get voltage when the card is off */ + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) + return -EINVAL; + + /* get the sclk */ + r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, + (void *)&mclk, &size); + if (r) + return r; + + return snprintf(buf, PAGE_SIZE, "%d\n", mclk * 10 * 1000); +} + +static ssize_t amdgpu_hwmon_show_mclk_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "mclk\n"); +} + +/** + * DOC: hwmon + * + * The amdgpu driver exposes the following sensor interfaces: + * + * - GPU temperature (via the on-die sensor) + * + * - GPU voltage + * + * - Northbridge voltage (APUs only) + * + * - GPU power + * + * - GPU fan + * + * - GPU gfx/compute engine clock + * + * - GPU memory clock (dGPU only) + * + * hwmon interfaces for GPU temperature: + * + * - temp[1-3]_input: the on die GPU temperature in millidegrees Celsius + * - temp2_input and temp3_input are supported on SOC15 dGPUs only + * + * - temp[1-3]_label: temperature channel label + * - temp2_label and temp3_label are supported on SOC15 dGPUs only + * + * - temp[1-3]_crit: temperature critical max value in millidegrees Celsius + * - temp2_crit and temp3_crit are supported on SOC15 dGPUs only + * + * - temp[1-3]_crit_hyst: temperature hysteresis for critical limit in millidegrees Celsius + * - temp2_crit_hyst and temp3_crit_hyst are supported on SOC15 dGPUs only + * + * - temp[1-3]_emergency: temperature emergency max value(asic shutdown) in millidegrees Celsius + * - these are supported on SOC15 dGPUs only + * + * hwmon interfaces for GPU voltage: + * + * - in0_input: the voltage on the GPU in millivolts + * + * - in1_input: the voltage on the Northbridge in millivolts + * + * hwmon interfaces for GPU power: + * + * - power1_average: average power used by the GPU in microWatts + * + * - power1_cap_min: minimum cap supported in microWatts + * + * - power1_cap_max: maximum cap supported in microWatts + * + * - power1_cap: selected power cap in microWatts + * + * hwmon interfaces for GPU fan: + * + * - pwm1: pulse width modulation fan level (0-255) + * + * - pwm1_enable: pulse width modulation fan control method (0: no fan speed control, 1: manual fan speed control using pwm interface, 2: automatic fan speed control) + * + * - pwm1_min: pulse width modulation fan control minimum level (0) + * + * - pwm1_max: pulse width modulation fan control maximum level (255) + * + * - fan1_min: an minimum value Unit: revolution/min (RPM) + * + * - fan1_max: an maxmum value Unit: revolution/max (RPM) + * + * - fan1_input: fan speed in RPM + * + * - fan[1-*]_target: Desired fan speed Unit: revolution/min (RPM) + * + * - fan[1-*]_enable: Enable or disable the sensors.1: Enable 0: Disable + * + * hwmon interfaces for GPU clocks: + * + * - freq1_input: the gfx/compute clock in hertz + * + * - freq2_input: the memory clock in hertz + * + * You can use hwmon tools like sensors to view this information on your system. + * + */ + +static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_EDGE); +static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 0); +static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 1); +static SENSOR_DEVICE_ATTR(temp1_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_EDGE); +static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_JUNCTION); +static SENSOR_DEVICE_ATTR(temp2_crit, S_IRUGO, amdgpu_hwmon_show_hotspot_temp_thresh, NULL, 0); +static SENSOR_DEVICE_ATTR(temp2_crit_hyst, S_IRUGO, amdgpu_hwmon_show_hotspot_temp_thresh, NULL, 1); +static SENSOR_DEVICE_ATTR(temp2_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_JUNCTION); +static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_MEM); +static SENSOR_DEVICE_ATTR(temp3_crit, S_IRUGO, amdgpu_hwmon_show_mem_temp_thresh, NULL, 0); +static SENSOR_DEVICE_ATTR(temp3_crit_hyst, S_IRUGO, amdgpu_hwmon_show_mem_temp_thresh, NULL, 1); +static SENSOR_DEVICE_ATTR(temp3_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_MEM); +static SENSOR_DEVICE_ATTR(temp1_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_EDGE); +static SENSOR_DEVICE_ATTR(temp2_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_JUNCTION); +static SENSOR_DEVICE_ATTR(temp3_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_MEM); +static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1, amdgpu_hwmon_set_pwm1, 0); +static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1_enable, amdgpu_hwmon_set_pwm1_enable, 0); +static SENSOR_DEVICE_ATTR(pwm1_min, S_IRUGO, amdgpu_hwmon_get_pwm1_min, NULL, 0); +static SENSOR_DEVICE_ATTR(pwm1_max, S_IRUGO, amdgpu_hwmon_get_pwm1_max, NULL, 0); +static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, amdgpu_hwmon_get_fan1_input, NULL, 0); +static SENSOR_DEVICE_ATTR(fan1_min, S_IRUGO, amdgpu_hwmon_get_fan1_min, NULL, 0); +static SENSOR_DEVICE_ATTR(fan1_max, S_IRUGO, amdgpu_hwmon_get_fan1_max, NULL, 0); +static SENSOR_DEVICE_ATTR(fan1_target, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_fan1_target, amdgpu_hwmon_set_fan1_target, 0); +static SENSOR_DEVICE_ATTR(fan1_enable, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_fan1_enable, amdgpu_hwmon_set_fan1_enable, 0); +static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO, amdgpu_hwmon_show_vddgfx, NULL, 0); +static SENSOR_DEVICE_ATTR(in0_label, S_IRUGO, amdgpu_hwmon_show_vddgfx_label, NULL, 0); +static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, amdgpu_hwmon_show_vddnb, NULL, 0); +static SENSOR_DEVICE_ATTR(in1_label, S_IRUGO, amdgpu_hwmon_show_vddnb_label, NULL, 0); +static SENSOR_DEVICE_ATTR(power1_average, S_IRUGO, amdgpu_hwmon_show_power_avg, NULL, 0); +static SENSOR_DEVICE_ATTR(power1_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 0); +static SENSOR_DEVICE_ATTR(power1_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 0); +static SENSOR_DEVICE_ATTR(power1_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 0); +static SENSOR_DEVICE_ATTR(freq1_input, S_IRUGO, amdgpu_hwmon_show_sclk, NULL, 0); +static SENSOR_DEVICE_ATTR(freq1_label, S_IRUGO, amdgpu_hwmon_show_sclk_label, NULL, 0); +static SENSOR_DEVICE_ATTR(freq2_input, S_IRUGO, amdgpu_hwmon_show_mclk, NULL, 0); +static SENSOR_DEVICE_ATTR(freq2_label, S_IRUGO, amdgpu_hwmon_show_mclk_label, NULL, 0); + +static struct attribute *hwmon_attributes[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_crit.dev_attr.attr, + &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp2_crit.dev_attr.attr, + &sensor_dev_attr_temp2_crit_hyst.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp3_crit.dev_attr.attr, + &sensor_dev_attr_temp3_crit_hyst.dev_attr.attr, + &sensor_dev_attr_temp1_emergency.dev_attr.attr, + &sensor_dev_attr_temp2_emergency.dev_attr.attr, + &sensor_dev_attr_temp3_emergency.dev_attr.attr, + &sensor_dev_attr_temp1_label.dev_attr.attr, + &sensor_dev_attr_temp2_label.dev_attr.attr, + &sensor_dev_attr_temp3_label.dev_attr.attr, + &sensor_dev_attr_pwm1.dev_attr.attr, + &sensor_dev_attr_pwm1_enable.dev_attr.attr, + &sensor_dev_attr_pwm1_min.dev_attr.attr, + &sensor_dev_attr_pwm1_max.dev_attr.attr, + &sensor_dev_attr_fan1_input.dev_attr.attr, + &sensor_dev_attr_fan1_min.dev_attr.attr, + &sensor_dev_attr_fan1_max.dev_attr.attr, + &sensor_dev_attr_fan1_target.dev_attr.attr, + &sensor_dev_attr_fan1_enable.dev_attr.attr, + &sensor_dev_attr_in0_input.dev_attr.attr, + &sensor_dev_attr_in0_label.dev_attr.attr, + &sensor_dev_attr_in1_input.dev_attr.attr, + &sensor_dev_attr_in1_label.dev_attr.attr, + &sensor_dev_attr_power1_average.dev_attr.attr, + &sensor_dev_attr_power1_cap_max.dev_attr.attr, + &sensor_dev_attr_power1_cap_min.dev_attr.attr, + &sensor_dev_attr_power1_cap.dev_attr.attr, + &sensor_dev_attr_freq1_input.dev_attr.attr, + &sensor_dev_attr_freq1_label.dev_attr.attr, + &sensor_dev_attr_freq2_input.dev_attr.attr, + &sensor_dev_attr_freq2_label.dev_attr.attr, + NULL +}; + +static umode_t hwmon_attributes_visible(struct kobject *kobj, + struct attribute *attr, int index) +{ + struct device *dev = kobj_to_dev(kobj); + struct amdgpu_device *adev = dev_get_drvdata(dev); + umode_t effective_mode = attr->mode; + + /* Skip fan attributes if fan is not present */ + if (adev->pm.no_fan && (attr == &sensor_dev_attr_pwm1.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_input.dev_attr.attr || + attr == &sensor_dev_attr_fan1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_max.dev_attr.attr || + attr == &sensor_dev_attr_fan1_target.dev_attr.attr || + attr == &sensor_dev_attr_fan1_enable.dev_attr.attr)) + return 0; + + /* Skip fan attributes on APU */ + if ((adev->flags & AMD_IS_APU) && + (attr == &sensor_dev_attr_pwm1.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_input.dev_attr.attr || + attr == &sensor_dev_attr_fan1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_max.dev_attr.attr || + attr == &sensor_dev_attr_fan1_target.dev_attr.attr || + attr == &sensor_dev_attr_fan1_enable.dev_attr.attr)) + return 0; + + /* Skip limit attributes if DPM is not enabled */ + if (!adev->pm.dpm_enabled && + (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr || + attr == &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr || + attr == &sensor_dev_attr_pwm1.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_input.dev_attr.attr || + attr == &sensor_dev_attr_fan1_min.dev_attr.attr || + attr == &sensor_dev_attr_fan1_max.dev_attr.attr || + attr == &sensor_dev_attr_fan1_target.dev_attr.attr || + attr == &sensor_dev_attr_fan1_enable.dev_attr.attr)) + return 0; + + if (!is_support_sw_smu(adev)) { + /* mask fan attributes if we have no bindings for this asic to expose */ + if ((!adev->powerplay.pp_funcs->get_fan_speed_percent && + attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */ + (!adev->powerplay.pp_funcs->get_fan_control_mode && + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */ + effective_mode &= ~S_IRUGO; + + if ((!adev->powerplay.pp_funcs->set_fan_speed_percent && + attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */ + (!adev->powerplay.pp_funcs->set_fan_control_mode && + attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */ + effective_mode &= ~S_IWUSR; + } + + if ((adev->flags & AMD_IS_APU) && + (attr == &sensor_dev_attr_power1_average.dev_attr.attr || + attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr || + attr == &sensor_dev_attr_power1_cap_min.dev_attr.attr|| + attr == &sensor_dev_attr_power1_cap.dev_attr.attr)) + return 0; + + if (!is_support_sw_smu(adev)) { + /* hide max/min values if we can't both query and manage the fan */ + if ((!adev->powerplay.pp_funcs->set_fan_speed_percent && + !adev->powerplay.pp_funcs->get_fan_speed_percent) && + (!adev->powerplay.pp_funcs->set_fan_speed_rpm && + !adev->powerplay.pp_funcs->get_fan_speed_rpm) && + (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr || + attr == &sensor_dev_attr_pwm1_min.dev_attr.attr)) + return 0; + + if ((!adev->powerplay.pp_funcs->set_fan_speed_rpm && + !adev->powerplay.pp_funcs->get_fan_speed_rpm) && + (attr == &sensor_dev_attr_fan1_max.dev_attr.attr || + attr == &sensor_dev_attr_fan1_min.dev_attr.attr)) + return 0; + } + + /* only APUs have vddnb */ + if (!(adev->flags & AMD_IS_APU) && + (attr == &sensor_dev_attr_in1_input.dev_attr.attr || + attr == &sensor_dev_attr_in1_label.dev_attr.attr)) + return 0; + + /* no mclk on APUs */ + if ((adev->flags & AMD_IS_APU) && + (attr == &sensor_dev_attr_freq2_input.dev_attr.attr || + attr == &sensor_dev_attr_freq2_label.dev_attr.attr)) + return 0; + + /* only SOC15 dGPUs support hotspot and mem temperatures */ + if (((adev->flags & AMD_IS_APU) || + adev->asic_type < CHIP_VEGA10) && + (attr == &sensor_dev_attr_temp2_crit.dev_attr.attr || + attr == &sensor_dev_attr_temp2_crit_hyst.dev_attr.attr || + attr == &sensor_dev_attr_temp3_crit.dev_attr.attr || + attr == &sensor_dev_attr_temp3_crit_hyst.dev_attr.attr || + attr == &sensor_dev_attr_temp1_emergency.dev_attr.attr || + attr == &sensor_dev_attr_temp2_emergency.dev_attr.attr || + attr == &sensor_dev_attr_temp3_emergency.dev_attr.attr || + attr == &sensor_dev_attr_temp2_input.dev_attr.attr || + attr == &sensor_dev_attr_temp3_input.dev_attr.attr || + attr == &sensor_dev_attr_temp2_label.dev_attr.attr || + attr == &sensor_dev_attr_temp3_label.dev_attr.attr)) + return 0; + + return effective_mode; +} + +static const struct attribute_group hwmon_attrgroup = { + .attrs = hwmon_attributes, + .is_visible = hwmon_attributes_visible, +}; + +static const struct attribute_group *hwmon_groups[] = { + &hwmon_attrgroup, + NULL +}; + +void amdgpu_dpm_thermal_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, + pm.dpm.thermal.work); + /* switch to the thermal state */ + enum amd_pm_state_type dpm_state = POWER_STATE_TYPE_INTERNAL_THERMAL; + int temp, size = sizeof(temp); + + if (!adev->pm.dpm_enabled) + return; + + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP, + (void *)&temp, &size)) { + if (temp < adev->pm.dpm.thermal.min_temp) + /* switch back the user state */ + dpm_state = adev->pm.dpm.user_state; + } else { + if (adev->pm.dpm.thermal.high_to_low) + /* switch back the user state */ + dpm_state = adev->pm.dpm.user_state; + } + mutex_lock(&adev->pm.mutex); + if (dpm_state == POWER_STATE_TYPE_INTERNAL_THERMAL) + adev->pm.dpm.thermal_active = true; + else + adev->pm.dpm.thermal_active = false; + adev->pm.dpm.state = dpm_state; + mutex_unlock(&adev->pm.mutex); + + amdgpu_pm_compute_clocks(adev); +} + +static struct amdgpu_ps *amdgpu_dpm_pick_power_state(struct amdgpu_device *adev, + enum amd_pm_state_type dpm_state) +{ + int i; + struct amdgpu_ps *ps; + u32 ui_class; + bool single_display = (adev->pm.dpm.new_active_crtc_count < 2) ? + true : false; + + /* check if the vblank period is too short to adjust the mclk */ + if (single_display && adev->powerplay.pp_funcs->vblank_too_short) { + if (amdgpu_dpm_vblank_too_short(adev)) + single_display = false; + } + + /* certain older asics have a separare 3D performance state, + * so try that first if the user selected performance + */ + if (dpm_state == POWER_STATE_TYPE_PERFORMANCE) + dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF; + /* balanced states don't exist at the moment */ + if (dpm_state == POWER_STATE_TYPE_BALANCED) + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + +restart_search: + /* Pick the best power state based on current conditions */ + for (i = 0; i < adev->pm.dpm.num_ps; i++) { + ps = &adev->pm.dpm.ps[i]; + ui_class = ps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK; + switch (dpm_state) { + /* user states */ + case POWER_STATE_TYPE_BATTERY: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_BALANCED: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_BALANCED) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + case POWER_STATE_TYPE_PERFORMANCE: + if (ui_class == ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { + if (ps->caps & ATOM_PPLIB_SINGLE_DISPLAY_ONLY) { + if (single_display) + return ps; + } else + return ps; + } + break; + /* internal states */ + case POWER_STATE_TYPE_INTERNAL_UVD: + if (adev->pm.dpm.uvd_ps) + return adev->pm.dpm.uvd_ps; + else + break; + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_HD2STATE) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_MVC) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_BOOT: + return adev->pm.dpm.boot_ps; + case POWER_STATE_TYPE_INTERNAL_THERMAL: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ACPI: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_ULV: + if (ps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) + return ps; + break; + case POWER_STATE_TYPE_INTERNAL_3DPERF: + if (ps->class & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE) + return ps; + break; + default: + break; + } + } + /* use a fallback state if we didn't match */ + switch (dpm_state) { + case POWER_STATE_TYPE_INTERNAL_UVD_SD: + dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_UVD_HD: + case POWER_STATE_TYPE_INTERNAL_UVD_HD2: + case POWER_STATE_TYPE_INTERNAL_UVD_MVC: + if (adev->pm.dpm.uvd_ps) { + return adev->pm.dpm.uvd_ps; + } else { + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + } + case POWER_STATE_TYPE_INTERNAL_THERMAL: + dpm_state = POWER_STATE_TYPE_INTERNAL_ACPI; + goto restart_search; + case POWER_STATE_TYPE_INTERNAL_ACPI: + dpm_state = POWER_STATE_TYPE_BATTERY; + goto restart_search; + case POWER_STATE_TYPE_BATTERY: + case POWER_STATE_TYPE_BALANCED: + case POWER_STATE_TYPE_INTERNAL_3DPERF: + dpm_state = POWER_STATE_TYPE_PERFORMANCE; + goto restart_search; + default: + break; + } + + return NULL; +} + +static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) +{ + struct amdgpu_ps *ps; + enum amd_pm_state_type dpm_state; + int ret; + bool equal = false; + + /* if dpm init failed */ + if (!adev->pm.dpm_enabled) + return; + + if (adev->pm.dpm.user_state != adev->pm.dpm.state) { + /* add other state override checks here */ + if ((!adev->pm.dpm.thermal_active) && + (!adev->pm.dpm.uvd_active)) + adev->pm.dpm.state = adev->pm.dpm.user_state; + } + dpm_state = adev->pm.dpm.state; + + ps = amdgpu_dpm_pick_power_state(adev, dpm_state); + if (ps) + adev->pm.dpm.requested_ps = ps; + else + return; + + if (amdgpu_dpm == 1 && adev->powerplay.pp_funcs->print_power_state) { + printk("switching from power state:\n"); + amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps); + printk("switching to power state:\n"); + amdgpu_dpm_print_power_state(adev, adev->pm.dpm.requested_ps); + } + + /* update whether vce is active */ + ps->vce_active = adev->pm.dpm.vce_active; + if (adev->powerplay.pp_funcs->display_configuration_changed) + amdgpu_dpm_display_configuration_changed(adev); + + ret = amdgpu_dpm_pre_set_power_state(adev); + if (ret) + return; + + if (adev->powerplay.pp_funcs->check_state_equal) { + if (0 != amdgpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)) + equal = false; + } + + if (equal) + return; + + amdgpu_dpm_set_power_state(adev); + amdgpu_dpm_post_set_power_state(adev); + + adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs; + adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count; + + if (adev->powerplay.pp_funcs->force_performance_level) { + if (adev->pm.dpm.thermal_active) { + enum amd_dpm_forced_level level = adev->pm.dpm.forced_level; + /* force low perf level for thermal */ + amdgpu_dpm_force_performance_level(adev, AMD_DPM_FORCED_LEVEL_LOW); + /* save the user's level */ + adev->pm.dpm.forced_level = level; + } else { + /* otherwise, user selected level */ + amdgpu_dpm_force_performance_level(adev, adev->pm.dpm.forced_level); + } + } +} + +void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable) +{ + int ret = 0; + if (is_support_sw_smu(adev)) { + ret = smu_dpm_set_power_gate(&adev->smu, AMD_IP_BLOCK_TYPE_UVD, enable); + if (ret) + DRM_ERROR("[SW SMU]: dpm enable uvd failed, state = %s, ret = %d. \n", + enable ? "true" : "false", ret); + } else if (adev->powerplay.pp_funcs->set_powergating_by_smu) { + /* enable/disable UVD */ + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_UVD, !enable); + mutex_unlock(&adev->pm.mutex); + } + /* enable/disable Low Memory PState for UVD (4k videos) */ + if (adev->asic_type == CHIP_STONEY && + adev->uvd.decode_image_width >= WIDTH_4K) { + struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle; + + if (hwmgr && hwmgr->hwmgr_func && + hwmgr->hwmgr_func->update_nbdpm_pstate) + hwmgr->hwmgr_func->update_nbdpm_pstate(hwmgr, + !enable, + true); + } +} + +void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable) +{ + int ret = 0; + if (is_support_sw_smu(adev)) { + ret = smu_dpm_set_power_gate(&adev->smu, AMD_IP_BLOCK_TYPE_VCE, enable); + if (ret) + DRM_ERROR("[SW SMU]: dpm enable vce failed, state = %s, ret = %d. \n", + enable ? "true" : "false", ret); + } else if (adev->powerplay.pp_funcs->set_powergating_by_smu) { + /* enable/disable VCE */ + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_VCE, !enable); + mutex_unlock(&adev->pm.mutex); + } +} + +void amdgpu_pm_print_power_states(struct amdgpu_device *adev) +{ + int i; + + if (adev->powerplay.pp_funcs->print_power_state == NULL) + return; + + for (i = 0; i < adev->pm.dpm.num_ps; i++) + amdgpu_dpm_print_power_state(adev, &adev->pm.dpm.ps[i]); + +} + +int amdgpu_pm_virt_sysfs_init(struct amdgpu_device *adev) +{ + int ret = 0; + + if (!(amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))) + return ret; + + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_sclk\n"); + return ret; + } + + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_mclk\n"); + return ret; + } + + ret = device_create_file(adev->dev, &dev_attr_power_dpm_force_performance_level); + if (ret) { + DRM_ERROR("failed to create device file for dpm state\n"); + return ret; + } + + return ret; +} + +void amdgpu_pm_virt_sysfs_fini(struct amdgpu_device *adev) +{ + if (!(amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))) + return; + + device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level); + device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); +} + +int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version) +{ + int r; + + if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->load_firmware) { + r = adev->powerplay.pp_funcs->load_firmware(adev->powerplay.pp_handle); + if (r) { + pr_err("smu firmware loading failed\n"); + return r; + } + *smu_version = adev->pm.fw_version; + } + return 0; +} + +int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) +{ + struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle; + int ret; + + if (adev->pm.sysfs_initialized) + return 0; + + if (adev->pm.dpm_enabled == 0) + return 0; + + adev->pm.int_hwmon_dev = hwmon_device_register_with_groups(adev->dev, + DRIVER_NAME, adev, + hwmon_groups); + if (IS_ERR(adev->pm.int_hwmon_dev)) { + ret = PTR_ERR(adev->pm.int_hwmon_dev); + dev_err(adev->dev, + "Unable to register hwmon device: %d\n", ret); + return ret; + } + + ret = device_create_file(adev->dev, &dev_attr_power_dpm_state); + if (ret) { + DRM_ERROR("failed to create device file for dpm state\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_power_dpm_force_performance_level); + if (ret) { + DRM_ERROR("failed to create device file for dpm state\n"); + return ret; + } + + + ret = device_create_file(adev->dev, &dev_attr_pp_num_states); + if (ret) { + DRM_ERROR("failed to create device file pp_num_states\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_cur_state); + if (ret) { + DRM_ERROR("failed to create device file pp_cur_state\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_force_state); + if (ret) { + DRM_ERROR("failed to create device file pp_force_state\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_table); + if (ret) { + DRM_ERROR("failed to create device file pp_table\n"); + return ret; + } + + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_sclk\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_mclk\n"); + return ret; + } + if (adev->asic_type >= CHIP_VEGA10) { + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_socclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_socclk\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_dcefclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_dcefclk\n"); + return ret; + } + } + if (adev->asic_type >= CHIP_VEGA20) { + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_fclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_fclk\n"); + return ret; + } + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_pcie\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_sclk_od); + if (ret) { + DRM_ERROR("failed to create device file pp_sclk_od\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_mclk_od); + if (ret) { + DRM_ERROR("failed to create device file pp_mclk_od\n"); + return ret; + } + ret = device_create_file(adev->dev, + &dev_attr_pp_power_profile_mode); + if (ret) { + DRM_ERROR("failed to create device file " + "pp_power_profile_mode\n"); + return ret; + } + if ((is_support_sw_smu(adev) && adev->smu.od_enabled) || + (!is_support_sw_smu(adev) && hwmgr->od_enabled)) { + ret = device_create_file(adev->dev, + &dev_attr_pp_od_clk_voltage); + if (ret) { + DRM_ERROR("failed to create device file " + "pp_od_clk_voltage\n"); + return ret; + } + } + ret = device_create_file(adev->dev, + &dev_attr_gpu_busy_percent); + if (ret) { + DRM_ERROR("failed to create device file " + "gpu_busy_level\n"); + return ret; + } + /* APU does not have its own dedicated memory */ + if (!(adev->flags & AMD_IS_APU) && + (adev->asic_type != CHIP_VEGA10)) { + ret = device_create_file(adev->dev, + &dev_attr_mem_busy_percent); + if (ret) { + DRM_ERROR("failed to create device file " + "mem_busy_percent\n"); + return ret; + } + } + /* PCIe Perf counters won't work on APU nodes */ + if (!(adev->flags & AMD_IS_APU)) { + ret = device_create_file(adev->dev, &dev_attr_pcie_bw); + if (ret) { + DRM_ERROR("failed to create device file pcie_bw\n"); + return ret; + } + } + if (adev->unique_id) + ret = device_create_file(adev->dev, &dev_attr_unique_id); + if (ret) { + DRM_ERROR("failed to create device file unique_id\n"); + return ret; + } + ret = amdgpu_debugfs_pm_init(adev); + if (ret) { + DRM_ERROR("Failed to register debugfs file for dpm!\n"); + return ret; + } + + if ((adev->asic_type >= CHIP_VEGA10) && + !(adev->flags & AMD_IS_APU)) { + ret = device_create_file(adev->dev, + &dev_attr_ppfeatures); + if (ret) { + DRM_ERROR("failed to create device file " + "ppfeatures\n"); + return ret; + } + } + + adev->pm.sysfs_initialized = true; + + return 0; +} + +void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) +{ + struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle; + + if (adev->pm.dpm_enabled == 0) + return; + + if (adev->pm.int_hwmon_dev) + hwmon_device_unregister(adev->pm.int_hwmon_dev); + device_remove_file(adev->dev, &dev_attr_power_dpm_state); + device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level); + + device_remove_file(adev->dev, &dev_attr_pp_num_states); + device_remove_file(adev->dev, &dev_attr_pp_cur_state); + device_remove_file(adev->dev, &dev_attr_pp_force_state); + device_remove_file(adev->dev, &dev_attr_pp_table); + + device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); + if (adev->asic_type >= CHIP_VEGA10) { + device_remove_file(adev->dev, &dev_attr_pp_dpm_socclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_dcefclk); + } + device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie); + if (adev->asic_type >= CHIP_VEGA20) + device_remove_file(adev->dev, &dev_attr_pp_dpm_fclk); + device_remove_file(adev->dev, &dev_attr_pp_sclk_od); + device_remove_file(adev->dev, &dev_attr_pp_mclk_od); + device_remove_file(adev->dev, + &dev_attr_pp_power_profile_mode); + if ((is_support_sw_smu(adev) && adev->smu.od_enabled) || + (!is_support_sw_smu(adev) && hwmgr->od_enabled)) + device_remove_file(adev->dev, + &dev_attr_pp_od_clk_voltage); + device_remove_file(adev->dev, &dev_attr_gpu_busy_percent); + if (!(adev->flags & AMD_IS_APU) && + (adev->asic_type != CHIP_VEGA10)) + device_remove_file(adev->dev, &dev_attr_mem_busy_percent); + if (!(adev->flags & AMD_IS_APU)) + device_remove_file(adev->dev, &dev_attr_pcie_bw); + if (adev->unique_id) + device_remove_file(adev->dev, &dev_attr_unique_id); + if ((adev->asic_type >= CHIP_VEGA10) && + !(adev->flags & AMD_IS_APU)) + device_remove_file(adev->dev, &dev_attr_ppfeatures); +} + +void amdgpu_pm_compute_clocks(struct amdgpu_device *adev) +{ + int i = 0; + + if (!adev->pm.dpm_enabled) + return; + + if (adev->mode_info.num_crtc) + amdgpu_display_bandwidth_update(adev); + + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (ring && ring->sched.ready) + amdgpu_fence_wait_empty(ring); + } + + if (is_support_sw_smu(adev)) { + struct smu_dpm_context *smu_dpm = &adev->smu.smu_dpm; + smu_handle_task(&adev->smu, + smu_dpm->dpm_level, + AMD_PP_TASK_DISPLAY_CONFIG_CHANGE); + } else { + if (adev->powerplay.pp_funcs->dispatch_tasks) { + if (!amdgpu_device_has_dc_support(adev)) { + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_get_active_displays(adev); + adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count; + adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev); + adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev); + /* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */ + if (adev->pm.pm_display_cfg.vrefresh > 120) + adev->pm.pm_display_cfg.min_vblank_time = 0; + if (adev->powerplay.pp_funcs->display_configuration_change) + adev->powerplay.pp_funcs->display_configuration_change( + adev->powerplay.pp_handle, + &adev->pm.pm_display_cfg); + mutex_unlock(&adev->pm.mutex); + } + amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL); + } else { + mutex_lock(&adev->pm.mutex); + amdgpu_dpm_get_active_displays(adev); + amdgpu_dpm_change_power_state_locked(adev); + mutex_unlock(&adev->pm.mutex); + } + } +} + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *adev) +{ + uint32_t value; + uint64_t value64; + uint32_t query = 0; + int size; + + /* GPU Clocks */ + size = sizeof(value); + seq_printf(m, "GFX Clocks and Power:\n"); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (MCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (SCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_STABLE_PSTATE_SCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (PSTATE_SCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_STABLE_PSTATE_MCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (PSTATE_MCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDGFX, (void *)&value, &size)) + seq_printf(m, "\t%u mV (VDDGFX)\n", value); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB, (void *)&value, &size)) + seq_printf(m, "\t%u mV (VDDNB)\n", value); + size = sizeof(uint32_t); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER, (void *)&query, &size)) + seq_printf(m, "\t%u.%u W (average GPU)\n", query >> 8, query & 0xff); + size = sizeof(value); + seq_printf(m, "\n"); + + /* GPU Temp */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP, (void *)&value, &size)) + seq_printf(m, "GPU Temperature: %u C\n", value/1000); + + /* GPU Load */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD, (void *)&value, &size)) + seq_printf(m, "GPU Load: %u %%\n", value); + /* MEM Load */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_LOAD, (void *)&value, &size)) + seq_printf(m, "MEM Load: %u %%\n", value); + + seq_printf(m, "\n"); + + /* SMC feature mask */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_ENABLED_SMC_FEATURES_MASK, (void *)&value64, &size)) + seq_printf(m, "SMC Feature Mask: 0x%016llx\n", value64); + + if (adev->asic_type > CHIP_VEGA20) { + /* VCN clocks */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCN_POWER_STATE, (void *)&value, &size)) { + if (!value) { + seq_printf(m, "VCN: Disabled\n"); + } else { + seq_printf(m, "VCN: Enabled\n"); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (DCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (VCLK)\n", value/100); + } + } + seq_printf(m, "\n"); + } else { + /* UVD clocks */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_POWER, (void *)&value, &size)) { + if (!value) { + seq_printf(m, "UVD: Disabled\n"); + } else { + seq_printf(m, "UVD: Enabled\n"); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (DCLK)\n", value/100); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (VCLK)\n", value/100); + } + } + seq_printf(m, "\n"); + + /* VCE clocks */ + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_POWER, (void *)&value, &size)) { + if (!value) { + seq_printf(m, "VCE: Disabled\n"); + } else { + seq_printf(m, "VCE: Enabled\n"); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_ECCLK, (void *)&value, &size)) + seq_printf(m, "\t%u MHz (ECCLK)\n", value/100); + } + } + } + + return 0; +} + +static void amdgpu_parse_cg_state(struct seq_file *m, u32 flags) +{ + int i; + + for (i = 0; clocks[i].flag; i++) + seq_printf(m, "\t%s: %s\n", clocks[i].name, + (flags & clocks[i].flag) ? "On" : "Off"); +} + +static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + struct drm_device *ddev = adev->ddev; + u32 flags = 0; + + amdgpu_device_ip_get_clockgating_state(adev, &flags); + seq_printf(m, "Clock Gating Flags Mask: 0x%x\n", flags); + amdgpu_parse_cg_state(m, flags); + seq_printf(m, "\n"); + + if (!adev->pm.dpm_enabled) { + seq_printf(m, "dpm not enabled\n"); + return 0; + } + if ((adev->flags & AMD_IS_PX) && + (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) { + seq_printf(m, "PX asic powered off\n"); + } else if (!is_support_sw_smu(adev) && adev->powerplay.pp_funcs->debugfs_print_current_performance_level) { + mutex_lock(&adev->pm.mutex); + if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) + adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m); + else + seq_printf(m, "Debugfs support not implemented for this asic\n"); + mutex_unlock(&adev->pm.mutex); + } else { + return amdgpu_debugfs_pm_info_pp(m, adev); + } + + return 0; +} + +static const struct drm_info_list amdgpu_pm_info_list[] = { + {"amdgpu_pm_info", amdgpu_debugfs_pm_info, 0, NULL}, +}; +#endif + +static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + return amdgpu_debugfs_add_files(adev, amdgpu_pm_info_list, ARRAY_SIZE(amdgpu_pm_info_list)); +#else + return 0; +#endif +}
diff --git a/amdgpu/amdgpu_pm.h b/amdgpu/amdgpu_pm.h new file mode 100644 index 0000000..ef31448 --- /dev/null +++ b/amdgpu/amdgpu_pm.h
@@ -0,0 +1,45 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_PM_H__ +#define __AMDGPU_PM_H__ + +struct cg_flag_name +{ + u32 flag; + const char *name; +}; + +void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev); +int amdgpu_pm_sysfs_init(struct amdgpu_device *adev); +int amdgpu_pm_virt_sysfs_init(struct amdgpu_device *adev); +void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev); +void amdgpu_pm_virt_sysfs_fini(struct amdgpu_device *adev); +void amdgpu_pm_print_power_states(struct amdgpu_device *adev); +int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version); +void amdgpu_pm_compute_clocks(struct amdgpu_device *adev); +void amdgpu_dpm_thermal_work_handler(struct work_struct *work); +void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable); +void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable); + +#endif
diff --git a/amdgpu/amdgpu_pmu.c b/amdgpu/amdgpu_pmu.c new file mode 100644 index 0000000..0e6dba9 --- /dev/null +++ b/amdgpu/amdgpu_pmu.c
@@ -0,0 +1,280 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Jonathan Kim <jonathan.kim@amd.com> + * + */ + +#include <linux/perf_event.h> +#include <linux/init.h> +#include "amdgpu.h" +#include "amdgpu_pmu.h" +#include "df_v3_6.h" + +#define PMU_NAME_SIZE 32 + +/* record to keep track of pmu entry per pmu type per device */ +struct amdgpu_pmu_entry { + struct list_head entry; + struct amdgpu_device *adev; + struct pmu pmu; + unsigned int pmu_perf_type; +}; + +static LIST_HEAD(amdgpu_pmu_list); + + +/* initialize perf counter */ +static int amdgpu_perf_event_init(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* test the event attr type check for PMU enumeration */ + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* update the hw_perf_event struct with config data */ + hwc->conf = event->attr.config; + + return 0; +} + +/* start perf counter */ +static void amdgpu_perf_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + struct amdgpu_pmu_entry *pe = container_of(event->pmu, + struct amdgpu_pmu_entry, + pmu); + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + hwc->state = 0; + + switch (pe->pmu_perf_type) { + case PERF_TYPE_AMDGPU_DF: + if (!(flags & PERF_EF_RELOAD)) + pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1); + + pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 0); + break; + default: + break; + } + + perf_event_update_userpage(event); + +} + +/* read perf counter */ +static void amdgpu_perf_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + struct amdgpu_pmu_entry *pe = container_of(event->pmu, + struct amdgpu_pmu_entry, + pmu); + + u64 count, prev; + + do { + prev = local64_read(&hwc->prev_count); + + switch (pe->pmu_perf_type) { + case PERF_TYPE_AMDGPU_DF: + pe->adev->df_funcs->pmc_get_count(pe->adev, hwc->conf, + &count); + break; + default: + count = 0; + break; + }; + } while (local64_cmpxchg(&hwc->prev_count, prev, count) != prev); + + local64_add(count - prev, &event->count); +} + +/* stop perf counter */ +static void amdgpu_perf_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + struct amdgpu_pmu_entry *pe = container_of(event->pmu, + struct amdgpu_pmu_entry, + pmu); + + if (hwc->state & PERF_HES_UPTODATE) + return; + + switch (pe->pmu_perf_type) { + case PERF_TYPE_AMDGPU_DF: + pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 0); + break; + default: + break; + }; + + WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); + hwc->state |= PERF_HES_STOPPED; + + if (hwc->state & PERF_HES_UPTODATE) + return; + + amdgpu_perf_read(event); + hwc->state |= PERF_HES_UPTODATE; +} + +/* add perf counter */ +static int amdgpu_perf_add(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + int retval; + + struct amdgpu_pmu_entry *pe = container_of(event->pmu, + struct amdgpu_pmu_entry, + pmu); + + event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + + switch (pe->pmu_perf_type) { + case PERF_TYPE_AMDGPU_DF: + retval = pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1); + break; + default: + return 0; + }; + + if (retval) + return retval; + + if (flags & PERF_EF_START) + amdgpu_perf_start(event, PERF_EF_RELOAD); + + return retval; + +} + +/* delete perf counter */ +static void amdgpu_perf_del(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + struct amdgpu_pmu_entry *pe = container_of(event->pmu, + struct amdgpu_pmu_entry, + pmu); + + amdgpu_perf_stop(event, PERF_EF_UPDATE); + + switch (pe->pmu_perf_type) { + case PERF_TYPE_AMDGPU_DF: + pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 1); + break; + default: + break; + }; + + perf_event_update_userpage(event); +} + +/* vega20 pmus */ + +/* init pmu tracking per pmu type */ +static int init_pmu_by_type(struct amdgpu_device *adev, + const struct attribute_group *attr_groups[], + char *pmu_type_name, char *pmu_file_prefix, + unsigned int pmu_perf_type, + unsigned int num_counters) +{ + char pmu_name[PMU_NAME_SIZE]; + struct amdgpu_pmu_entry *pmu_entry; + int ret = 0; + + pmu_entry = kzalloc(sizeof(struct amdgpu_pmu_entry), GFP_KERNEL); + + if (!pmu_entry) + return -ENOMEM; + + pmu_entry->adev = adev; + pmu_entry->pmu = (struct pmu){ + .event_init = amdgpu_perf_event_init, + .add = amdgpu_perf_add, + .del = amdgpu_perf_del, + .start = amdgpu_perf_start, + .stop = amdgpu_perf_stop, + .read = amdgpu_perf_read, + .task_ctx_nr = perf_invalid_context, + }; + + pmu_entry->pmu.attr_groups = attr_groups; + pmu_entry->pmu_perf_type = pmu_perf_type; + snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d", + pmu_file_prefix, adev->ddev->primary->index); + + ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1); + + if (ret) { + kfree(pmu_entry); + pr_warn("Error initializing AMDGPU %s PMUs.\n", pmu_type_name); + return ret; + } + + pr_info("Detected AMDGPU %s Counters. # of Counters = %d.\n", + pmu_type_name, num_counters); + + list_add_tail(&pmu_entry->entry, &amdgpu_pmu_list); + + return 0; +} + +/* init amdgpu_pmu */ +int amdgpu_pmu_init(struct amdgpu_device *adev) +{ + int ret = 0; + + switch (adev->asic_type) { + case CHIP_VEGA20: + /* init df */ + ret = init_pmu_by_type(adev, df_v3_6_attr_groups, + "DF", "amdgpu_df", PERF_TYPE_AMDGPU_DF, + DF_V3_6_MAX_COUNTERS); + + /* other pmu types go here*/ + break; + default: + return 0; + } + + return 0; +} + + +/* destroy all pmu data associated with target device */ +void amdgpu_pmu_fini(struct amdgpu_device *adev) +{ + struct amdgpu_pmu_entry *pe, *temp; + + list_for_each_entry_safe(pe, temp, &amdgpu_pmu_list, entry) { + if (pe->adev == adev) { + list_del(&pe->entry); + perf_pmu_unregister(&pe->pmu); + kfree(pe); + } + } +}
diff --git a/amdgpu/amdgpu_pmu.h b/amdgpu/amdgpu_pmu.h new file mode 100644 index 0000000..7dddb71 --- /dev/null +++ b/amdgpu/amdgpu_pmu.h
@@ -0,0 +1,37 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Jonathan Kim <jonathan.kim@amd.com> + * + */ + +#ifndef _AMDGPU_PMU_H_ +#define _AMDGPU_PMU_H_ + +enum amdgpu_pmu_perf_type { + PERF_TYPE_AMDGPU_DF = 0, + PERF_TYPE_AMDGPU_MAX +}; + +int amdgpu_pmu_init(struct amdgpu_device *adev); +void amdgpu_pmu_fini(struct amdgpu_device *adev); + +#endif /* _AMDGPU_PMU_H_ */
diff --git a/amdgpu/amdgpu_psp.c b/amdgpu/amdgpu_psp.c new file mode 100644 index 0000000..c027e5e --- /dev/null +++ b/amdgpu/amdgpu_psp.c
@@ -0,0 +1,1331 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Huang Rui + * + */ + +#include <linux/firmware.h> + +#include "amdgpu.h" +#include "amdgpu_psp.h" +#include "amdgpu_ucode.h" +#include "soc15_common.h" +#include "psp_v3_1.h" +#include "psp_v10_0.h" +#include "psp_v11_0.h" + +static void psp_set_funcs(struct amdgpu_device *adev); + +static int psp_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + + psp_set_funcs(adev); + + switch (adev->asic_type) { + case CHIP_VEGA10: + case CHIP_VEGA12: + psp_v3_1_set_psp_funcs(psp); + psp->autoload_supported = false; + break; + case CHIP_RAVEN: + psp_v10_0_set_psp_funcs(psp); + psp->autoload_supported = false; + break; + case CHIP_VEGA20: + psp_v11_0_set_psp_funcs(psp); + psp->autoload_supported = false; + break; + case CHIP_NAVI10: + psp_v11_0_set_psp_funcs(psp); + psp->autoload_supported = true; + break; + default: + return -EINVAL; + } + + psp->adev = adev; + + return 0; +} + +static int psp_sw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + int ret; + + ret = psp_init_microcode(psp); + if (ret) { + DRM_ERROR("Failed to load psp firmware!\n"); + return ret; + } + + return 0; +} + +static int psp_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + release_firmware(adev->psp.sos_fw); + adev->psp.sos_fw = NULL; + release_firmware(adev->psp.asd_fw); + adev->psp.asd_fw = NULL; + if (adev->psp.ta_fw) { + release_firmware(adev->psp.ta_fw); + adev->psp.ta_fw = NULL; + } + return 0; +} + +int psp_wait_for(struct psp_context *psp, uint32_t reg_index, + uint32_t reg_val, uint32_t mask, bool check_changed) +{ + uint32_t val; + int i; + struct amdgpu_device *adev = psp->adev; + + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32(reg_index); + if (check_changed) { + if (val != reg_val) + return 0; + } else { + if ((val & mask) == reg_val) + return 0; + } + udelay(1); + } + + return -ETIME; +} + +static int +psp_cmd_submit_buf(struct psp_context *psp, + struct amdgpu_firmware_info *ucode, + struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr) +{ + int ret; + int index; + int timeout = 2000; + + mutex_lock(&psp->mutex); + + memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE); + + memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp)); + + index = atomic_inc_return(&psp->fence_value); + ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr, + fence_mc_addr, index); + if (ret) { + atomic_dec(&psp->fence_value); + mutex_unlock(&psp->mutex); + return ret; + } + + while (*((unsigned int *)psp->fence_buf) != index) { + if (--timeout == 0) + break; + msleep(1); + } + + /* In some cases, psp response status is not 0 even there is no + * problem while the command is submitted. Some version of PSP FW + * doesn't write 0 to that field. + * So here we would like to only print a warning instead of an error + * during psp initialization to avoid breaking hw_init and it doesn't + * return -EINVAL. + */ + if (psp->cmd_buf_mem->resp.status || !timeout) { + if (ucode) + DRM_WARN("failed to load ucode id (%d) ", + ucode->ucode_id); + DRM_WARN("psp command failed and response status is (%d)\n", + psp->cmd_buf_mem->resp.status); + if (!timeout) { + mutex_unlock(&psp->mutex); + return -EINVAL; + } + } + + /* get xGMI session id from response buffer */ + cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id; + + if (ucode) { + ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo; + ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi; + } + mutex_unlock(&psp->mutex); + + return ret; +} + +static void psp_prep_tmr_cmd_buf(struct psp_context *psp, + struct psp_gfx_cmd_resp *cmd, + uint64_t tmr_mc, uint32_t size) +{ + if (psp_support_vmr_ring(psp)) + cmd->cmd_id = GFX_CMD_ID_SETUP_VMR; + else + cmd->cmd_id = GFX_CMD_ID_SETUP_TMR; + cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc); + cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc); + cmd->cmd.cmd_setup_tmr.buf_size = size; +} + +static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint64_t pri_buf_mc, uint32_t size) +{ + cmd->cmd_id = GFX_CMD_ID_LOAD_TOC; + cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc); + cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc); + cmd->cmd.cmd_load_toc.toc_size = size; +} + +/* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */ +static int psp_load_toc(struct psp_context *psp, + uint32_t *tmr_size) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + /* Copy toc to psp firmware private buffer */ + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size); + + psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + if (!ret) + *tmr_size = psp->cmd_buf_mem->resp.tmr_size; + kfree(cmd); + return ret; +} + +/* Set up Trusted Memory Region */ +static int psp_tmr_init(struct psp_context *psp) +{ + int ret; + int tmr_size; + + /* + * According to HW engineer, they prefer the TMR address be "naturally + * aligned" , e.g. the start address be an integer divide of TMR size. + * + * Note: this memory need be reserved till the driver + * uninitializes. + */ + tmr_size = PSP_TMR_SIZE; + + /* For ASICs support RLC autoload, psp will parse the toc + * and calculate the total size of TMR needed */ + if (psp->toc_start_addr && + psp->toc_bin_size && + psp->fw_pri_buf) { + ret = psp_load_toc(psp, &tmr_size); + if (ret) { + DRM_ERROR("Failed to load toc\n"); + return ret; + } + } + + ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf); + + return ret; +} + +static int psp_tmr_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, + amdgpu_bo_size(psp->tmr_bo)); + DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n", + amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + if (ret) + goto failed; + + kfree(cmd); + + return 0; + +failed: + kfree(cmd); + return ret; +} + +static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint64_t asd_mc, uint64_t asd_mc_shared, + uint32_t size, uint32_t shared_size) +{ + cmd->cmd_id = GFX_CMD_ID_LOAD_ASD; + cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc); + cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc); + cmd->cmd.cmd_load_ta.app_len = size; + + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size; +} + +static int psp_asd_init(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for shared ASD <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->asd_shared_bo, + &psp->asd_shared_mc_addr, + &psp->asd_shared_buf); + + return ret; +} + +static int psp_asd_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* If PSP version doesn't match ASD version, asd loading will be failed. + * add workaround to bypass it for sriov now. + * TODO: add version check to make it common + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size); + + psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr, + psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint32_t id, uint32_t value) +{ + cmd->cmd_id = GFX_CMD_ID_PROG_REG; + cmd->cmd.cmd_setup_reg_prog.reg_value = value; + cmd->cmd.cmd_setup_reg_prog.reg_id = id; +} + +int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg, + uint32_t value) +{ + struct psp_gfx_cmd_resp *cmd = NULL; + int ret = 0; + + if (reg >= PSP_REG_LAST) + return -EINVAL; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_reg_prog_cmd_buf(cmd, reg, value); + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + kfree(cmd); + return ret; +} + +static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared, + uint32_t xgmi_ta_size, uint32_t shared_size) +{ + cmd->cmd_id = GFX_CMD_ID_LOAD_TA; + cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc); + cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc); + cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size; + + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size; +} + +static int psp_xgmi_init_shared_buf(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for xgmi ta <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->xgmi_context.xgmi_shared_bo, + &psp->xgmi_context.xgmi_shared_mc_addr, + &psp->xgmi_context.xgmi_shared_buf); + + return ret; +} + +static int psp_xgmi_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the loading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size); + + psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, + psp->xgmi_context.xgmi_shared_mc_addr, + psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + if (!ret) { + psp->xgmi_context.initialized = 1; + psp->xgmi_context.session_id = cmd->resp.session_id; + } + + kfree(cmd); + + return ret; +} + +static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint32_t xgmi_session_id) +{ + cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA; + cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id; +} + +static int psp_xgmi_unload(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the unloading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint32_t ta_cmd_id, + uint32_t xgmi_session_id) +{ + cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD; + cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id; + cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id; + /* Note: cmd_invoke_cmd.buf is not used for now */ +} + +int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the loading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id, + psp->xgmi_context.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static int psp_xgmi_terminate(struct psp_context *psp) +{ + int ret; + + if (!psp->xgmi_context.initialized) + return 0; + + ret = psp_xgmi_unload(psp); + if (ret) + return ret; + + psp->xgmi_context.initialized = 0; + + /* free xgmi shared memory */ + amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo, + &psp->xgmi_context.xgmi_shared_mc_addr, + &psp->xgmi_context.xgmi_shared_buf); + + return 0; +} + +static int psp_xgmi_initialize(struct psp_context *psp) +{ + struct ta_xgmi_shared_memory *xgmi_cmd; + int ret; + + if (!psp->adev->psp.ta_fw) + return -ENOENT; + + if (!psp->xgmi_context.initialized) { + ret = psp_xgmi_init_shared_buf(psp); + if (ret) + return ret; + } + + /* Load XGMI TA */ + ret = psp_xgmi_load(psp); + if (ret) + return ret; + + /* Initialize XGMI session */ + xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf); + memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); + xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE; + + ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id); + + return ret; +} + +// ras begin +static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint64_t ras_ta_mc, uint64_t ras_mc_shared, + uint32_t ras_ta_size, uint32_t shared_size) +{ + cmd->cmd_id = GFX_CMD_ID_LOAD_TA; + cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc); + cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc); + cmd->cmd.cmd_load_ta.app_len = ras_ta_size; + + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared); + cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size; +} + +static int psp_ras_init_shared_buf(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for ras ta <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->ras.ras_shared_bo, + &psp->ras.ras_shared_mc_addr, + &psp->ras.ras_shared_buf); + + return ret; +} + +static int psp_ras_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the loading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size); + + psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, + psp->ras.ras_shared_mc_addr, + psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + if (!ret) { + psp->ras.ras_initialized = 1; + psp->ras.session_id = cmd->resp.session_id; + } + + kfree(cmd); + + return ret; +} + +static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint32_t ras_session_id) +{ + cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA; + cmd->cmd.cmd_unload_ta.session_id = ras_session_id; +} + +static int psp_ras_unload(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the unloading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd, + uint32_t ta_cmd_id, + uint32_t ras_session_id) +{ + cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD; + cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id; + cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id; + /* Note: cmd_invoke_cmd.buf is not used for now */ +} + +int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + /* + * TODO: bypass the loading in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id, + psp->ras.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +int psp_ras_enable_features(struct psp_context *psp, + union ta_ras_cmd_input *info, bool enable) +{ + struct ta_ras_shared_memory *ras_cmd; + int ret; + + if (!psp->ras.ras_initialized) + return -EINVAL; + + ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; + memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory)); + + if (enable) + ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES; + else + ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES; + + ras_cmd->ras_in_message = *info; + + ret = psp_ras_invoke(psp, ras_cmd->cmd_id); + if (ret) + return -EINVAL; + + return ras_cmd->ras_status; +} + +static int psp_ras_terminate(struct psp_context *psp) +{ + int ret; + + if (!psp->ras.ras_initialized) + return 0; + + ret = psp_ras_unload(psp); + if (ret) + return ret; + + psp->ras.ras_initialized = 0; + + /* free ras shared memory */ + amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo, + &psp->ras.ras_shared_mc_addr, + &psp->ras.ras_shared_buf); + + return 0; +} + +static int psp_ras_initialize(struct psp_context *psp) +{ + int ret; + + if (!psp->ras.ras_initialized) { + ret = psp_ras_init_shared_buf(psp); + if (ret) + return ret; + } + + ret = psp_ras_load(psp); + if (ret) + return ret; + + return 0; +} +// ras end + +static int psp_hw_start(struct psp_context *psp) +{ + struct amdgpu_device *adev = psp->adev; + int ret; + + if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) { + if (psp->kdb_bin_size && + (psp->funcs->bootloader_load_kdb != NULL)) { + ret = psp_bootloader_load_kdb(psp); + if (ret) { + DRM_ERROR("PSP load kdb failed!\n"); + return ret; + } + } + + ret = psp_bootloader_load_sysdrv(psp); + if (ret) { + DRM_ERROR("PSP load sysdrv failed!\n"); + return ret; + } + + ret = psp_bootloader_load_sos(psp); + if (ret) { + DRM_ERROR("PSP load sos failed!\n"); + return ret; + } + } + + ret = psp_ring_create(psp, PSP_RING_TYPE__KM); + if (ret) { + DRM_ERROR("PSP create ring failed!\n"); + return ret; + } + + ret = psp_tmr_init(psp); + if (ret) { + DRM_ERROR("PSP tmr init failed!\n"); + return ret; + } + + ret = psp_tmr_load(psp); + if (ret) { + DRM_ERROR("PSP load tmr failed!\n"); + return ret; + } + + ret = psp_asd_init(psp); + if (ret) { + DRM_ERROR("PSP asd init failed!\n"); + return ret; + } + + ret = psp_asd_load(psp); + if (ret) { + DRM_ERROR("PSP load asd failed!\n"); + return ret; + } + + if (adev->gmc.xgmi.num_physical_nodes > 1) { + ret = psp_xgmi_initialize(psp); + /* Warning the XGMI seesion initialize failure + * Instead of stop driver initialization + */ + if (ret) + dev_err(psp->adev->dev, + "XGMI: Failed to initialize XGMI session\n"); + } + + + if (psp->adev->psp.ta_fw) { + ret = psp_ras_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "RAS: Failed to initialize RAS\n"); + } + + return 0; +} + +static int psp_get_fw_type(struct amdgpu_firmware_info *ucode, + enum psp_gfx_fw_type *type) +{ + switch (ucode->ucode_id) { + case AMDGPU_UCODE_ID_SDMA0: + *type = GFX_FW_TYPE_SDMA0; + break; + case AMDGPU_UCODE_ID_SDMA1: + *type = GFX_FW_TYPE_SDMA1; + break; + case AMDGPU_UCODE_ID_CP_CE: + *type = GFX_FW_TYPE_CP_CE; + break; + case AMDGPU_UCODE_ID_CP_PFP: + *type = GFX_FW_TYPE_CP_PFP; + break; + case AMDGPU_UCODE_ID_CP_ME: + *type = GFX_FW_TYPE_CP_ME; + break; + case AMDGPU_UCODE_ID_CP_MEC1: + *type = GFX_FW_TYPE_CP_MEC; + break; + case AMDGPU_UCODE_ID_CP_MEC1_JT: + *type = GFX_FW_TYPE_CP_MEC_ME1; + break; + case AMDGPU_UCODE_ID_CP_MEC2: + *type = GFX_FW_TYPE_CP_MEC; + break; + case AMDGPU_UCODE_ID_CP_MEC2_JT: + *type = GFX_FW_TYPE_CP_MEC_ME2; + break; + case AMDGPU_UCODE_ID_RLC_G: + *type = GFX_FW_TYPE_RLC_G; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM; + break; + case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM: + *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM; + break; + case AMDGPU_UCODE_ID_SMC: + *type = GFX_FW_TYPE_SMU; + break; + case AMDGPU_UCODE_ID_UVD: + *type = GFX_FW_TYPE_UVD; + break; + case AMDGPU_UCODE_ID_UVD1: + *type = GFX_FW_TYPE_UVD1; + break; + case AMDGPU_UCODE_ID_VCE: + *type = GFX_FW_TYPE_VCE; + break; + case AMDGPU_UCODE_ID_VCN: + *type = GFX_FW_TYPE_VCN; + break; + case AMDGPU_UCODE_ID_DMCU_ERAM: + *type = GFX_FW_TYPE_DMCU_ERAM; + break; + case AMDGPU_UCODE_ID_DMCU_INTV: + *type = GFX_FW_TYPE_DMCU_ISR; + break; + case AMDGPU_UCODE_ID_VCN0_RAM: + *type = GFX_FW_TYPE_VCN0_RAM; + break; + case AMDGPU_UCODE_ID_VCN1_RAM: + *type = GFX_FW_TYPE_VCN1_RAM; + break; + case AMDGPU_UCODE_ID_MAXIMUM: + default: + return -EINVAL; + } + + return 0; +} + +static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, + struct psp_gfx_cmd_resp *cmd) +{ + int ret; + uint64_t fw_mem_mc_addr = ucode->mc_addr; + + memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); + + cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; + cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); + cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); + cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; + + ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); + if (ret) + DRM_ERROR("Unknown firmware type\n"); + + return ret; +} + +static int psp_execute_np_fw_load(struct psp_context *psp, + struct amdgpu_firmware_info *ucode) +{ + int ret = 0; + + ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd); + if (ret) + return ret; + + ret = psp_cmd_submit_buf(psp, ucode, psp->cmd, + psp->fence_buf_mc_addr); + + return ret; +} + +static int psp_np_fw_load(struct psp_context *psp) +{ + int i, ret; + struct amdgpu_firmware_info *ucode; + struct amdgpu_device* adev = psp->adev; + + if (psp->autoload_supported) { + ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC]; + if (!ucode->fw) + goto out; + + ret = psp_execute_np_fw_load(psp, ucode); + if (ret) + return ret; + } + +out: + for (i = 0; i < adev->firmware.max_ucodes; i++) { + ucode = &adev->firmware.ucode[i]; + if (!ucode->fw) + continue; + + if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC && + (psp_smu_reload_quirk(psp) || psp->autoload_supported)) + continue; + if (amdgpu_sriov_vf(adev) && + (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0 + || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 + || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G)) + /*skip ucode loading in SRIOV VF */ + continue; + if (psp->autoload_supported && + (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT || + ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)) + /* skip mec JT when autoload is enabled */ + continue; + + ret = psp_execute_np_fw_load(psp, ucode); + if (ret) + return ret; + + /* Start rlc autoload after psp recieved all the gfx firmware */ + if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) { + ret = psp_rlc_autoload(psp); + if (ret) { + DRM_ERROR("Failed to start rlc autoload\n"); + return ret; + } + } +#if 0 + /* check if firmware loaded sucessfully */ + if (!amdgpu_psp_check_fw_loading_status(adev, i)) + return -EINVAL; +#endif + } + + return 0; +} + +static int psp_load_fw(struct amdgpu_device *adev) +{ + int ret; + struct psp_context *psp = &adev->psp; + + if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) { + psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */ + goto skip_memalloc; + } + + psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!psp->cmd) + return -ENOMEM; + + /* this fw pri bo is not used under SRIOV */ + if (!amdgpu_sriov_vf(psp->adev)) { + ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG, + AMDGPU_GEM_DOMAIN_GTT, + &psp->fw_pri_bo, + &psp->fw_pri_mc_addr, + &psp->fw_pri_buf); + if (ret) + goto failed; + } + + ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &psp->fence_buf_bo, + &psp->fence_buf_mc_addr, + &psp->fence_buf); + if (ret) + goto failed; + + ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, + (void **)&psp->cmd_buf_mem); + if (ret) + goto failed; + + memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE); + + ret = psp_ring_init(psp, PSP_RING_TYPE__KM); + if (ret) { + DRM_ERROR("PSP ring init failed!\n"); + goto failed; + } + +skip_memalloc: + ret = psp_hw_start(psp); + if (ret) + goto failed; + + ret = psp_np_fw_load(psp); + if (ret) + goto failed; + + return 0; + +failed: + /* + * all cleanup jobs (xgmi terminate, ras terminate, + * ring destroy, cmd/fence/fw buffers destory, + * psp->cmd destory) are delayed to psp_hw_fini + */ + return ret; +} + +static int psp_hw_init(void *handle) +{ + int ret; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + mutex_lock(&adev->firmware.mutex); + /* + * This sequence is just used on hw_init only once, no need on + * resume. + */ + ret = amdgpu_ucode_init_bo(adev); + if (ret) + goto failed; + + ret = psp_load_fw(adev); + if (ret) { + DRM_ERROR("PSP firmware loading failed\n"); + goto failed; + } + + mutex_unlock(&adev->firmware.mutex); + return 0; + +failed: + adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT; + mutex_unlock(&adev->firmware.mutex); + return -EINVAL; +} + +static int psp_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + + if (adev->gmc.xgmi.num_physical_nodes > 1 && + psp->xgmi_context.initialized == 1) + psp_xgmi_terminate(psp); + + if (psp->adev->psp.ta_fw) + psp_ras_terminate(psp); + + psp_ring_destroy(psp, PSP_RING_TYPE__KM); + + amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf); + amdgpu_bo_free_kernel(&psp->fw_pri_bo, + &psp->fw_pri_mc_addr, &psp->fw_pri_buf); + amdgpu_bo_free_kernel(&psp->fence_buf_bo, + &psp->fence_buf_mc_addr, &psp->fence_buf); + amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr, + &psp->asd_shared_buf); + amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, + (void **)&psp->cmd_buf_mem); + + kfree(psp->cmd); + psp->cmd = NULL; + + return 0; +} + +static int psp_suspend(void *handle) +{ + int ret; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + + if (adev->gmc.xgmi.num_physical_nodes > 1 && + psp->xgmi_context.initialized == 1) { + ret = psp_xgmi_terminate(psp); + if (ret) { + DRM_ERROR("Failed to terminate xgmi ta\n"); + return ret; + } + } + + if (psp->adev->psp.ta_fw) { + ret = psp_ras_terminate(psp); + if (ret) { + DRM_ERROR("Failed to terminate ras ta\n"); + return ret; + } + } + + ret = psp_ring_stop(psp, PSP_RING_TYPE__KM); + if (ret) { + DRM_ERROR("PSP ring stop failed\n"); + return ret; + } + + return 0; +} + +static int psp_resume(void *handle) +{ + int ret; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct psp_context *psp = &adev->psp; + + DRM_INFO("PSP is resuming...\n"); + + mutex_lock(&adev->firmware.mutex); + + ret = psp_hw_start(psp); + if (ret) + goto failed; + + ret = psp_np_fw_load(psp); + if (ret) + goto failed; + + mutex_unlock(&adev->firmware.mutex); + + return 0; + +failed: + DRM_ERROR("PSP resume failed\n"); + mutex_unlock(&adev->firmware.mutex); + return ret; +} + +int psp_gpu_reset(struct amdgpu_device *adev) +{ + int ret; + + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) + return 0; + + mutex_lock(&adev->psp.mutex); + ret = psp_mode1_reset(&adev->psp); + mutex_unlock(&adev->psp.mutex); + + return ret; +} + +int psp_rlc_autoload_start(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC; + + ret = psp_cmd_submit_buf(psp, NULL, cmd, + psp->fence_buf_mc_addr); + kfree(cmd); + return ret; +} + +int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx, + uint64_t cmd_gpu_addr, int cmd_size) +{ + struct amdgpu_firmware_info ucode = {0}; + + ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM : + AMDGPU_UCODE_ID_VCN0_RAM; + ucode.mc_addr = cmd_gpu_addr; + ucode.ucode_size = cmd_size; + + return psp_execute_np_fw_load(&adev->psp, &ucode); +} + +static bool psp_check_fw_loading_status(struct amdgpu_device *adev, + enum AMDGPU_UCODE_ID ucode_type) +{ + struct amdgpu_firmware_info *ucode = NULL; + + if (!adev->firmware.fw_size) + return false; + + ucode = &adev->firmware.ucode[ucode_type]; + if (!ucode->fw || !ucode->ucode_size) + return false; + + return psp_compare_sram_data(&adev->psp, ucode, ucode_type); +} + +static int psp_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int psp_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +const struct amd_ip_funcs psp_ip_funcs = { + .name = "psp", + .early_init = psp_early_init, + .late_init = NULL, + .sw_init = psp_sw_init, + .sw_fini = psp_sw_fini, + .hw_init = psp_hw_init, + .hw_fini = psp_hw_fini, + .suspend = psp_suspend, + .resume = psp_resume, + .is_idle = NULL, + .check_soft_reset = NULL, + .wait_for_idle = NULL, + .soft_reset = NULL, + .set_clockgating_state = psp_set_clockgating_state, + .set_powergating_state = psp_set_powergating_state, +}; + +static const struct amdgpu_psp_funcs psp_funcs = { + .check_fw_loading_status = psp_check_fw_loading_status, +}; + +static void psp_set_funcs(struct amdgpu_device *adev) +{ + if (NULL == adev->firmware.funcs) + adev->firmware.funcs = &psp_funcs; +} + +const struct amdgpu_ip_block_version psp_v3_1_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_PSP, + .major = 3, + .minor = 1, + .rev = 0, + .funcs = &psp_ip_funcs, +}; + +const struct amdgpu_ip_block_version psp_v10_0_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_PSP, + .major = 10, + .minor = 0, + .rev = 0, + .funcs = &psp_ip_funcs, +}; + +const struct amdgpu_ip_block_version psp_v11_0_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_PSP, + .major = 11, + .minor = 0, + .rev = 0, + .funcs = &psp_ip_funcs, +};
diff --git a/amdgpu/amdgpu_psp.h b/amdgpu/amdgpu_psp.h new file mode 100644 index 0000000..e0fc2a7 --- /dev/null +++ b/amdgpu/amdgpu_psp.h
@@ -0,0 +1,289 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Huang Rui + * + */ +#ifndef __AMDGPU_PSP_H__ +#define __AMDGPU_PSP_H__ + +#include "amdgpu.h" +#include "psp_gfx_if.h" +#include "ta_xgmi_if.h" +#include "ta_ras_if.h" + +#define PSP_FENCE_BUFFER_SIZE 0x1000 +#define PSP_CMD_BUFFER_SIZE 0x1000 +#define PSP_ASD_SHARED_MEM_SIZE 0x4000 +#define PSP_XGMI_SHARED_MEM_SIZE 0x4000 +#define PSP_RAS_SHARED_MEM_SIZE 0x4000 +#define PSP_1_MEG 0x100000 +#define PSP_TMR_SIZE 0x400000 + +struct psp_context; +struct psp_xgmi_node_info; +struct psp_xgmi_topology_info; + +enum psp_bootloader_cmd { + PSP_BL__LOAD_SYSDRV = 0x10000, + PSP_BL__LOAD_SOSDRV = 0x20000, + PSP_BL__LOAD_KEY_DATABASE = 0x80000, +}; + +enum psp_ring_type +{ + PSP_RING_TYPE__INVALID = 0, + /* + * These values map to the way the PSP kernel identifies the + * rings. + */ + PSP_RING_TYPE__UM = 1, /* User mode ring (formerly called RBI) */ + PSP_RING_TYPE__KM = 2 /* Kernel mode ring (formerly called GPCOM) */ +}; + +struct psp_ring +{ + enum psp_ring_type ring_type; + struct psp_gfx_rb_frame *ring_mem; + uint64_t ring_mem_mc_addr; + void *ring_mem_handle; + uint32_t ring_size; +}; + +/* More registers may will be supported */ +enum psp_reg_prog_id { + PSP_REG_IH_RB_CNTL = 0, /* register IH_RB_CNTL */ + PSP_REG_IH_RB_CNTL_RING1 = 1, /* register IH_RB_CNTL_RING1 */ + PSP_REG_IH_RB_CNTL_RING2 = 2, /* register IH_RB_CNTL_RING2 */ + PSP_REG_LAST +}; + +struct psp_funcs +{ + int (*init_microcode)(struct psp_context *psp); + int (*bootloader_load_kdb)(struct psp_context *psp); + int (*bootloader_load_sysdrv)(struct psp_context *psp); + int (*bootloader_load_sos)(struct psp_context *psp); + int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type); + int (*ring_create)(struct psp_context *psp, + enum psp_ring_type ring_type); + int (*ring_stop)(struct psp_context *psp, + enum psp_ring_type ring_type); + int (*ring_destroy)(struct psp_context *psp, + enum psp_ring_type ring_type); + int (*cmd_submit)(struct psp_context *psp, + struct amdgpu_firmware_info *ucode, + uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr, + int index); + bool (*compare_sram_data)(struct psp_context *psp, + struct amdgpu_firmware_info *ucode, + enum AMDGPU_UCODE_ID ucode_type); + bool (*smu_reload_quirk)(struct psp_context *psp); + int (*mode1_reset)(struct psp_context *psp); + int (*xgmi_get_node_id)(struct psp_context *psp, uint64_t *node_id); + int (*xgmi_get_hive_id)(struct psp_context *psp, uint64_t *hive_id); + int (*xgmi_get_topology_info)(struct psp_context *psp, int number_devices, + struct psp_xgmi_topology_info *topology); + int (*xgmi_set_topology_info)(struct psp_context *psp, int number_devices, + struct psp_xgmi_topology_info *topology); + bool (*support_vmr_ring)(struct psp_context *psp); + int (*ras_trigger_error)(struct psp_context *psp, + struct ta_ras_trigger_error_input *info); + int (*ras_cure_posion)(struct psp_context *psp, uint64_t *mode_ptr); + int (*rlc_autoload_start)(struct psp_context *psp); +}; + +#define AMDGPU_XGMI_MAX_CONNECTED_NODES 64 +struct psp_xgmi_node_info { + uint64_t node_id; + uint8_t num_hops; + uint8_t is_sharing_enabled; + enum ta_xgmi_assigned_sdma_engine sdma_engine; +}; + +struct psp_xgmi_topology_info { + uint32_t num_nodes; + struct psp_xgmi_node_info nodes[AMDGPU_XGMI_MAX_CONNECTED_NODES]; +}; + +struct psp_xgmi_context { + uint8_t initialized; + uint32_t session_id; + struct amdgpu_bo *xgmi_shared_bo; + uint64_t xgmi_shared_mc_addr; + void *xgmi_shared_buf; + struct psp_xgmi_topology_info top_info; +}; + +struct psp_ras_context { + /*ras fw*/ + bool ras_initialized; + uint32_t session_id; + struct amdgpu_bo *ras_shared_bo; + uint64_t ras_shared_mc_addr; + void *ras_shared_buf; + struct amdgpu_ras *ras; +}; + +struct psp_context +{ + struct amdgpu_device *adev; + struct psp_ring km_ring; + struct psp_gfx_cmd_resp *cmd; + + const struct psp_funcs *funcs; + + /* firmware buffer */ + struct amdgpu_bo *fw_pri_bo; + uint64_t fw_pri_mc_addr; + void *fw_pri_buf; + + /* sos firmware */ + const struct firmware *sos_fw; + uint32_t sos_fw_version; + uint32_t sos_feature_version; + uint32_t sys_bin_size; + uint32_t sos_bin_size; + uint32_t toc_bin_size; + uint32_t kdb_bin_size; + uint8_t *sys_start_addr; + uint8_t *sos_start_addr; + uint8_t *toc_start_addr; + uint8_t *kdb_start_addr; + + /* tmr buffer */ + struct amdgpu_bo *tmr_bo; + uint64_t tmr_mc_addr; + void *tmr_buf; + + /* asd firmware and buffer */ + const struct firmware *asd_fw; + uint32_t asd_fw_version; + uint32_t asd_feature_version; + uint32_t asd_ucode_size; + uint8_t *asd_start_addr; + struct amdgpu_bo *asd_shared_bo; + uint64_t asd_shared_mc_addr; + void *asd_shared_buf; + + /* fence buffer */ + struct amdgpu_bo *fence_buf_bo; + uint64_t fence_buf_mc_addr; + void *fence_buf; + + /* cmd buffer */ + struct amdgpu_bo *cmd_buf_bo; + uint64_t cmd_buf_mc_addr; + struct psp_gfx_cmd_resp *cmd_buf_mem; + + /* fence value associated with cmd buffer */ + atomic_t fence_value; + /* flag to mark whether gfx fw autoload is supported or not */ + bool autoload_supported; + + /* xgmi ta firmware and buffer */ + const struct firmware *ta_fw; + uint32_t ta_fw_version; + uint32_t ta_xgmi_ucode_version; + uint32_t ta_xgmi_ucode_size; + uint8_t *ta_xgmi_start_addr; + uint32_t ta_ras_ucode_version; + uint32_t ta_ras_ucode_size; + uint8_t *ta_ras_start_addr; + struct psp_xgmi_context xgmi_context; + struct psp_ras_context ras; + struct mutex mutex; +}; + +struct amdgpu_psp_funcs { + bool (*check_fw_loading_status)(struct amdgpu_device *adev, + enum AMDGPU_UCODE_ID); +}; + + +#define psp_ring_init(psp, type) (psp)->funcs->ring_init((psp), (type)) +#define psp_ring_create(psp, type) (psp)->funcs->ring_create((psp), (type)) +#define psp_ring_stop(psp, type) (psp)->funcs->ring_stop((psp), (type)) +#define psp_ring_destroy(psp, type) ((psp)->funcs->ring_destroy((psp), (type))) +#define psp_cmd_submit(psp, ucode, cmd_mc, fence_mc, index) \ + (psp)->funcs->cmd_submit((psp), (ucode), (cmd_mc), (fence_mc), (index)) +#define psp_compare_sram_data(psp, ucode, type) \ + (psp)->funcs->compare_sram_data((psp), (ucode), (type)) +#define psp_init_microcode(psp) \ + ((psp)->funcs->init_microcode ? (psp)->funcs->init_microcode((psp)) : 0) +#define psp_bootloader_load_kdb(psp) \ + ((psp)->funcs->bootloader_load_kdb ? (psp)->funcs->bootloader_load_kdb((psp)) : 0) +#define psp_bootloader_load_sysdrv(psp) \ + ((psp)->funcs->bootloader_load_sysdrv ? (psp)->funcs->bootloader_load_sysdrv((psp)) : 0) +#define psp_bootloader_load_sos(psp) \ + ((psp)->funcs->bootloader_load_sos ? (psp)->funcs->bootloader_load_sos((psp)) : 0) +#define psp_smu_reload_quirk(psp) \ + ((psp)->funcs->smu_reload_quirk ? (psp)->funcs->smu_reload_quirk((psp)) : false) +#define psp_support_vmr_ring(psp) \ + ((psp)->funcs->support_vmr_ring ? (psp)->funcs->support_vmr_ring((psp)) : false) +#define psp_mode1_reset(psp) \ + ((psp)->funcs->mode1_reset ? (psp)->funcs->mode1_reset((psp)) : false) +#define psp_xgmi_get_node_id(psp, node_id) \ + ((psp)->funcs->xgmi_get_node_id ? (psp)->funcs->xgmi_get_node_id((psp), (node_id)) : -EINVAL) +#define psp_xgmi_get_hive_id(psp, hive_id) \ + ((psp)->funcs->xgmi_get_hive_id ? (psp)->funcs->xgmi_get_hive_id((psp), (hive_id)) : -EINVAL) +#define psp_xgmi_get_topology_info(psp, num_device, topology) \ + ((psp)->funcs->xgmi_get_topology_info ? \ + (psp)->funcs->xgmi_get_topology_info((psp), (num_device), (topology)) : -EINVAL) +#define psp_xgmi_set_topology_info(psp, num_device, topology) \ + ((psp)->funcs->xgmi_set_topology_info ? \ + (psp)->funcs->xgmi_set_topology_info((psp), (num_device), (topology)) : -EINVAL) +#define psp_rlc_autoload(psp) \ + ((psp)->funcs->rlc_autoload_start ? (psp)->funcs->rlc_autoload_start((psp)) : 0) + +#define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i)) + +#define psp_ras_trigger_error(psp, info) \ + ((psp)->funcs->ras_trigger_error ? \ + (psp)->funcs->ras_trigger_error((psp), (info)) : -EINVAL) +#define psp_ras_cure_posion(psp, addr) \ + ((psp)->funcs->ras_cure_posion ? \ + (psp)->funcs->ras_cure_posion(psp, (addr)) : -EINVAL) + +extern const struct amd_ip_funcs psp_ip_funcs; + +extern const struct amdgpu_ip_block_version psp_v3_1_ip_block; +extern int psp_wait_for(struct psp_context *psp, uint32_t reg_index, + uint32_t field_val, uint32_t mask, bool check_changed); + +extern const struct amdgpu_ip_block_version psp_v10_0_ip_block; + +int psp_gpu_reset(struct amdgpu_device *adev); +int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx, + uint64_t cmd_gpu_addr, int cmd_size); + +int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id); + +int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id); +int psp_ras_enable_features(struct psp_context *psp, + union ta_ras_cmd_input *info, bool enable); + +int psp_rlc_autoload_start(struct psp_context *psp); + +extern const struct amdgpu_ip_block_version psp_v11_0_ip_block; +int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg, + uint32_t value); +#endif
diff --git a/amdgpu/amdgpu_ras.c b/amdgpu/amdgpu_ras.c new file mode 100644 index 0000000..fac7aa2 --- /dev/null +++ b/amdgpu/amdgpu_ras.c
@@ -0,0 +1,1685 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include <linux/debugfs.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/uaccess.h> + +#include "amdgpu.h" +#include "amdgpu_ras.h" +#include "amdgpu_atomfirmware.h" + +struct ras_ih_data { + /* interrupt bottom half */ + struct work_struct ih_work; + int inuse; + /* IP callback */ + ras_ih_cb cb; + /* full of entries */ + unsigned char *ring; + unsigned int ring_size; + unsigned int element_size; + unsigned int aligned_element_size; + unsigned int rptr; + unsigned int wptr; +}; + +struct ras_fs_data { + char sysfs_name[32]; + char debugfs_name[32]; +}; + +struct ras_err_data { + unsigned long ue_count; + unsigned long ce_count; +}; + +struct ras_err_handler_data { + /* point to bad pages array */ + struct { + unsigned long bp; + struct amdgpu_bo *bo; + } *bps; + /* the count of entries */ + int count; + /* the space can place new entries */ + int space_left; + /* last reserved entry's index + 1 */ + int last_reserved; +}; + +struct ras_manager { + struct ras_common_if head; + /* reference count */ + int use; + /* ras block link */ + struct list_head node; + /* the device */ + struct amdgpu_device *adev; + /* debugfs */ + struct dentry *ent; + /* sysfs */ + struct device_attribute sysfs_attr; + int attr_inuse; + + /* fs node name */ + struct ras_fs_data fs_data; + + /* IH data */ + struct ras_ih_data ih_data; + + struct ras_err_data err_data; +}; + +struct ras_badpage { + unsigned int bp; + unsigned int size; + unsigned int flags; +}; + +const char *ras_error_string[] = { + "none", + "parity", + "single_correctable", + "multi_uncorrectable", + "poison", +}; + +const char *ras_block_string[] = { + "umc", + "sdma", + "gfx", + "mmhub", + "athub", + "pcie_bif", + "hdp", + "xgmi_wafl", + "df", + "smn", + "sem", + "mp0", + "mp1", + "fuse", +}; + +#define ras_err_str(i) (ras_error_string[ffs(i)]) +#define ras_block_str(i) (ras_block_string[i]) + +#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS 1 +#define AMDGPU_RAS_FLAG_INIT_NEED_RESET 2 +#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) + +static int amdgpu_ras_reserve_vram(struct amdgpu_device *adev, + uint64_t offset, uint64_t size, + struct amdgpu_bo **bo_ptr); +static int amdgpu_ras_release_vram(struct amdgpu_device *adev, + struct amdgpu_bo **bo_ptr); + +static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; + struct ras_query_if info = { + .head = obj->head, + }; + ssize_t s; + char val[128]; + + if (amdgpu_ras_error_query(obj->adev, &info)) + return -EINVAL; + + s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", + "ue", info.ue_count, + "ce", info.ce_count); + if (*pos >= s) + return 0; + + s -= *pos; + s = min_t(u64, s, size); + + + if (copy_to_user(buf, &val[*pos], s)) + return -EINVAL; + + *pos += s; + + return s; +} + +static const struct file_operations amdgpu_ras_debugfs_ops = { + .owner = THIS_MODULE, + .read = amdgpu_ras_debugfs_read, + .write = NULL, + .llseek = default_llseek +}; + +static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { + *block_id = i; + if (strcmp(name, ras_block_str(i)) == 0) + return 0; + } + return -EINVAL; +} + +static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, + const char __user *buf, size_t size, + loff_t *pos, struct ras_debug_if *data) +{ + ssize_t s = min_t(u64, 64, size); + char str[65]; + char block_name[33]; + char err[9] = "ue"; + int op = -1; + int block_id; + u64 address, value; + + if (*pos) + return -EINVAL; + *pos = size; + + memset(str, 0, sizeof(str)); + memset(data, 0, sizeof(*data)); + + if (copy_from_user(str, buf, s)) + return -EINVAL; + + if (sscanf(str, "disable %32s", block_name) == 1) + op = 0; + else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) + op = 1; + else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) + op = 2; + else if (str[0] && str[1] && str[2] && str[3]) + /* ascii string, but commands are not matched. */ + return -EINVAL; + + if (op != -1) { + if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) + return -EINVAL; + + data->head.block = block_id; + data->head.type = memcmp("ue", err, 2) == 0 ? + AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE : + AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; + data->op = op; + + if (op == 2) { + if (sscanf(str, "%*s %*s %*s %llu %llu", + &address, &value) != 2) + if (sscanf(str, "%*s %*s %*s 0x%llx 0x%llx", + &address, &value) != 2) + return -EINVAL; + data->inject.address = address; + data->inject.value = value; + } + } else { + if (size < sizeof(*data)) + return -EINVAL; + + if (copy_from_user(data, buf, sizeof(*data))) + return -EINVAL; + } + + return 0; +} +/** + * DOC: AMDGPU RAS debugfs control interface + * + * It accepts struct ras_debug_if who has two members. + * + * First member: ras_debug_if::head or ras_debug_if::inject. + * + * head is used to indicate which IP block will be under control. + * + * head has four members, they are block, type, sub_block_index, name. + * block: which IP will be under control. + * type: what kind of error will be enabled/disabled/injected. + * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. + * name: the name of IP. + * + * inject has two more members than head, they are address, value. + * As their names indicate, inject operation will write the + * value to the address. + * + * Second member: struct ras_debug_if::op. + * It has three kinds of operations. + * 0: disable RAS on the block. Take ::head as its data. + * 1: enable RAS on the block. Take ::head as its data. + * 2: inject errors on the block. Take ::inject as its data. + * + * How to use the interface? + * programs: + * copy the struct ras_debug_if in your codes and initialize it. + * write the struct to the control node. + * + * bash: + * echo op block [error [address value]] > .../ras/ras_ctrl + * op: disable, enable, inject + * disable: only block is needed + * enable: block and error are needed + * inject: error, address, value are needed + * block: umc, smda, gfx, ......... + * see ras_block_string[] for details + * error: ue, ce + * ue: multi_uncorrectable + * ce: single_correctable + * + * here are some examples for bash commands, + * echo inject umc ue 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl + * echo inject umc ce 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl + * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl + * + * How to check the result? + * + * For disable/enable, please check ras features at + * /sys/class/drm/card[0/1/2...]/device/ras/features + * + * For inject, please check corresponding err count at + * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count + * + * NOTE: operation is only allowed on blocks which are supported. + * Please check ras mask at /sys/module/amdgpu/parameters/ras_mask + */ +static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; + struct ras_debug_if data; + struct amdgpu_bo *bo; + int ret = 0; + + ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); + if (ret) + return -EINVAL; + + if (!amdgpu_ras_is_supported(adev, data.head.block)) + return -EINVAL; + + switch (data.op) { + case 0: + ret = amdgpu_ras_feature_enable(adev, &data.head, 0); + break; + case 1: + ret = amdgpu_ras_feature_enable(adev, &data.head, 1); + break; + case 2: + ret = amdgpu_ras_reserve_vram(adev, + data.inject.address, PAGE_SIZE, &bo); + if (ret) { + /* address was offset, now it is absolute.*/ + data.inject.address += adev->gmc.vram_start; + if (data.inject.address > adev->gmc.vram_end) + break; + } else + data.inject.address = amdgpu_bo_gpu_offset(bo); + ret = amdgpu_ras_error_inject(adev, &data.inject); + amdgpu_ras_release_vram(adev, &bo); + break; + default: + ret = -EINVAL; + break; + }; + + if (ret) + return -EINVAL; + + return size; +} + +static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { + .owner = THIS_MODULE, + .read = NULL, + .write = amdgpu_ras_debugfs_ctrl_write, + .llseek = default_llseek +}; + +static ssize_t amdgpu_ras_sysfs_read(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); + struct ras_query_if info = { + .head = obj->head, + }; + + if (amdgpu_ras_error_query(obj->adev, &info)) + return -EINVAL; + + return snprintf(buf, PAGE_SIZE, "%s: %lu\n%s: %lu\n", + "ue", info.ue_count, + "ce", info.ce_count); +} + +/* obj begin */ + +#define get_obj(obj) do { (obj)->use++; } while (0) +#define alive_obj(obj) ((obj)->use) + +static inline void put_obj(struct ras_manager *obj) +{ + if (obj && --obj->use == 0) + list_del(&obj->node); + if (obj && obj->use < 0) { + DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", obj->head.name); + } +} + +/* make one obj and return it. */ +static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj; + + if (!con) + return NULL; + + if (head->block >= AMDGPU_RAS_BLOCK_COUNT) + return NULL; + + obj = &con->objs[head->block]; + /* already exist. return obj? */ + if (alive_obj(obj)) + return NULL; + + obj->head = *head; + obj->adev = adev; + list_add(&obj->node, &con->head); + get_obj(obj); + + return obj; +} + +/* return an obj equal to head, or the first when head is NULL */ +static struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj; + int i; + + if (!con) + return NULL; + + if (head) { + if (head->block >= AMDGPU_RAS_BLOCK_COUNT) + return NULL; + + obj = &con->objs[head->block]; + + if (alive_obj(obj)) { + WARN_ON(head->block != obj->head.block); + return obj; + } + } else { + for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { + obj = &con->objs[i]; + if (alive_obj(obj)) { + WARN_ON(i != obj->head.block); + return obj; + } + } + } + + return NULL; +} +/* obj end */ + +/* feature ctl begin */ +static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + return con->hw_supported & BIT(head->block); +} + +static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + return con->features & BIT(head->block); +} + +/* + * if obj is not created, then create one. + * set feature enable flag. + */ +static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, + struct ras_common_if *head, int enable) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); + + /* If hardware does not support ras, then do not create obj. + * But if hardware support ras, we can create the obj. + * Ras framework checks con->hw_supported to see if it need do + * corresponding initialization. + * IP checks con->support to see if it need disable ras. + */ + if (!amdgpu_ras_is_feature_allowed(adev, head)) + return 0; + if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head))) + return 0; + + if (enable) { + if (!obj) { + obj = amdgpu_ras_create_obj(adev, head); + if (!obj) + return -EINVAL; + } else { + /* In case we create obj somewhere else */ + get_obj(obj); + } + con->features |= BIT(head->block); + } else { + if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { + con->features &= ~BIT(head->block); + put_obj(obj); + } + } + + return 0; +} + +/* wrapper of psp_ras_enable_features */ +int amdgpu_ras_feature_enable(struct amdgpu_device *adev, + struct ras_common_if *head, bool enable) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + union ta_ras_cmd_input info; + int ret; + + if (!con) + return -EINVAL; + + if (!enable) { + info.disable_features = (struct ta_ras_disable_features_input) { + .block_id = amdgpu_ras_block_to_ta(head->block), + .error_type = amdgpu_ras_error_to_ta(head->type), + }; + } else { + info.enable_features = (struct ta_ras_enable_features_input) { + .block_id = amdgpu_ras_block_to_ta(head->block), + .error_type = amdgpu_ras_error_to_ta(head->type), + }; + } + + /* Do not enable if it is not allowed. */ + WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head)); + /* Are we alerady in that state we are going to set? */ + if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head))) + return 0; + + ret = psp_ras_enable_features(&adev->psp, &info, enable); + if (ret) { + DRM_ERROR("RAS ERROR: %s %s feature failed ret %d\n", + enable ? "enable":"disable", + ras_block_str(head->block), + ret); + if (ret == TA_RAS_STATUS__RESET_NEEDED) + return -EAGAIN; + return -EINVAL; + } + + /* setup the obj */ + __amdgpu_ras_feature_enable(adev, head, enable); + + return 0; +} + +/* Only used in device probe stage and called only once. */ +int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, + struct ras_common_if *head, bool enable) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + int ret; + + if (!con) + return -EINVAL; + + if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { + if (enable) { + /* There is no harm to issue a ras TA cmd regardless of + * the currecnt ras state. + * If current state == target state, it will do nothing + * But sometimes it requests driver to reset and repost + * with error code -EAGAIN. + */ + ret = amdgpu_ras_feature_enable(adev, head, 1); + /* With old ras TA, we might fail to enable ras. + * Log it and just setup the object. + * TODO need remove this WA in the future. + */ + if (ret == -EINVAL) { + ret = __amdgpu_ras_feature_enable(adev, head, 1); + if (!ret) + DRM_INFO("RAS INFO: %s setup object\n", + ras_block_str(head->block)); + } + } else { + /* setup the object then issue a ras TA disable cmd.*/ + ret = __amdgpu_ras_feature_enable(adev, head, 1); + if (ret) + return ret; + + ret = amdgpu_ras_feature_enable(adev, head, 0); + } + } else + ret = amdgpu_ras_feature_enable(adev, head, enable); + + return ret; +} + +static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, + bool bypass) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &con->head, node) { + /* bypass psp. + * aka just release the obj and corresponding flags + */ + if (bypass) { + if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) + break; + } else { + if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) + break; + } + } + + return con->features; +} + +static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, + bool bypass) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + int ras_block_count = AMDGPU_RAS_BLOCK_COUNT; + int i; + const enum amdgpu_ras_error_type default_ras_type = + AMDGPU_RAS_ERROR__NONE; + + for (i = 0; i < ras_block_count; i++) { + struct ras_common_if head = { + .block = i, + .type = default_ras_type, + .sub_block_index = 0, + }; + strcpy(head.name, ras_block_str(i)); + if (bypass) { + /* + * bypass psp. vbios enable ras for us. + * so just create the obj + */ + if (__amdgpu_ras_feature_enable(adev, &head, 1)) + break; + } else { + if (amdgpu_ras_feature_enable(adev, &head, 1)) + break; + } + } + + return con->features; +} +/* feature ctl end */ + +/* query/inject/cure begin */ +int amdgpu_ras_error_query(struct amdgpu_device *adev, + struct ras_query_if *info) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); + + if (!obj) + return -EINVAL; + /* TODO might read the register to read the count */ + + info->ue_count = obj->err_data.ue_count; + info->ce_count = obj->err_data.ce_count; + + return 0; +} + +/* wrapper of psp_ras_trigger_error */ +int amdgpu_ras_error_inject(struct amdgpu_device *adev, + struct ras_inject_if *info) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); + struct ta_ras_trigger_error_input block_info = { + .block_id = amdgpu_ras_block_to_ta(info->head.block), + .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), + .sub_block_index = info->head.sub_block_index, + .address = info->address, + .value = info->value, + }; + int ret = 0; + + if (!obj) + return -EINVAL; + + if (block_info.block_id != TA_RAS_BLOCK__UMC) { + DRM_INFO("%s error injection is not supported yet\n", + ras_block_str(info->head.block)); + return -EINVAL; + } + + ret = psp_ras_trigger_error(&adev->psp, &block_info); + if (ret) + DRM_ERROR("RAS ERROR: inject %s error failed ret %d\n", + ras_block_str(info->head.block), + ret); + + return ret; +} + +int amdgpu_ras_error_cure(struct amdgpu_device *adev, + struct ras_cure_if *info) +{ + /* psp fw has no cure interface for now. */ + return 0; +} + +/* get the total error counts on all IPs */ +int amdgpu_ras_query_error_count(struct amdgpu_device *adev, + bool is_ce) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj; + struct ras_err_data data = {0, 0}; + + if (!con) + return -EINVAL; + + list_for_each_entry(obj, &con->head, node) { + struct ras_query_if info = { + .head = obj->head, + }; + + if (amdgpu_ras_error_query(adev, &info)) + return -EINVAL; + + data.ce_count += info.ce_count; + data.ue_count += info.ue_count; + } + + return is_ce ? data.ce_count : data.ue_count; +} +/* query/inject/cure end */ + + +/* sysfs begin */ + +static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, + struct ras_badpage **bps, unsigned int *count); + +static char *amdgpu_ras_badpage_flags_str(unsigned int flags) +{ + switch (flags) { + case 0: + return "R"; + case 1: + return "P"; + case 2: + default: + return "F"; + }; +} + +/* + * DOC: ras sysfs gpu_vram_bad_pages interface + * + * It allows user to read the bad pages of vram on the gpu through + * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages + * + * It outputs multiple lines, and each line stands for one gpu page. + * + * The format of one line is below, + * gpu pfn : gpu page size : flags + * + * gpu pfn and gpu page size are printed in hex format. + * flags can be one of below character, + * R: reserved, this gpu page is reserved and not able to use. + * P: pending for reserve, this gpu page is marked as bad, will be reserved + * in next window of page_reserve. + * F: unable to reserve. this gpu page can't be reserved due to some reasons. + * + * examples: + * 0x00000001 : 0x00001000 : R + * 0x00000002 : 0x00001000 : P + */ + +static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, + struct kobject *kobj, struct bin_attribute *attr, + char *buf, loff_t ppos, size_t count) +{ + struct amdgpu_ras *con = + container_of(attr, struct amdgpu_ras, badpages_attr); + struct amdgpu_device *adev = con->adev; + const unsigned int element_size = + sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; + unsigned int start = div64_ul(ppos + element_size - 1, element_size); + unsigned int end = div64_ul(ppos + count - 1, element_size); + ssize_t s = 0; + struct ras_badpage *bps = NULL; + unsigned int bps_count = 0; + + memset(buf, 0, count); + + if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) + return 0; + + for (; start < end && start < bps_count; start++) + s += scnprintf(&buf[s], element_size + 1, + "0x%08x : 0x%08x : %1s\n", + bps[start].bp, + bps[start].size, + amdgpu_ras_badpage_flags_str(bps[start].flags)); + + kfree(bps); + + return s; +} + +static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct amdgpu_ras *con = + container_of(attr, struct amdgpu_ras, features_attr); + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + struct ras_common_if head; + int ras_block_count = AMDGPU_RAS_BLOCK_COUNT; + int i; + ssize_t s; + struct ras_manager *obj; + + s = scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features); + + for (i = 0; i < ras_block_count; i++) { + head.block = i; + + if (amdgpu_ras_is_feature_enabled(adev, &head)) { + obj = amdgpu_ras_find_obj(adev, &head); + s += scnprintf(&buf[s], PAGE_SIZE - s, + "%s: %s\n", + ras_block_str(i), + ras_err_str(obj->head.type)); + } else + s += scnprintf(&buf[s], PAGE_SIZE - s, + "%s: disabled\n", + ras_block_str(i)); + } + + return s; +} + +static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct attribute *attrs[] = { + &con->features_attr.attr, + NULL + }; + struct bin_attribute *bin_attrs[] = { + &con->badpages_attr, + NULL + }; + struct attribute_group group = { + .name = "ras", + .attrs = attrs, + .bin_attrs = bin_attrs, + }; + + con->features_attr = (struct device_attribute) { + .attr = { + .name = "features", + .mode = S_IRUGO, + }, + .show = amdgpu_ras_sysfs_features_read, + }; + + con->badpages_attr = (struct bin_attribute) { + .attr = { + .name = "gpu_vram_bad_pages", + .mode = S_IRUGO, + }, + .size = 0, + .private = NULL, + .read = amdgpu_ras_sysfs_badpages_read, + }; + + sysfs_attr_init(attrs[0]); + sysfs_bin_attr_init(bin_attrs[0]); + + return sysfs_create_group(&adev->dev->kobj, &group); +} + +static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct attribute *attrs[] = { + &con->features_attr.attr, + NULL + }; + struct bin_attribute *bin_attrs[] = { + &con->badpages_attr, + NULL + }; + struct attribute_group group = { + .name = "ras", + .attrs = attrs, + .bin_attrs = bin_attrs, + }; + + sysfs_remove_group(&adev->dev->kobj, &group); + + return 0; +} + +int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, + struct ras_fs_if *head) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); + + if (!obj || obj->attr_inuse) + return -EINVAL; + + get_obj(obj); + + memcpy(obj->fs_data.sysfs_name, + head->sysfs_name, + sizeof(obj->fs_data.sysfs_name)); + + obj->sysfs_attr = (struct device_attribute){ + .attr = { + .name = obj->fs_data.sysfs_name, + .mode = S_IRUGO, + }, + .show = amdgpu_ras_sysfs_read, + }; + sysfs_attr_init(&obj->sysfs_attr.attr); + + if (sysfs_add_file_to_group(&adev->dev->kobj, + &obj->sysfs_attr.attr, + "ras")) { + put_obj(obj); + return -EINVAL; + } + + obj->attr_inuse = 1; + + return 0; +} + +int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); + + if (!obj || !obj->attr_inuse) + return -EINVAL; + + sysfs_remove_file_from_group(&adev->dev->kobj, + &obj->sysfs_attr.attr, + "ras"); + obj->attr_inuse = 0; + put_obj(obj); + + return 0; +} + +static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &con->head, node) { + amdgpu_ras_sysfs_remove(adev, &obj->head); + } + + amdgpu_ras_sysfs_remove_feature_node(adev); + + return 0; +} +/* sysfs end */ + +/* debugfs begin */ +static void amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct drm_minor *minor = adev->ddev->primary; + + con->dir = debugfs_create_dir("ras", minor->debugfs_root); + con->ent = debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, con->dir, + adev, &amdgpu_ras_debugfs_ctrl_ops); +} + +void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, + struct ras_fs_if *head) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); + + if (!obj || obj->ent) + return; + + get_obj(obj); + + memcpy(obj->fs_data.debugfs_name, + head->debugfs_name, + sizeof(obj->fs_data.debugfs_name)); + + obj->ent = debugfs_create_file(obj->fs_data.debugfs_name, + S_IWUGO | S_IRUGO, con->dir, obj, + &amdgpu_ras_debugfs_ops); +} + +void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev, + struct ras_common_if *head) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); + + if (!obj || !obj->ent) + return; + + debugfs_remove(obj->ent); + obj->ent = NULL; + put_obj(obj); +} + +static void amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &con->head, node) { + amdgpu_ras_debugfs_remove(adev, &obj->head); + } + + debugfs_remove(con->ent); + debugfs_remove(con->dir); + con->dir = NULL; + con->ent = NULL; +} +/* debugfs end */ + +/* ras fs */ + +static int amdgpu_ras_fs_init(struct amdgpu_device *adev) +{ + amdgpu_ras_sysfs_create_feature_node(adev); + amdgpu_ras_debugfs_create_ctrl_node(adev); + + return 0; +} + +static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) +{ + amdgpu_ras_debugfs_remove_all(adev); + amdgpu_ras_sysfs_remove_all(adev); + return 0; +} +/* ras fs end */ + +/* ih begin */ +static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) +{ + struct ras_ih_data *data = &obj->ih_data; + struct amdgpu_iv_entry entry; + int ret; + + while (data->rptr != data->wptr) { + rmb(); + memcpy(&entry, &data->ring[data->rptr], + data->element_size); + + wmb(); + data->rptr = (data->aligned_element_size + + data->rptr) % data->ring_size; + + /* Let IP handle its data, maybe we need get the output + * from the callback to udpate the error type/count, etc + */ + if (data->cb) { + ret = data->cb(obj->adev, &entry); + /* ue will trigger an interrupt, and in that case + * we need do a reset to recovery the whole system. + * But leave IP do that recovery, here we just dispatch + * the error. + */ + if (ret == AMDGPU_RAS_UE) { + obj->err_data.ue_count++; + } + /* Might need get ce count by register, but not all IP + * saves ce count, some IP just use one bit or two bits + * to indicate ce happened. + */ + } + } +} + +static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) +{ + struct ras_ih_data *data = + container_of(work, struct ras_ih_data, ih_work); + struct ras_manager *obj = + container_of(data, struct ras_manager, ih_data); + + amdgpu_ras_interrupt_handler(obj); +} + +int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, + struct ras_dispatch_if *info) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); + struct ras_ih_data *data = &obj->ih_data; + + if (!obj) + return -EINVAL; + + if (data->inuse == 0) + return 0; + + /* Might be overflow... */ + memcpy(&data->ring[data->wptr], info->entry, + data->element_size); + + wmb(); + data->wptr = (data->aligned_element_size + + data->wptr) % data->ring_size; + + schedule_work(&data->ih_work); + + return 0; +} + +int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, + struct ras_ih_if *info) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); + struct ras_ih_data *data; + + if (!obj) + return -EINVAL; + + data = &obj->ih_data; + if (data->inuse == 0) + return 0; + + cancel_work_sync(&data->ih_work); + + kfree(data->ring); + memset(data, 0, sizeof(*data)); + put_obj(obj); + + return 0; +} + +int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, + struct ras_ih_if *info) +{ + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); + struct ras_ih_data *data; + + if (!obj) { + /* in case we registe the IH before enable ras feature */ + obj = amdgpu_ras_create_obj(adev, &info->head); + if (!obj) + return -EINVAL; + } else + get_obj(obj); + + data = &obj->ih_data; + /* add the callback.etc */ + *data = (struct ras_ih_data) { + .inuse = 0, + .cb = info->cb, + .element_size = sizeof(struct amdgpu_iv_entry), + .rptr = 0, + .wptr = 0, + }; + + INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); + + data->aligned_element_size = ALIGN(data->element_size, 8); + /* the ring can store 64 iv entries. */ + data->ring_size = 64 * data->aligned_element_size; + data->ring = kmalloc(data->ring_size, GFP_KERNEL); + if (!data->ring) { + put_obj(obj); + return -ENOMEM; + } + + /* IH is ready */ + data->inuse = 1; + + return 0; +} + +static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &con->head, node) { + struct ras_ih_if info = { + .head = obj->head, + }; + amdgpu_ras_interrupt_remove_handler(adev, &info); + } + + return 0; +} +/* ih end */ + +/* recovery begin */ + +/* return 0 on success. + * caller need free bps. + */ +static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, + struct ras_badpage **bps, unsigned int *count) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data *data; + int i = 0; + int ret = 0; + + if (!con || !con->eh_data || !bps || !count) + return -EINVAL; + + mutex_lock(&con->recovery_lock); + data = con->eh_data; + if (!data || data->count == 0) { + *bps = NULL; + goto out; + } + + *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL); + if (!*bps) { + ret = -ENOMEM; + goto out; + } + + for (; i < data->count; i++) { + (*bps)[i] = (struct ras_badpage){ + .bp = data->bps[i].bp, + .size = AMDGPU_GPU_PAGE_SIZE, + .flags = 0, + }; + + if (data->last_reserved <= i) + (*bps)[i].flags = 1; + else if (data->bps[i].bo == NULL) + (*bps)[i].flags = 2; + } + + *count = data->count; +out: + mutex_unlock(&con->recovery_lock); + return ret; +} + +static void amdgpu_ras_do_recovery(struct work_struct *work) +{ + struct amdgpu_ras *ras = + container_of(work, struct amdgpu_ras, recovery_work); + + amdgpu_device_gpu_recover(ras->adev, 0); + atomic_set(&ras->in_recovery, 0); +} + +static int amdgpu_ras_release_vram(struct amdgpu_device *adev, + struct amdgpu_bo **bo_ptr) +{ + /* no need to free it actually. */ + amdgpu_bo_free_kernel(bo_ptr, NULL, NULL); + return 0; +} + +/* reserve vram with size@offset */ +static int amdgpu_ras_reserve_vram(struct amdgpu_device *adev, + uint64_t offset, uint64_t size, + struct amdgpu_bo **bo_ptr) +{ + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_bo_param bp; + int r = 0; + int i; + struct amdgpu_bo *bo; + + if (bo_ptr) + *bo_ptr = NULL; + memset(&bp, 0, sizeof(bp)); + bp.size = size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; + bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | + AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) + return -EINVAL; + + r = amdgpu_bo_reserve(bo, false); + if (r) + goto error_reserve; + + offset = ALIGN(offset, PAGE_SIZE); + for (i = 0; i < bo->placement.num_placement; ++i) { + bo->placements[i].fpfn = offset >> PAGE_SHIFT; + bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; + } + + ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem); + r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.mem, &ctx); + if (r) + goto error_pin; + + r = amdgpu_bo_pin_restricted(bo, + AMDGPU_GEM_DOMAIN_VRAM, + offset, + offset + size); + if (r) + goto error_pin; + + if (bo_ptr) + *bo_ptr = bo; + + amdgpu_bo_unreserve(bo); + return r; + +error_pin: + amdgpu_bo_unreserve(bo); +error_reserve: + amdgpu_bo_unref(&bo); + return r; +} + +/* alloc/realloc bps array */ +static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, + struct ras_err_handler_data *data, int pages) +{ + unsigned int old_space = data->count + data->space_left; + unsigned int new_space = old_space + pages; + unsigned int align_space = ALIGN(new_space, 1024); + void *tmp = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL); + + if (!tmp) + return -ENOMEM; + + if (data->bps) { + memcpy(tmp, data->bps, + data->count * sizeof(*data->bps)); + kfree(data->bps); + } + + data->bps = tmp; + data->space_left += align_space - old_space; + return 0; +} + +/* it deal with vram only. */ +int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, + unsigned long *bps, int pages) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data *data; + int i = pages; + int ret = 0; + + if (!con || !con->eh_data || !bps || pages <= 0) + return 0; + + mutex_lock(&con->recovery_lock); + data = con->eh_data; + if (!data) + goto out; + + if (data->space_left <= pages) + if (amdgpu_ras_realloc_eh_data_space(adev, data, pages)) { + ret = -ENOMEM; + goto out; + } + + while (i--) + data->bps[data->count++].bp = bps[i]; + + data->space_left -= pages; +out: + mutex_unlock(&con->recovery_lock); + + return ret; +} + +/* called in gpu recovery/init */ +int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data *data; + uint64_t bp; + struct amdgpu_bo *bo; + int i; + + if (!con || !con->eh_data) + return 0; + + mutex_lock(&con->recovery_lock); + data = con->eh_data; + if (!data) + goto out; + /* reserve vram at driver post stage. */ + for (i = data->last_reserved; i < data->count; i++) { + bp = data->bps[i].bp; + + if (amdgpu_ras_reserve_vram(adev, bp << PAGE_SHIFT, + PAGE_SIZE, &bo)) + DRM_ERROR("RAS ERROR: reserve vram %llx fail\n", bp); + + data->bps[i].bo = bo; + data->last_reserved = i + 1; + } +out: + mutex_unlock(&con->recovery_lock); + return 0; +} + +/* called when driver unload */ +static int amdgpu_ras_release_bad_pages(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data *data; + struct amdgpu_bo *bo; + int i; + + if (!con || !con->eh_data) + return 0; + + mutex_lock(&con->recovery_lock); + data = con->eh_data; + if (!data) + goto out; + + for (i = data->last_reserved - 1; i >= 0; i--) { + bo = data->bps[i].bo; + + amdgpu_ras_release_vram(adev, &bo); + + data->bps[i].bo = bo; + data->last_reserved = i; + } +out: + mutex_unlock(&con->recovery_lock); + return 0; +} + +static int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev) +{ + /* TODO + * write the array to eeprom when SMU disabled. + */ + return 0; +} + +static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) +{ + /* TODO + * read the array to eeprom when SMU disabled. + */ + return 0; +} + +static int amdgpu_ras_recovery_init(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data **data = &con->eh_data; + + *data = kmalloc(sizeof(**data), + GFP_KERNEL|__GFP_ZERO); + if (!*data) + return -ENOMEM; + + mutex_init(&con->recovery_lock); + INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); + atomic_set(&con->in_recovery, 0); + con->adev = adev; + + amdgpu_ras_load_bad_pages(adev); + amdgpu_ras_reserve_bad_pages(adev); + + return 0; +} + +static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_err_handler_data *data = con->eh_data; + + cancel_work_sync(&con->recovery_work); + amdgpu_ras_save_bad_pages(adev); + amdgpu_ras_release_bad_pages(adev); + + mutex_lock(&con->recovery_lock); + con->eh_data = NULL; + kfree(data->bps); + kfree(data); + mutex_unlock(&con->recovery_lock); + + return 0; +} +/* recovery end */ + +/* return 0 if ras will reset gpu and repost.*/ +int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev, + unsigned int block) +{ + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); + + if (!ras) + return -EINVAL; + + ras->flags |= AMDGPU_RAS_FLAG_INIT_NEED_RESET; + return 0; +} + +/* + * check hardware's ras ability which will be saved in hw_supported. + * if hardware does not support ras, we can skip some ras initializtion and + * forbid some ras operations from IP. + * if software itself, say boot parameter, limit the ras ability. We still + * need allow IP do some limited operations, like disable. In such case, + * we have to initialize ras as normal. but need check if operation is + * allowed or not in each function. + */ +static void amdgpu_ras_check_supported(struct amdgpu_device *adev, + uint32_t *hw_supported, uint32_t *supported) +{ + *hw_supported = 0; + *supported = 0; + + if (amdgpu_sriov_vf(adev) || + adev->asic_type != CHIP_VEGA20) + return; + + if (adev->is_atom_fw && + (amdgpu_atomfirmware_mem_ecc_supported(adev) || + amdgpu_atomfirmware_sram_ecc_supported(adev))) + *hw_supported = AMDGPU_RAS_BLOCK_MASK; + + *supported = amdgpu_ras_enable == 0 ? + 0 : *hw_supported & amdgpu_ras_mask; +} + +int amdgpu_ras_init(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + if (con) + return 0; + + con = kmalloc(sizeof(struct amdgpu_ras) + + sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT, + GFP_KERNEL|__GFP_ZERO); + if (!con) + return -ENOMEM; + + con->objs = (struct ras_manager *)(con + 1); + + amdgpu_ras_set_context(adev, con); + + amdgpu_ras_check_supported(adev, &con->hw_supported, + &con->supported); + if (!con->hw_supported) { + amdgpu_ras_set_context(adev, NULL); + kfree(con); + return 0; + } + + con->features = 0; + INIT_LIST_HEAD(&con->head); + /* Might need get this flag from vbios. */ + con->flags = RAS_DEFAULT_FLAGS; + + if (amdgpu_ras_recovery_init(adev)) + goto recovery_out; + + amdgpu_ras_mask &= AMDGPU_RAS_BLOCK_MASK; + + if (amdgpu_ras_fs_init(adev)) + goto fs_out; + + DRM_INFO("RAS INFO: ras initialized successfully, " + "hardware ability[%x] ras_mask[%x]\n", + con->hw_supported, con->supported); + return 0; +fs_out: + amdgpu_ras_recovery_fini(adev); +recovery_out: + amdgpu_ras_set_context(adev, NULL); + kfree(con); + + return -EINVAL; +} + +/* do some init work after IP late init as dependence. + * and it runs in resume/gpu reset/booting up cases. + */ +void amdgpu_ras_resume(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct ras_manager *obj, *tmp; + + if (!con) + return; + + if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { + /* Set up all other IPs which are not implemented. There is a + * tricky thing that IP's actual ras error type should be + * MULTI_UNCORRECTABLE, but as driver does not handle it, so + * ERROR_NONE make sense anyway. + */ + amdgpu_ras_enable_all_features(adev, 1); + + /* We enable ras on all hw_supported block, but as boot + * parameter might disable some of them and one or more IP has + * not implemented yet. So we disable them on behalf. + */ + list_for_each_entry_safe(obj, tmp, &con->head, node) { + if (!amdgpu_ras_is_supported(adev, obj->head.block)) { + amdgpu_ras_feature_enable(adev, &obj->head, 0); + /* there should be no any reference. */ + WARN_ON(alive_obj(obj)); + } + } + } + + if (con->flags & AMDGPU_RAS_FLAG_INIT_NEED_RESET) { + con->flags &= ~AMDGPU_RAS_FLAG_INIT_NEED_RESET; + /* setup ras obj state as disabled. + * for init_by_vbios case. + * if we want to enable ras, just enable it in a normal way. + * If we want do disable it, need setup ras obj as enabled, + * then issue another TA disable cmd. + * See feature_enable_on_boot + */ + amdgpu_ras_disable_all_features(adev, 1); + amdgpu_ras_reset_gpu(adev, 0); + } +} + +void amdgpu_ras_suspend(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + if (!con) + return; + + amdgpu_ras_disable_all_features(adev, 0); + /* Make sure all ras objects are disabled. */ + if (con->features) + amdgpu_ras_disable_all_features(adev, 1); +} + +/* do some fini work before IP fini as dependence */ +int amdgpu_ras_pre_fini(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + if (!con) + return 0; + + /* Need disable ras on all IPs here before ip [hw/sw]fini */ + amdgpu_ras_disable_all_features(adev, 0); + amdgpu_ras_recovery_fini(adev); + return 0; +} + +int amdgpu_ras_fini(struct amdgpu_device *adev) +{ + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + + if (!con) + return 0; + + amdgpu_ras_fs_fini(adev); + amdgpu_ras_interrupt_remove_all(adev); + + WARN(con->features, "Feature mask is not cleared"); + + if (con->features) + amdgpu_ras_disable_all_features(adev, 1); + + amdgpu_ras_set_context(adev, NULL); + kfree(con); + + return 0; +}
diff --git a/amdgpu/amdgpu_ras.h b/amdgpu/amdgpu_ras.h new file mode 100644 index 0000000..b284119 --- /dev/null +++ b/amdgpu/amdgpu_ras.h
@@ -0,0 +1,299 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _AMDGPU_RAS_H +#define _AMDGPU_RAS_H + +#include <linux/debugfs.h> +#include <linux/list.h> +#include "amdgpu.h" +#include "amdgpu_psp.h" +#include "ta_ras_if.h" + +enum amdgpu_ras_block { + AMDGPU_RAS_BLOCK__UMC = 0, + AMDGPU_RAS_BLOCK__SDMA, + AMDGPU_RAS_BLOCK__GFX, + AMDGPU_RAS_BLOCK__MMHUB, + AMDGPU_RAS_BLOCK__ATHUB, + AMDGPU_RAS_BLOCK__PCIE_BIF, + AMDGPU_RAS_BLOCK__HDP, + AMDGPU_RAS_BLOCK__XGMI_WAFL, + AMDGPU_RAS_BLOCK__DF, + AMDGPU_RAS_BLOCK__SMN, + AMDGPU_RAS_BLOCK__SEM, + AMDGPU_RAS_BLOCK__MP0, + AMDGPU_RAS_BLOCK__MP1, + AMDGPU_RAS_BLOCK__FUSE, + + AMDGPU_RAS_BLOCK__LAST +}; + +#define AMDGPU_RAS_BLOCK_COUNT AMDGPU_RAS_BLOCK__LAST +#define AMDGPU_RAS_BLOCK_MASK ((1ULL << AMDGPU_RAS_BLOCK_COUNT) - 1) + +enum amdgpu_ras_error_type { + AMDGPU_RAS_ERROR__NONE = 0, + AMDGPU_RAS_ERROR__PARITY = 1, + AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE = 2, + AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE = 4, + AMDGPU_RAS_ERROR__POISON = 8, +}; + +enum amdgpu_ras_ret { + AMDGPU_RAS_SUCCESS = 0, + AMDGPU_RAS_FAIL, + AMDGPU_RAS_UE, + AMDGPU_RAS_CE, + AMDGPU_RAS_PT, +}; + +struct ras_common_if { + enum amdgpu_ras_block block; + enum amdgpu_ras_error_type type; + uint32_t sub_block_index; + /* block name */ + char name[32]; +}; + +typedef int (*ras_ih_cb)(struct amdgpu_device *adev, + struct amdgpu_iv_entry *entry); + +struct amdgpu_ras { + /* ras infrastructure */ + /* for ras itself. */ + uint32_t hw_supported; + /* for IP to check its ras ability. */ + uint32_t supported; + uint32_t features; + struct list_head head; + /* debugfs */ + struct dentry *dir; + /* debugfs ctrl */ + struct dentry *ent; + /* sysfs */ + struct device_attribute features_attr; + struct bin_attribute badpages_attr; + /* block array */ + struct ras_manager *objs; + + /* gpu recovery */ + struct work_struct recovery_work; + atomic_t in_recovery; + struct amdgpu_device *adev; + /* error handler data */ + struct ras_err_handler_data *eh_data; + struct mutex recovery_lock; + + uint32_t flags; +}; + +/* interfaces for IP */ + +struct ras_fs_if { + struct ras_common_if head; + char sysfs_name[32]; + char debugfs_name[32]; +}; + +struct ras_query_if { + struct ras_common_if head; + unsigned long ue_count; + unsigned long ce_count; +}; + +struct ras_inject_if { + struct ras_common_if head; + uint64_t address; + uint64_t value; +}; + +struct ras_cure_if { + struct ras_common_if head; + uint64_t address; +}; + +struct ras_ih_if { + struct ras_common_if head; + ras_ih_cb cb; +}; + +struct ras_dispatch_if { + struct ras_common_if head; + struct amdgpu_iv_entry *entry; +}; + +struct ras_debug_if { + union { + struct ras_common_if head; + struct ras_inject_if inject; + }; + int op; +}; +/* work flow + * vbios + * 1: ras feature enable (enabled by default) + * psp + * 2: ras framework init (in ip_init) + * IP + * 3: IH add + * 4: debugfs/sysfs create + * 5: query/inject + * 6: debugfs/sysfs remove + * 7: IH remove + * 8: feature disable + */ + +#define amdgpu_ras_get_context(adev) ((adev)->psp.ras.ras) +#define amdgpu_ras_set_context(adev, ras_con) ((adev)->psp.ras.ras = (ras_con)) + +/* check if ras is supported on block, say, sdma, gfx */ +static inline int amdgpu_ras_is_supported(struct amdgpu_device *adev, + unsigned int block) +{ + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); + + if (block >= AMDGPU_RAS_BLOCK_COUNT) + return 0; + return ras && (ras->supported & (1 << block)); +} + +int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev, + unsigned int block); + +void amdgpu_ras_resume(struct amdgpu_device *adev); +void amdgpu_ras_suspend(struct amdgpu_device *adev); + +int amdgpu_ras_query_error_count(struct amdgpu_device *adev, + bool is_ce); + +/* error handling functions */ +int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, + unsigned long *bps, int pages); + +int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev); + +static inline int amdgpu_ras_reset_gpu(struct amdgpu_device *adev, + bool is_baco) +{ + struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); + + if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) + schedule_work(&ras->recovery_work); + return 0; +} + +static inline enum ta_ras_block +amdgpu_ras_block_to_ta(enum amdgpu_ras_block block) { + switch (block) { + case AMDGPU_RAS_BLOCK__UMC: + return TA_RAS_BLOCK__UMC; + case AMDGPU_RAS_BLOCK__SDMA: + return TA_RAS_BLOCK__SDMA; + case AMDGPU_RAS_BLOCK__GFX: + return TA_RAS_BLOCK__GFX; + case AMDGPU_RAS_BLOCK__MMHUB: + return TA_RAS_BLOCK__MMHUB; + case AMDGPU_RAS_BLOCK__ATHUB: + return TA_RAS_BLOCK__ATHUB; + case AMDGPU_RAS_BLOCK__PCIE_BIF: + return TA_RAS_BLOCK__PCIE_BIF; + case AMDGPU_RAS_BLOCK__HDP: + return TA_RAS_BLOCK__HDP; + case AMDGPU_RAS_BLOCK__XGMI_WAFL: + return TA_RAS_BLOCK__XGMI_WAFL; + case AMDGPU_RAS_BLOCK__DF: + return TA_RAS_BLOCK__DF; + case AMDGPU_RAS_BLOCK__SMN: + return TA_RAS_BLOCK__SMN; + case AMDGPU_RAS_BLOCK__SEM: + return TA_RAS_BLOCK__SEM; + case AMDGPU_RAS_BLOCK__MP0: + return TA_RAS_BLOCK__MP0; + case AMDGPU_RAS_BLOCK__MP1: + return TA_RAS_BLOCK__MP1; + case AMDGPU_RAS_BLOCK__FUSE: + return TA_RAS_BLOCK__FUSE; + default: + WARN_ONCE(1, "RAS ERROR: unexpected block id %d\n", block); + return TA_RAS_BLOCK__UMC; + } +} + +static inline enum ta_ras_error_type +amdgpu_ras_error_to_ta(enum amdgpu_ras_error_type error) { + switch (error) { + case AMDGPU_RAS_ERROR__NONE: + return TA_RAS_ERROR__NONE; + case AMDGPU_RAS_ERROR__PARITY: + return TA_RAS_ERROR__PARITY; + case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: + return TA_RAS_ERROR__SINGLE_CORRECTABLE; + case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: + return TA_RAS_ERROR__MULTI_UNCORRECTABLE; + case AMDGPU_RAS_ERROR__POISON: + return TA_RAS_ERROR__POISON; + default: + WARN_ONCE(1, "RAS ERROR: unexpected error type %d\n", error); + return TA_RAS_ERROR__NONE; + } +} + +/* called in ip_init and ip_fini */ +int amdgpu_ras_init(struct amdgpu_device *adev); +int amdgpu_ras_fini(struct amdgpu_device *adev); +int amdgpu_ras_pre_fini(struct amdgpu_device *adev); + +int amdgpu_ras_feature_enable(struct amdgpu_device *adev, + struct ras_common_if *head, bool enable); + +int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, + struct ras_common_if *head, bool enable); + +int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, + struct ras_fs_if *head); + +int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, + struct ras_common_if *head); + +void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, + struct ras_fs_if *head); + +void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev, + struct ras_common_if *head); + +int amdgpu_ras_error_query(struct amdgpu_device *adev, + struct ras_query_if *info); + +int amdgpu_ras_error_inject(struct amdgpu_device *adev, + struct ras_inject_if *info); + +int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, + struct ras_ih_if *info); + +int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, + struct ras_ih_if *info); + +int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, + struct ras_dispatch_if *info); +#endif
diff --git a/amdgpu/amdgpu_ring.c b/amdgpu/amdgpu_ring.c new file mode 100644 index 0000000..e5c83e1 --- /dev/null +++ b/amdgpu/amdgpu_ring.c
@@ -0,0 +1,541 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + * Christian König + */ +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/debugfs.h> + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atom.h" + +/* + * Rings + * Most engines on the GPU are fed via ring buffers. Ring + * buffers are areas of GPU accessible memory that the host + * writes commands into and the GPU reads commands out of. + * There is a rptr (read pointer) that determines where the + * GPU is currently reading, and a wptr (write pointer) + * which determines where the host has written. When the + * pointers are equal, the ring is idle. When the host + * writes commands to the ring buffer, it increments the + * wptr. The GPU then starts fetching commands and executes + * them until the pointers are equal again. + */ +static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, + struct amdgpu_ring *ring); +static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring); + +/** + * amdgpu_ring_alloc - allocate space on the ring buffer + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring structure holding ring information + * @ndw: number of dwords to allocate in the ring buffer + * + * Allocate @ndw dwords in the ring buffer (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw) +{ + /* Align requested size with padding so unlock_commit can + * pad safely */ + ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask; + + /* Make sure we aren't trying to allocate more space + * than the maximum for one submission + */ + if (WARN_ON_ONCE(ndw > ring->max_dw)) + return -ENOMEM; + + ring->count_dw = ndw; + ring->wptr_old = ring->wptr; + + if (ring->funcs->begin_use) + ring->funcs->begin_use(ring); + + return 0; +} + +/** amdgpu_ring_insert_nop - insert NOP packets + * + * @ring: amdgpu_ring structure holding ring information + * @count: the number of NOP packets to insert + * + * This is the generic insert_nop function for rings except SDMA + */ +void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + int i; + + for (i = 0; i < count; i++) + amdgpu_ring_write(ring, ring->funcs->nop); +} + +/** amdgpu_ring_generic_pad_ib - pad IB with NOP packets + * + * @ring: amdgpu_ring structure holding ring information + * @ib: IB to add NOP packets to + * + * This is the generic pad_ib function for rings except SDMA + */ +void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) +{ + while (ib->length_dw & ring->funcs->align_mask) + ib->ptr[ib->length_dw++] = ring->funcs->nop; +} + +/** + * amdgpu_ring_commit - tell the GPU to execute the new + * commands on the ring buffer + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring structure holding ring information + * + * Update the wptr (write pointer) to tell the GPU to + * execute new commands on the ring buffer (all asics). + */ +void amdgpu_ring_commit(struct amdgpu_ring *ring) +{ + uint32_t count; + + /* We pad to match fetch size */ + count = ring->funcs->align_mask + 1 - + (ring->wptr & ring->funcs->align_mask); + count %= ring->funcs->align_mask + 1; + ring->funcs->insert_nop(ring, count); + + mb(); + amdgpu_ring_set_wptr(ring); + + if (ring->funcs->end_use) + ring->funcs->end_use(ring); +} + +/** + * amdgpu_ring_undo - reset the wptr + * + * @ring: amdgpu_ring structure holding ring information + * + * Reset the driver's copy of the wptr (all asics). + */ +void amdgpu_ring_undo(struct amdgpu_ring *ring) +{ + ring->wptr = ring->wptr_old; + + if (ring->funcs->end_use) + ring->funcs->end_use(ring); +} + +/** + * amdgpu_ring_priority_put - restore a ring's priority + * + * @ring: amdgpu_ring structure holding the information + * @priority: target priority + * + * Release a request for executing at @priority + */ +void amdgpu_ring_priority_put(struct amdgpu_ring *ring, + enum drm_sched_priority priority) +{ + int i; + + if (!ring->funcs->set_priority) + return; + + if (atomic_dec_return(&ring->num_jobs[priority]) > 0) + return; + + /* no need to restore if the job is already at the lowest priority */ + if (priority == DRM_SCHED_PRIORITY_NORMAL) + return; + + mutex_lock(&ring->priority_mutex); + /* something higher prio is executing, no need to decay */ + if (ring->priority > priority) + goto out_unlock; + + /* decay priority to the next level with a job available */ + for (i = priority; i >= DRM_SCHED_PRIORITY_MIN; i--) { + if (i == DRM_SCHED_PRIORITY_NORMAL + || atomic_read(&ring->num_jobs[i])) { + ring->priority = i; + ring->funcs->set_priority(ring, i); + break; + } + } + +out_unlock: + mutex_unlock(&ring->priority_mutex); +} + +/** + * amdgpu_ring_priority_get - change the ring's priority + * + * @ring: amdgpu_ring structure holding the information + * @priority: target priority + * + * Request a ring's priority to be raised to @priority (refcounted). + */ +void amdgpu_ring_priority_get(struct amdgpu_ring *ring, + enum drm_sched_priority priority) +{ + if (!ring->funcs->set_priority) + return; + + if (atomic_inc_return(&ring->num_jobs[priority]) <= 0) + return; + + mutex_lock(&ring->priority_mutex); + if (priority <= ring->priority) + goto out_unlock; + + ring->priority = priority; + ring->funcs->set_priority(ring, priority); + +out_unlock: + mutex_unlock(&ring->priority_mutex); +} + +/** + * amdgpu_ring_init - init driver ring struct. + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring structure holding ring information + * @max_ndw: maximum number of dw for ring alloc + * @nop: nop packet for this ring + * + * Initialize the driver information for the selected ring (all asics). + * Returns 0 on success, error on failure. + */ +int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, + unsigned max_dw, struct amdgpu_irq_src *irq_src, + unsigned irq_type) +{ + int r, i; + int sched_hw_submission = amdgpu_sched_hw_submission; + + /* Set the hw submission limit higher for KIQ because + * it's used for a number of gfx/compute tasks by both + * KFD and KGD which may have outstanding fences and + * it doesn't really use the gpu scheduler anyway; + * KIQ tasks get submitted directly to the ring. + */ + if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) + sched_hw_submission = max(sched_hw_submission, 256); + else if (ring == &adev->sdma.instance[0].page) + sched_hw_submission = 256; + + if (ring->adev == NULL) { + if (adev->num_rings >= AMDGPU_MAX_RINGS) + return -EINVAL; + + ring->adev = adev; + ring->idx = adev->num_rings++; + adev->rings[ring->idx] = ring; + r = amdgpu_fence_driver_init_ring(ring, sched_hw_submission); + if (r) + return r; + } + + r = amdgpu_device_wb_get(adev, &ring->rptr_offs); + if (r) { + dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r); + return r; + } + + r = amdgpu_device_wb_get(adev, &ring->wptr_offs); + if (r) { + dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r); + return r; + } + + r = amdgpu_device_wb_get(adev, &ring->fence_offs); + if (r) { + dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r); + return r; + } + + r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs); + if (r) { + dev_err(adev->dev, + "(%d) ring trail_fence_offs wb alloc failed\n", r); + return r; + } + ring->trail_fence_gpu_addr = + adev->wb.gpu_addr + (ring->trail_fence_offs * 4); + ring->trail_fence_cpu_addr = &adev->wb.wb[ring->trail_fence_offs]; + + r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs); + if (r) { + dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r); + return r; + } + ring->cond_exe_gpu_addr = adev->wb.gpu_addr + (ring->cond_exe_offs * 4); + ring->cond_exe_cpu_addr = &adev->wb.wb[ring->cond_exe_offs]; + /* always set cond_exec_polling to CONTINUE */ + *ring->cond_exe_cpu_addr = 1; + + r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); + if (r) { + dev_err(adev->dev, "failed initializing fences (%d).\n", r); + return r; + } + + ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission); + + ring->buf_mask = (ring->ring_size / 4) - 1; + ring->ptr_mask = ring->funcs->support_64bit_ptrs ? + 0xffffffffffffffff : ring->buf_mask; + /* Allocate ring buffer */ + if (ring->ring_obj == NULL) { + r = amdgpu_bo_create_kernel(adev, ring->ring_size + ring->funcs->extra_dw, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, + &ring->ring_obj, + &ring->gpu_addr, + (void **)&ring->ring); + if (r) { + dev_err(adev->dev, "(%d) ring create failed\n", r); + return r; + } + amdgpu_ring_clear_ring(ring); + } + + ring->max_dw = max_dw; + ring->priority = DRM_SCHED_PRIORITY_NORMAL; + mutex_init(&ring->priority_mutex); + + for (i = 0; i < DRM_SCHED_PRIORITY_MAX; ++i) + atomic_set(&ring->num_jobs[i], 0); + + if (amdgpu_debugfs_ring_init(adev, ring)) { + DRM_ERROR("Failed to register debugfs file for rings !\n"); + } + + return 0; +} + +/** + * amdgpu_ring_fini - tear down the driver ring struct. + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring structure holding ring information + * + * Tear down the driver information for the selected ring (all asics). + */ +void amdgpu_ring_fini(struct amdgpu_ring *ring) +{ + ring->sched.ready = false; + + /* Not to finish a ring which is not initialized */ + if (!(ring->adev) || !(ring->adev->rings[ring->idx])) + return; + + amdgpu_device_wb_free(ring->adev, ring->rptr_offs); + amdgpu_device_wb_free(ring->adev, ring->wptr_offs); + + amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs); + amdgpu_device_wb_free(ring->adev, ring->fence_offs); + + amdgpu_bo_free_kernel(&ring->ring_obj, + &ring->gpu_addr, + (void **)&ring->ring); + + amdgpu_debugfs_ring_fini(ring); + + dma_fence_put(ring->vmid_wait); + ring->vmid_wait = NULL; + ring->me = 0; + + ring->adev->rings[ring->idx] = NULL; +} + +/** + * amdgpu_ring_emit_reg_write_reg_wait_helper - ring helper + * + * @adev: amdgpu_device pointer + * @reg0: register to write + * @reg1: register to wait on + * @ref: reference value to write/wait on + * @mask: mask to wait on + * + * Helper for rings that don't support write and wait in a + * single oneshot packet. + */ +void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring, + uint32_t reg0, uint32_t reg1, + uint32_t ref, uint32_t mask) +{ + amdgpu_ring_emit_wreg(ring, reg0, ref); + amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask); +} + +/** + * amdgpu_ring_soft_recovery - try to soft recover a ring lockup + * + * @ring: ring to try the recovery on + * @vmid: VMID we try to get going again + * @fence: timedout fence + * + * Tries to get a ring proceeding again when it is stuck. + */ +bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, + struct dma_fence *fence) +{ + ktime_t deadline = ktime_add_us(ktime_get(), 10000); + + if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) + return false; + + atomic_inc(&ring->adev->gpu_reset_counter); + while (!dma_fence_is_signaled(fence) && + ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0) + ring->funcs->soft_recovery(ring, vmid); + + return dma_fence_is_signaled(fence); +} + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +/* Layout of file is 12 bytes consisting of + * - rptr + * - wptr + * - driver's copy of wptr + * + * followed by n-words of ring data + */ +static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_ring *ring = file_inode(f)->i_private; + int r, i; + uint32_t value, result, early[3]; + + if (*pos & 3 || size & 3) + return -EINVAL; + + result = 0; + + if (*pos < 12) { + early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask; + early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask; + early[2] = ring->wptr & ring->buf_mask; + for (i = *pos / 4; i < 3 && size; i++) { + r = put_user(early[i], (uint32_t *)buf); + if (r) + return r; + buf += 4; + result += 4; + size -= 4; + *pos += 4; + } + } + + while (size) { + if (*pos >= (ring->ring_size + 12)) + return result; + + value = ring->ring[(*pos - 12)/4]; + r = put_user(value, (uint32_t*)buf); + if (r) + return r; + buf += 4; + result += 4; + size -= 4; + *pos += 4; + } + + return result; +} + +static const struct file_operations amdgpu_debugfs_ring_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_ring_read, + .llseek = default_llseek +}; + +#endif + +static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ +#if defined(CONFIG_DEBUG_FS) + struct drm_minor *minor = adev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + char name[32]; + + sprintf(name, "amdgpu_ring_%s", ring->name); + + ent = debugfs_create_file(name, + S_IFREG | S_IRUGO, root, + ring, &amdgpu_debugfs_ring_fops); + if (!ent) + return -ENOMEM; + + i_size_write(ent->d_inode, ring->ring_size + 12); + ring->ent = ent; +#endif + return 0; +} + +static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring) +{ +#if defined(CONFIG_DEBUG_FS) + debugfs_remove(ring->ent); +#endif +} + +/** + * amdgpu_ring_test_helper - tests ring and set sched readiness status + * + * @ring: ring to try the recovery on + * + * Tests ring and set sched readiness status + * + * Returns 0 on success, error on failure. + */ +int amdgpu_ring_test_helper(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + int r; + + r = amdgpu_ring_test_ring(ring); + if (r) + DRM_DEV_ERROR(adev->dev, "ring %s test failed (%d)\n", + ring->name, r); + else + DRM_DEV_DEBUG(adev->dev, "ring test on %s succeeded\n", + ring->name); + + ring->sched.ready = !r; + return r; +}
diff --git a/amdgpu/amdgpu_ring.h b/amdgpu/amdgpu_ring.h new file mode 100644 index 0000000..4410c97 --- /dev/null +++ b/amdgpu/amdgpu_ring.h
@@ -0,0 +1,331 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ +#ifndef __AMDGPU_RING_H__ +#define __AMDGPU_RING_H__ + +#include <drm/amdgpu_drm.h> +#include <drm/gpu_scheduler.h> +#include <drm/drm_print.h> + +/* max number of rings */ +#define AMDGPU_MAX_RINGS 24 +#define AMDGPU_MAX_GFX_RINGS 2 +#define AMDGPU_MAX_COMPUTE_RINGS 8 +#define AMDGPU_MAX_VCE_RINGS 3 +#define AMDGPU_MAX_UVD_ENC_RINGS 2 + +/* some special values for the owner field */ +#define AMDGPU_FENCE_OWNER_UNDEFINED ((void *)0ul) +#define AMDGPU_FENCE_OWNER_VM ((void *)1ul) +#define AMDGPU_FENCE_OWNER_KFD ((void *)2ul) + +#define AMDGPU_FENCE_FLAG_64BIT (1 << 0) +#define AMDGPU_FENCE_FLAG_INT (1 << 1) +#define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) + +#define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched) + +enum amdgpu_ring_type { + AMDGPU_RING_TYPE_GFX, + AMDGPU_RING_TYPE_COMPUTE, + AMDGPU_RING_TYPE_SDMA, + AMDGPU_RING_TYPE_UVD, + AMDGPU_RING_TYPE_VCE, + AMDGPU_RING_TYPE_KIQ, + AMDGPU_RING_TYPE_UVD_ENC, + AMDGPU_RING_TYPE_VCN_DEC, + AMDGPU_RING_TYPE_VCN_ENC, + AMDGPU_RING_TYPE_VCN_JPEG +}; + +struct amdgpu_device; +struct amdgpu_ring; +struct amdgpu_ib; +struct amdgpu_cs_parser; +struct amdgpu_job; + +/* + * Fences. + */ +struct amdgpu_fence_driver { + uint64_t gpu_addr; + volatile uint32_t *cpu_addr; + /* sync_seq is protected by ring emission lock */ + uint32_t sync_seq; + atomic_t last_seq; + bool initialized; + struct amdgpu_irq_src *irq_src; + unsigned irq_type; + struct timer_list fallback_timer; + unsigned num_fences_mask; + spinlock_t lock; + struct dma_fence **fences; +}; + +int amdgpu_fence_driver_init(struct amdgpu_device *adev); +void amdgpu_fence_driver_fini(struct amdgpu_device *adev); +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); + +int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, + unsigned num_hw_submission); +int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, + struct amdgpu_irq_src *irq_src, + unsigned irq_type); +void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); +void amdgpu_fence_driver_resume(struct amdgpu_device *adev); +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, + unsigned flags); +int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s); +bool amdgpu_fence_process(struct amdgpu_ring *ring); +int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); +signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring, + uint32_t wait_seq, + signed long timeout); +unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); + +/* + * Rings. + */ + +/* provided by hw blocks that expose a ring buffer for commands */ +struct amdgpu_ring_funcs { + enum amdgpu_ring_type type; + uint32_t align_mask; + u32 nop; + bool support_64bit_ptrs; + bool no_user_fence; + unsigned vmhub; + unsigned extra_dw; + + /* ring read/write ptr handling */ + u64 (*get_rptr)(struct amdgpu_ring *ring); + u64 (*get_wptr)(struct amdgpu_ring *ring); + void (*set_wptr)(struct amdgpu_ring *ring); + /* validating and patching of IBs */ + int (*parse_cs)(struct amdgpu_cs_parser *p, uint32_t ib_idx); + int (*patch_cs_in_place)(struct amdgpu_cs_parser *p, uint32_t ib_idx); + /* constants to calculate how many DW are needed for an emit */ + unsigned emit_frame_size; + unsigned emit_ib_size; + /* command emit functions */ + void (*emit_ib)(struct amdgpu_ring *ring, + struct amdgpu_job *job, + struct amdgpu_ib *ib, + uint32_t flags); + void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr, + uint64_t seq, unsigned flags); + void (*emit_pipeline_sync)(struct amdgpu_ring *ring); + void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid, + uint64_t pd_addr); + void (*emit_hdp_flush)(struct amdgpu_ring *ring); + void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid, + uint32_t gds_base, uint32_t gds_size, + uint32_t gws_base, uint32_t gws_size, + uint32_t oa_base, uint32_t oa_size); + /* testing functions */ + int (*test_ring)(struct amdgpu_ring *ring); + int (*test_ib)(struct amdgpu_ring *ring, long timeout); + /* insert NOP packets */ + void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count); + void (*insert_start)(struct amdgpu_ring *ring); + void (*insert_end)(struct amdgpu_ring *ring); + /* pad the indirect buffer to the necessary number of dw */ + void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib); + unsigned (*init_cond_exec)(struct amdgpu_ring *ring); + void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset); + /* note usage for clock and power gating */ + void (*begin_use)(struct amdgpu_ring *ring); + void (*end_use)(struct amdgpu_ring *ring); + void (*emit_switch_buffer) (struct amdgpu_ring *ring); + void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags); + void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg); + void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val); + void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg, + uint32_t val, uint32_t mask); + void (*emit_reg_write_reg_wait)(struct amdgpu_ring *ring, + uint32_t reg0, uint32_t reg1, + uint32_t ref, uint32_t mask); + void (*emit_tmz)(struct amdgpu_ring *ring, bool start); + /* priority functions */ + void (*set_priority) (struct amdgpu_ring *ring, + enum drm_sched_priority priority); + /* Try to soft recover the ring to make the fence signal */ + void (*soft_recovery)(struct amdgpu_ring *ring, unsigned vmid); + int (*preempt_ib)(struct amdgpu_ring *ring); +}; + +struct amdgpu_ring { + struct amdgpu_device *adev; + const struct amdgpu_ring_funcs *funcs; + struct amdgpu_fence_driver fence_drv; + struct drm_gpu_scheduler sched; + + struct amdgpu_bo *ring_obj; + volatile uint32_t *ring; + unsigned rptr_offs; + u64 wptr; + u64 wptr_old; + unsigned ring_size; + unsigned max_dw; + int count_dw; + uint64_t gpu_addr; + uint64_t ptr_mask; + uint32_t buf_mask; + u32 idx; + u32 me; + u32 pipe; + u32 queue; + struct amdgpu_bo *mqd_obj; + uint64_t mqd_gpu_addr; + void *mqd_ptr; + uint64_t eop_gpu_addr; + u32 doorbell_index; + bool use_doorbell; + bool use_pollmem; + unsigned wptr_offs; + unsigned fence_offs; + uint64_t current_ctx; + char name[16]; + u32 trail_seq; + unsigned trail_fence_offs; + u64 trail_fence_gpu_addr; + volatile u32 *trail_fence_cpu_addr; + unsigned cond_exe_offs; + u64 cond_exe_gpu_addr; + volatile u32 *cond_exe_cpu_addr; + unsigned vm_inv_eng; + struct dma_fence *vmid_wait; + bool has_compute_vm_bug; + + atomic_t num_jobs[DRM_SCHED_PRIORITY_MAX]; + struct mutex priority_mutex; + /* protected by priority_mutex */ + int priority; + +#if defined(CONFIG_DEBUG_FS) + struct dentry *ent; +#endif +}; + +#define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib))) +#define amdgpu_ring_patch_cs_in_place(r, p, ib) ((r)->funcs->patch_cs_in_place((p), (ib))) +#define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r)) +#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t)) +#define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r)) +#define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) +#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) +#define amdgpu_ring_emit_ib(r, job, ib, flags) ((r)->funcs->emit_ib((r), (job), (ib), (flags))) +#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r)) +#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr)) +#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) +#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as)) +#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r)) +#define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r)) +#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d)) +#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d)) +#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v)) +#define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m)) +#define amdgpu_ring_emit_reg_write_reg_wait(r, d0, d1, v, m) (r)->funcs->emit_reg_write_reg_wait((r), (d0), (d1), (v), (m)) +#define amdgpu_ring_emit_tmz(r, b) (r)->funcs->emit_tmz((r), (b)) +#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib))) +#define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r)) +#define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o)) +#define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r) + +int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw); +void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count); +void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib); +void amdgpu_ring_commit(struct amdgpu_ring *ring); +void amdgpu_ring_undo(struct amdgpu_ring *ring); +void amdgpu_ring_priority_get(struct amdgpu_ring *ring, + enum drm_sched_priority priority); +void amdgpu_ring_priority_put(struct amdgpu_ring *ring, + enum drm_sched_priority priority); +int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, + unsigned ring_size, struct amdgpu_irq_src *irq_src, + unsigned irq_type); +void amdgpu_ring_fini(struct amdgpu_ring *ring); +void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring, + uint32_t reg0, uint32_t val0, + uint32_t reg1, uint32_t val1); +bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, + struct dma_fence *fence); + +static inline void amdgpu_ring_set_preempt_cond_exec(struct amdgpu_ring *ring, + bool cond_exec) +{ + *ring->cond_exe_cpu_addr = cond_exec; +} + +static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring) +{ + int i = 0; + while (i <= ring->buf_mask) + ring->ring[i++] = ring->funcs->nop; + +} + +static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v) +{ + if (ring->count_dw <= 0) + DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n"); + ring->ring[ring->wptr++ & ring->buf_mask] = v; + ring->wptr &= ring->ptr_mask; + ring->count_dw--; +} + +static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, + void *src, int count_dw) +{ + unsigned occupied, chunk1, chunk2; + void *dst; + + if (unlikely(ring->count_dw < count_dw)) + DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n"); + + occupied = ring->wptr & ring->buf_mask; + dst = (void *)&ring->ring[occupied]; + chunk1 = ring->buf_mask + 1 - occupied; + chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1; + chunk2 = count_dw - chunk1; + chunk1 <<= 2; + chunk2 <<= 2; + + if (chunk1) + memcpy(dst, src, chunk1); + + if (chunk2) { + src += chunk1; + dst = (void *)ring->ring; + memcpy(dst, src, chunk2); + } + + ring->wptr += count_dw; + ring->wptr &= ring->ptr_mask; + ring->count_dw -= count_dw; +} + +int amdgpu_ring_test_helper(struct amdgpu_ring *ring); + +#endif
diff --git a/amdgpu/amdgpu_rlc.c b/amdgpu/amdgpu_rlc.c new file mode 100644 index 0000000..c8793e6 --- /dev/null +++ b/amdgpu/amdgpu_rlc.c
@@ -0,0 +1,282 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include <linux/firmware.h> +#include "amdgpu.h" +#include "amdgpu_gfx.h" +#include "amdgpu_rlc.h" + +/** + * amdgpu_gfx_rlc_enter_safe_mode - Set RLC into safe mode + * + * @adev: amdgpu_device pointer + * + * Set RLC enter into safe mode if RLC is enabled and haven't in safe mode. + */ +void amdgpu_gfx_rlc_enter_safe_mode(struct amdgpu_device *adev) +{ + if (adev->gfx.rlc.in_safe_mode) + return; + + /* if RLC is not enabled, do nothing */ + if (!adev->gfx.rlc.funcs->is_rlc_enabled(adev)) + return; + + if (adev->cg_flags & + (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_3D_CGCG)) { + adev->gfx.rlc.funcs->set_safe_mode(adev); + adev->gfx.rlc.in_safe_mode = true; + } +} + +/** + * amdgpu_gfx_rlc_exit_safe_mode - Set RLC out of safe mode + * + * @adev: amdgpu_device pointer + * + * Set RLC exit safe mode if RLC is enabled and have entered into safe mode. + */ +void amdgpu_gfx_rlc_exit_safe_mode(struct amdgpu_device *adev) +{ + if (!(adev->gfx.rlc.in_safe_mode)) + return; + + /* if RLC is not enabled, do nothing */ + if (!adev->gfx.rlc.funcs->is_rlc_enabled(adev)) + return; + + if (adev->cg_flags & + (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_3D_CGCG)) { + adev->gfx.rlc.funcs->unset_safe_mode(adev); + adev->gfx.rlc.in_safe_mode = false; + } +} + +/** + * amdgpu_gfx_rlc_init_sr - Init save restore block + * + * @adev: amdgpu_device pointer + * @dws: the size of save restore block + * + * Allocate and setup value to save restore block of rlc. + * Returns 0 on succeess or negative error code if allocate failed. + */ +int amdgpu_gfx_rlc_init_sr(struct amdgpu_device *adev, u32 dws) +{ + const u32 *src_ptr; + volatile u32 *dst_ptr; + u32 i; + int r; + + /* allocate save restore block */ + r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.rlc.save_restore_obj, + &adev->gfx.rlc.save_restore_gpu_addr, + (void **)&adev->gfx.rlc.sr_ptr); + if (r) { + dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r); + amdgpu_gfx_rlc_fini(adev); + return r; + } + + /* write the sr buffer */ + src_ptr = adev->gfx.rlc.reg_list; + dst_ptr = adev->gfx.rlc.sr_ptr; + for (i = 0; i < adev->gfx.rlc.reg_list_size; i++) + dst_ptr[i] = cpu_to_le32(src_ptr[i]); + amdgpu_bo_kunmap(adev->gfx.rlc.save_restore_obj); + amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj); + + return 0; +} + +/** + * amdgpu_gfx_rlc_init_csb - Init clear state block + * + * @adev: amdgpu_device pointer + * + * Allocate and setup value to clear state block of rlc. + * Returns 0 on succeess or negative error code if allocate failed. + */ +int amdgpu_gfx_rlc_init_csb(struct amdgpu_device *adev) +{ + volatile u32 *dst_ptr; + u32 dws; + int r; + + /* allocate clear state block */ + adev->gfx.rlc.clear_state_size = dws = adev->gfx.rlc.funcs->get_csb_size(adev); + r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.rlc.clear_state_obj, + &adev->gfx.rlc.clear_state_gpu_addr, + (void **)&adev->gfx.rlc.cs_ptr); + if (r) { + dev_err(adev->dev, "(%d) failed to create rlc csb bo\n", r); + amdgpu_gfx_rlc_fini(adev); + return r; + } + + /* set up the cs buffer */ + dst_ptr = adev->gfx.rlc.cs_ptr; + adev->gfx.rlc.funcs->get_csb_buffer(adev, dst_ptr); + amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj); + amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj); + amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj); + + return 0; +} + +/** + * amdgpu_gfx_rlc_init_cpt - Init cp table + * + * @adev: amdgpu_device pointer + * + * Allocate and setup value to cp table of rlc. + * Returns 0 on succeess or negative error code if allocate failed. + */ +int amdgpu_gfx_rlc_init_cpt(struct amdgpu_device *adev) +{ + int r; + + r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.rlc.cp_table_obj, + &adev->gfx.rlc.cp_table_gpu_addr, + (void **)&adev->gfx.rlc.cp_table_ptr); + if (r) { + dev_err(adev->dev, "(%d) failed to create cp table bo\n", r); + amdgpu_gfx_rlc_fini(adev); + return r; + } + + /* set up the cp table */ + amdgpu_gfx_rlc_setup_cp_table(adev); + amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj); + amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj); + + return 0; +} + +/** + * amdgpu_gfx_rlc_setup_cp_table - setup cp the buffer of cp table + * + * @adev: amdgpu_device pointer + * + * Write cp firmware data into cp table. + */ +void amdgpu_gfx_rlc_setup_cp_table(struct amdgpu_device *adev) +{ + const __le32 *fw_data; + volatile u32 *dst_ptr; + int me, i, max_me; + u32 bo_offset = 0; + u32 table_offset, table_size; + + max_me = adev->gfx.rlc.funcs->get_cp_table_num(adev); + + /* write the cp table buffer */ + dst_ptr = adev->gfx.rlc.cp_table_ptr; + for (me = 0; me < max_me; me++) { + if (me == 0) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; + fw_data = (const __le32 *) + (adev->gfx.ce_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 1) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; + fw_data = (const __le32 *) + (adev->gfx.pfp_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 2) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; + fw_data = (const __le32 *) + (adev->gfx.me_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 3) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; + fw_data = (const __le32 *) + (adev->gfx.mec_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 4) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data; + fw_data = (const __le32 *) + (adev->gfx.mec2_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } + + for (i = 0; i < table_size; i ++) { + dst_ptr[bo_offset + i] = + cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); + } + + bo_offset += table_size; + } +} + +/** + * amdgpu_gfx_rlc_fini - Free BO which used for RLC + * + * @adev: amdgpu_device pointer + * + * Free three BO which is used for rlc_save_restore_block, rlc_clear_state_block + * and rlc_jump_table_block. + */ +void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev) +{ + /* save restore block */ + if (adev->gfx.rlc.save_restore_obj) { + amdgpu_bo_free_kernel(&adev->gfx.rlc.save_restore_obj, + &adev->gfx.rlc.save_restore_gpu_addr, + (void **)&adev->gfx.rlc.sr_ptr); + } + + /* clear state block */ + amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, + &adev->gfx.rlc.clear_state_gpu_addr, + (void **)&adev->gfx.rlc.cs_ptr); + + /* jump table block */ + amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, + &adev->gfx.rlc.cp_table_gpu_addr, + (void **)&adev->gfx.rlc.cp_table_ptr); +}
diff --git a/amdgpu/amdgpu_rlc.h b/amdgpu/amdgpu_rlc.h new file mode 100644 index 0000000..d3d4707 --- /dev/null +++ b/amdgpu/amdgpu_rlc.h
@@ -0,0 +1,196 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_RLC_H__ +#define __AMDGPU_RLC_H__ + +#include "clearstate_defs.h" + +/* firmware ID used in rlc toc */ +typedef enum _FIRMWARE_ID_ { + FIRMWARE_ID_INVALID = 0, + FIRMWARE_ID_RLC_G_UCODE = 1, + FIRMWARE_ID_RLC_TOC = 2, + FIRMWARE_ID_RLCG_SCRATCH = 3, + FIRMWARE_ID_RLC_SRM_ARAM = 4, + FIRMWARE_ID_RLC_SRM_INDEX_ADDR = 5, + FIRMWARE_ID_RLC_SRM_INDEX_DATA = 6, + FIRMWARE_ID_RLC_P_UCODE = 7, + FIRMWARE_ID_RLC_V_UCODE = 8, + FIRMWARE_ID_RLX6_UCODE = 9, + FIRMWARE_ID_RLX6_DRAM_BOOT = 10, + FIRMWARE_ID_GLOBAL_TAP_DELAYS = 11, + FIRMWARE_ID_SE0_TAP_DELAYS = 12, + FIRMWARE_ID_SE1_TAP_DELAYS = 13, + FIRMWARE_ID_GLOBAL_SE0_SE1_SKEW_DELAYS = 14, + FIRMWARE_ID_SDMA0_UCODE = 15, + FIRMWARE_ID_SDMA0_JT = 16, + FIRMWARE_ID_SDMA1_UCODE = 17, + FIRMWARE_ID_SDMA1_JT = 18, + FIRMWARE_ID_CP_CE = 19, + FIRMWARE_ID_CP_PFP = 20, + FIRMWARE_ID_CP_ME = 21, + FIRMWARE_ID_CP_MEC = 22, + FIRMWARE_ID_CP_MES = 23, + FIRMWARE_ID_MES_STACK = 24, + FIRMWARE_ID_RLC_SRM_DRAM_SR = 25, + FIRMWARE_ID_RLCG_SCRATCH_SR = 26, + FIRMWARE_ID_RLCP_SCRATCH_SR = 27, + FIRMWARE_ID_RLCV_SCRATCH_SR = 28, + FIRMWARE_ID_RLX6_DRAM_SR = 29, + FIRMWARE_ID_SDMA0_PG_CONTEXT = 30, + FIRMWARE_ID_SDMA1_PG_CONTEXT = 31, + FIRMWARE_ID_GLOBAL_MUX_SELECT_RAM = 32, + FIRMWARE_ID_SE0_MUX_SELECT_RAM = 33, + FIRMWARE_ID_SE1_MUX_SELECT_RAM = 34, + FIRMWARE_ID_ACCUM_CTRL_RAM = 35, + FIRMWARE_ID_RLCP_CAM = 36, + FIRMWARE_ID_RLC_SPP_CAM_EXT = 37, + FIRMWARE_ID_MAX = 38, +} FIRMWARE_ID; + +typedef struct _RLC_TABLE_OF_CONTENT { + union { + unsigned int DW0; + struct { + unsigned int offset : 25; + unsigned int id : 7; + }; + }; + + union { + unsigned int DW1; + struct { + unsigned int load_at_boot : 1; + unsigned int load_at_vddgfx : 1; + unsigned int load_at_reset : 1; + unsigned int memory_destination : 2; + unsigned int vfflr_image_code : 4; + unsigned int load_mode_direct : 1; + unsigned int save_for_vddgfx : 1; + unsigned int save_for_vfflr : 1; + unsigned int reserved : 1; + unsigned int signed_source : 1; + unsigned int size : 18; + }; + }; + + union { + unsigned int DW2; + struct { + unsigned int indirect_addr_reg : 16; + unsigned int index : 16; + }; + }; + + union { + unsigned int DW3; + struct { + unsigned int indirect_data_reg : 16; + unsigned int indirect_start_offset : 16; + }; + }; +} RLC_TABLE_OF_CONTENT; + +#define RLC_TOC_MAX_SIZE 64 + +struct amdgpu_rlc_funcs { + bool (*is_rlc_enabled)(struct amdgpu_device *adev); + void (*set_safe_mode)(struct amdgpu_device *adev); + void (*unset_safe_mode)(struct amdgpu_device *adev); + int (*init)(struct amdgpu_device *adev); + u32 (*get_csb_size)(struct amdgpu_device *adev); + void (*get_csb_buffer)(struct amdgpu_device *adev, volatile u32 *buffer); + int (*get_cp_table_num)(struct amdgpu_device *adev); + int (*resume)(struct amdgpu_device *adev); + void (*stop)(struct amdgpu_device *adev); + void (*reset)(struct amdgpu_device *adev); + void (*start)(struct amdgpu_device *adev); +}; + +struct amdgpu_rlc { + /* for power gating */ + struct amdgpu_bo *save_restore_obj; + uint64_t save_restore_gpu_addr; + volatile uint32_t *sr_ptr; + const u32 *reg_list; + u32 reg_list_size; + /* for clear state */ + struct amdgpu_bo *clear_state_obj; + uint64_t clear_state_gpu_addr; + volatile uint32_t *cs_ptr; + const struct cs_section_def *cs_data; + u32 clear_state_size; + /* for cp tables */ + struct amdgpu_bo *cp_table_obj; + uint64_t cp_table_gpu_addr; + volatile uint32_t *cp_table_ptr; + u32 cp_table_size; + + /* safe mode for updating CG/PG state */ + bool in_safe_mode; + const struct amdgpu_rlc_funcs *funcs; + + /* for firmware data */ + u32 save_and_restore_offset; + u32 clear_state_descriptor_offset; + u32 avail_scratch_ram_locations; + u32 reg_restore_list_size; + u32 reg_list_format_start; + u32 reg_list_format_separate_start; + u32 starting_offsets_start; + u32 reg_list_format_size_bytes; + u32 reg_list_size_bytes; + u32 reg_list_format_direct_reg_list_length; + u32 save_restore_list_cntl_size_bytes; + u32 save_restore_list_gpm_size_bytes; + u32 save_restore_list_srm_size_bytes; + + u32 *register_list_format; + u32 *register_restore; + u8 *save_restore_list_cntl; + u8 *save_restore_list_gpm; + u8 *save_restore_list_srm; + + bool is_rlc_v2_1; + + /* for rlc autoload */ + struct amdgpu_bo *rlc_autoload_bo; + u64 rlc_autoload_gpu_addr; + void *rlc_autoload_ptr; + + /* rlc toc buffer */ + struct amdgpu_bo *rlc_toc_bo; + uint64_t rlc_toc_gpu_addr; + void *rlc_toc_buf; +}; + +void amdgpu_gfx_rlc_enter_safe_mode(struct amdgpu_device *adev); +void amdgpu_gfx_rlc_exit_safe_mode(struct amdgpu_device *adev); +int amdgpu_gfx_rlc_init_sr(struct amdgpu_device *adev, u32 dws); +int amdgpu_gfx_rlc_init_csb(struct amdgpu_device *adev); +int amdgpu_gfx_rlc_init_cpt(struct amdgpu_device *adev); +void amdgpu_gfx_rlc_setup_cp_table(struct amdgpu_device *adev); +void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_sa.c b/amdgpu/amdgpu_sa.c new file mode 100644 index 0000000..0bd1d4f --- /dev/null +++ b/amdgpu/amdgpu_sa.c
@@ -0,0 +1,398 @@ +/* + * Copyright 2011 Red Hat Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Jerome Glisse <glisse@freedesktop.org> + */ +/* Algorithm: + * + * We store the last allocated bo in "hole", we always try to allocate + * after the last allocated bo. Principle is that in a linear GPU ring + * progression was is after last is the oldest bo we allocated and thus + * the first one that should no longer be in use by the GPU. + * + * If it's not the case we skip over the bo after last to the closest + * done bo if such one exist. If none exist and we are not asked to + * block we report failure to allocate. + * + * If we are asked to block we wait on all the oldest fence of all + * rings. We just wait for any of those fence to complete. + */ + +#include "amdgpu.h" + +static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo); +static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager); + +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev, + struct amdgpu_sa_manager *sa_manager, + unsigned size, u32 align, u32 domain) +{ + int i, r; + + init_waitqueue_head(&sa_manager->wq); + sa_manager->bo = NULL; + sa_manager->size = size; + sa_manager->domain = domain; + sa_manager->align = align; + sa_manager->hole = &sa_manager->olist; + INIT_LIST_HEAD(&sa_manager->olist); + for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) + INIT_LIST_HEAD(&sa_manager->flist[i]); + + r = amdgpu_bo_create_kernel(adev, size, align, domain, &sa_manager->bo, + &sa_manager->gpu_addr, &sa_manager->cpu_ptr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r); + return r; + } + + memset(sa_manager->cpu_ptr, 0, sa_manager->size); + return r; +} + +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev, + struct amdgpu_sa_manager *sa_manager) +{ + struct amdgpu_sa_bo *sa_bo, *tmp; + + if (sa_manager->bo == NULL) { + dev_err(adev->dev, "no bo for sa manager\n"); + return; + } + + if (!list_empty(&sa_manager->olist)) { + sa_manager->hole = &sa_manager->olist, + amdgpu_sa_bo_try_free(sa_manager); + if (!list_empty(&sa_manager->olist)) { + dev_err(adev->dev, "sa_manager is not empty, clearing anyway\n"); + } + } + list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) { + amdgpu_sa_bo_remove_locked(sa_bo); + } + + amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr); + sa_manager->size = 0; +} + +static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo) +{ + struct amdgpu_sa_manager *sa_manager = sa_bo->manager; + if (sa_manager->hole == &sa_bo->olist) { + sa_manager->hole = sa_bo->olist.prev; + } + list_del_init(&sa_bo->olist); + list_del_init(&sa_bo->flist); + dma_fence_put(sa_bo->fence); + kfree(sa_bo); +} + +static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager) +{ + struct amdgpu_sa_bo *sa_bo, *tmp; + + if (sa_manager->hole->next == &sa_manager->olist) + return; + + sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist); + list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) { + if (sa_bo->fence == NULL || + !dma_fence_is_signaled(sa_bo->fence)) { + return; + } + amdgpu_sa_bo_remove_locked(sa_bo); + } +} + +static inline unsigned amdgpu_sa_bo_hole_soffset(struct amdgpu_sa_manager *sa_manager) +{ + struct list_head *hole = sa_manager->hole; + + if (hole != &sa_manager->olist) { + return list_entry(hole, struct amdgpu_sa_bo, olist)->eoffset; + } + return 0; +} + +static inline unsigned amdgpu_sa_bo_hole_eoffset(struct amdgpu_sa_manager *sa_manager) +{ + struct list_head *hole = sa_manager->hole; + + if (hole->next != &sa_manager->olist) { + return list_entry(hole->next, struct amdgpu_sa_bo, olist)->soffset; + } + return sa_manager->size; +} + +static bool amdgpu_sa_bo_try_alloc(struct amdgpu_sa_manager *sa_manager, + struct amdgpu_sa_bo *sa_bo, + unsigned size, unsigned align) +{ + unsigned soffset, eoffset, wasted; + + soffset = amdgpu_sa_bo_hole_soffset(sa_manager); + eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager); + wasted = (align - (soffset % align)) % align; + + if ((eoffset - soffset) >= (size + wasted)) { + soffset += wasted; + + sa_bo->manager = sa_manager; + sa_bo->soffset = soffset; + sa_bo->eoffset = soffset + size; + list_add(&sa_bo->olist, sa_manager->hole); + INIT_LIST_HEAD(&sa_bo->flist); + sa_manager->hole = &sa_bo->olist; + return true; + } + return false; +} + +/** + * amdgpu_sa_event - Check if we can stop waiting + * + * @sa_manager: pointer to the sa_manager + * @size: number of bytes we want to allocate + * @align: alignment we need to match + * + * Check if either there is a fence we can wait for or + * enough free memory to satisfy the allocation directly + */ +static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager, + unsigned size, unsigned align) +{ + unsigned soffset, eoffset, wasted; + int i; + + for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) + if (!list_empty(&sa_manager->flist[i])) + return true; + + soffset = amdgpu_sa_bo_hole_soffset(sa_manager); + eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager); + wasted = (align - (soffset % align)) % align; + + if ((eoffset - soffset) >= (size + wasted)) { + return true; + } + + return false; +} + +static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, + struct dma_fence **fences, + unsigned *tries) +{ + struct amdgpu_sa_bo *best_bo = NULL; + unsigned i, soffset, best, tmp; + + /* if hole points to the end of the buffer */ + if (sa_manager->hole->next == &sa_manager->olist) { + /* try again with its beginning */ + sa_manager->hole = &sa_manager->olist; + return true; + } + + soffset = amdgpu_sa_bo_hole_soffset(sa_manager); + /* to handle wrap around we add sa_manager->size */ + best = sa_manager->size * 2; + /* go over all fence list and try to find the closest sa_bo + * of the current last + */ + for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) { + struct amdgpu_sa_bo *sa_bo; + + fences[i] = NULL; + + if (list_empty(&sa_manager->flist[i])) + continue; + + sa_bo = list_first_entry(&sa_manager->flist[i], + struct amdgpu_sa_bo, flist); + + if (!dma_fence_is_signaled(sa_bo->fence)) { + fences[i] = sa_bo->fence; + continue; + } + + /* limit the number of tries each ring gets */ + if (tries[i] > 2) { + continue; + } + + tmp = sa_bo->soffset; + if (tmp < soffset) { + /* wrap around, pretend it's after */ + tmp += sa_manager->size; + } + tmp -= soffset; + if (tmp < best) { + /* this sa bo is the closest one */ + best = tmp; + best_bo = sa_bo; + } + } + + if (best_bo) { + uint32_t idx = best_bo->fence->context; + + idx %= AMDGPU_SA_NUM_FENCE_LISTS; + ++tries[idx]; + sa_manager->hole = best_bo->olist.prev; + + /* we knew that this one is signaled, + so it's save to remote it */ + amdgpu_sa_bo_remove_locked(best_bo); + return true; + } + return false; +} + +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager, + struct amdgpu_sa_bo **sa_bo, + unsigned size, unsigned align) +{ + struct dma_fence *fences[AMDGPU_SA_NUM_FENCE_LISTS]; + unsigned tries[AMDGPU_SA_NUM_FENCE_LISTS]; + unsigned count; + int i, r; + signed long t; + + if (WARN_ON_ONCE(align > sa_manager->align)) + return -EINVAL; + + if (WARN_ON_ONCE(size > sa_manager->size)) + return -EINVAL; + + *sa_bo = kmalloc(sizeof(struct amdgpu_sa_bo), GFP_KERNEL); + if (!(*sa_bo)) + return -ENOMEM; + (*sa_bo)->manager = sa_manager; + (*sa_bo)->fence = NULL; + INIT_LIST_HEAD(&(*sa_bo)->olist); + INIT_LIST_HEAD(&(*sa_bo)->flist); + + spin_lock(&sa_manager->wq.lock); + do { + for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) + tries[i] = 0; + + do { + amdgpu_sa_bo_try_free(sa_manager); + + if (amdgpu_sa_bo_try_alloc(sa_manager, *sa_bo, + size, align)) { + spin_unlock(&sa_manager->wq.lock); + return 0; + } + + /* see if we can skip over some allocations */ + } while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries)); + + for (i = 0, count = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) + if (fences[i]) + fences[count++] = dma_fence_get(fences[i]); + + if (count) { + spin_unlock(&sa_manager->wq.lock); + t = dma_fence_wait_any_timeout(fences, count, false, + MAX_SCHEDULE_TIMEOUT, + NULL); + for (i = 0; i < count; ++i) + dma_fence_put(fences[i]); + + r = (t > 0) ? 0 : t; + spin_lock(&sa_manager->wq.lock); + } else { + /* if we have nothing to wait for block */ + r = wait_event_interruptible_locked( + sa_manager->wq, + amdgpu_sa_event(sa_manager, size, align) + ); + } + + } while (!r); + + spin_unlock(&sa_manager->wq.lock); + kfree(*sa_bo); + *sa_bo = NULL; + return r; +} + +void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, + struct dma_fence *fence) +{ + struct amdgpu_sa_manager *sa_manager; + + if (sa_bo == NULL || *sa_bo == NULL) { + return; + } + + sa_manager = (*sa_bo)->manager; + spin_lock(&sa_manager->wq.lock); + if (fence && !dma_fence_is_signaled(fence)) { + uint32_t idx; + + (*sa_bo)->fence = dma_fence_get(fence); + idx = fence->context % AMDGPU_SA_NUM_FENCE_LISTS; + list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]); + } else { + amdgpu_sa_bo_remove_locked(*sa_bo); + } + wake_up_all_locked(&sa_manager->wq); + spin_unlock(&sa_manager->wq.lock); + *sa_bo = NULL; +} + +#if defined(CONFIG_DEBUG_FS) + +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, + struct seq_file *m) +{ + struct amdgpu_sa_bo *i; + + spin_lock(&sa_manager->wq.lock); + list_for_each_entry(i, &sa_manager->olist, olist) { + uint64_t soffset = i->soffset + sa_manager->gpu_addr; + uint64_t eoffset = i->eoffset + sa_manager->gpu_addr; + if (&i->olist == sa_manager->hole) { + seq_printf(m, ">"); + } else { + seq_printf(m, " "); + } + seq_printf(m, "[0x%010llx 0x%010llx] size %8lld", + soffset, eoffset, eoffset - soffset); + + if (i->fence) + seq_printf(m, " protected by 0x%016llx on context %llu", + i->fence->seqno, i->fence->context); + + seq_printf(m, "\n"); + } + spin_unlock(&sa_manager->wq.lock); +} +#endif
diff --git a/amdgpu/amdgpu_sched.c b/amdgpu/amdgpu_sched.c new file mode 100644 index 0000000..c799691 --- /dev/null +++ b/amdgpu/amdgpu_sched.c
@@ -0,0 +1,145 @@ +/* + * Copyright 2017 Valve Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Andres Rodriguez <andresx7@gmail.com> + */ + +#include <linux/fdtable.h> +#include <linux/file.h> +#include <linux/pid.h> + +#include <drm/amdgpu_drm.h> + +#include "amdgpu.h" + +#include "amdgpu_vm.h" + +enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority) +{ + switch (amdgpu_priority) { + case AMDGPU_CTX_PRIORITY_VERY_HIGH: + return DRM_SCHED_PRIORITY_HIGH_HW; + case AMDGPU_CTX_PRIORITY_HIGH: + return DRM_SCHED_PRIORITY_HIGH_SW; + case AMDGPU_CTX_PRIORITY_NORMAL: + return DRM_SCHED_PRIORITY_NORMAL; + case AMDGPU_CTX_PRIORITY_LOW: + case AMDGPU_CTX_PRIORITY_VERY_LOW: + return DRM_SCHED_PRIORITY_LOW; + case AMDGPU_CTX_PRIORITY_UNSET: + return DRM_SCHED_PRIORITY_UNSET; + default: + WARN(1, "Invalid context priority %d\n", amdgpu_priority); + return DRM_SCHED_PRIORITY_INVALID; + } +} + +static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, + int fd, + enum drm_sched_priority priority) +{ + struct fd f = fdget(fd); + struct amdgpu_fpriv *fpriv; + struct amdgpu_ctx *ctx; + uint32_t id; + int r; + + if (!f.file) + return -EINVAL; + + r = amdgpu_file_to_fpriv(f.file, &fpriv); + if (r) { + fdput(f); + return r; + } + + idr_for_each_entry(&fpriv->ctx_mgr.ctx_handles, ctx, id) + amdgpu_ctx_priority_override(ctx, priority); + + fdput(f); + return 0; +} + +static int amdgpu_sched_context_priority_override(struct amdgpu_device *adev, + int fd, + unsigned ctx_id, + enum drm_sched_priority priority) +{ + struct fd f = fdget(fd); + struct amdgpu_fpriv *fpriv; + struct amdgpu_ctx *ctx; + int r; + + if (!f.file) + return -EINVAL; + + r = amdgpu_file_to_fpriv(f.file, &fpriv); + if (r) { + fdput(f); + return r; + } + + ctx = amdgpu_ctx_get(fpriv, ctx_id); + + if (!ctx) { + fdput(f); + return -EINVAL; + } + + amdgpu_ctx_priority_override(ctx, priority); + amdgpu_ctx_put(ctx); + fdput(f); + + return 0; +} + +int amdgpu_sched_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + union drm_amdgpu_sched *args = data; + struct amdgpu_device *adev = dev->dev_private; + enum drm_sched_priority priority; + int r; + + priority = amdgpu_to_sched_priority(args->in.priority); + if (priority == DRM_SCHED_PRIORITY_INVALID) + return -EINVAL; + + switch (args->in.op) { + case AMDGPU_SCHED_OP_PROCESS_PRIORITY_OVERRIDE: + r = amdgpu_sched_process_priority_override(adev, + args->in.fd, + priority); + break; + case AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE: + r = amdgpu_sched_context_priority_override(adev, + args->in.fd, + args->in.ctx_id, + priority); + break; + default: + DRM_ERROR("Invalid sched op specified: %d\n", args->in.op); + r = -EINVAL; + break; + } + + return r; +}
diff --git a/amdgpu/amdgpu_sched.h b/amdgpu/amdgpu_sched.h new file mode 100644 index 0000000..12299fd --- /dev/null +++ b/amdgpu/amdgpu_sched.h
@@ -0,0 +1,37 @@ +/* + * Copyright 2017 Valve Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Andres Rodriguez <andresx7@gmail.com> + */ + +#ifndef __AMDGPU_SCHED_H__ +#define __AMDGPU_SCHED_H__ + +enum drm_sched_priority; + +struct drm_device; +struct drm_file; + +enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority); +int amdgpu_sched_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +#endif // __AMDGPU_SCHED_H__
diff --git a/amdgpu/amdgpu_sdma.c b/amdgpu/amdgpu_sdma.c new file mode 100644 index 0000000..5c13c50 --- /dev/null +++ b/amdgpu/amdgpu_sdma.c
@@ -0,0 +1,85 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu.h" +#include "amdgpu_sdma.h" + +#define AMDGPU_CSA_SDMA_SIZE 64 +/* SDMA CSA reside in the 3rd page of CSA */ +#define AMDGPU_CSA_SDMA_OFFSET (4096 * 2) + +/* + * GPU SDMA IP block helpers function. + */ + +struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) + if (ring == &adev->sdma.instance[i].ring || + ring == &adev->sdma.instance[i].page) + return &adev->sdma.instance[i]; + + return NULL; +} + +int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index) +{ + struct amdgpu_device *adev = ring->adev; + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) { + if (ring == &adev->sdma.instance[i].ring || + ring == &adev->sdma.instance[i].page) { + *index = i; + return 0; + } + } + + return -EINVAL; +} + +uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, + unsigned vmid) +{ + struct amdgpu_device *adev = ring->adev; + uint64_t csa_mc_addr; + uint32_t index = 0; + int r; + + if (vmid == 0 || !amdgpu_mcbp) + return 0; + + r = amdgpu_sdma_get_index_from_ring(ring, &index); + + if (r || index > 31) + csa_mc_addr = 0; + else + csa_mc_addr = amdgpu_csa_vaddr(adev) + + AMDGPU_CSA_SDMA_OFFSET + + index * AMDGPU_CSA_SDMA_SIZE; + + return csa_mc_addr; +}
diff --git a/amdgpu/amdgpu_sdma.h b/amdgpu/amdgpu_sdma.h new file mode 100644 index 0000000..35dd152 --- /dev/null +++ b/amdgpu/amdgpu_sdma.h
@@ -0,0 +1,101 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_SDMA_H__ +#define __AMDGPU_SDMA_H__ + +/* max number of IP instances */ +#define AMDGPU_MAX_SDMA_INSTANCES 2 + +enum amdgpu_sdma_irq { + AMDGPU_SDMA_IRQ_INSTANCE0 = 0, + AMDGPU_SDMA_IRQ_INSTANCE1, + AMDGPU_SDMA_IRQ_LAST +}; + +struct amdgpu_sdma_instance { + /* SDMA firmware */ + const struct firmware *fw; + uint32_t fw_version; + uint32_t feature_version; + + struct amdgpu_ring ring; + struct amdgpu_ring page; + bool burst_nop; +}; + +struct amdgpu_sdma { + struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES]; + struct amdgpu_irq_src trap_irq; + struct amdgpu_irq_src illegal_inst_irq; + struct amdgpu_irq_src ecc_irq; + int num_instances; + uint32_t srbm_soft_reset; + bool has_page_queue; + struct ras_common_if *ras_if; +}; + +/* + * Provided by hw blocks that can move/clear data. e.g., gfx or sdma + * But currently, we use sdma to move data. + */ +struct amdgpu_buffer_funcs { + /* maximum bytes in a single operation */ + uint32_t copy_max_bytes; + + /* number of dw to reserve per operation */ + unsigned copy_num_dw; + + /* used for buffer migration */ + void (*emit_copy_buffer)(struct amdgpu_ib *ib, + /* src addr in bytes */ + uint64_t src_offset, + /* dst addr in bytes */ + uint64_t dst_offset, + /* number of byte to transfer */ + uint32_t byte_count); + + /* maximum bytes in a single operation */ + uint32_t fill_max_bytes; + + /* number of dw to reserve per operation */ + unsigned fill_num_dw; + + /* used for buffer clearing */ + void (*emit_fill_buffer)(struct amdgpu_ib *ib, + /* value to write to memory */ + uint32_t src_data, + /* dst addr in bytes */ + uint64_t dst_offset, + /* number of byte to fill */ + uint32_t byte_count); +}; + +#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) + +struct amdgpu_sdma_instance * +amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring); +int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index); +uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, unsigned vmid); +#endif
diff --git a/amdgpu/amdgpu_socbb.h b/amdgpu/amdgpu_socbb.h new file mode 100644 index 0000000..f4176cb --- /dev/null +++ b/amdgpu/amdgpu_socbb.h
@@ -0,0 +1,82 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_SOCBB_H__ +#define __AMDGPU_SOCBB_H__ + +struct gpu_info_voltage_scaling_v1_0 { + uint32_t state; + uint32_t dscclk_mhz; + uint32_t dcfclk_mhz; + uint32_t socclk_mhz; + uint32_t dram_speed_mts; + uint32_t fabricclk_mhz; + uint32_t dispclk_mhz; + uint32_t phyclk_mhz; + uint32_t dppclk_mhz; +}; + +struct gpu_info_soc_bounding_box_v1_0 { + uint32_t sr_exit_time_us; + uint32_t sr_enter_plus_exit_time_us; + uint32_t urgent_latency_us; + uint32_t urgent_latency_pixel_data_only_us; + uint32_t urgent_latency_pixel_mixed_with_vm_data_us; + uint32_t urgent_latency_vm_data_only_us; + uint32_t writeback_latency_us; + uint32_t ideal_dram_bw_after_urgent_percent; + uint32_t pct_ideal_dram_sdp_bw_after_urgent_pixel_only; // PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly + uint32_t pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm; + uint32_t pct_ideal_dram_sdp_bw_after_urgent_vm_only; + uint32_t max_avg_sdp_bw_use_normal_percent; + uint32_t max_avg_dram_bw_use_normal_percent; + uint32_t max_request_size_bytes; + uint32_t downspread_percent; + uint32_t dram_page_open_time_ns; + uint32_t dram_rw_turnaround_time_ns; + uint32_t dram_return_buffer_per_channel_bytes; + uint32_t dram_channel_width_bytes; + uint32_t fabric_datapath_to_dcn_data_return_bytes; + uint32_t dcn_downspread_percent; + uint32_t dispclk_dppclk_vco_speed_mhz; + uint32_t dfs_vco_period_ps; + uint32_t urgent_out_of_order_return_per_channel_pixel_only_bytes; + uint32_t urgent_out_of_order_return_per_channel_pixel_and_vm_bytes; + uint32_t urgent_out_of_order_return_per_channel_vm_only_bytes; + uint32_t round_trip_ping_latency_dcfclk_cycles; + uint32_t urgent_out_of_order_return_per_channel_bytes; + uint32_t channel_interleave_bytes; + uint32_t num_banks; + uint32_t num_chans; + uint32_t vmm_page_size_bytes; + uint32_t dram_clock_change_latency_us; + uint32_t writeback_dram_clock_change_latency_us; + uint32_t return_bus_width_bytes; + uint32_t voltage_override; + uint32_t xfc_bus_transport_time_us; + uint32_t xfc_xbuf_latency_tolerance_us; + uint32_t use_urgent_burst_bw; + uint32_t num_states; + struct gpu_info_voltage_scaling_v1_0 clock_limits[8]; +}; + +#endif
diff --git a/amdgpu/amdgpu_sync.c b/amdgpu/amdgpu_sync.c new file mode 100644 index 0000000..9828f3c --- /dev/null +++ b/amdgpu/amdgpu_sync.c
@@ -0,0 +1,424 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König <christian.koenig@amd.com> + */ + +#include "amdgpu.h" +#include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" + +struct amdgpu_sync_entry { + struct hlist_node node; + struct dma_fence *fence; + bool explicit; +}; + +static struct kmem_cache *amdgpu_sync_slab; + +/** + * amdgpu_sync_create - zero init sync object + * + * @sync: sync object to initialize + * + * Just clear the sync object for now. + */ +void amdgpu_sync_create(struct amdgpu_sync *sync) +{ + hash_init(sync->fences); + sync->last_vm_update = NULL; +} + +/** + * amdgpu_sync_same_dev - test if fence belong to us + * + * @adev: amdgpu device to use for the test + * @f: fence to test + * + * Test if the fence was issued by us. + */ +static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, + struct dma_fence *f) +{ + struct drm_sched_fence *s_fence = to_drm_sched_fence(f); + + if (s_fence) { + struct amdgpu_ring *ring; + + ring = container_of(s_fence->sched, struct amdgpu_ring, sched); + return ring->adev == adev; + } + + return false; +} + +/** + * amdgpu_sync_get_owner - extract the owner of a fence + * + * @fence: fence get the owner from + * + * Extract who originally created the fence. + */ +static void *amdgpu_sync_get_owner(struct dma_fence *f) +{ + struct drm_sched_fence *s_fence; + struct amdgpu_amdkfd_fence *kfd_fence; + + if (!f) + return AMDGPU_FENCE_OWNER_UNDEFINED; + + s_fence = to_drm_sched_fence(f); + if (s_fence) + return s_fence->owner; + + kfd_fence = to_amdgpu_amdkfd_fence(f); + if (kfd_fence) + return AMDGPU_FENCE_OWNER_KFD; + + return AMDGPU_FENCE_OWNER_UNDEFINED; +} + +/** + * amdgpu_sync_keep_later - Keep the later fence + * + * @keep: existing fence to test + * @fence: new fence + * + * Either keep the existing fence or the new one, depending which one is later. + */ +static void amdgpu_sync_keep_later(struct dma_fence **keep, + struct dma_fence *fence) +{ + if (*keep && dma_fence_is_later(*keep, fence)) + return; + + dma_fence_put(*keep); + *keep = dma_fence_get(fence); +} + +/** + * amdgpu_sync_add_later - add the fence to the hash + * + * @sync: sync object to add the fence to + * @f: fence to add + * + * Tries to add the fence to an existing hash entry. Returns true when an entry + * was found, false otherwise. + */ +static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f, bool explicit) +{ + struct amdgpu_sync_entry *e; + + hash_for_each_possible(sync->fences, e, node, f->context) { + if (unlikely(e->fence->context != f->context)) + continue; + + amdgpu_sync_keep_later(&e->fence, f); + + /* Preserve eplicit flag to not loose pipe line sync */ + e->explicit |= explicit; + + return true; + } + return false; +} + +/** + * amdgpu_sync_fence - remember to sync to this fence + * + * @sync: sync object to add fence to + * @fence: fence to sync to + * + */ +int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, + struct dma_fence *f, bool explicit) +{ + struct amdgpu_sync_entry *e; + + if (!f) + return 0; + if (amdgpu_sync_same_dev(adev, f) && + amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM) + amdgpu_sync_keep_later(&sync->last_vm_update, f); + + if (amdgpu_sync_add_later(sync, f, explicit)) + return 0; + + e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); + if (!e) + return -ENOMEM; + + e->explicit = explicit; + + hash_add(sync->fences, &e->node, f->context); + e->fence = dma_fence_get(f); + return 0; +} + +/** + * amdgpu_sync_resv - sync to a reservation object + * + * @sync: sync object to add fences from reservation object to + * @resv: reservation object with embedded fence + * @explicit_sync: true if we should only sync to the exclusive fence + * + * Sync to the fence + */ +int amdgpu_sync_resv(struct amdgpu_device *adev, + struct amdgpu_sync *sync, + struct reservation_object *resv, + void *owner, bool explicit_sync) +{ + struct reservation_object_list *flist; + struct dma_fence *f; + void *fence_owner; + unsigned i; + int r = 0; + + if (resv == NULL) + return -EINVAL; + + /* always sync to the exclusive fence */ + f = reservation_object_get_excl(resv); + r = amdgpu_sync_fence(adev, sync, f, false); + + flist = reservation_object_get_list(resv); + if (!flist || r) + return r; + + for (i = 0; i < flist->shared_count; ++i) { + f = rcu_dereference_protected(flist->shared[i], + reservation_object_held(resv)); + /* We only want to trigger KFD eviction fences on + * evict or move jobs. Skip KFD fences otherwise. + */ + fence_owner = amdgpu_sync_get_owner(f); + if (fence_owner == AMDGPU_FENCE_OWNER_KFD && + owner != AMDGPU_FENCE_OWNER_UNDEFINED) + continue; + + if (amdgpu_sync_same_dev(adev, f)) { + /* VM updates are only interesting + * for other VM updates and moves. + */ + if ((owner != AMDGPU_FENCE_OWNER_UNDEFINED) && + (fence_owner != AMDGPU_FENCE_OWNER_UNDEFINED) && + ((owner == AMDGPU_FENCE_OWNER_VM) != + (fence_owner == AMDGPU_FENCE_OWNER_VM))) + continue; + + /* Ignore fence from the same owner and explicit one as + * long as it isn't undefined. + */ + if (owner != AMDGPU_FENCE_OWNER_UNDEFINED && + (fence_owner == owner || explicit_sync)) + continue; + } + + r = amdgpu_sync_fence(adev, sync, f, false); + if (r) + break; + } + return r; +} + +/** + * amdgpu_sync_peek_fence - get the next fence not signaled yet + * + * @sync: the sync object + * @ring: optional ring to use for test + * + * Returns the next fence not signaled yet without removing it from the sync + * object. + */ +struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, + struct amdgpu_ring *ring) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + int i; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + struct dma_fence *f = e->fence; + struct drm_sched_fence *s_fence = to_drm_sched_fence(f); + + if (dma_fence_is_signaled(f)) { + hash_del(&e->node); + dma_fence_put(f); + kmem_cache_free(amdgpu_sync_slab, e); + continue; + } + if (ring && s_fence) { + /* For fences from the same ring it is sufficient + * when they are scheduled. + */ + if (s_fence->sched == &ring->sched) { + if (dma_fence_is_signaled(&s_fence->scheduled)) + continue; + + return &s_fence->scheduled; + } + } + + return f; + } + + return NULL; +} + +/** + * amdgpu_sync_get_fence - get the next fence from the sync object + * + * @sync: sync object to use + * @explicit: true if the next fence is explicit + * + * Get and removes the next fence from the sync object not signaled yet. + */ +struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + struct dma_fence *f; + int i; + hash_for_each_safe(sync->fences, i, tmp, e, node) { + + f = e->fence; + if (explicit) + *explicit = e->explicit; + + hash_del(&e->node); + kmem_cache_free(amdgpu_sync_slab, e); + + if (!dma_fence_is_signaled(f)) + return f; + + dma_fence_put(f); + } + return NULL; +} + +/** + * amdgpu_sync_clone - clone a sync object + * + * @source: sync object to clone + * @clone: pointer to destination sync object + * + * Adds references to all unsignaled fences in @source to @clone. Also + * removes signaled fences from @source while at it. + */ +int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + struct dma_fence *f; + int i, r; + + hash_for_each_safe(source->fences, i, tmp, e, node) { + f = e->fence; + if (!dma_fence_is_signaled(f)) { + r = amdgpu_sync_fence(NULL, clone, f, e->explicit); + if (r) + return r; + } else { + hash_del(&e->node); + dma_fence_put(f); + kmem_cache_free(amdgpu_sync_slab, e); + } + } + + dma_fence_put(clone->last_vm_update); + clone->last_vm_update = dma_fence_get(source->last_vm_update); + + return 0; +} + +int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + int i, r; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + r = dma_fence_wait(e->fence, intr); + if (r) + return r; + + hash_del(&e->node); + dma_fence_put(e->fence); + kmem_cache_free(amdgpu_sync_slab, e); + } + + return 0; +} + +/** + * amdgpu_sync_free - free the sync object + * + * @sync: sync object to use + * + * Free the sync object. + */ +void amdgpu_sync_free(struct amdgpu_sync *sync) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + unsigned i; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + hash_del(&e->node); + dma_fence_put(e->fence); + kmem_cache_free(amdgpu_sync_slab, e); + } + + dma_fence_put(sync->last_vm_update); +} + +/** + * amdgpu_sync_init - init sync object subsystem + * + * Allocate the slab allocator. + */ +int amdgpu_sync_init(void) +{ + amdgpu_sync_slab = kmem_cache_create( + "amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!amdgpu_sync_slab) + return -ENOMEM; + + return 0; +} + +/** + * amdgpu_sync_fini - fini sync object subsystem + * + * Free the slab allocator. + */ +void amdgpu_sync_fini(void) +{ + kmem_cache_destroy(amdgpu_sync_slab); +}
diff --git a/amdgpu/amdgpu_sync.h b/amdgpu/amdgpu_sync.h new file mode 100644 index 0000000..10cf23a --- /dev/null +++ b/amdgpu/amdgpu_sync.h
@@ -0,0 +1,59 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ +#ifndef __AMDGPU_SYNC_H__ +#define __AMDGPU_SYNC_H__ + +#include <linux/hashtable.h> + +struct dma_fence; +struct reservation_object; +struct amdgpu_device; +struct amdgpu_ring; + +/* + * Container for fences used to sync command submissions. + */ +struct amdgpu_sync { + DECLARE_HASHTABLE(fences, 4); + struct dma_fence *last_vm_update; +}; + +void amdgpu_sync_create(struct amdgpu_sync *sync); +int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, + struct dma_fence *f, bool explicit); +int amdgpu_sync_resv(struct amdgpu_device *adev, + struct amdgpu_sync *sync, + struct reservation_object *resv, + void *owner, + bool explicit_sync); +struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, + struct amdgpu_ring *ring); +struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit); +int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone); +int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr); +void amdgpu_sync_free(struct amdgpu_sync *sync); +int amdgpu_sync_init(void); +void amdgpu_sync_fini(void); + +#endif
diff --git a/amdgpu/amdgpu_trace.h b/amdgpu/amdgpu_trace.h new file mode 100644 index 0000000..77674a7 --- /dev/null +++ b/amdgpu/amdgpu_trace.h
@@ -0,0 +1,497 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#if !defined(_AMDGPU_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _AMDGPU_TRACE_H_ + +#include <linux/stringify.h> +#include <linux/types.h> +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM amdgpu +#define TRACE_INCLUDE_FILE amdgpu_trace + +#define AMDGPU_JOB_GET_TIMELINE_NAME(job) \ + job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished) + +TRACE_EVENT(amdgpu_mm_rreg, + TP_PROTO(unsigned did, uint32_t reg, uint32_t value), + TP_ARGS(did, reg, value), + TP_STRUCT__entry( + __field(unsigned, did) + __field(uint32_t, reg) + __field(uint32_t, value) + ), + TP_fast_assign( + __entry->did = did; + __entry->reg = reg; + __entry->value = value; + ), + TP_printk("0x%04lx, 0x%08lx, 0x%08lx", + (unsigned long)__entry->did, + (unsigned long)__entry->reg, + (unsigned long)__entry->value) +); + +TRACE_EVENT(amdgpu_mm_wreg, + TP_PROTO(unsigned did, uint32_t reg, uint32_t value), + TP_ARGS(did, reg, value), + TP_STRUCT__entry( + __field(unsigned, did) + __field(uint32_t, reg) + __field(uint32_t, value) + ), + TP_fast_assign( + __entry->did = did; + __entry->reg = reg; + __entry->value = value; + ), + TP_printk("0x%04lx, 0x%08lx, 0x%08lx", + (unsigned long)__entry->did, + (unsigned long)__entry->reg, + (unsigned long)__entry->value) +); + +TRACE_EVENT(amdgpu_iv, + TP_PROTO(unsigned ih, struct amdgpu_iv_entry *iv), + TP_ARGS(ih, iv), + TP_STRUCT__entry( + __field(unsigned, ih) + __field(unsigned, client_id) + __field(unsigned, src_id) + __field(unsigned, ring_id) + __field(unsigned, vmid) + __field(unsigned, vmid_src) + __field(uint64_t, timestamp) + __field(unsigned, timestamp_src) + __field(unsigned, pasid) + __array(unsigned, src_data, 4) + ), + TP_fast_assign( + __entry->ih = ih; + __entry->client_id = iv->client_id; + __entry->src_id = iv->src_id; + __entry->ring_id = iv->ring_id; + __entry->vmid = iv->vmid; + __entry->vmid_src = iv->vmid_src; + __entry->timestamp = iv->timestamp; + __entry->timestamp_src = iv->timestamp_src; + __entry->pasid = iv->pasid; + __entry->src_data[0] = iv->src_data[0]; + __entry->src_data[1] = iv->src_data[1]; + __entry->src_data[2] = iv->src_data[2]; + __entry->src_data[3] = iv->src_data[3]; + ), + TP_printk("ih:%u client_id:%u src_id:%u ring:%u vmid:%u " + "timestamp: %llu pasid:%u src_data: %08x %08x %08x %08x", + __entry->ih, __entry->client_id, __entry->src_id, + __entry->ring_id, __entry->vmid, + __entry->timestamp, __entry->pasid, + __entry->src_data[0], __entry->src_data[1], + __entry->src_data[2], __entry->src_data[3]) +); + + +TRACE_EVENT(amdgpu_bo_create, + TP_PROTO(struct amdgpu_bo *bo), + TP_ARGS(bo), + TP_STRUCT__entry( + __field(struct amdgpu_bo *, bo) + __field(u32, pages) + __field(u32, type) + __field(u32, prefer) + __field(u32, allow) + __field(u32, visible) + ), + + TP_fast_assign( + __entry->bo = bo; + __entry->pages = bo->tbo.num_pages; + __entry->type = bo->tbo.mem.mem_type; + __entry->prefer = bo->preferred_domains; + __entry->allow = bo->allowed_domains; + __entry->visible = bo->flags; + ), + + TP_printk("bo=%p, pages=%u, type=%d, preferred=%d, allowed=%d, visible=%d", + __entry->bo, __entry->pages, __entry->type, + __entry->prefer, __entry->allow, __entry->visible) +); + +TRACE_EVENT(amdgpu_cs, + TP_PROTO(struct amdgpu_cs_parser *p, int i), + TP_ARGS(p, i), + TP_STRUCT__entry( + __field(struct amdgpu_bo_list *, bo_list) + __field(u32, ring) + __field(u32, dw) + __field(u32, fences) + ), + + TP_fast_assign( + __entry->bo_list = p->bo_list; + __entry->ring = to_amdgpu_ring(p->entity->rq->sched)->idx; + __entry->dw = p->job->ibs[i].length_dw; + __entry->fences = amdgpu_fence_count_emitted( + to_amdgpu_ring(p->entity->rq->sched)); + ), + TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u", + __entry->bo_list, __entry->ring, __entry->dw, + __entry->fences) +); + +TRACE_EVENT(amdgpu_cs_ioctl, + TP_PROTO(struct amdgpu_job *job), + TP_ARGS(job), + TP_STRUCT__entry( + __field(uint64_t, sched_job_id) + __string(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job)) + __field(unsigned int, context) + __field(unsigned int, seqno) + __field(struct dma_fence *, fence) + __field(char *, ring_name) + __field(u32, num_ibs) + ), + + TP_fast_assign( + __entry->sched_job_id = job->base.id; + __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job)) + __entry->context = job->base.s_fence->finished.context; + __entry->seqno = job->base.s_fence->finished.seqno; + __entry->ring_name = to_amdgpu_ring(job->base.sched)->name; + __entry->num_ibs = job->num_ibs; + ), + TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u", + __entry->sched_job_id, __get_str(timeline), __entry->context, + __entry->seqno, __entry->ring_name, __entry->num_ibs) +); + +TRACE_EVENT(amdgpu_sched_run_job, + TP_PROTO(struct amdgpu_job *job), + TP_ARGS(job), + TP_STRUCT__entry( + __field(uint64_t, sched_job_id) + __string(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job)) + __field(unsigned int, context) + __field(unsigned int, seqno) + __field(char *, ring_name) + __field(u32, num_ibs) + ), + + TP_fast_assign( + __entry->sched_job_id = job->base.id; + __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job)) + __entry->context = job->base.s_fence->finished.context; + __entry->seqno = job->base.s_fence->finished.seqno; + __entry->ring_name = to_amdgpu_ring(job->base.sched)->name; + __entry->num_ibs = job->num_ibs; + ), + TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u", + __entry->sched_job_id, __get_str(timeline), __entry->context, + __entry->seqno, __entry->ring_name, __entry->num_ibs) +); + + +TRACE_EVENT(amdgpu_vm_grab_id, + TP_PROTO(struct amdgpu_vm *vm, struct amdgpu_ring *ring, + struct amdgpu_job *job), + TP_ARGS(vm, ring, job), + TP_STRUCT__entry( + __field(u32, pasid) + __string(ring, ring->name) + __field(u32, ring) + __field(u32, vmid) + __field(u32, vm_hub) + __field(u64, pd_addr) + __field(u32, needs_flush) + ), + + TP_fast_assign( + __entry->pasid = vm->pasid; + __assign_str(ring, ring->name) + __entry->vmid = job->vmid; + __entry->vm_hub = ring->funcs->vmhub, + __entry->pd_addr = job->vm_pd_addr; + __entry->needs_flush = job->vm_needs_flush; + ), + TP_printk("pasid=%d, ring=%s, id=%u, hub=%u, pd_addr=%010Lx needs_flush=%u", + __entry->pasid, __get_str(ring), __entry->vmid, + __entry->vm_hub, __entry->pd_addr, __entry->needs_flush) +); + +TRACE_EVENT(amdgpu_vm_bo_map, + TP_PROTO(struct amdgpu_bo_va *bo_va, + struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(bo_va, mapping), + TP_STRUCT__entry( + __field(struct amdgpu_bo *, bo) + __field(long, start) + __field(long, last) + __field(u64, offset) + __field(u64, flags) + ), + + TP_fast_assign( + __entry->bo = bo_va ? bo_va->base.bo : NULL; + __entry->start = mapping->start; + __entry->last = mapping->last; + __entry->offset = mapping->offset; + __entry->flags = mapping->flags; + ), + TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%llx", + __entry->bo, __entry->start, __entry->last, + __entry->offset, __entry->flags) +); + +TRACE_EVENT(amdgpu_vm_bo_unmap, + TP_PROTO(struct amdgpu_bo_va *bo_va, + struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(bo_va, mapping), + TP_STRUCT__entry( + __field(struct amdgpu_bo *, bo) + __field(long, start) + __field(long, last) + __field(u64, offset) + __field(u64, flags) + ), + + TP_fast_assign( + __entry->bo = bo_va ? bo_va->base.bo : NULL; + __entry->start = mapping->start; + __entry->last = mapping->last; + __entry->offset = mapping->offset; + __entry->flags = mapping->flags; + ), + TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%llx", + __entry->bo, __entry->start, __entry->last, + __entry->offset, __entry->flags) +); + +DECLARE_EVENT_CLASS(amdgpu_vm_mapping, + TP_PROTO(struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(mapping), + TP_STRUCT__entry( + __field(u64, soffset) + __field(u64, eoffset) + __field(u64, flags) + ), + + TP_fast_assign( + __entry->soffset = mapping->start; + __entry->eoffset = mapping->last + 1; + __entry->flags = mapping->flags; + ), + TP_printk("soffs=%010llx, eoffs=%010llx, flags=%llx", + __entry->soffset, __entry->eoffset, __entry->flags) +); + +DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_update, + TP_PROTO(struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(mapping) +); + +DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_mapping, + TP_PROTO(struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(mapping) +); + +DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_cs, + TP_PROTO(struct amdgpu_bo_va_mapping *mapping), + TP_ARGS(mapping) +); + +TRACE_EVENT(amdgpu_vm_set_ptes, + TP_PROTO(uint64_t pe, uint64_t addr, unsigned count, + uint32_t incr, uint64_t flags), + TP_ARGS(pe, addr, count, incr, flags), + TP_STRUCT__entry( + __field(u64, pe) + __field(u64, addr) + __field(u32, count) + __field(u32, incr) + __field(u64, flags) + ), + + TP_fast_assign( + __entry->pe = pe; + __entry->addr = addr; + __entry->count = count; + __entry->incr = incr; + __entry->flags = flags; + ), + TP_printk("pe=%010Lx, addr=%010Lx, incr=%u, flags=%llx, count=%u", + __entry->pe, __entry->addr, __entry->incr, + __entry->flags, __entry->count) +); + +TRACE_EVENT(amdgpu_vm_copy_ptes, + TP_PROTO(uint64_t pe, uint64_t src, unsigned count), + TP_ARGS(pe, src, count), + TP_STRUCT__entry( + __field(u64, pe) + __field(u64, src) + __field(u32, count) + ), + + TP_fast_assign( + __entry->pe = pe; + __entry->src = src; + __entry->count = count; + ), + TP_printk("pe=%010Lx, src=%010Lx, count=%u", + __entry->pe, __entry->src, __entry->count) +); + +TRACE_EVENT(amdgpu_vm_flush, + TP_PROTO(struct amdgpu_ring *ring, unsigned vmid, + uint64_t pd_addr), + TP_ARGS(ring, vmid, pd_addr), + TP_STRUCT__entry( + __string(ring, ring->name) + __field(u32, vmid) + __field(u32, vm_hub) + __field(u64, pd_addr) + ), + + TP_fast_assign( + __assign_str(ring, ring->name) + __entry->vmid = vmid; + __entry->vm_hub = ring->funcs->vmhub; + __entry->pd_addr = pd_addr; + ), + TP_printk("ring=%s, id=%u, hub=%u, pd_addr=%010Lx", + __get_str(ring), __entry->vmid, + __entry->vm_hub,__entry->pd_addr) +); + +DECLARE_EVENT_CLASS(amdgpu_pasid, + TP_PROTO(unsigned pasid), + TP_ARGS(pasid), + TP_STRUCT__entry( + __field(unsigned, pasid) + ), + TP_fast_assign( + __entry->pasid = pasid; + ), + TP_printk("pasid=%u", __entry->pasid) +); + +DEFINE_EVENT(amdgpu_pasid, amdgpu_pasid_allocated, + TP_PROTO(unsigned pasid), + TP_ARGS(pasid) +); + +DEFINE_EVENT(amdgpu_pasid, amdgpu_pasid_freed, + TP_PROTO(unsigned pasid), + TP_ARGS(pasid) +); + +TRACE_EVENT(amdgpu_bo_list_set, + TP_PROTO(struct amdgpu_bo_list *list, struct amdgpu_bo *bo), + TP_ARGS(list, bo), + TP_STRUCT__entry( + __field(struct amdgpu_bo_list *, list) + __field(struct amdgpu_bo *, bo) + __field(u64, bo_size) + ), + + TP_fast_assign( + __entry->list = list; + __entry->bo = bo; + __entry->bo_size = amdgpu_bo_size(bo); + ), + TP_printk("list=%p, bo=%p, bo_size=%Ld", + __entry->list, + __entry->bo, + __entry->bo_size) +); + +TRACE_EVENT(amdgpu_cs_bo_status, + TP_PROTO(uint64_t total_bo, uint64_t total_size), + TP_ARGS(total_bo, total_size), + TP_STRUCT__entry( + __field(u64, total_bo) + __field(u64, total_size) + ), + + TP_fast_assign( + __entry->total_bo = total_bo; + __entry->total_size = total_size; + ), + TP_printk("total_bo_size=%Ld, total_bo_count=%Ld", + __entry->total_bo, __entry->total_size) +); + +TRACE_EVENT(amdgpu_bo_move, + TP_PROTO(struct amdgpu_bo* bo, uint32_t new_placement, uint32_t old_placement), + TP_ARGS(bo, new_placement, old_placement), + TP_STRUCT__entry( + __field(struct amdgpu_bo *, bo) + __field(u64, bo_size) + __field(u32, new_placement) + __field(u32, old_placement) + ), + + TP_fast_assign( + __entry->bo = bo; + __entry->bo_size = amdgpu_bo_size(bo); + __entry->new_placement = new_placement; + __entry->old_placement = old_placement; + ), + TP_printk("bo=%p, from=%d, to=%d, size=%Ld", + __entry->bo, __entry->old_placement, + __entry->new_placement, __entry->bo_size) +); + +TRACE_EVENT(amdgpu_ib_pipe_sync, + TP_PROTO(struct amdgpu_job *sched_job, struct dma_fence *fence), + TP_ARGS(sched_job, fence), + TP_STRUCT__entry( + __field(const char *,name) + __field(uint64_t, id) + __field(struct dma_fence *, fence) + __field(uint64_t, ctx) + __field(unsigned, seqno) + ), + + TP_fast_assign( + __entry->name = sched_job->base.sched->name; + __entry->id = sched_job->base.id; + __entry->fence = fence; + __entry->ctx = fence->context; + __entry->seqno = fence->seqno; + ), + TP_printk("job ring=%s, id=%llu, need pipe sync to fence=%p, context=%llu, seq=%u", + __entry->name, __entry->id, + __entry->fence, __entry->ctx, + __entry->seqno) +); + +#undef AMDGPU_JOB_GET_TIMELINE_NAME +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/amd/amdgpu +#include <trace/define_trace.h>
diff --git a/amdgpu/amdgpu_trace_points.c b/amdgpu/amdgpu_trace_points.c new file mode 100644 index 0000000..57c6c39 --- /dev/null +++ b/amdgpu/amdgpu_trace_points.c
@@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT +/* Copyright Red Hat Inc 2010. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author : Dave Airlie <airlied@redhat.com> + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" + +#define CREATE_TRACE_POINTS +#include "amdgpu_trace.h"
diff --git a/amdgpu/amdgpu_ttm.c b/amdgpu/amdgpu_ttm.c new file mode 100644 index 0000000..e51b48a --- /dev/null +++ b/amdgpu/amdgpu_ttm.c
@@ -0,0 +1,2505 @@ +/* + * Copyright 2009 Jerome Glisse. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Jerome Glisse <glisse@freedesktop.org> + * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> + * Dave Airlie + */ + +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/hmm.h> +#include <linux/pagemap.h> +#include <linux/sched/task.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/swap.h> +#include <linux/swiotlb.h> + +#include <drm/ttm/ttm_bo_api.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_module.h> +#include <drm/ttm/ttm_page_alloc.h> + +#include <drm/drm_debugfs.h> +#include <drm/amdgpu_drm.h> + +#include "amdgpu.h" +#include "amdgpu_object.h" +#include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_sdma.h" +#include "bif/bif_4_1_d.h" + +static int amdgpu_map_buffer(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem, unsigned num_pages, + uint64_t offset, unsigned window, + struct amdgpu_ring *ring, + uint64_t *addr); + +static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev); +static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); + +static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) +{ + return 0; +} + +/** + * amdgpu_init_mem_type - Initialize a memory manager for a specific type of + * memory request. + * + * @bdev: The TTM BO device object (contains a reference to amdgpu_device) + * @type: The type of memory requested + * @man: The memory type manager for each domain + * + * This is called by ttm_bo_init_mm() when a buffer object is being + * initialized. + */ +static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, + struct ttm_mem_type_manager *man) +{ + struct amdgpu_device *adev; + + adev = amdgpu_ttm_adev(bdev); + + switch (type) { + case TTM_PL_SYSTEM: + /* System memory */ + man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; + man->available_caching = TTM_PL_MASK_CACHING; + man->default_caching = TTM_PL_FLAG_CACHED; + break; + case TTM_PL_TT: + /* GTT memory */ + man->func = &amdgpu_gtt_mgr_func; + man->gpu_offset = adev->gmc.gart_start; + man->available_caching = TTM_PL_MASK_CACHING; + man->default_caching = TTM_PL_FLAG_CACHED; + man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; + break; + case TTM_PL_VRAM: + /* "On-card" video ram */ + man->func = &amdgpu_vram_mgr_func; + man->gpu_offset = adev->gmc.vram_start; + man->flags = TTM_MEMTYPE_FLAG_FIXED | + TTM_MEMTYPE_FLAG_MAPPABLE; + man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; + man->default_caching = TTM_PL_FLAG_WC; + break; + case AMDGPU_PL_GDS: + case AMDGPU_PL_GWS: + case AMDGPU_PL_OA: + /* On-chip GDS memory*/ + man->func = &ttm_bo_manager_func; + man->gpu_offset = 0; + man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA; + man->available_caching = TTM_PL_FLAG_UNCACHED; + man->default_caching = TTM_PL_FLAG_UNCACHED; + break; + default: + DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); + return -EINVAL; + } + return 0; +} + +/** + * amdgpu_evict_flags - Compute placement flags + * + * @bo: The buffer object to evict + * @placement: Possible destination(s) for evicted BO + * + * Fill in placement data when ttm_bo_evict() is called + */ +static void amdgpu_evict_flags(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct amdgpu_bo *abo; + static const struct ttm_place placements = { + .fpfn = 0, + .lpfn = 0, + .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM + }; + + /* Don't handle scatter gather BOs */ + if (bo->type == ttm_bo_type_sg) { + placement->num_placement = 0; + placement->num_busy_placement = 0; + return; + } + + /* Object isn't an AMDGPU object so ignore */ + if (!amdgpu_bo_is_amdgpu_bo(bo)) { + placement->placement = &placements; + placement->busy_placement = &placements; + placement->num_placement = 1; + placement->num_busy_placement = 1; + return; + } + + abo = ttm_to_amdgpu_bo(bo); + switch (bo->mem.mem_type) { + case AMDGPU_PL_GDS: + case AMDGPU_PL_GWS: + case AMDGPU_PL_OA: + placement->num_placement = 0; + placement->num_busy_placement = 0; + return; + + case TTM_PL_VRAM: + if (!adev->mman.buffer_funcs_enabled) { + /* Move to system memory */ + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); + } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && + amdgpu_bo_in_cpu_visible_vram(abo)) { + + /* Try evicting to the CPU inaccessible part of VRAM + * first, but only set GTT as busy placement, so this + * BO will be evicted to GTT rather than causing other + * BOs to be evicted from VRAM + */ + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT); + abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; + abo->placements[0].lpfn = 0; + abo->placement.busy_placement = &abo->placements[1]; + abo->placement.num_busy_placement = 1; + } else { + /* Move to GTT memory */ + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); + } + break; + case TTM_PL_TT: + default: + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); + break; + } + *placement = abo->placement; +} + +/** + * amdgpu_verify_access - Verify access for a mmap call + * + * @bo: The buffer object to map + * @filp: The file pointer from the process performing the mmap + * + * This is called by ttm_bo_mmap() to verify whether a process + * has the right to mmap a BO to their process space. + */ +static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) +{ + struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); + + /* + * Don't verify access for KFD BOs. They don't have a GEM + * object associated with them. + */ + if (abo->kfd_bo) + return 0; + + if (amdgpu_ttm_tt_get_usermm(bo->ttm)) + return -EPERM; + return drm_vma_node_verify_access(&abo->gem_base.vma_node, + filp->private_data); +} + +/** + * amdgpu_move_null - Register memory for a buffer object + * + * @bo: The bo to assign the memory to + * @new_mem: The memory to be assigned. + * + * Assign the memory from new_mem to the memory of the buffer object bo. + */ +static void amdgpu_move_null(struct ttm_buffer_object *bo, + struct ttm_mem_reg *new_mem) +{ + struct ttm_mem_reg *old_mem = &bo->mem; + + BUG_ON(old_mem->mm_node != NULL); + *old_mem = *new_mem; + new_mem->mm_node = NULL; +} + +/** + * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer. + * + * @bo: The bo to assign the memory to. + * @mm_node: Memory manager node for drm allocator. + * @mem: The region where the bo resides. + * + */ +static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo, + struct drm_mm_node *mm_node, + struct ttm_mem_reg *mem) +{ + uint64_t addr = 0; + + if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) { + addr = mm_node->start << PAGE_SHIFT; + addr += bo->bdev->man[mem->mem_type].gpu_offset; + } + return addr; +} + +/** + * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to + * @offset. It also modifies the offset to be within the drm_mm_node returned + * + * @mem: The region where the bo resides. + * @offset: The offset that drm_mm_node is used for finding. + * + */ +static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem, + unsigned long *offset) +{ + struct drm_mm_node *mm_node = mem->mm_node; + + while (*offset >= (mm_node->size << PAGE_SHIFT)) { + *offset -= (mm_node->size << PAGE_SHIFT); + ++mm_node; + } + return mm_node; +} + +/** + * amdgpu_copy_ttm_mem_to_mem - Helper function for copy + * + * The function copies @size bytes from {src->mem + src->offset} to + * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a + * move and different for a BO to BO copy. + * + * @f: Returns the last fence if multiple jobs are submitted. + */ +int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, + struct amdgpu_copy_mem *src, + struct amdgpu_copy_mem *dst, + uint64_t size, + struct reservation_object *resv, + struct dma_fence **f) +{ + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + struct drm_mm_node *src_mm, *dst_mm; + uint64_t src_node_start, dst_node_start, src_node_size, + dst_node_size, src_page_offset, dst_page_offset; + struct dma_fence *fence = NULL; + int r = 0; + const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE * + AMDGPU_GPU_PAGE_SIZE); + + if (!adev->mman.buffer_funcs_enabled) { + DRM_ERROR("Trying to move memory with ring turned off.\n"); + return -EINVAL; + } + + src_mm = amdgpu_find_mm_node(src->mem, &src->offset); + src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) + + src->offset; + src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset; + src_page_offset = src_node_start & (PAGE_SIZE - 1); + + dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset); + dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) + + dst->offset; + dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset; + dst_page_offset = dst_node_start & (PAGE_SIZE - 1); + + mutex_lock(&adev->mman.gtt_window_lock); + + while (size) { + unsigned long cur_size; + uint64_t from = src_node_start, to = dst_node_start; + struct dma_fence *next; + + /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst + * begins at an offset, then adjust the size accordingly + */ + cur_size = min3(min(src_node_size, dst_node_size), size, + GTT_MAX_BYTES); + if (cur_size + src_page_offset > GTT_MAX_BYTES || + cur_size + dst_page_offset > GTT_MAX_BYTES) + cur_size -= max(src_page_offset, dst_page_offset); + + /* Map only what needs to be accessed. Map src to window 0 and + * dst to window 1 + */ + if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) { + r = amdgpu_map_buffer(src->bo, src->mem, + PFN_UP(cur_size + src_page_offset), + src_node_start, 0, ring, + &from); + if (r) + goto error; + /* Adjust the offset because amdgpu_map_buffer returns + * start of mapped page + */ + from += src_page_offset; + } + + if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) { + r = amdgpu_map_buffer(dst->bo, dst->mem, + PFN_UP(cur_size + dst_page_offset), + dst_node_start, 1, ring, + &to); + if (r) + goto error; + to += dst_page_offset; + } + + r = amdgpu_copy_buffer(ring, from, to, cur_size, + resv, &next, false, true); + if (r) + goto error; + + dma_fence_put(fence); + fence = next; + + size -= cur_size; + if (!size) + break; + + src_node_size -= cur_size; + if (!src_node_size) { + src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm, + src->mem); + src_node_size = (src_mm->size << PAGE_SHIFT); + src_page_offset = 0; + } else { + src_node_start += cur_size; + src_page_offset = src_node_start & (PAGE_SIZE - 1); + } + dst_node_size -= cur_size; + if (!dst_node_size) { + dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm, + dst->mem); + dst_node_size = (dst_mm->size << PAGE_SHIFT); + dst_page_offset = 0; + } else { + dst_node_start += cur_size; + dst_page_offset = dst_node_start & (PAGE_SIZE - 1); + } + } +error: + mutex_unlock(&adev->mman.gtt_window_lock); + if (f) + *f = dma_fence_get(fence); + dma_fence_put(fence); + return r; +} + +/** + * amdgpu_move_blit - Copy an entire buffer to another buffer + * + * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to + * help move buffers to and from VRAM. + */ +static int amdgpu_move_blit(struct ttm_buffer_object *bo, + bool evict, bool no_wait_gpu, + struct ttm_mem_reg *new_mem, + struct ttm_mem_reg *old_mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct amdgpu_copy_mem src, dst; + struct dma_fence *fence = NULL; + int r; + + src.bo = bo; + dst.bo = bo; + src.mem = old_mem; + dst.mem = new_mem; + src.offset = 0; + dst.offset = 0; + + r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst, + new_mem->num_pages << PAGE_SHIFT, + bo->resv, &fence); + if (r) + goto error; + + /* Always block for VM page tables before committing the new location */ + if (bo->type == ttm_bo_type_kernel) + r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem); + else + r = ttm_bo_pipeline_move(bo, fence, evict, new_mem); + dma_fence_put(fence); + return r; + +error: + if (fence) + dma_fence_wait(fence, false); + dma_fence_put(fence); + return r; +} + +/** + * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer + * + * Called by amdgpu_bo_move(). + */ +static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_mem_reg *new_mem) +{ + struct amdgpu_device *adev; + struct ttm_mem_reg *old_mem = &bo->mem; + struct ttm_mem_reg tmp_mem; + struct ttm_place placements; + struct ttm_placement placement; + int r; + + adev = amdgpu_ttm_adev(bo->bdev); + + /* create space/pages for new_mem in GTT space */ + tmp_mem = *new_mem; + tmp_mem.mm_node = NULL; + placement.num_placement = 1; + placement.placement = &placements; + placement.num_busy_placement = 1; + placement.busy_placement = &placements; + placements.fpfn = 0; + placements.lpfn = 0; + placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; + r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); + if (unlikely(r)) { + pr_err("Failed to find GTT space for blit from VRAM\n"); + return r; + } + + /* set caching flags */ + r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); + if (unlikely(r)) { + goto out_cleanup; + } + + /* Bind the memory to the GTT space */ + r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx); + if (unlikely(r)) { + goto out_cleanup; + } + + /* blit VRAM to GTT */ + r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem); + if (unlikely(r)) { + goto out_cleanup; + } + + /* move BO (in tmp_mem) to new_mem */ + r = ttm_bo_move_ttm(bo, ctx, new_mem); +out_cleanup: + ttm_bo_mem_put(bo, &tmp_mem); + return r; +} + +/** + * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM + * + * Called by amdgpu_bo_move(). + */ +static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_mem_reg *new_mem) +{ + struct amdgpu_device *adev; + struct ttm_mem_reg *old_mem = &bo->mem; + struct ttm_mem_reg tmp_mem; + struct ttm_placement placement; + struct ttm_place placements; + int r; + + adev = amdgpu_ttm_adev(bo->bdev); + + /* make space in GTT for old_mem buffer */ + tmp_mem = *new_mem; + tmp_mem.mm_node = NULL; + placement.num_placement = 1; + placement.placement = &placements; + placement.num_busy_placement = 1; + placement.busy_placement = &placements; + placements.fpfn = 0; + placements.lpfn = 0; + placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; + r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); + if (unlikely(r)) { + pr_err("Failed to find GTT space for blit to VRAM\n"); + return r; + } + + /* move/bind old memory to GTT space */ + r = ttm_bo_move_ttm(bo, ctx, &tmp_mem); + if (unlikely(r)) { + goto out_cleanup; + } + + /* copy to VRAM */ + r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem); + if (unlikely(r)) { + goto out_cleanup; + } +out_cleanup: + ttm_bo_mem_put(bo, &tmp_mem); + return r; +} + +/** + * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy + * + * Called by amdgpu_bo_move() + */ +static bool amdgpu_mem_visible(struct amdgpu_device *adev, + struct ttm_mem_reg *mem) +{ + struct drm_mm_node *nodes = mem->mm_node; + + if (mem->mem_type == TTM_PL_SYSTEM || + mem->mem_type == TTM_PL_TT) + return true; + if (mem->mem_type != TTM_PL_VRAM) + return false; + + /* ttm_mem_reg_ioremap only supports contiguous memory */ + if (nodes->size != mem->num_pages) + return false; + + return ((nodes->start + nodes->size) << PAGE_SHIFT) + <= adev->gmc.visible_vram_size; +} + +/** + * amdgpu_bo_move - Move a buffer object to a new memory location + * + * Called by ttm_bo_handle_move_mem() + */ +static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_mem_reg *new_mem) +{ + struct amdgpu_device *adev; + struct amdgpu_bo *abo; + struct ttm_mem_reg *old_mem = &bo->mem; + int r; + + /* Can't move a pinned BO */ + abo = ttm_to_amdgpu_bo(bo); + if (WARN_ON_ONCE(abo->pin_count > 0)) + return -EINVAL; + + adev = amdgpu_ttm_adev(bo->bdev); + + if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { + amdgpu_move_null(bo, new_mem); + return 0; + } + if ((old_mem->mem_type == TTM_PL_TT && + new_mem->mem_type == TTM_PL_SYSTEM) || + (old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_TT)) { + /* bind is enough */ + amdgpu_move_null(bo, new_mem); + return 0; + } + if (old_mem->mem_type == AMDGPU_PL_GDS || + old_mem->mem_type == AMDGPU_PL_GWS || + old_mem->mem_type == AMDGPU_PL_OA || + new_mem->mem_type == AMDGPU_PL_GDS || + new_mem->mem_type == AMDGPU_PL_GWS || + new_mem->mem_type == AMDGPU_PL_OA) { + /* Nothing to save here */ + amdgpu_move_null(bo, new_mem); + return 0; + } + + if (!adev->mman.buffer_funcs_enabled) { + r = -ENODEV; + goto memcpy; + } + + if (old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM) { + r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem); + } else if (old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_VRAM) { + r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem); + } else { + r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, + new_mem, old_mem); + } + + if (r) { +memcpy: + /* Check that all memory is CPU accessible */ + if (!amdgpu_mem_visible(adev, old_mem) || + !amdgpu_mem_visible(adev, new_mem)) { + pr_err("Move buffer fallback to memcpy unavailable\n"); + return r; + } + + r = ttm_bo_move_memcpy(bo, ctx, new_mem); + if (r) + return r; + } + + if (bo->type == ttm_bo_type_device && + new_mem->mem_type == TTM_PL_VRAM && + old_mem->mem_type != TTM_PL_VRAM) { + /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU + * accesses the BO after it's moved. + */ + abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + } + + /* update statistics */ + atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); + return 0; +} + +/** + * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault + * + * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault() + */ +static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); + struct drm_mm_node *mm_node = mem->mm_node; + + mem->bus.addr = NULL; + mem->bus.offset = 0; + mem->bus.size = mem->num_pages << PAGE_SHIFT; + mem->bus.base = 0; + mem->bus.is_iomem = false; + if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) + return -EINVAL; + switch (mem->mem_type) { + case TTM_PL_SYSTEM: + /* system memory */ + return 0; + case TTM_PL_TT: + break; + case TTM_PL_VRAM: + mem->bus.offset = mem->start << PAGE_SHIFT; + /* check if it's visible */ + if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size) + return -EINVAL; + /* Only physically contiguous buffers apply. In a contiguous + * buffer, size of the first mm_node would match the number of + * pages in ttm_mem_reg. + */ + if (adev->mman.aper_base_kaddr && + (mm_node->size == mem->num_pages)) + mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + + mem->bus.offset; + + mem->bus.base = adev->gmc.aper_base; + mem->bus.is_iomem = true; + break; + default: + return -EINVAL; + } + return 0; +} + +static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) +{ +} + +static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct drm_mm_node *mm; + unsigned long offset = (page_offset << PAGE_SHIFT); + + mm = amdgpu_find_mm_node(&bo->mem, &offset); + return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + + (offset >> PAGE_SHIFT); +} + +/* + * TTM backend functions. + */ +struct amdgpu_ttm_tt { + struct ttm_dma_tt ttm; + u64 offset; + uint64_t userptr; + struct task_struct *usertask; + uint32_t userflags; +#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) + struct hmm_range *range; +#endif +}; + +/** + * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user + * memory and start HMM tracking CPU page table update + * + * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only + * once afterwards to stop HMM tracking + */ +#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) + +#define MAX_RETRY_HMM_RANGE_FAULT 16 + +int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) +{ + struct hmm_mirror *mirror = bo->mn ? &bo->mn->mirror : NULL; + struct ttm_tt *ttm = bo->tbo.ttm; + struct amdgpu_ttm_tt *gtt = (void *)ttm; + struct mm_struct *mm = gtt->usertask->mm; + unsigned long start = gtt->userptr; + struct vm_area_struct *vma; + struct hmm_range *range; + unsigned long i; + uint64_t *pfns; + int retry = 0; + int r = 0; + + if (!mm) /* Happens during process shutdown */ + return -ESRCH; + + if (unlikely(!mirror)) { + DRM_DEBUG_DRIVER("Failed to get hmm_mirror\n"); + r = -EFAULT; + goto out; + } + + vma = find_vma(mm, start); + if (unlikely(!vma || start < vma->vm_start)) { + r = -EFAULT; + goto out; + } + if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && + vma->vm_file)) { + r = -EPERM; + goto out; + } + + range = kzalloc(sizeof(*range), GFP_KERNEL); + if (unlikely(!range)) { + r = -ENOMEM; + goto out; + } + + pfns = kvmalloc_array(ttm->num_pages, sizeof(*pfns), GFP_KERNEL); + if (unlikely(!pfns)) { + r = -ENOMEM; + goto out_free_ranges; + } + + amdgpu_hmm_init_range(range); + range->default_flags = range->flags[HMM_PFN_VALID]; + range->default_flags |= amdgpu_ttm_tt_is_readonly(ttm) ? + 0 : range->flags[HMM_PFN_WRITE]; + range->pfn_flags_mask = 0; + range->pfns = pfns; + hmm_range_register(range, mirror, start, + start + ttm->num_pages * PAGE_SIZE, PAGE_SHIFT); + +retry: + /* + * Just wait for range to be valid, safe to ignore return value as we + * will use the return value of hmm_range_fault() below under the + * mmap_sem to ascertain the validity of the range. + */ + hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT); + + down_read(&mm->mmap_sem); + + r = hmm_range_fault(range, true); + if (unlikely(r < 0)) { + if (likely(r == -EAGAIN)) { + /* + * return -EAGAIN, mmap_sem is dropped + */ + if (retry++ < MAX_RETRY_HMM_RANGE_FAULT) + goto retry; + else + pr_err("Retry hmm fault too many times\n"); + } + + goto out_up_read; + } + + up_read(&mm->mmap_sem); + + for (i = 0; i < ttm->num_pages; i++) { + pages[i] = hmm_device_entry_to_page(range, pfns[i]); + if (unlikely(!pages[i])) { + pr_err("Page fault failed for pfn[%lu] = 0x%llx\n", + i, pfns[i]); + r = -ENOMEM; + + goto out_free_pfns; + } + } + + gtt->range = range; + + return 0; + +out_up_read: + if (likely(r != -EAGAIN)) + up_read(&mm->mmap_sem); +out_free_pfns: + hmm_range_unregister(range); + kvfree(pfns); +out_free_ranges: + kfree(range); +out: + return r; +} + +/** + * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change + * Check if the pages backing this ttm range have been invalidated + * + * Returns: true if pages are still valid + */ +bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + bool r = false; + + if (!gtt || !gtt->userptr) + return false; + + DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n", + gtt->userptr, ttm->num_pages); + + WARN_ONCE(!gtt->range || !gtt->range->pfns, + "No user pages to check\n"); + + if (gtt->range) { + r = hmm_range_valid(gtt->range); + hmm_range_unregister(gtt->range); + + kvfree(gtt->range->pfns); + kfree(gtt->range); + gtt->range = NULL; + } + + return r; +} +#endif + +/** + * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary. + * + * Called by amdgpu_cs_list_validate(). This creates the page list + * that backs user memory and will ultimately be mapped into the device + * address space. + */ +void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages) +{ + unsigned long i; + + for (i = 0; i < ttm->num_pages; ++i) + ttm->pages[i] = pages ? pages[i] : NULL; +} + +/** + * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages + * + * Called by amdgpu_ttm_backend_bind() + **/ +static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void *)ttm; + unsigned nents; + int r; + + int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); + enum dma_data_direction direction = write ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + + /* Allocate an SG array and squash pages into it */ + r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, + ttm->num_pages << PAGE_SHIFT, + GFP_KERNEL); + if (r) + goto release_sg; + + /* Map SG to device */ + r = -ENOMEM; + nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction); + if (nents != ttm->sg->nents) + goto release_sg; + + /* convert SG to linear array of pages and dma addresses */ + drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, + gtt->ttm.dma_address, ttm->num_pages); + + return 0; + +release_sg: + kfree(ttm->sg); + return r; +} + +/** + * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages + */ +static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); + enum dma_data_direction direction = write ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + + /* double check that we don't free the table twice */ + if (!ttm->sg->sgl) + return; + + /* unmap the pages mapped to the device */ + dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction); + + sg_free_table(ttm->sg); + +#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) + if (gtt->range && + ttm->pages[0] == hmm_device_entry_to_page(gtt->range, + gtt->range->pfns[0])) + WARN_ONCE(1, "Missing get_user_page_done\n"); +#endif +} + +int amdgpu_ttm_gart_bind(struct amdgpu_device *adev, + struct ttm_buffer_object *tbo, + uint64_t flags) +{ + struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo); + struct ttm_tt *ttm = tbo->ttm; + struct amdgpu_ttm_tt *gtt = (void *)ttm; + int r; + + if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) { + uint64_t page_idx = 1; + + r = amdgpu_gart_bind(adev, gtt->offset, page_idx, + ttm->pages, gtt->ttm.dma_address, flags); + if (r) + goto gart_bind_fail; + + /* Patch mtype of the second part BO */ + flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; + flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC); + + r = amdgpu_gart_bind(adev, + gtt->offset + (page_idx << PAGE_SHIFT), + ttm->num_pages - page_idx, + &ttm->pages[page_idx], + &(gtt->ttm.dma_address[page_idx]), flags); + } else { + r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, + ttm->pages, gtt->ttm.dma_address, flags); + } + +gart_bind_fail: + if (r) + DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", + ttm->num_pages, gtt->offset); + + return r; +} + +/** + * amdgpu_ttm_backend_bind - Bind GTT memory + * + * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem(). + * This handles binding GTT memory to the device address space. + */ +static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm, + struct ttm_mem_reg *bo_mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void*)ttm; + uint64_t flags; + int r = 0; + + if (gtt->userptr) { + r = amdgpu_ttm_tt_pin_userptr(ttm); + if (r) { + DRM_ERROR("failed to pin userptr\n"); + return r; + } + } + if (!ttm->num_pages) { + WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", + ttm->num_pages, bo_mem, ttm); + } + + if (bo_mem->mem_type == AMDGPU_PL_GDS || + bo_mem->mem_type == AMDGPU_PL_GWS || + bo_mem->mem_type == AMDGPU_PL_OA) + return -EINVAL; + + if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) { + gtt->offset = AMDGPU_BO_INVALID_OFFSET; + return 0; + } + + /* compute PTE flags relevant to this BO memory */ + flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem); + + /* bind pages into GART page tables */ + gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; + r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, + ttm->pages, gtt->ttm.dma_address, flags); + + if (r) + DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", + ttm->num_pages, gtt->offset); + return r; +} + +/** + * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object + */ +int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_ttm_tt *gtt = (void*)bo->ttm; + struct ttm_mem_reg tmp; + struct ttm_placement placement; + struct ttm_place placements; + uint64_t addr, flags; + int r; + + if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET) + return 0; + + addr = amdgpu_gmc_agp_addr(bo); + if (addr != AMDGPU_BO_INVALID_OFFSET) { + bo->mem.start = addr >> PAGE_SHIFT; + } else { + + /* allocate GART space */ + tmp = bo->mem; + tmp.mm_node = NULL; + placement.num_placement = 1; + placement.placement = &placements; + placement.num_busy_placement = 1; + placement.busy_placement = &placements; + placements.fpfn = 0; + placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; + placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) | + TTM_PL_FLAG_TT; + + r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx); + if (unlikely(r)) + return r; + + /* compute PTE flags for this buffer object */ + flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp); + + /* Bind pages */ + gtt->offset = (u64)tmp.start << PAGE_SHIFT; + r = amdgpu_ttm_gart_bind(adev, bo, flags); + if (unlikely(r)) { + ttm_bo_mem_put(bo, &tmp); + return r; + } + + ttm_bo_mem_put(bo, &bo->mem); + bo->mem = tmp; + } + + bo->offset = (bo->mem.start << PAGE_SHIFT) + + bo->bdev->man[bo->mem.mem_type].gpu_offset; + + return 0; +} + +/** + * amdgpu_ttm_recover_gart - Rebind GTT pages + * + * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to + * rebind GTT pages during a GPU reset. + */ +int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); + uint64_t flags; + int r; + + if (!tbo->ttm) + return 0; + + flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem); + r = amdgpu_ttm_gart_bind(adev, tbo, flags); + + return r; +} + +/** + * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages + * + * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and + * ttm_tt_destroy(). + */ +static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void *)ttm; + int r; + + /* if the pages have userptr pinning then clear that first */ + if (gtt->userptr) + amdgpu_ttm_tt_unpin_userptr(ttm); + + if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) + return 0; + + /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */ + r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); + if (r) + DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n", + gtt->ttm.ttm.num_pages, gtt->offset); + return r; +} + +static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt->usertask) + put_task_struct(gtt->usertask); + + ttm_dma_tt_fini(>t->ttm); + kfree(gtt); +} + +static struct ttm_backend_func amdgpu_backend_func = { + .bind = &amdgpu_ttm_backend_bind, + .unbind = &amdgpu_ttm_backend_unbind, + .destroy = &amdgpu_ttm_backend_destroy, +}; + +/** + * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO + * + * @bo: The buffer object to create a GTT ttm_tt object around + * + * Called by ttm_tt_create(). + */ +static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, + uint32_t page_flags) +{ + struct amdgpu_device *adev; + struct amdgpu_ttm_tt *gtt; + + adev = amdgpu_ttm_adev(bo->bdev); + + gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL); + if (gtt == NULL) { + return NULL; + } + gtt->ttm.ttm.func = &amdgpu_backend_func; + + /* allocate space for the uninitialized page entries */ + if (ttm_sg_tt_init(>t->ttm, bo, page_flags)) { + kfree(gtt); + return NULL; + } + return >t->ttm.ttm; +} + +/** + * amdgpu_ttm_tt_populate - Map GTT pages visible to the device + * + * Map the pages of a ttm_tt object to an address space visible + * to the underlying device. + */ +static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm, + struct ttm_operation_ctx *ctx) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void *)ttm; + bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); + + /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */ + if (gtt && gtt->userptr) { + ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); + if (!ttm->sg) + return -ENOMEM; + + ttm->page_flags |= TTM_PAGE_FLAG_SG; + ttm->state = tt_unbound; + return 0; + } + + if (slave && ttm->sg) { + drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, + gtt->ttm.dma_address, + ttm->num_pages); + ttm->state = tt_unbound; + return 0; + } + +#ifdef CONFIG_SWIOTLB + if (adev->need_swiotlb && swiotlb_nr_tbl()) { + return ttm_dma_populate(>t->ttm, adev->dev, ctx); + } +#endif + + /* fall back to generic helper to populate the page array + * and map them to the device */ + return ttm_populate_and_map_pages(adev->dev, >t->ttm, ctx); +} + +/** + * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays + * + * Unmaps pages of a ttm_tt object from the device address space and + * unpopulates the page array backing it. + */ +static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm) +{ + struct amdgpu_device *adev; + struct amdgpu_ttm_tt *gtt = (void *)ttm; + bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); + + if (gtt && gtt->userptr) { + amdgpu_ttm_tt_set_user_pages(ttm, NULL); + kfree(ttm->sg); + ttm->page_flags &= ~TTM_PAGE_FLAG_SG; + return; + } + + if (slave) + return; + + adev = amdgpu_ttm_adev(ttm->bdev); + +#ifdef CONFIG_SWIOTLB + if (adev->need_swiotlb && swiotlb_nr_tbl()) { + ttm_dma_unpopulate(>t->ttm, adev->dev); + return; + } +#endif + + /* fall back to generic helper to unmap and unpopulate array */ + ttm_unmap_and_unpopulate_pages(adev->dev, >t->ttm); +} + +/** + * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current + * task + * + * @ttm: The ttm_tt object to bind this userptr object to + * @addr: The address in the current tasks VM space to use + * @flags: Requirements of userptr object. + * + * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages + * to current task + */ +int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, + uint32_t flags) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt == NULL) + return -EINVAL; + + gtt->userptr = addr; + gtt->userflags = flags; + + if (gtt->usertask) + put_task_struct(gtt->usertask); + gtt->usertask = current->group_leader; + get_task_struct(gtt->usertask); + + return 0; +} + +/** + * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object + */ +struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt == NULL) + return NULL; + + if (gtt->usertask == NULL) + return NULL; + + return gtt->usertask->mm; +} + +/** + * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an + * address range for the current task. + * + */ +bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, + unsigned long end) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + unsigned long size; + + if (gtt == NULL || !gtt->userptr) + return false; + + /* Return false if no part of the ttm_tt object lies within + * the range + */ + size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE; + if (gtt->userptr > end || gtt->userptr + size <= start) + return false; + + return true; +} + +/** + * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr? + */ +bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt == NULL || !gtt->userptr) + return false; + + return true; +} + +/** + * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only? + */ +bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + + if (gtt == NULL) + return false; + + return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); +} + +/** + * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object + * + * @ttm: The ttm_tt object to compute the flags for + * @mem: The memory registry backing this ttm_tt object + * + * Figure out the flags to use for a VM PDE (Page Directory Entry). + */ +uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem) +{ + uint64_t flags = 0; + + if (mem && mem->mem_type != TTM_PL_SYSTEM) + flags |= AMDGPU_PTE_VALID; + + if (mem && mem->mem_type == TTM_PL_TT) { + flags |= AMDGPU_PTE_SYSTEM; + + if (ttm->caching_state == tt_cached) + flags |= AMDGPU_PTE_SNOOPED; + } + + return flags; +} + +/** + * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object + * + * @ttm: The ttm_tt object to compute the flags for + * @mem: The memory registry backing this ttm_tt object + + * Figure out the flags to use for a VM PTE (Page Table Entry). + */ +uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, + struct ttm_mem_reg *mem) +{ + uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); + + flags |= adev->gart.gart_pte_flags; + flags |= AMDGPU_PTE_READABLE; + + if (!amdgpu_ttm_tt_is_readonly(ttm)) + flags |= AMDGPU_PTE_WRITEABLE; + + return flags; +} + +/** + * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer + * object. + * + * Return true if eviction is sensible. Called by ttm_mem_evict_first() on + * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until + * it can find space for a new object and by ttm_bo_force_list_clean() which is + * used to clean out a memory space. + */ +static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, + const struct ttm_place *place) +{ + unsigned long num_pages = bo->mem.num_pages; + struct drm_mm_node *node = bo->mem.mm_node; + struct reservation_object_list *flist; + struct dma_fence *f; + int i; + + /* Don't evict VM page tables while they are busy, otherwise we can't + * cleanly handle page faults. + */ + if (bo->type == ttm_bo_type_kernel && + !reservation_object_test_signaled_rcu(bo->resv, true)) + return false; + + /* If bo is a KFD BO, check if the bo belongs to the current process. + * If true, then return false as any KFD process needs all its BOs to + * be resident to run successfully + */ + flist = reservation_object_get_list(bo->resv); + if (flist) { + for (i = 0; i < flist->shared_count; ++i) { + f = rcu_dereference_protected(flist->shared[i], + reservation_object_held(bo->resv)); + if (amdkfd_fence_check_mm(f, current->mm)) + return false; + } + } + + switch (bo->mem.mem_type) { + case TTM_PL_TT: + return true; + + case TTM_PL_VRAM: + /* Check each drm MM node individually */ + while (num_pages) { + if (place->fpfn < (node->start + node->size) && + !(place->lpfn && place->lpfn <= node->start)) + return true; + + num_pages -= node->size; + ++node; + } + return false; + + default: + break; + } + + return ttm_bo_eviction_valuable(bo, place); +} + +/** + * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object. + * + * @bo: The buffer object to read/write + * @offset: Offset into buffer object + * @buf: Secondary buffer to write/read from + * @len: Length in bytes of access + * @write: true if writing + * + * This is used to access VRAM that backs a buffer object via MMIO + * access for debugging purposes. + */ +static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo, + unsigned long offset, + void *buf, int len, int write) +{ + struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); + struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); + struct drm_mm_node *nodes; + uint32_t value = 0; + int ret = 0; + uint64_t pos; + unsigned long flags; + + if (bo->mem.mem_type != TTM_PL_VRAM) + return -EIO; + + nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset); + pos = (nodes->start << PAGE_SHIFT) + offset; + + while (len && pos < adev->gmc.mc_vram_size) { + uint64_t aligned_pos = pos & ~(uint64_t)3; + uint32_t bytes = 4 - (pos & 3); + uint32_t shift = (pos & 3) * 8; + uint32_t mask = 0xffffffff << shift; + + if (len < bytes) { + mask &= 0xffffffff >> (bytes - len) * 8; + bytes = len; + } + + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000); + WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31); + if (!write || mask != 0xffffffff) + value = RREG32_NO_KIQ(mmMM_DATA); + if (write) { + value &= ~mask; + value |= (*(uint32_t *)buf << shift) & mask; + WREG32_NO_KIQ(mmMM_DATA, value); + } + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + if (!write) { + value = (value & mask) >> shift; + memcpy(buf, &value, bytes); + } + + ret += bytes; + buf = (uint8_t *)buf + bytes; + pos += bytes; + len -= bytes; + if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) { + ++nodes; + pos = (nodes->start << PAGE_SHIFT); + } + } + + return ret; +} + +static struct ttm_bo_driver amdgpu_bo_driver = { + .ttm_tt_create = &amdgpu_ttm_tt_create, + .ttm_tt_populate = &amdgpu_ttm_tt_populate, + .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate, + .invalidate_caches = &amdgpu_invalidate_caches, + .init_mem_type = &amdgpu_init_mem_type, + .eviction_valuable = amdgpu_ttm_bo_eviction_valuable, + .evict_flags = &amdgpu_evict_flags, + .move = &amdgpu_bo_move, + .verify_access = &amdgpu_verify_access, + .move_notify = &amdgpu_bo_move_notify, + .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify, + .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, + .io_mem_free = &amdgpu_ttm_io_mem_free, + .io_mem_pfn = amdgpu_ttm_io_mem_pfn, + .access_memory = &amdgpu_ttm_access_memory, + .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify +}; + +/* + * Firmware Reservation functions + */ +/** + * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram + * + * @adev: amdgpu_device pointer + * + * free fw reserved vram if it has been reserved. + */ +static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) +{ + amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo, + NULL, &adev->fw_vram_usage.va); +} + +/** + * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw + * + * @adev: amdgpu_device pointer + * + * create bo vram reservation from fw. + */ +static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) +{ + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_bo_param bp; + int r = 0; + int i; + u64 vram_size = adev->gmc.visible_vram_size; + u64 offset = adev->fw_vram_usage.start_offset; + u64 size = adev->fw_vram_usage.size; + struct amdgpu_bo *bo; + + memset(&bp, 0, sizeof(bp)); + bp.size = adev->fw_vram_usage.size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; + bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | + AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; + bp.type = ttm_bo_type_kernel; + bp.resv = NULL; + adev->fw_vram_usage.va = NULL; + adev->fw_vram_usage.reserved_bo = NULL; + + if (adev->fw_vram_usage.size > 0 && + adev->fw_vram_usage.size <= vram_size) { + + r = amdgpu_bo_create(adev, &bp, + &adev->fw_vram_usage.reserved_bo); + if (r) + goto error_create; + + r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false); + if (r) + goto error_reserve; + + /* remove the original mem node and create a new one at the + * request position + */ + bo = adev->fw_vram_usage.reserved_bo; + offset = ALIGN(offset, PAGE_SIZE); + for (i = 0; i < bo->placement.num_placement; ++i) { + bo->placements[i].fpfn = offset >> PAGE_SHIFT; + bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; + } + + ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem); + r = ttm_bo_mem_space(&bo->tbo, &bo->placement, + &bo->tbo.mem, &ctx); + if (r) + goto error_pin; + + r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo, + AMDGPU_GEM_DOMAIN_VRAM, + adev->fw_vram_usage.start_offset, + (adev->fw_vram_usage.start_offset + + adev->fw_vram_usage.size)); + if (r) + goto error_pin; + r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo, + &adev->fw_vram_usage.va); + if (r) + goto error_kmap; + + amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo); + } + return r; + +error_kmap: + amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo); +error_pin: + amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo); +error_reserve: + amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo); +error_create: + adev->fw_vram_usage.va = NULL; + adev->fw_vram_usage.reserved_bo = NULL; + return r; +} +/** + * amdgpu_ttm_init - Init the memory management (ttm) as well as various + * gtt/vram related fields. + * + * This initializes all of the memory space pools that the TTM layer + * will need such as the GTT space (system memory mapped to the device), + * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which + * can be mapped per VMID. + */ +int amdgpu_ttm_init(struct amdgpu_device *adev) +{ + uint64_t gtt_size; + int r; + u64 vis_vram_limit; + + mutex_init(&adev->mman.gtt_window_lock); + + /* No others user of address space so set it to 0 */ + r = ttm_bo_device_init(&adev->mman.bdev, + &amdgpu_bo_driver, + adev->ddev->anon_inode->i_mapping, + adev->need_dma32); + if (r) { + DRM_ERROR("failed initializing buffer object driver(%d).\n", r); + return r; + } + adev->mman.initialized = true; + + /* We opt to avoid OOM on system pages allocations */ + adev->mman.bdev.no_retry = true; + + /* Initialize VRAM pool with all of VRAM divided into pages */ + r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM, + adev->gmc.real_vram_size >> PAGE_SHIFT); + if (r) { + DRM_ERROR("Failed initializing VRAM heap.\n"); + return r; + } + + /* Reduce size of CPU-visible VRAM if requested */ + vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024; + if (amdgpu_vis_vram_limit > 0 && + vis_vram_limit <= adev->gmc.visible_vram_size) + adev->gmc.visible_vram_size = vis_vram_limit; + + /* Change the size here instead of the init above so only lpfn is affected */ + amdgpu_ttm_set_buffer_funcs_status(adev, false); +#ifdef CONFIG_64BIT + adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, + adev->gmc.visible_vram_size); +#endif + + /* + *The reserved vram for firmware must be pinned to the specified + *place on the VRAM, so reserve it early. + */ + r = amdgpu_ttm_fw_reserve_vram_init(adev); + if (r) { + return r; + } + + /* allocate memory as required for VGA + * This is used for VGA emulation and pre-OS scanout buffers to + * avoid display artifacts while transitioning between pre-OS + * and driver. */ + r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->stolen_vga_memory, + NULL, NULL); + if (r) + return r; + DRM_INFO("amdgpu: %uM of VRAM memory ready\n", + (unsigned) (adev->gmc.real_vram_size / (1024 * 1024))); + + /* Compute GTT size, either bsaed on 3/4th the size of RAM size + * or whatever the user passed on module init */ + if (amdgpu_gtt_size == -1) { + struct sysinfo si; + + si_meminfo(&si); + gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20), + adev->gmc.mc_vram_size), + ((uint64_t)si.totalram * si.mem_unit * 3/4)); + } + else + gtt_size = (uint64_t)amdgpu_gtt_size << 20; + + /* Initialize GTT memory pool */ + r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT); + if (r) { + DRM_ERROR("Failed initializing GTT heap.\n"); + return r; + } + DRM_INFO("amdgpu: %uM of GTT memory ready.\n", + (unsigned)(gtt_size / (1024 * 1024))); + + /* Initialize various on-chip memory pools */ + r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS, + adev->gds.gds_size); + if (r) { + DRM_ERROR("Failed initializing GDS heap.\n"); + return r; + } + + r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS, + adev->gds.gws_size); + if (r) { + DRM_ERROR("Failed initializing gws heap.\n"); + return r; + } + + r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA, + adev->gds.oa_size); + if (r) { + DRM_ERROR("Failed initializing oa heap.\n"); + return r; + } + + /* Register debugfs entries for amdgpu_ttm */ + r = amdgpu_ttm_debugfs_init(adev); + if (r) { + DRM_ERROR("Failed to init debugfs\n"); + return r; + } + return 0; +} + +/** + * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm + */ +void amdgpu_ttm_late_init(struct amdgpu_device *adev) +{ + /* return the VGA stolen memory (if any) back to VRAM */ + amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); +} + +/** + * amdgpu_ttm_fini - De-initialize the TTM memory pools + */ +void amdgpu_ttm_fini(struct amdgpu_device *adev) +{ + if (!adev->mman.initialized) + return; + + amdgpu_ttm_debugfs_fini(adev); + amdgpu_ttm_fw_reserve_vram_fini(adev); + if (adev->mman.aper_base_kaddr) + iounmap(adev->mman.aper_base_kaddr); + adev->mman.aper_base_kaddr = NULL; + + ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM); + ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT); + ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS); + ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS); + ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA); + ttm_bo_device_release(&adev->mman.bdev); + adev->mman.initialized = false; + DRM_INFO("amdgpu: ttm finalized\n"); +} + +/** + * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions + * + * @adev: amdgpu_device pointer + * @enable: true when we can use buffer functions. + * + * Enable/disable use of buffer functions during suspend/resume. This should + * only be called at bootup or when userspace isn't running. + */ +void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) +{ + struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM]; + uint64_t size; + int r; + + if (!adev->mman.initialized || adev->in_gpu_reset || + adev->mman.buffer_funcs_enabled == enable) + return; + + if (enable) { + struct amdgpu_ring *ring; + struct drm_sched_rq *rq; + + ring = adev->mman.buffer_funcs_ring; + rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL]; + r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL); + if (r) { + DRM_ERROR("Failed setting up TTM BO move entity (%d)\n", + r); + return; + } + } else { + drm_sched_entity_destroy(&adev->mman.entity); + dma_fence_put(man->move); + man->move = NULL; + } + + /* this just adjusts TTM size idea, which sets lpfn to the correct value */ + if (enable) + size = adev->gmc.real_vram_size; + else + size = adev->gmc.visible_vram_size; + man->size = size >> PAGE_SHIFT; + adev->mman.buffer_funcs_enabled = enable; +} + +int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_file *file_priv = filp->private_data; + struct amdgpu_device *adev = file_priv->minor->dev->dev_private; + + if (adev == NULL) + return -EINVAL; + + return ttm_bo_mmap(filp, vma, &adev->mman.bdev); +} + +static int amdgpu_map_buffer(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem, unsigned num_pages, + uint64_t offset, unsigned window, + struct amdgpu_ring *ring, + uint64_t *addr) +{ + struct amdgpu_ttm_tt *gtt = (void *)bo->ttm; + struct amdgpu_device *adev = ring->adev; + struct ttm_tt *ttm = bo->ttm; + struct amdgpu_job *job; + unsigned num_dw, num_bytes; + dma_addr_t *dma_address; + struct dma_fence *fence; + uint64_t src_addr, dst_addr; + uint64_t flags; + int r; + + BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < + AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); + + *addr = adev->gmc.gart_start; + *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE * + AMDGPU_GPU_PAGE_SIZE; + + num_dw = adev->mman.buffer_funcs->copy_num_dw; + while (num_dw & 0x7) + num_dw++; + + num_bytes = num_pages * 8; + + r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job); + if (r) + return r; + + src_addr = num_dw * 4; + src_addr += job->ibs[0].gpu_addr; + + dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); + dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8; + amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, + dst_addr, num_bytes); + + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + WARN_ON(job->ibs[0].length_dw > num_dw); + + dma_address = >t->ttm.dma_address[offset >> PAGE_SHIFT]; + flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem); + r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags, + &job->ibs[0].ptr[num_dw]); + if (r) + goto error_free; + + r = amdgpu_job_submit(job, &adev->mman.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, &fence); + if (r) + goto error_free; + + dma_fence_put(fence); + + return r; + +error_free: + amdgpu_job_free(job); + return r; +} + +int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, + uint64_t dst_offset, uint32_t byte_count, + struct reservation_object *resv, + struct dma_fence **fence, bool direct_submit, + bool vm_needs_flush) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_job *job; + + uint32_t max_bytes; + unsigned num_loops, num_dw; + unsigned i; + int r; + + if (direct_submit && !ring->sched.ready) { + DRM_ERROR("Trying to move memory with ring turned off.\n"); + return -EINVAL; + } + + max_bytes = adev->mman.buffer_funcs->copy_max_bytes; + num_loops = DIV_ROUND_UP(byte_count, max_bytes); + num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw; + + /* for IB padding */ + while (num_dw & 0x7) + num_dw++; + + r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job); + if (r) + return r; + + if (vm_needs_flush) { + job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); + job->vm_needs_flush = true; + } + if (resv) { + r = amdgpu_sync_resv(adev, &job->sync, resv, + AMDGPU_FENCE_OWNER_UNDEFINED, + false); + if (r) { + DRM_ERROR("sync failed (%d).\n", r); + goto error_free; + } + } + + for (i = 0; i < num_loops; i++) { + uint32_t cur_size_in_bytes = min(byte_count, max_bytes); + + amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, + dst_offset, cur_size_in_bytes); + + src_offset += cur_size_in_bytes; + dst_offset += cur_size_in_bytes; + byte_count -= cur_size_in_bytes; + } + + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + WARN_ON(job->ibs[0].length_dw > num_dw); + if (direct_submit) + r = amdgpu_job_submit_direct(job, ring, fence); + else + r = amdgpu_job_submit(job, &adev->mman.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, fence); + if (r) + goto error_free; + + return r; + +error_free: + amdgpu_job_free(job); + DRM_ERROR("Error scheduling IBs (%d)\n", r); + return r; +} + +int amdgpu_fill_buffer(struct amdgpu_bo *bo, + uint32_t src_data, + struct reservation_object *resv, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes; + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + + struct drm_mm_node *mm_node; + unsigned long num_pages; + unsigned int num_loops, num_dw; + + struct amdgpu_job *job; + int r; + + if (!adev->mman.buffer_funcs_enabled) { + DRM_ERROR("Trying to clear memory with ring turned off.\n"); + return -EINVAL; + } + + if (bo->tbo.mem.mem_type == TTM_PL_TT) { + r = amdgpu_ttm_alloc_gart(&bo->tbo); + if (r) + return r; + } + + num_pages = bo->tbo.num_pages; + mm_node = bo->tbo.mem.mm_node; + num_loops = 0; + while (num_pages) { + uint64_t byte_count = mm_node->size << PAGE_SHIFT; + + num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes); + num_pages -= mm_node->size; + ++mm_node; + } + num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw; + + /* for IB padding */ + num_dw += 64; + + r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job); + if (r) + return r; + + if (resv) { + r = amdgpu_sync_resv(adev, &job->sync, resv, + AMDGPU_FENCE_OWNER_UNDEFINED, false); + if (r) { + DRM_ERROR("sync failed (%d).\n", r); + goto error_free; + } + } + + num_pages = bo->tbo.num_pages; + mm_node = bo->tbo.mem.mm_node; + + while (num_pages) { + uint64_t byte_count = mm_node->size << PAGE_SHIFT; + uint64_t dst_addr; + + dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem); + while (byte_count) { + uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count, + max_bytes); + + amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, + dst_addr, cur_size_in_bytes); + + dst_addr += cur_size_in_bytes; + byte_count -= cur_size_in_bytes; + } + + num_pages -= mm_node->size; + ++mm_node; + } + + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + WARN_ON(job->ibs[0].length_dw > num_dw); + r = amdgpu_job_submit(job, &adev->mman.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, fence); + if (r) + goto error_free; + + return 0; + +error_free: + amdgpu_job_free(job); + return r; +} + +#if defined(CONFIG_DEBUG_FS) + +static int amdgpu_mm_dump_table(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + unsigned ttm_pl = (uintptr_t)node->info_ent->data; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl]; + struct drm_printer p = drm_seq_file_printer(m); + + man->func->debug(man, &p); + return 0; +} + +static const struct drm_info_list amdgpu_ttm_debugfs_list[] = { + {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM}, + {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT}, + {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS}, + {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS}, + {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA}, + {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, +#ifdef CONFIG_SWIOTLB + {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} +#endif +}; + +/** + * amdgpu_ttm_vram_read - Linear read access to VRAM + * + * Accesses VRAM via MMIO for debugging purposes. + */ +static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + if (*pos >= adev->gmc.mc_vram_size) + return -ENXIO; + + while (size) { + unsigned long flags; + uint32_t value; + + if (*pos >= adev->gmc.mc_vram_size) + return result; + + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000); + WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31); + value = RREG32_NO_KIQ(mmMM_DATA); + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + + r = put_user(value, (uint32_t *)buf); + if (r) + return r; + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +/** + * amdgpu_ttm_vram_write - Linear write access to VRAM + * + * Accesses VRAM via MMIO for debugging purposes. + */ +static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + if (*pos >= adev->gmc.mc_vram_size) + return -ENXIO; + + while (size) { + unsigned long flags; + uint32_t value; + + if (*pos >= adev->gmc.mc_vram_size) + return result; + + r = get_user(value, (uint32_t *)buf); + if (r) + return r; + + spin_lock_irqsave(&adev->mmio_idx_lock, flags); + WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000); + WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31); + WREG32_NO_KIQ(mmMM_DATA, value); + spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + return result; +} + +static const struct file_operations amdgpu_ttm_vram_fops = { + .owner = THIS_MODULE, + .read = amdgpu_ttm_vram_read, + .write = amdgpu_ttm_vram_write, + .llseek = default_llseek, +}; + +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + +/** + * amdgpu_ttm_gtt_read - Linear read access to GTT memory + */ +static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + ssize_t result = 0; + int r; + + while (size) { + loff_t p = *pos / PAGE_SIZE; + unsigned off = *pos & ~PAGE_MASK; + size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); + struct page *page; + void *ptr; + + if (p >= adev->gart.num_cpu_pages) + return result; + + page = adev->gart.pages[p]; + if (page) { + ptr = kmap(page); + ptr += off; + + r = copy_to_user(buf, ptr, cur_size); + kunmap(adev->gart.pages[p]); + } else + r = clear_user(buf, cur_size); + + if (r) + return -EFAULT; + + result += cur_size; + buf += cur_size; + *pos += cur_size; + size -= cur_size; + } + + return result; +} + +static const struct file_operations amdgpu_ttm_gtt_fops = { + .owner = THIS_MODULE, + .read = amdgpu_ttm_gtt_read, + .llseek = default_llseek +}; + +#endif + +/** + * amdgpu_iomem_read - Virtual read access to GPU mapped memory + * + * This function is used to read memory that has been mapped to the + * GPU and the known addresses are not physical addresses but instead + * bus addresses (e.g., what you'd put in an IB or ring buffer). + */ +static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + struct iommu_domain *dom; + ssize_t result = 0; + int r; + + /* retrieve the IOMMU domain if any for this device */ + dom = iommu_get_domain_for_dev(adev->dev); + + while (size) { + phys_addr_t addr = *pos & PAGE_MASK; + loff_t off = *pos & ~PAGE_MASK; + size_t bytes = PAGE_SIZE - off; + unsigned long pfn; + struct page *p; + void *ptr; + + bytes = bytes < size ? bytes : size; + + /* Translate the bus address to a physical address. If + * the domain is NULL it means there is no IOMMU active + * and the address translation is the identity + */ + addr = dom ? iommu_iova_to_phys(dom, addr) : addr; + + pfn = addr >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return -EPERM; + + p = pfn_to_page(pfn); + if (p->mapping != adev->mman.bdev.dev_mapping) + return -EPERM; + + ptr = kmap(p); + r = copy_to_user(buf, ptr + off, bytes); + kunmap(p); + if (r) + return -EFAULT; + + size -= bytes; + *pos += bytes; + result += bytes; + } + + return result; +} + +/** + * amdgpu_iomem_write - Virtual write access to GPU mapped memory + * + * This function is used to write memory that has been mapped to the + * GPU and the known addresses are not physical addresses but instead + * bus addresses (e.g., what you'd put in an IB or ring buffer). + */ +static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = file_inode(f)->i_private; + struct iommu_domain *dom; + ssize_t result = 0; + int r; + + dom = iommu_get_domain_for_dev(adev->dev); + + while (size) { + phys_addr_t addr = *pos & PAGE_MASK; + loff_t off = *pos & ~PAGE_MASK; + size_t bytes = PAGE_SIZE - off; + unsigned long pfn; + struct page *p; + void *ptr; + + bytes = bytes < size ? bytes : size; + + addr = dom ? iommu_iova_to_phys(dom, addr) : addr; + + pfn = addr >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return -EPERM; + + p = pfn_to_page(pfn); + if (p->mapping != adev->mman.bdev.dev_mapping) + return -EPERM; + + ptr = kmap(p); + r = copy_from_user(ptr + off, buf, bytes); + kunmap(p); + if (r) + return -EFAULT; + + size -= bytes; + *pos += bytes; + result += bytes; + } + + return result; +} + +static const struct file_operations amdgpu_ttm_iomem_fops = { + .owner = THIS_MODULE, + .read = amdgpu_iomem_read, + .write = amdgpu_iomem_write, + .llseek = default_llseek +}; + +static const struct { + char *name; + const struct file_operations *fops; + int domain; +} ttm_debugfs_entries[] = { + { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM }, +#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS + { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT }, +#endif + { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM }, +}; + +#endif + +static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + unsigned count; + + struct drm_minor *minor = adev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + + for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) { + ent = debugfs_create_file( + ttm_debugfs_entries[count].name, + S_IFREG | S_IRUGO, root, + adev, + ttm_debugfs_entries[count].fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM) + i_size_write(ent->d_inode, adev->gmc.mc_vram_size); + else if (ttm_debugfs_entries[count].domain == TTM_PL_TT) + i_size_write(ent->d_inode, adev->gmc.gart_size); + adev->mman.debugfs_entries[count] = ent; + } + + count = ARRAY_SIZE(amdgpu_ttm_debugfs_list); + +#ifdef CONFIG_SWIOTLB + if (!(adev->need_swiotlb && swiotlb_nr_tbl())) + --count; +#endif + + return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count); +#else + return 0; +#endif +} + +static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + unsigned i; + + for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++) + debugfs_remove(adev->mman.debugfs_entries[i]); +#endif +}
diff --git a/amdgpu/amdgpu_ttm.h b/amdgpu/amdgpu_ttm.h new file mode 100644 index 0000000..caa76c6 --- /dev/null +++ b/amdgpu/amdgpu_ttm.h
@@ -0,0 +1,134 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_TTM_H__ +#define __AMDGPU_TTM_H__ + +#include "amdgpu.h" +#include <drm/gpu_scheduler.h> + +#define AMDGPU_PL_GDS (TTM_PL_PRIV + 0) +#define AMDGPU_PL_GWS (TTM_PL_PRIV + 1) +#define AMDGPU_PL_OA (TTM_PL_PRIV + 2) + +#define AMDGPU_PL_FLAG_GDS (TTM_PL_FLAG_PRIV << 0) +#define AMDGPU_PL_FLAG_GWS (TTM_PL_FLAG_PRIV << 1) +#define AMDGPU_PL_FLAG_OA (TTM_PL_FLAG_PRIV << 2) + +#define AMDGPU_GTT_MAX_TRANSFER_SIZE 512 +#define AMDGPU_GTT_NUM_TRANSFER_WINDOWS 2 + +struct amdgpu_mman { + struct ttm_bo_device bdev; + bool mem_global_referenced; + bool initialized; + void __iomem *aper_base_kaddr; + +#if defined(CONFIG_DEBUG_FS) + struct dentry *debugfs_entries[8]; +#endif + + /* buffer handling */ + const struct amdgpu_buffer_funcs *buffer_funcs; + struct amdgpu_ring *buffer_funcs_ring; + bool buffer_funcs_enabled; + + struct mutex gtt_window_lock; + /* Scheduler entity for buffer moves */ + struct drm_sched_entity entity; +}; + +struct amdgpu_copy_mem { + struct ttm_buffer_object *bo; + struct ttm_mem_reg *mem; + unsigned long offset; +}; + +extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func; +extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func; + +bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem); +uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man); +int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man); + +u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo); +uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man); +uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man); + +int amdgpu_ttm_init(struct amdgpu_device *adev); +void amdgpu_ttm_late_init(struct amdgpu_device *adev); +void amdgpu_ttm_fini(struct amdgpu_device *adev); +void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, + bool enable); + +int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, + uint64_t dst_offset, uint32_t byte_count, + struct reservation_object *resv, + struct dma_fence **fence, bool direct_submit, + bool vm_needs_flush); +int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, + struct amdgpu_copy_mem *src, + struct amdgpu_copy_mem *dst, + uint64_t size, + struct reservation_object *resv, + struct dma_fence **f); +int amdgpu_fill_buffer(struct amdgpu_bo *bo, + uint32_t src_data, + struct reservation_object *resv, + struct dma_fence **fence); + +int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma); +int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo); +int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo); + +#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) +int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages); +bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm); +#else +static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, + struct page **pages) +{ + return -EPERM; +} +static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm) +{ + return false; +} +#endif + +void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages); +int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, + uint32_t flags); +bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm); +struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm); +bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, + unsigned long end); +bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm, + int *last_invalidated); +bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm); +bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm); +uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem); +uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, + struct ttm_mem_reg *mem); + +#endif
diff --git a/amdgpu/amdgpu_ucode.c b/amdgpu/amdgpu_ucode.c new file mode 100644 index 0000000..bfaa0ea --- /dev/null +++ b/amdgpu/amdgpu_ucode.c
@@ -0,0 +1,600 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/firmware.h> +#include <linux/slab.h> +#include <linux/module.h> + +#include "amdgpu.h" +#include "amdgpu_ucode.h" + +static void amdgpu_ucode_print_common_hdr(const struct common_firmware_header *hdr) +{ + DRM_DEBUG("size_bytes: %u\n", le32_to_cpu(hdr->size_bytes)); + DRM_DEBUG("header_size_bytes: %u\n", le32_to_cpu(hdr->header_size_bytes)); + DRM_DEBUG("header_version_major: %u\n", le16_to_cpu(hdr->header_version_major)); + DRM_DEBUG("header_version_minor: %u\n", le16_to_cpu(hdr->header_version_minor)); + DRM_DEBUG("ip_version_major: %u\n", le16_to_cpu(hdr->ip_version_major)); + DRM_DEBUG("ip_version_minor: %u\n", le16_to_cpu(hdr->ip_version_minor)); + DRM_DEBUG("ucode_version: 0x%08x\n", le32_to_cpu(hdr->ucode_version)); + DRM_DEBUG("ucode_size_bytes: %u\n", le32_to_cpu(hdr->ucode_size_bytes)); + DRM_DEBUG("ucode_array_offset_bytes: %u\n", + le32_to_cpu(hdr->ucode_array_offset_bytes)); + DRM_DEBUG("crc32: 0x%08x\n", le32_to_cpu(hdr->crc32)); +} + +void amdgpu_ucode_print_mc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("MC\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct mc_firmware_header_v1_0 *mc_hdr = + container_of(hdr, struct mc_firmware_header_v1_0, header); + + DRM_DEBUG("io_debug_size_bytes: %u\n", + le32_to_cpu(mc_hdr->io_debug_size_bytes)); + DRM_DEBUG("io_debug_array_offset_bytes: %u\n", + le32_to_cpu(mc_hdr->io_debug_array_offset_bytes)); + } else { + DRM_ERROR("Unknown MC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void amdgpu_ucode_print_smc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("SMC\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct smc_firmware_header_v1_0 *smc_hdr = + container_of(hdr, struct smc_firmware_header_v1_0, header); + + DRM_DEBUG("ucode_start_addr: %u\n", le32_to_cpu(smc_hdr->ucode_start_addr)); + } else if (version_major == 2) { + const struct smc_firmware_header_v1_0 *v1_hdr = + container_of(hdr, struct smc_firmware_header_v1_0, header); + const struct smc_firmware_header_v2_0 *v2_hdr = + container_of(v1_hdr, struct smc_firmware_header_v2_0, v1_0); + + DRM_INFO("ppt_offset_bytes: %u\n", le32_to_cpu(v2_hdr->ppt_offset_bytes)); + DRM_INFO("ppt_size_bytes: %u\n", le32_to_cpu(v2_hdr->ppt_size_bytes)); + } else { + DRM_ERROR("Unknown SMC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void amdgpu_ucode_print_gfx_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("GFX\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct gfx_firmware_header_v1_0 *gfx_hdr = + container_of(hdr, struct gfx_firmware_header_v1_0, header); + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(gfx_hdr->ucode_feature_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(gfx_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(gfx_hdr->jt_size)); + } else { + DRM_ERROR("Unknown GFX ucode version: %u.%u\n", version_major, version_minor); + } +} + +void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("RLC\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct rlc_firmware_header_v1_0 *rlc_hdr = + container_of(hdr, struct rlc_firmware_header_v1_0, header); + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr->ucode_feature_version)); + DRM_DEBUG("save_and_restore_offset: %u\n", + le32_to_cpu(rlc_hdr->save_and_restore_offset)); + DRM_DEBUG("clear_state_descriptor_offset: %u\n", + le32_to_cpu(rlc_hdr->clear_state_descriptor_offset)); + DRM_DEBUG("avail_scratch_ram_locations: %u\n", + le32_to_cpu(rlc_hdr->avail_scratch_ram_locations)); + DRM_DEBUG("master_pkt_description_offset: %u\n", + le32_to_cpu(rlc_hdr->master_pkt_description_offset)); + } else if (version_major == 2) { + const struct rlc_firmware_header_v2_0 *rlc_hdr = + container_of(hdr, struct rlc_firmware_header_v2_0, header); + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr->ucode_feature_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(rlc_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(rlc_hdr->jt_size)); + DRM_DEBUG("save_and_restore_offset: %u\n", + le32_to_cpu(rlc_hdr->save_and_restore_offset)); + DRM_DEBUG("clear_state_descriptor_offset: %u\n", + le32_to_cpu(rlc_hdr->clear_state_descriptor_offset)); + DRM_DEBUG("avail_scratch_ram_locations: %u\n", + le32_to_cpu(rlc_hdr->avail_scratch_ram_locations)); + DRM_DEBUG("reg_restore_list_size: %u\n", + le32_to_cpu(rlc_hdr->reg_restore_list_size)); + DRM_DEBUG("reg_list_format_start: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_start)); + DRM_DEBUG("reg_list_format_separate_start: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_start)); + DRM_DEBUG("starting_offsets_start: %u\n", + le32_to_cpu(rlc_hdr->starting_offsets_start)); + DRM_DEBUG("reg_list_format_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_size_bytes)); + DRM_DEBUG("reg_list_format_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); + DRM_DEBUG("reg_list_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_size_bytes)); + DRM_DEBUG("reg_list_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); + DRM_DEBUG("reg_list_format_separate_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_size_bytes)); + DRM_DEBUG("reg_list_format_separate_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes)); + DRM_DEBUG("reg_list_separate_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes)); + DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes)); + if (version_minor == 1) { + const struct rlc_firmware_header_v2_1 *v2_1 = + container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0); + DRM_DEBUG("reg_list_format_direct_reg_list_length: %u\n", + le32_to_cpu(v2_1->reg_list_format_direct_reg_list_length)); + DRM_DEBUG("save_restore_list_cntl_ucode_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_cntl_ucode_ver)); + DRM_DEBUG("save_restore_list_cntl_feature_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_cntl_feature_ver)); + DRM_DEBUG("save_restore_list_cntl_size_bytes %u\n", + le32_to_cpu(v2_1->save_restore_list_cntl_size_bytes)); + DRM_DEBUG("save_restore_list_cntl_offset_bytes: %u\n", + le32_to_cpu(v2_1->save_restore_list_cntl_offset_bytes)); + DRM_DEBUG("save_restore_list_gpm_ucode_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_gpm_ucode_ver)); + DRM_DEBUG("save_restore_list_gpm_feature_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_gpm_feature_ver)); + DRM_DEBUG("save_restore_list_gpm_size_bytes %u\n", + le32_to_cpu(v2_1->save_restore_list_gpm_size_bytes)); + DRM_DEBUG("save_restore_list_gpm_offset_bytes: %u\n", + le32_to_cpu(v2_1->save_restore_list_gpm_offset_bytes)); + DRM_DEBUG("save_restore_list_srm_ucode_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_srm_ucode_ver)); + DRM_DEBUG("save_restore_list_srm_feature_ver: %u\n", + le32_to_cpu(v2_1->save_restore_list_srm_feature_ver)); + DRM_DEBUG("save_restore_list_srm_size_bytes %u\n", + le32_to_cpu(v2_1->save_restore_list_srm_size_bytes)); + DRM_DEBUG("save_restore_list_srm_offset_bytes: %u\n", + le32_to_cpu(v2_1->save_restore_list_srm_offset_bytes)); + } + } else { + DRM_ERROR("Unknown RLC ucode version: %u.%u\n", version_major, version_minor); + } +} + +void amdgpu_ucode_print_sdma_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("SDMA\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct sdma_firmware_header_v1_0 *sdma_hdr = + container_of(hdr, struct sdma_firmware_header_v1_0, header); + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(sdma_hdr->ucode_feature_version)); + DRM_DEBUG("ucode_change_version: %u\n", + le32_to_cpu(sdma_hdr->ucode_change_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(sdma_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(sdma_hdr->jt_size)); + if (version_minor >= 1) { + const struct sdma_firmware_header_v1_1 *sdma_v1_1_hdr = + container_of(sdma_hdr, struct sdma_firmware_header_v1_1, v1_0); + DRM_DEBUG("digest_size: %u\n", le32_to_cpu(sdma_v1_1_hdr->digest_size)); + } + } else { + DRM_ERROR("Unknown SDMA ucode version: %u.%u\n", + version_major, version_minor); + } +} + +void amdgpu_ucode_print_psp_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("PSP\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct psp_firmware_header_v1_0 *psp_hdr = + container_of(hdr, struct psp_firmware_header_v1_0, header); + + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(psp_hdr->ucode_feature_version)); + DRM_DEBUG("sos_offset_bytes: %u\n", + le32_to_cpu(psp_hdr->sos_offset_bytes)); + DRM_DEBUG("sos_size_bytes: %u\n", + le32_to_cpu(psp_hdr->sos_size_bytes)); + if (version_minor == 1) { + const struct psp_firmware_header_v1_1 *psp_hdr_v1_1 = + container_of(psp_hdr, struct psp_firmware_header_v1_1, v1_0); + DRM_DEBUG("toc_header_version: %u\n", + le32_to_cpu(psp_hdr_v1_1->toc_header_version)); + DRM_DEBUG("toc_offset_bytes: %u\n", + le32_to_cpu(psp_hdr_v1_1->toc_offset_bytes)); + DRM_DEBUG("toc_size_bytes: %u\n", + le32_to_cpu(psp_hdr_v1_1->toc_size_bytes)); + DRM_DEBUG("kdb_header_version: %u\n", + le32_to_cpu(psp_hdr_v1_1->kdb_header_version)); + DRM_DEBUG("kdb_offset_bytes: %u\n", + le32_to_cpu(psp_hdr_v1_1->kdb_offset_bytes)); + DRM_DEBUG("kdb_size_bytes: %u\n", + le32_to_cpu(psp_hdr_v1_1->kdb_size_bytes)); + } + } else { + DRM_ERROR("Unknown PSP ucode version: %u.%u\n", + version_major, version_minor); + } +} + +void amdgpu_ucode_print_gpu_info_hdr(const struct common_firmware_header *hdr) +{ + uint16_t version_major = le16_to_cpu(hdr->header_version_major); + uint16_t version_minor = le16_to_cpu(hdr->header_version_minor); + + DRM_DEBUG("GPU_INFO\n"); + amdgpu_ucode_print_common_hdr(hdr); + + if (version_major == 1) { + const struct gpu_info_firmware_header_v1_0 *gpu_info_hdr = + container_of(hdr, struct gpu_info_firmware_header_v1_0, header); + + DRM_DEBUG("version_major: %u\n", + le16_to_cpu(gpu_info_hdr->version_major)); + DRM_DEBUG("version_minor: %u\n", + le16_to_cpu(gpu_info_hdr->version_minor)); + } else { + DRM_ERROR("Unknown gpu_info ucode version: %u.%u\n", version_major, version_minor); + } +} + +int amdgpu_ucode_validate(const struct firmware *fw) +{ + const struct common_firmware_header *hdr = + (const struct common_firmware_header *)fw->data; + + if (fw->size == le32_to_cpu(hdr->size_bytes)) + return 0; + + return -EINVAL; +} + +bool amdgpu_ucode_hdr_version(union amdgpu_firmware_header *hdr, + uint16_t hdr_major, uint16_t hdr_minor) +{ + if ((hdr->common.header_version_major == hdr_major) && + (hdr->common.header_version_minor == hdr_minor)) + return false; + return true; +} + +enum amdgpu_firmware_load_type +amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type) +{ + switch (adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_SI + case CHIP_TAHITI: + case CHIP_PITCAIRN: + case CHIP_VERDE: + case CHIP_OLAND: + case CHIP_HAINAN: + return AMDGPU_FW_LOAD_DIRECT; +#endif +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + case CHIP_KAVERI: + case CHIP_KABINI: + case CHIP_HAWAII: + case CHIP_MULLINS: + return AMDGPU_FW_LOAD_DIRECT; +#endif + case CHIP_TOPAZ: + case CHIP_TONGA: + case CHIP_FIJI: + case CHIP_CARRIZO: + case CHIP_STONEY: + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + case CHIP_VEGAM: + return AMDGPU_FW_LOAD_SMU; + case CHIP_VEGA10: + case CHIP_RAVEN: + case CHIP_VEGA12: + case CHIP_VEGA20: + case CHIP_NAVI10: + if (!load_type) + return AMDGPU_FW_LOAD_DIRECT; + else + return AMDGPU_FW_LOAD_PSP; + default: + DRM_ERROR("Unknown firmware load type\n"); + } + + return AMDGPU_FW_LOAD_DIRECT; +} + +#define FW_VERSION_ATTR(name, mode, field) \ +static ssize_t show_##name(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + struct drm_device *ddev = dev_get_drvdata(dev); \ + struct amdgpu_device *adev = ddev->dev_private; \ + \ + return snprintf(buf, PAGE_SIZE, "0x%08x\n", adev->field); \ +} \ +static DEVICE_ATTR(name, mode, show_##name, NULL) + +FW_VERSION_ATTR(vce_fw_version, 0444, vce.fw_version); +FW_VERSION_ATTR(uvd_fw_version, 0444, uvd.fw_version); +FW_VERSION_ATTR(mc_fw_version, 0444, gmc.fw_version); +FW_VERSION_ATTR(me_fw_version, 0444, gfx.me_fw_version); +FW_VERSION_ATTR(pfp_fw_version, 0444, gfx.pfp_fw_version); +FW_VERSION_ATTR(ce_fw_version, 0444, gfx.ce_fw_version); +FW_VERSION_ATTR(rlc_fw_version, 0444, gfx.rlc_fw_version); +FW_VERSION_ATTR(rlc_srlc_fw_version, 0444, gfx.rlc_srlc_fw_version); +FW_VERSION_ATTR(rlc_srlg_fw_version, 0444, gfx.rlc_srlg_fw_version); +FW_VERSION_ATTR(rlc_srls_fw_version, 0444, gfx.rlc_srls_fw_version); +FW_VERSION_ATTR(mec_fw_version, 0444, gfx.mec_fw_version); +FW_VERSION_ATTR(mec2_fw_version, 0444, gfx.mec2_fw_version); +FW_VERSION_ATTR(sos_fw_version, 0444, psp.sos_fw_version); +FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd_fw_version); +FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ta_fw_version); +FW_VERSION_ATTR(ta_xgmi_fw_version, 0444, psp.ta_fw_version); +FW_VERSION_ATTR(smc_fw_version, 0444, pm.fw_version); +FW_VERSION_ATTR(sdma_fw_version, 0444, sdma.instance[0].fw_version); +FW_VERSION_ATTR(sdma2_fw_version, 0444, sdma.instance[1].fw_version); +FW_VERSION_ATTR(vcn_fw_version, 0444, vcn.fw_version); +FW_VERSION_ATTR(dmcu_fw_version, 0444, dm.dmcu_fw_version); + +static struct attribute *fw_attrs[] = { + &dev_attr_vce_fw_version.attr, &dev_attr_uvd_fw_version.attr, + &dev_attr_mc_fw_version.attr, &dev_attr_me_fw_version.attr, + &dev_attr_pfp_fw_version.attr, &dev_attr_ce_fw_version.attr, + &dev_attr_rlc_fw_version.attr, &dev_attr_rlc_srlc_fw_version.attr, + &dev_attr_rlc_srlg_fw_version.attr, &dev_attr_rlc_srls_fw_version.attr, + &dev_attr_mec_fw_version.attr, &dev_attr_mec2_fw_version.attr, + &dev_attr_sos_fw_version.attr, &dev_attr_asd_fw_version.attr, + &dev_attr_ta_ras_fw_version.attr, &dev_attr_ta_xgmi_fw_version.attr, + &dev_attr_smc_fw_version.attr, &dev_attr_sdma_fw_version.attr, + &dev_attr_sdma2_fw_version.attr, &dev_attr_vcn_fw_version.attr, + &dev_attr_dmcu_fw_version.attr, NULL +}; + +static const struct attribute_group fw_attr_group = { + .name = "fw_version", + .attrs = fw_attrs +}; + +int amdgpu_ucode_sysfs_init(struct amdgpu_device *adev) +{ + return sysfs_create_group(&adev->dev->kobj, &fw_attr_group); +} + +void amdgpu_ucode_sysfs_fini(struct amdgpu_device *adev) +{ + sysfs_remove_group(&adev->dev->kobj, &fw_attr_group); +} + +static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev, + struct amdgpu_firmware_info *ucode, + uint64_t mc_addr, void *kptr) +{ + const struct common_firmware_header *header = NULL; + const struct gfx_firmware_header_v1_0 *cp_hdr = NULL; + const struct dmcu_firmware_header_v1_0 *dmcu_hdr = NULL; + + if (NULL == ucode->fw) + return 0; + + ucode->mc_addr = mc_addr; + ucode->kaddr = kptr; + + if (ucode->ucode_id == AMDGPU_UCODE_ID_STORAGE) + return 0; + + header = (const struct common_firmware_header *)ucode->fw->data; + cp_hdr = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + dmcu_hdr = (const struct dmcu_firmware_header_v1_0 *)ucode->fw->data; + + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP || + (ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC1 && + ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC2 && + ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC1_JT && + ucode->ucode_id != AMDGPU_UCODE_ID_CP_MEC2_JT && + ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL && + ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM && + ucode->ucode_id != AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM && + ucode->ucode_id != AMDGPU_UCODE_ID_DMCU_ERAM && + ucode->ucode_id != AMDGPU_UCODE_ID_DMCU_INTV)) { + ucode->ucode_size = le32_to_cpu(header->ucode_size_bytes); + + memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data + + le32_to_cpu(header->ucode_array_offset_bytes)), + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1 || + ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2) { + ucode->ucode_size = le32_to_cpu(header->ucode_size_bytes) - + le32_to_cpu(cp_hdr->jt_size) * 4; + + memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data + + le32_to_cpu(header->ucode_array_offset_bytes)), + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT || + ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT) { + ucode->ucode_size = le32_to_cpu(cp_hdr->jt_size) * 4; + + memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data + + le32_to_cpu(header->ucode_array_offset_bytes) + + le32_to_cpu(cp_hdr->jt_offset) * 4), + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_DMCU_ERAM) { + ucode->ucode_size = le32_to_cpu(header->ucode_size_bytes) - + le32_to_cpu(dmcu_hdr->intv_size_bytes); + + memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data + + le32_to_cpu(header->ucode_array_offset_bytes)), + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_DMCU_INTV) { + ucode->ucode_size = le32_to_cpu(dmcu_hdr->intv_size_bytes); + + memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data + + le32_to_cpu(header->ucode_array_offset_bytes) + + le32_to_cpu(dmcu_hdr->intv_offset_bytes)), + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL) { + ucode->ucode_size = adev->gfx.rlc.save_restore_list_cntl_size_bytes; + memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_cntl, + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM) { + ucode->ucode_size = adev->gfx.rlc.save_restore_list_gpm_size_bytes; + memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_gpm, + ucode->ucode_size); + } else if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) { + ucode->ucode_size = adev->gfx.rlc.save_restore_list_srm_size_bytes; + memcpy(ucode->kaddr, adev->gfx.rlc.save_restore_list_srm, + ucode->ucode_size); + } + + return 0; +} + +static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode, + uint64_t mc_addr, void *kptr) +{ + const struct gfx_firmware_header_v1_0 *header = NULL; + const struct common_firmware_header *comm_hdr = NULL; + uint8_t* src_addr = NULL; + uint8_t* dst_addr = NULL; + + if (NULL == ucode->fw) + return 0; + + comm_hdr = (const struct common_firmware_header *)ucode->fw->data; + header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + dst_addr = ucode->kaddr + + ALIGN(le32_to_cpu(comm_hdr->ucode_size_bytes), + PAGE_SIZE); + src_addr = (uint8_t *)ucode->fw->data + + le32_to_cpu(comm_hdr->ucode_array_offset_bytes) + + (le32_to_cpu(header->jt_offset) * 4); + memcpy(dst_addr, src_addr, le32_to_cpu(header->jt_size) * 4); + + return 0; +} + +int amdgpu_ucode_create_bo(struct amdgpu_device *adev) +{ + if (adev->firmware.load_type != AMDGPU_FW_LOAD_DIRECT) { + amdgpu_bo_create_kernel(adev, adev->firmware.fw_size, PAGE_SIZE, + amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT, + &adev->firmware.fw_buf, + &adev->firmware.fw_buf_mc, + &adev->firmware.fw_buf_ptr); + if (!adev->firmware.fw_buf) { + dev_err(adev->dev, "failed to create kernel buffer for firmware.fw_buf\n"); + return -ENOMEM; + } else if (amdgpu_sriov_vf(adev)) { + memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size); + } + } + return 0; +} + +void amdgpu_ucode_free_bo(struct amdgpu_device *adev) +{ + if (adev->firmware.load_type != AMDGPU_FW_LOAD_DIRECT) + amdgpu_bo_free_kernel(&adev->firmware.fw_buf, + &adev->firmware.fw_buf_mc, + &adev->firmware.fw_buf_ptr); +} + +int amdgpu_ucode_init_bo(struct amdgpu_device *adev) +{ + uint64_t fw_offset = 0; + int i; + struct amdgpu_firmware_info *ucode = NULL; + + /* for baremetal, the ucode is allocated in gtt, so don't need to fill the bo when reset/suspend */ + if (!amdgpu_sriov_vf(adev) && (adev->in_gpu_reset || adev->in_suspend)) + return 0; + /* + * if SMU loaded firmware, it needn't add SMC, UVD, and VCE + * ucode info here + */ + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + if (amdgpu_sriov_vf(adev)) + adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM - 3; + else + adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM - 4; + } else { + adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM; + } + + for (i = 0; i < adev->firmware.max_ucodes; i++) { + ucode = &adev->firmware.ucode[i]; + if (ucode->fw) { + amdgpu_ucode_init_single_fw(adev, ucode, adev->firmware.fw_buf_mc + fw_offset, + adev->firmware.fw_buf_ptr + fw_offset); + if (i == AMDGPU_UCODE_ID_CP_MEC1 && + adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + const struct gfx_firmware_header_v1_0 *cp_hdr; + cp_hdr = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + amdgpu_ucode_patch_jt(ucode, adev->firmware.fw_buf_mc + fw_offset, + adev->firmware.fw_buf_ptr + fw_offset); + fw_offset += ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); + } + fw_offset += ALIGN(ucode->ucode_size, PAGE_SIZE); + } + } + return 0; +}
diff --git a/amdgpu/amdgpu_ucode.h b/amdgpu/amdgpu_ucode.h new file mode 100644 index 0000000..c1fb6dc --- /dev/null +++ b/amdgpu/amdgpu_ucode.h
@@ -0,0 +1,370 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_UCODE_H__ +#define __AMDGPU_UCODE_H__ + +#include "amdgpu_socbb.h" + +struct common_firmware_header { + uint32_t size_bytes; /* size of the entire header+image(s) in bytes */ + uint32_t header_size_bytes; /* size of just the header in bytes */ + uint16_t header_version_major; /* header version */ + uint16_t header_version_minor; /* header version */ + uint16_t ip_version_major; /* IP version */ + uint16_t ip_version_minor; /* IP version */ + uint32_t ucode_version; + uint32_t ucode_size_bytes; /* size of ucode in bytes */ + uint32_t ucode_array_offset_bytes; /* payload offset from the start of the header */ + uint32_t crc32; /* crc32 checksum of the payload */ +}; + +/* version_major=1, version_minor=0 */ +struct mc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t io_debug_size_bytes; /* size of debug array in dwords */ + uint32_t io_debug_array_offset_bytes; /* payload offset from the start of the header */ +}; + +/* version_major=1, version_minor=0 */ +struct smc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_start_addr; +}; + +/* version_major=2, version_minor=0 */ +struct smc_firmware_header_v2_0 { + struct smc_firmware_header_v1_0 v1_0; + uint32_t ppt_offset_bytes; /* soft pptable offset */ + uint32_t ppt_size_bytes; /* soft pptable size */ +}; + +struct smc_soft_pptable_entry { + uint32_t id; + uint32_t ppt_offset_bytes; + uint32_t ppt_size_bytes; +}; + +/* version_major=2, version_minor=1 */ +struct smc_firmware_header_v2_1 { + struct smc_firmware_header_v1_0 v1_0; + uint32_t pptable_count; + uint32_t pptable_entry_offset; +}; + +/* version_major=1, version_minor=0 */ +struct psp_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t sos_offset_bytes; + uint32_t sos_size_bytes; +}; + +/* version_major=1, version_minor=1 */ +struct psp_firmware_header_v1_1 { + struct psp_firmware_header_v1_0 v1_0; + uint32_t toc_header_version; + uint32_t toc_offset_bytes; + uint32_t toc_size_bytes; + uint32_t kdb_header_version; + uint32_t kdb_offset_bytes; + uint32_t kdb_size_bytes; +}; + +/* version_major=1, version_minor=0 */ +struct ta_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ta_xgmi_ucode_version; + uint32_t ta_xgmi_offset_bytes; + uint32_t ta_xgmi_size_bytes; + uint32_t ta_ras_ucode_version; + uint32_t ta_ras_offset_bytes; + uint32_t ta_ras_size_bytes; +}; + +/* version_major=1, version_minor=0 */ +struct gfx_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t jt_offset; /* jt location */ + uint32_t jt_size; /* size of jt */ +}; + +/* version_major=1, version_minor=0 */ +struct mes_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t mes_ucode_version; + uint32_t mes_ucode_size_bytes; + uint32_t mes_ucode_offset_bytes; + uint32_t mes_ucode_data_version; + uint32_t mes_ucode_data_size_bytes; + uint32_t mes_ucode_data_offset_bytes; + uint32_t mes_uc_start_addr_lo; + uint32_t mes_uc_start_addr_hi; + uint32_t mes_data_start_addr_lo; + uint32_t mes_data_start_addr_hi; +}; + +/* version_major=1, version_minor=0 */ +struct rlc_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t save_and_restore_offset; + uint32_t clear_state_descriptor_offset; + uint32_t avail_scratch_ram_locations; + uint32_t master_pkt_description_offset; +}; + +/* version_major=2, version_minor=0 */ +struct rlc_firmware_header_v2_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t jt_offset; /* jt location */ + uint32_t jt_size; /* size of jt */ + uint32_t save_and_restore_offset; + uint32_t clear_state_descriptor_offset; + uint32_t avail_scratch_ram_locations; + uint32_t reg_restore_list_size; + uint32_t reg_list_format_start; + uint32_t reg_list_format_separate_start; + uint32_t starting_offsets_start; + uint32_t reg_list_format_size_bytes; /* size of reg list format array in bytes */ + uint32_t reg_list_format_array_offset_bytes; /* payload offset from the start of the header */ + uint32_t reg_list_size_bytes; /* size of reg list array in bytes */ + uint32_t reg_list_array_offset_bytes; /* payload offset from the start of the header */ + uint32_t reg_list_format_separate_size_bytes; /* size of reg list format array in bytes */ + uint32_t reg_list_format_separate_array_offset_bytes; /* payload offset from the start of the header */ + uint32_t reg_list_separate_size_bytes; /* size of reg list array in bytes */ + uint32_t reg_list_separate_array_offset_bytes; /* payload offset from the start of the header */ +}; + +/* version_major=2, version_minor=1 */ +struct rlc_firmware_header_v2_1 { + struct rlc_firmware_header_v2_0 v2_0; + uint32_t reg_list_format_direct_reg_list_length; /* length of direct reg list format array */ + uint32_t save_restore_list_cntl_ucode_ver; + uint32_t save_restore_list_cntl_feature_ver; + uint32_t save_restore_list_cntl_size_bytes; + uint32_t save_restore_list_cntl_offset_bytes; + uint32_t save_restore_list_gpm_ucode_ver; + uint32_t save_restore_list_gpm_feature_ver; + uint32_t save_restore_list_gpm_size_bytes; + uint32_t save_restore_list_gpm_offset_bytes; + uint32_t save_restore_list_srm_ucode_ver; + uint32_t save_restore_list_srm_feature_ver; + uint32_t save_restore_list_srm_size_bytes; + uint32_t save_restore_list_srm_offset_bytes; +}; + +/* version_major=1, version_minor=0 */ +struct sdma_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t ucode_feature_version; + uint32_t ucode_change_version; + uint32_t jt_offset; /* jt location */ + uint32_t jt_size; /* size of jt */ +}; + +/* version_major=1, version_minor=1 */ +struct sdma_firmware_header_v1_1 { + struct sdma_firmware_header_v1_0 v1_0; + uint32_t digest_size; +}; + +/* gpu info payload */ +struct gpu_info_firmware_v1_0 { + uint32_t gc_num_se; + uint32_t gc_num_cu_per_sh; + uint32_t gc_num_sh_per_se; + uint32_t gc_num_rb_per_se; + uint32_t gc_num_tccs; + uint32_t gc_num_gprs; + uint32_t gc_num_max_gs_thds; + uint32_t gc_gs_table_depth; + uint32_t gc_gsprim_buff_depth; + uint32_t gc_parameter_cache_depth; + uint32_t gc_double_offchip_lds_buffer; + uint32_t gc_wave_size; + uint32_t gc_max_waves_per_simd; + uint32_t gc_max_scratch_slots_per_cu; + uint32_t gc_lds_size; +}; + +struct gpu_info_firmware_v1_1 { + struct gpu_info_firmware_v1_0 v1_0; + uint32_t num_sc_per_sh; + uint32_t num_packer_per_sc; +}; + +/* gpu info payload + * version_major=1, version_minor=1 */ +struct gpu_info_firmware_v1_2 { + struct gpu_info_firmware_v1_1 v1_1; + struct gpu_info_soc_bounding_box_v1_0 soc_bounding_box; +}; + +/* version_major=1, version_minor=0 */ +struct gpu_info_firmware_header_v1_0 { + struct common_firmware_header header; + uint16_t version_major; /* version */ + uint16_t version_minor; /* version */ +}; + +/* version_major=1, version_minor=0 */ +struct dmcu_firmware_header_v1_0 { + struct common_firmware_header header; + uint32_t intv_offset_bytes; /* interrupt vectors offset from end of header, in bytes */ + uint32_t intv_size_bytes; /* size of interrupt vectors, in bytes */ +}; + +/* header is fixed size */ +union amdgpu_firmware_header { + struct common_firmware_header common; + struct mc_firmware_header_v1_0 mc; + struct smc_firmware_header_v1_0 smc; + struct smc_firmware_header_v2_0 smc_v2_0; + struct psp_firmware_header_v1_0 psp; + struct psp_firmware_header_v1_1 psp_v1_1; + struct ta_firmware_header_v1_0 ta; + struct gfx_firmware_header_v1_0 gfx; + struct rlc_firmware_header_v1_0 rlc; + struct rlc_firmware_header_v2_0 rlc_v2_0; + struct rlc_firmware_header_v2_1 rlc_v2_1; + struct sdma_firmware_header_v1_0 sdma; + struct sdma_firmware_header_v1_1 sdma_v1_1; + struct gpu_info_firmware_header_v1_0 gpu_info; + struct dmcu_firmware_header_v1_0 dmcu; + uint8_t raw[0x100]; +}; + +/* + * fw loading support + */ +enum AMDGPU_UCODE_ID { + AMDGPU_UCODE_ID_SDMA0 = 0, + AMDGPU_UCODE_ID_SDMA1, + AMDGPU_UCODE_ID_CP_CE, + AMDGPU_UCODE_ID_CP_PFP, + AMDGPU_UCODE_ID_CP_ME, + AMDGPU_UCODE_ID_CP_MEC1, + AMDGPU_UCODE_ID_CP_MEC1_JT, + AMDGPU_UCODE_ID_CP_MEC2, + AMDGPU_UCODE_ID_CP_MEC2_JT, + AMDGPU_UCODE_ID_CP_MES, + AMDGPU_UCODE_ID_CP_MES_DATA, + AMDGPU_UCODE_ID_RLC_G, + AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL, + AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM, + AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM, + AMDGPU_UCODE_ID_STORAGE, + AMDGPU_UCODE_ID_SMC, + AMDGPU_UCODE_ID_UVD, + AMDGPU_UCODE_ID_UVD1, + AMDGPU_UCODE_ID_VCE, + AMDGPU_UCODE_ID_VCN, + AMDGPU_UCODE_ID_DMCU_ERAM, + AMDGPU_UCODE_ID_DMCU_INTV, + AMDGPU_UCODE_ID_VCN0_RAM, + AMDGPU_UCODE_ID_VCN1_RAM, + AMDGPU_UCODE_ID_MAXIMUM, +}; + +/* engine firmware status */ +enum AMDGPU_UCODE_STATUS { + AMDGPU_UCODE_STATUS_INVALID, + AMDGPU_UCODE_STATUS_NOT_LOADED, + AMDGPU_UCODE_STATUS_LOADED, +}; + +enum amdgpu_firmware_load_type { + AMDGPU_FW_LOAD_DIRECT = 0, + AMDGPU_FW_LOAD_SMU, + AMDGPU_FW_LOAD_PSP, + AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO, +}; + +/* conform to smu_ucode_xfer_cz.h */ +#define AMDGPU_SDMA0_UCODE_LOADED 0x00000001 +#define AMDGPU_SDMA1_UCODE_LOADED 0x00000002 +#define AMDGPU_CPCE_UCODE_LOADED 0x00000004 +#define AMDGPU_CPPFP_UCODE_LOADED 0x00000008 +#define AMDGPU_CPME_UCODE_LOADED 0x00000010 +#define AMDGPU_CPMEC1_UCODE_LOADED 0x00000020 +#define AMDGPU_CPMEC2_UCODE_LOADED 0x00000040 +#define AMDGPU_CPRLC_UCODE_LOADED 0x00000100 + +/* amdgpu firmware info */ +struct amdgpu_firmware_info { + /* ucode ID */ + enum AMDGPU_UCODE_ID ucode_id; + /* request_firmware */ + const struct firmware *fw; + /* starting mc address */ + uint64_t mc_addr; + /* kernel linear address */ + void *kaddr; + /* ucode_size_bytes */ + uint32_t ucode_size; + /* starting tmr mc address */ + uint32_t tmr_mc_addr_lo; + uint32_t tmr_mc_addr_hi; +}; + +struct amdgpu_firmware { + struct amdgpu_firmware_info ucode[AMDGPU_UCODE_ID_MAXIMUM]; + enum amdgpu_firmware_load_type load_type; + struct amdgpu_bo *fw_buf; + unsigned int fw_size; + unsigned int max_ucodes; + /* firmwares are loaded by psp instead of smu from vega10 */ + const struct amdgpu_psp_funcs *funcs; + struct amdgpu_bo *rbuf; + struct mutex mutex; + + /* gpu info firmware data pointer */ + const struct firmware *gpu_info_fw; + + void *fw_buf_ptr; + uint64_t fw_buf_mc; +}; + +void amdgpu_ucode_print_mc_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_smc_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_gfx_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_sdma_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_psp_hdr(const struct common_firmware_header *hdr); +void amdgpu_ucode_print_gpu_info_hdr(const struct common_firmware_header *hdr); +int amdgpu_ucode_validate(const struct firmware *fw); +bool amdgpu_ucode_hdr_version(union amdgpu_firmware_header *hdr, + uint16_t hdr_major, uint16_t hdr_minor); + +int amdgpu_ucode_init_bo(struct amdgpu_device *adev); +int amdgpu_ucode_create_bo(struct amdgpu_device *adev); +int amdgpu_ucode_sysfs_init(struct amdgpu_device *adev); +void amdgpu_ucode_free_bo(struct amdgpu_device *adev); +void amdgpu_ucode_sysfs_fini(struct amdgpu_device *adev); + +enum amdgpu_firmware_load_type +amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type); + +#endif
diff --git a/amdgpu/amdgpu_uvd.c b/amdgpu/amdgpu_uvd.c new file mode 100644 index 0000000..5b2fea3 --- /dev/null +++ b/amdgpu/amdgpu_uvd.c
@@ -0,0 +1,1292 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König <deathsimple@vodafone.de> + */ + +#include <linux/firmware.h> +#include <linux/module.h> + +#include <drm/drm.h> + +#include "amdgpu.h" +#include "amdgpu_pm.h" +#include "amdgpu_uvd.h" +#include "cikd.h" +#include "uvd/uvd_4_2_d.h" + +/* 1 second timeout */ +#define UVD_IDLE_TIMEOUT msecs_to_jiffies(1000) + +/* Firmware versions for VI */ +#define FW_1_65_10 ((1 << 24) | (65 << 16) | (10 << 8)) +#define FW_1_87_11 ((1 << 24) | (87 << 16) | (11 << 8)) +#define FW_1_87_12 ((1 << 24) | (87 << 16) | (12 << 8)) +#define FW_1_37_15 ((1 << 24) | (37 << 16) | (15 << 8)) + +/* Polaris10/11 firmware version */ +#define FW_1_66_16 ((1 << 24) | (66 << 16) | (16 << 8)) + +/* Firmware Names */ +#ifdef CONFIG_DRM_AMDGPU_CIK +#define FIRMWARE_BONAIRE "amdgpu/bonaire_uvd.bin" +#define FIRMWARE_KABINI "amdgpu/kabini_uvd.bin" +#define FIRMWARE_KAVERI "amdgpu/kaveri_uvd.bin" +#define FIRMWARE_HAWAII "amdgpu/hawaii_uvd.bin" +#define FIRMWARE_MULLINS "amdgpu/mullins_uvd.bin" +#endif +#define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin" +#define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin" +#define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin" +#define FIRMWARE_STONEY "amdgpu/stoney_uvd.bin" +#define FIRMWARE_POLARIS10 "amdgpu/polaris10_uvd.bin" +#define FIRMWARE_POLARIS11 "amdgpu/polaris11_uvd.bin" +#define FIRMWARE_POLARIS12 "amdgpu/polaris12_uvd.bin" +#define FIRMWARE_VEGAM "amdgpu/vegam_uvd.bin" + +#define FIRMWARE_VEGA10 "amdgpu/vega10_uvd.bin" +#define FIRMWARE_VEGA12 "amdgpu/vega12_uvd.bin" +#define FIRMWARE_VEGA20 "amdgpu/vega20_uvd.bin" + +/* These are common relative offsets for all asics, from uvd_7_0_offset.h, */ +#define UVD_GPCOM_VCPU_CMD 0x03c3 +#define UVD_GPCOM_VCPU_DATA0 0x03c4 +#define UVD_GPCOM_VCPU_DATA1 0x03c5 +#define UVD_NO_OP 0x03ff +#define UVD_BASE_SI 0x3800 + +/** + * amdgpu_uvd_cs_ctx - Command submission parser context + * + * Used for emulating virtual memory support on UVD 4.2. + */ +struct amdgpu_uvd_cs_ctx { + struct amdgpu_cs_parser *parser; + unsigned reg, count; + unsigned data0, data1; + unsigned idx; + unsigned ib_idx; + + /* does the IB has a msg command */ + bool has_msg_cmd; + + /* minimum buffer sizes */ + unsigned *buf_sizes; +}; + +#ifdef CONFIG_DRM_AMDGPU_CIK +MODULE_FIRMWARE(FIRMWARE_BONAIRE); +MODULE_FIRMWARE(FIRMWARE_KABINI); +MODULE_FIRMWARE(FIRMWARE_KAVERI); +MODULE_FIRMWARE(FIRMWARE_HAWAII); +MODULE_FIRMWARE(FIRMWARE_MULLINS); +#endif +MODULE_FIRMWARE(FIRMWARE_TONGA); +MODULE_FIRMWARE(FIRMWARE_CARRIZO); +MODULE_FIRMWARE(FIRMWARE_FIJI); +MODULE_FIRMWARE(FIRMWARE_STONEY); +MODULE_FIRMWARE(FIRMWARE_POLARIS10); +MODULE_FIRMWARE(FIRMWARE_POLARIS11); +MODULE_FIRMWARE(FIRMWARE_POLARIS12); +MODULE_FIRMWARE(FIRMWARE_VEGAM); + +MODULE_FIRMWARE(FIRMWARE_VEGA10); +MODULE_FIRMWARE(FIRMWARE_VEGA12); +MODULE_FIRMWARE(FIRMWARE_VEGA20); + +static void amdgpu_uvd_idle_work_handler(struct work_struct *work); + +int amdgpu_uvd_sw_init(struct amdgpu_device *adev) +{ + unsigned long bo_size; + const char *fw_name; + const struct common_firmware_header *hdr; + unsigned family_id; + int i, j, r; + + INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler); + + switch (adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + fw_name = FIRMWARE_BONAIRE; + break; + case CHIP_KABINI: + fw_name = FIRMWARE_KABINI; + break; + case CHIP_KAVERI: + fw_name = FIRMWARE_KAVERI; + break; + case CHIP_HAWAII: + fw_name = FIRMWARE_HAWAII; + break; + case CHIP_MULLINS: + fw_name = FIRMWARE_MULLINS; + break; +#endif + case CHIP_TONGA: + fw_name = FIRMWARE_TONGA; + break; + case CHIP_FIJI: + fw_name = FIRMWARE_FIJI; + break; + case CHIP_CARRIZO: + fw_name = FIRMWARE_CARRIZO; + break; + case CHIP_STONEY: + fw_name = FIRMWARE_STONEY; + break; + case CHIP_POLARIS10: + fw_name = FIRMWARE_POLARIS10; + break; + case CHIP_POLARIS11: + fw_name = FIRMWARE_POLARIS11; + break; + case CHIP_POLARIS12: + fw_name = FIRMWARE_POLARIS12; + break; + case CHIP_VEGA10: + fw_name = FIRMWARE_VEGA10; + break; + case CHIP_VEGA12: + fw_name = FIRMWARE_VEGA12; + break; + case CHIP_VEGAM: + fw_name = FIRMWARE_VEGAM; + break; + case CHIP_VEGA20: + fw_name = FIRMWARE_VEGA20; + break; + default: + return -EINVAL; + } + + r = request_firmware(&adev->uvd.fw, fw_name, adev->dev); + if (r) { + dev_err(adev->dev, "amdgpu_uvd: Can't load firmware \"%s\"\n", + fw_name); + return r; + } + + r = amdgpu_ucode_validate(adev->uvd.fw); + if (r) { + dev_err(adev->dev, "amdgpu_uvd: Can't validate firmware \"%s\"\n", + fw_name); + release_firmware(adev->uvd.fw); + adev->uvd.fw = NULL; + return r; + } + + /* Set the default UVD handles that the firmware can handle */ + adev->uvd.max_handles = AMDGPU_DEFAULT_UVD_HANDLES; + + hdr = (const struct common_firmware_header *)adev->uvd.fw->data; + family_id = le32_to_cpu(hdr->ucode_version) & 0xff; + + if (adev->asic_type < CHIP_VEGA20) { + unsigned version_major, version_minor; + + version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff; + version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff; + DRM_INFO("Found UVD firmware Version: %hu.%hu Family ID: %hu\n", + version_major, version_minor, family_id); + + /* + * Limit the number of UVD handles depending on microcode major + * and minor versions. The firmware version which has 40 UVD + * instances support is 1.80. So all subsequent versions should + * also have the same support. + */ + if ((version_major > 0x01) || + ((version_major == 0x01) && (version_minor >= 0x50))) + adev->uvd.max_handles = AMDGPU_MAX_UVD_HANDLES; + + adev->uvd.fw_version = ((version_major << 24) | (version_minor << 16) | + (family_id << 8)); + + if ((adev->asic_type == CHIP_POLARIS10 || + adev->asic_type == CHIP_POLARIS11) && + (adev->uvd.fw_version < FW_1_66_16)) + DRM_ERROR("POLARIS10/11 UVD firmware version %hu.%hu is too old.\n", + version_major, version_minor); + } else { + unsigned int enc_major, enc_minor, dec_minor; + + dec_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff; + enc_minor = (le32_to_cpu(hdr->ucode_version) >> 24) & 0x3f; + enc_major = (le32_to_cpu(hdr->ucode_version) >> 30) & 0x3; + DRM_INFO("Found UVD firmware ENC: %hu.%hu DEC: .%hu Family ID: %hu\n", + enc_major, enc_minor, dec_minor, family_id); + + adev->uvd.max_handles = AMDGPU_MAX_UVD_HANDLES; + + adev->uvd.fw_version = le32_to_cpu(hdr->ucode_version); + } + + bo_size = AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE + + AMDGPU_UVD_SESSION_SIZE * adev->uvd.max_handles; + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) + bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); + + for (j = 0; j < adev->uvd.num_uvd_inst; j++) { + if (adev->uvd.harvest_config & (1 << j)) + continue; + r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->uvd.inst[j].vcpu_bo, + &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r); + return r; + } + } + + for (i = 0; i < adev->uvd.max_handles; ++i) { + atomic_set(&adev->uvd.handles[i], 0); + adev->uvd.filp[i] = NULL; + } + + /* from uvd v5.0 HW addressing capacity increased to 64 bits */ + if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) + adev->uvd.address_64_bit = true; + + switch (adev->asic_type) { + case CHIP_TONGA: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_65_10; + break; + case CHIP_CARRIZO: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_11; + break; + case CHIP_FIJI: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_12; + break; + case CHIP_STONEY: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_37_15; + break; + default: + adev->uvd.use_ctx_buf = adev->asic_type >= CHIP_POLARIS10; + } + + return 0; +} + +int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) +{ + int i, j; + + drm_sched_entity_destroy(&adev->uvd.entity); + + for (j = 0; j < adev->uvd.num_uvd_inst; ++j) { + if (adev->uvd.harvest_config & (1 << j)) + continue; + kvfree(adev->uvd.inst[j].saved_bo); + + amdgpu_bo_free_kernel(&adev->uvd.inst[j].vcpu_bo, + &adev->uvd.inst[j].gpu_addr, + (void **)&adev->uvd.inst[j].cpu_addr); + + amdgpu_ring_fini(&adev->uvd.inst[j].ring); + + for (i = 0; i < AMDGPU_MAX_UVD_ENC_RINGS; ++i) + amdgpu_ring_fini(&adev->uvd.inst[j].ring_enc[i]); + } + release_firmware(adev->uvd.fw); + + return 0; +} + +/** + * amdgpu_uvd_entity_init - init entity + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_uvd_entity_init(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring; + struct drm_sched_rq *rq; + int r; + + ring = &adev->uvd.inst[0].ring; + rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; + r = drm_sched_entity_init(&adev->uvd.entity, &rq, 1, NULL); + if (r) { + DRM_ERROR("Failed setting up UVD kernel entity.\n"); + return r; + } + + return 0; +} + +int amdgpu_uvd_suspend(struct amdgpu_device *adev) +{ + unsigned size; + void *ptr; + int i, j; + + cancel_delayed_work_sync(&adev->uvd.idle_work); + + /* only valid for physical mode */ + if (adev->asic_type < CHIP_POLARIS10) { + for (i = 0; i < adev->uvd.max_handles; ++i) + if (atomic_read(&adev->uvd.handles[i])) + break; + + if (i == adev->uvd.max_handles) + return 0; + } + + for (j = 0; j < adev->uvd.num_uvd_inst; ++j) { + if (adev->uvd.harvest_config & (1 << j)) + continue; + if (adev->uvd.inst[j].vcpu_bo == NULL) + continue; + + size = amdgpu_bo_size(adev->uvd.inst[j].vcpu_bo); + ptr = adev->uvd.inst[j].cpu_addr; + + adev->uvd.inst[j].saved_bo = kvmalloc(size, GFP_KERNEL); + if (!adev->uvd.inst[j].saved_bo) + return -ENOMEM; + + memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size); + } + return 0; +} + +int amdgpu_uvd_resume(struct amdgpu_device *adev) +{ + unsigned size; + void *ptr; + int i; + + for (i = 0; i < adev->uvd.num_uvd_inst; i++) { + if (adev->uvd.harvest_config & (1 << i)) + continue; + if (adev->uvd.inst[i].vcpu_bo == NULL) + return -EINVAL; + + size = amdgpu_bo_size(adev->uvd.inst[i].vcpu_bo); + ptr = adev->uvd.inst[i].cpu_addr; + + if (adev->uvd.inst[i].saved_bo != NULL) { + memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size); + kvfree(adev->uvd.inst[i].saved_bo); + adev->uvd.inst[i].saved_bo = NULL; + } else { + const struct common_firmware_header *hdr; + unsigned offset; + + hdr = (const struct common_firmware_header *)adev->uvd.fw->data; + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + offset = le32_to_cpu(hdr->ucode_array_offset_bytes); + memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset, + le32_to_cpu(hdr->ucode_size_bytes)); + size -= le32_to_cpu(hdr->ucode_size_bytes); + ptr += le32_to_cpu(hdr->ucode_size_bytes); + } + memset_io(ptr, 0, size); + /* to restore uvd fence seq */ + amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring); + } + } + return 0; +} + +void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) +{ + struct amdgpu_ring *ring = &adev->uvd.inst[0].ring; + int i, r; + + for (i = 0; i < adev->uvd.max_handles; ++i) { + uint32_t handle = atomic_read(&adev->uvd.handles[i]); + + if (handle != 0 && adev->uvd.filp[i] == filp) { + struct dma_fence *fence; + + r = amdgpu_uvd_get_destroy_msg(ring, handle, false, + &fence); + if (r) { + DRM_ERROR("Error destroying UVD %d!\n", r); + continue; + } + + dma_fence_wait(fence, false); + dma_fence_put(fence); + + adev->uvd.filp[i] = NULL; + atomic_set(&adev->uvd.handles[i], 0); + } + } +} + +static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo) +{ + int i; + for (i = 0; i < abo->placement.num_placement; ++i) { + abo->placements[i].fpfn = 0 >> PAGE_SHIFT; + abo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT; + } +} + +static u64 amdgpu_uvd_get_addr_from_ctx(struct amdgpu_uvd_cs_ctx *ctx) +{ + uint32_t lo, hi; + uint64_t addr; + + lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0); + hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1); + addr = ((uint64_t)lo) | (((uint64_t)hi) << 32); + + return addr; +} + +/** + * amdgpu_uvd_cs_pass1 - first parsing round + * + * @ctx: UVD parser context + * + * Make sure UVD message and feedback buffers are in VRAM and + * nobody is violating an 256MB boundary. + */ +static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) +{ + struct ttm_operation_ctx tctx = { false, false }; + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *bo; + uint32_t cmd; + uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx); + int r = 0; + + r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping); + if (r) { + DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr); + return r; + } + + if (!ctx->parser->adev->uvd.address_64_bit) { + /* check if it's a message or feedback command */ + cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1; + if (cmd == 0x0 || cmd == 0x3) { + /* yes, force it into VRAM */ + uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; + amdgpu_bo_placement_from_domain(bo, domain); + } + amdgpu_uvd_force_into_uvd_segment(bo); + + r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx); + } + + return r; +} + +/** + * amdgpu_uvd_cs_msg_decode - handle UVD decode message + * + * @msg: pointer to message structure + * @buf_sizes: returned buffer sizes + * + * Peek into the decode message and calculate the necessary buffer sizes. + */ +static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, + unsigned buf_sizes[]) +{ + unsigned stream_type = msg[4]; + unsigned width = msg[6]; + unsigned height = msg[7]; + unsigned dpb_size = msg[9]; + unsigned pitch = msg[28]; + unsigned level = msg[57]; + + unsigned width_in_mb = width / 16; + unsigned height_in_mb = ALIGN(height / 16, 2); + unsigned fs_in_mb = width_in_mb * height_in_mb; + + unsigned image_size, tmp, min_dpb_size, num_dpb_buffer; + unsigned min_ctx_size = ~0; + + image_size = width * height; + image_size += image_size / 2; + image_size = ALIGN(image_size, 1024); + + switch (stream_type) { + case 0: /* H264 */ + switch(level) { + case 30: + num_dpb_buffer = 8100 / fs_in_mb; + break; + case 31: + num_dpb_buffer = 18000 / fs_in_mb; + break; + case 32: + num_dpb_buffer = 20480 / fs_in_mb; + break; + case 41: + num_dpb_buffer = 32768 / fs_in_mb; + break; + case 42: + num_dpb_buffer = 34816 / fs_in_mb; + break; + case 50: + num_dpb_buffer = 110400 / fs_in_mb; + break; + case 51: + num_dpb_buffer = 184320 / fs_in_mb; + break; + default: + num_dpb_buffer = 184320 / fs_in_mb; + break; + } + num_dpb_buffer++; + if (num_dpb_buffer > 17) + num_dpb_buffer = 17; + + /* reference picture buffer */ + min_dpb_size = image_size * num_dpb_buffer; + + /* macroblock context buffer */ + min_dpb_size += width_in_mb * height_in_mb * num_dpb_buffer * 192; + + /* IT surface buffer */ + min_dpb_size += width_in_mb * height_in_mb * 32; + break; + + case 1: /* VC1 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + + /* CONTEXT_BUFFER */ + min_dpb_size += width_in_mb * height_in_mb * 128; + + /* IT surface buffer */ + min_dpb_size += width_in_mb * 64; + + /* DB surface buffer */ + min_dpb_size += width_in_mb * 128; + + /* BP */ + tmp = max(width_in_mb, height_in_mb); + min_dpb_size += ALIGN(tmp * 7 * 16, 64); + break; + + case 3: /* MPEG2 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + break; + + case 4: /* MPEG4 */ + + /* reference picture buffer */ + min_dpb_size = image_size * 3; + + /* CM */ + min_dpb_size += width_in_mb * height_in_mb * 64; + + /* IT surface buffer */ + min_dpb_size += ALIGN(width_in_mb * height_in_mb * 32, 64); + break; + + case 7: /* H264 Perf */ + switch(level) { + case 30: + num_dpb_buffer = 8100 / fs_in_mb; + break; + case 31: + num_dpb_buffer = 18000 / fs_in_mb; + break; + case 32: + num_dpb_buffer = 20480 / fs_in_mb; + break; + case 41: + num_dpb_buffer = 32768 / fs_in_mb; + break; + case 42: + num_dpb_buffer = 34816 / fs_in_mb; + break; + case 50: + num_dpb_buffer = 110400 / fs_in_mb; + break; + case 51: + num_dpb_buffer = 184320 / fs_in_mb; + break; + default: + num_dpb_buffer = 184320 / fs_in_mb; + break; + } + num_dpb_buffer++; + if (num_dpb_buffer > 17) + num_dpb_buffer = 17; + + /* reference picture buffer */ + min_dpb_size = image_size * num_dpb_buffer; + + if (!adev->uvd.use_ctx_buf){ + /* macroblock context buffer */ + min_dpb_size += + width_in_mb * height_in_mb * num_dpb_buffer * 192; + + /* IT surface buffer */ + min_dpb_size += width_in_mb * height_in_mb * 32; + } else { + /* macroblock context buffer */ + min_ctx_size = + width_in_mb * height_in_mb * num_dpb_buffer * 192; + } + break; + + case 8: /* MJPEG */ + min_dpb_size = 0; + break; + + case 16: /* H265 */ + image_size = (ALIGN(width, 16) * ALIGN(height, 16) * 3) / 2; + image_size = ALIGN(image_size, 256); + + num_dpb_buffer = (le32_to_cpu(msg[59]) & 0xff) + 2; + min_dpb_size = image_size * num_dpb_buffer; + min_ctx_size = ((width + 255) / 16) * ((height + 255) / 16) + * 16 * num_dpb_buffer + 52 * 1024; + break; + + default: + DRM_ERROR("UVD codec not handled %d!\n", stream_type); + return -EINVAL; + } + + if (width > pitch) { + DRM_ERROR("Invalid UVD decoding target pitch!\n"); + return -EINVAL; + } + + if (dpb_size < min_dpb_size) { + DRM_ERROR("Invalid dpb_size in UVD message (%d / %d)!\n", + dpb_size, min_dpb_size); + return -EINVAL; + } + + buf_sizes[0x1] = dpb_size; + buf_sizes[0x2] = image_size; + buf_sizes[0x4] = min_ctx_size; + /* store image width to adjust nb memory pstate */ + adev->uvd.decode_image_width = width; + return 0; +} + +/** + * amdgpu_uvd_cs_msg - handle UVD message + * + * @ctx: UVD parser context + * @bo: buffer object containing the message + * @offset: offset into the buffer object + * + * Peek into the UVD message and extract the session id. + * Make sure that we don't open up to many sessions. + */ +static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx, + struct amdgpu_bo *bo, unsigned offset) +{ + struct amdgpu_device *adev = ctx->parser->adev; + int32_t *msg, msg_type, handle; + void *ptr; + long r; + int i; + + if (offset & 0x3F) { + DRM_ERROR("UVD messages must be 64 byte aligned!\n"); + return -EINVAL; + } + + r = amdgpu_bo_kmap(bo, &ptr); + if (r) { + DRM_ERROR("Failed mapping the UVD) message (%ld)!\n", r); + return r; + } + + msg = ptr + offset; + + msg_type = msg[1]; + handle = msg[2]; + + if (handle == 0) { + DRM_ERROR("Invalid UVD handle!\n"); + return -EINVAL; + } + + switch (msg_type) { + case 0: + /* it's a create msg, calc image size (width * height) */ + amdgpu_bo_kunmap(bo); + + /* try to alloc a new handle */ + for (i = 0; i < adev->uvd.max_handles; ++i) { + if (atomic_read(&adev->uvd.handles[i]) == handle) { + DRM_ERROR(")Handle 0x%x already in use!\n", + handle); + return -EINVAL; + } + + if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) { + adev->uvd.filp[i] = ctx->parser->filp; + return 0; + } + } + + DRM_ERROR("No more free UVD handles!\n"); + return -ENOSPC; + + case 1: + /* it's a decode msg, calc buffer sizes */ + r = amdgpu_uvd_cs_msg_decode(adev, msg, ctx->buf_sizes); + amdgpu_bo_kunmap(bo); + if (r) + return r; + + /* validate the handle */ + for (i = 0; i < adev->uvd.max_handles; ++i) { + if (atomic_read(&adev->uvd.handles[i]) == handle) { + if (adev->uvd.filp[i] != ctx->parser->filp) { + DRM_ERROR("UVD handle collision detected!\n"); + return -EINVAL; + } + return 0; + } + } + + DRM_ERROR("Invalid UVD handle 0x%x!\n", handle); + return -ENOENT; + + case 2: + /* it's a destroy msg, free the handle */ + for (i = 0; i < adev->uvd.max_handles; ++i) + atomic_cmpxchg(&adev->uvd.handles[i], handle, 0); + amdgpu_bo_kunmap(bo); + return 0; + + default: + DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type); + return -EINVAL; + } + BUG(); + return -EINVAL; +} + +/** + * amdgpu_uvd_cs_pass2 - second parsing round + * + * @ctx: UVD parser context + * + * Patch buffer addresses, make sure buffer sizes are correct. + */ +static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx) +{ + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *bo; + uint32_t cmd; + uint64_t start, end; + uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx); + int r; + + r = amdgpu_cs_find_mapping(ctx->parser, addr, &bo, &mapping); + if (r) { + DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr); + return r; + } + + start = amdgpu_bo_gpu_offset(bo); + + end = (mapping->last + 1 - mapping->start); + end = end * AMDGPU_GPU_PAGE_SIZE + start; + + addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE; + start += addr; + + amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data0, + lower_32_bits(start)); + amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data1, + upper_32_bits(start)); + + cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1; + if (cmd < 0x4) { + if ((end - start) < ctx->buf_sizes[cmd]) { + DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, + (unsigned)(end - start), + ctx->buf_sizes[cmd]); + return -EINVAL; + } + + } else if (cmd == 0x206) { + if ((end - start) < ctx->buf_sizes[4]) { + DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, + (unsigned)(end - start), + ctx->buf_sizes[4]); + return -EINVAL; + } + } else if ((cmd != 0x100) && (cmd != 0x204)) { + DRM_ERROR("invalid UVD command %X!\n", cmd); + return -EINVAL; + } + + if (!ctx->parser->adev->uvd.address_64_bit) { + if ((start >> 28) != ((end - 1) >> 28)) { + DRM_ERROR("reloc %LX-%LX crossing 256MB boundary!\n", + start, end); + return -EINVAL; + } + + if ((cmd == 0 || cmd == 0x3) && + (start >> 28) != (ctx->parser->adev->uvd.inst->gpu_addr >> 28)) { + DRM_ERROR("msg/fb buffer %LX-%LX out of 256MB segment!\n", + start, end); + return -EINVAL; + } + } + + if (cmd == 0) { + ctx->has_msg_cmd = true; + r = amdgpu_uvd_cs_msg(ctx, bo, addr); + if (r) + return r; + } else if (!ctx->has_msg_cmd) { + DRM_ERROR("Message needed before other commands are send!\n"); + return -EINVAL; + } + + return 0; +} + +/** + * amdgpu_uvd_cs_reg - parse register writes + * + * @ctx: UVD parser context + * @cb: callback function + * + * Parse the register writes, call cb on each complete command. + */ +static int amdgpu_uvd_cs_reg(struct amdgpu_uvd_cs_ctx *ctx, + int (*cb)(struct amdgpu_uvd_cs_ctx *ctx)) +{ + struct amdgpu_ib *ib = &ctx->parser->job->ibs[ctx->ib_idx]; + int i, r; + + ctx->idx++; + for (i = 0; i <= ctx->count; ++i) { + unsigned reg = ctx->reg + i; + + if (ctx->idx >= ib->length_dw) { + DRM_ERROR("Register command after end of CS!\n"); + return -EINVAL; + } + + switch (reg) { + case mmUVD_GPCOM_VCPU_DATA0: + ctx->data0 = ctx->idx; + break; + case mmUVD_GPCOM_VCPU_DATA1: + ctx->data1 = ctx->idx; + break; + case mmUVD_GPCOM_VCPU_CMD: + r = cb(ctx); + if (r) + return r; + break; + case mmUVD_ENGINE_CNTL: + case mmUVD_NO_OP: + break; + default: + DRM_ERROR("Invalid reg 0x%X!\n", reg); + return -EINVAL; + } + ctx->idx++; + } + return 0; +} + +/** + * amdgpu_uvd_cs_packets - parse UVD packets + * + * @ctx: UVD parser context + * @cb: callback function + * + * Parse the command stream packets. + */ +static int amdgpu_uvd_cs_packets(struct amdgpu_uvd_cs_ctx *ctx, + int (*cb)(struct amdgpu_uvd_cs_ctx *ctx)) +{ + struct amdgpu_ib *ib = &ctx->parser->job->ibs[ctx->ib_idx]; + int r; + + for (ctx->idx = 0 ; ctx->idx < ib->length_dw; ) { + uint32_t cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx); + unsigned type = CP_PACKET_GET_TYPE(cmd); + switch (type) { + case PACKET_TYPE0: + ctx->reg = CP_PACKET0_GET_REG(cmd); + ctx->count = CP_PACKET_GET_COUNT(cmd); + r = amdgpu_uvd_cs_reg(ctx, cb); + if (r) + return r; + break; + case PACKET_TYPE2: + ++ctx->idx; + break; + default: + DRM_ERROR("Unknown packet type %d !\n", type); + return -EINVAL; + } + } + return 0; +} + +/** + * amdgpu_uvd_ring_parse_cs - UVD command submission parser + * + * @parser: Command submission parser context + * + * Parse the command stream, patch in addresses as necessary. + */ +int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx) +{ + struct amdgpu_uvd_cs_ctx ctx = {}; + unsigned buf_sizes[] = { + [0x00000000] = 2048, + [0x00000001] = 0xFFFFFFFF, + [0x00000002] = 0xFFFFFFFF, + [0x00000003] = 2048, + [0x00000004] = 0xFFFFFFFF, + }; + struct amdgpu_ib *ib = &parser->job->ibs[ib_idx]; + int r; + + parser->job->vm = NULL; + ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); + + if (ib->length_dw % 16) { + DRM_ERROR("UVD IB length (%d) not 16 dwords aligned!\n", + ib->length_dw); + return -EINVAL; + } + + ctx.parser = parser; + ctx.buf_sizes = buf_sizes; + ctx.ib_idx = ib_idx; + + /* first round only required on chips without UVD 64 bit address support */ + if (!parser->adev->uvd.address_64_bit) { + /* first round, make sure the buffers are actually in the UVD segment */ + r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass1); + if (r) + return r; + } + + /* second round, patch buffer addresses into the command stream */ + r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass2); + if (r) + return r; + + if (!ctx.has_msg_cmd) { + DRM_ERROR("UVD-IBs need a msg command!\n"); + return -EINVAL; + } + + return 0; +} + +static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, + bool direct, struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct dma_fence *f = NULL; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + uint32_t data[4]; + uint64_t addr; + long r; + int i; + unsigned offset_idx = 0; + unsigned offset[3] = { UVD_BASE_SI, 0, 0 }; + + amdgpu_bo_kunmap(bo); + amdgpu_bo_unpin(bo); + + if (!ring->adev->uvd.address_64_bit) { + struct ttm_operation_ctx ctx = { true, false }; + + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); + amdgpu_uvd_force_into_uvd_segment(bo); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (r) + goto err; + } + + r = amdgpu_job_alloc_with_ib(adev, 64, &job); + if (r) + goto err; + + if (adev->asic_type >= CHIP_VEGA10) { + offset_idx = 1 + ring->me; + offset[1] = adev->reg_offset[UVD_HWIP][0][1]; + offset[2] = adev->reg_offset[UVD_HWIP][1][1]; + } + + data[0] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA0, 0); + data[1] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_DATA1, 0); + data[2] = PACKET0(offset[offset_idx] + UVD_GPCOM_VCPU_CMD, 0); + data[3] = PACKET0(offset[offset_idx] + UVD_NO_OP, 0); + + ib = &job->ibs[0]; + addr = amdgpu_bo_gpu_offset(bo); + ib->ptr[0] = data[0]; + ib->ptr[1] = addr; + ib->ptr[2] = data[1]; + ib->ptr[3] = addr >> 32; + ib->ptr[4] = data[2]; + ib->ptr[5] = 0; + for (i = 6; i < 16; i += 2) { + ib->ptr[i] = data[3]; + ib->ptr[i+1] = 0; + } + ib->length_dw = 16; + + if (direct) { + r = reservation_object_wait_timeout_rcu(bo->tbo.resv, + true, false, + msecs_to_jiffies(10)); + if (r == 0) + r = -ETIMEDOUT; + if (r < 0) + goto err_free; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err_free; + } else { + r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv, + AMDGPU_FENCE_OWNER_UNDEFINED, false); + if (r) + goto err_free; + + r = amdgpu_job_submit(job, &adev->uvd.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, &f); + if (r) + goto err_free; + } + + amdgpu_bo_fence(bo, f, false); + amdgpu_bo_unreserve(bo); + amdgpu_bo_unref(&bo); + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + + return 0; + +err_free: + amdgpu_job_free(job); + +err: + amdgpu_bo_unreserve(bo); + amdgpu_bo_unref(&bo); + return r; +} + +/* multiple fence commands without any stream commands in between can + crash the vcpu so just try to emmit a dummy create/destroy msg to + avoid this */ +int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_bo *bo = NULL; + uint32_t *msg; + int r, i; + + r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL, (void **)&msg); + if (r) + return r; + + /* stitch together an UVD create msg */ + msg[0] = cpu_to_le32(0x00000de4); + msg[1] = cpu_to_le32(0x00000000); + msg[2] = cpu_to_le32(handle); + msg[3] = cpu_to_le32(0x00000000); + msg[4] = cpu_to_le32(0x00000000); + msg[5] = cpu_to_le32(0x00000000); + msg[6] = cpu_to_le32(0x00000000); + msg[7] = cpu_to_le32(0x00000780); + msg[8] = cpu_to_le32(0x00000440); + msg[9] = cpu_to_le32(0x00000000); + msg[10] = cpu_to_le32(0x01b37000); + for (i = 11; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + return amdgpu_uvd_send_msg(ring, bo, true, fence); +} + +int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + bool direct, struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_bo *bo = NULL; + uint32_t *msg; + int r, i; + + r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL, (void **)&msg); + if (r) + return r; + + /* stitch together an UVD destroy msg */ + msg[0] = cpu_to_le32(0x00000de4); + msg[1] = cpu_to_le32(0x00000002); + msg[2] = cpu_to_le32(handle); + msg[3] = cpu_to_le32(0x00000000); + for (i = 4; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + return amdgpu_uvd_send_msg(ring, bo, direct, fence); +} + +static void amdgpu_uvd_idle_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, uvd.idle_work.work); + unsigned fences = 0, i, j; + + for (i = 0; i < adev->uvd.num_uvd_inst; ++i) { + if (adev->uvd.harvest_config & (1 << i)) + continue; + fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring); + for (j = 0; j < adev->uvd.num_enc_rings; ++j) { + fences += amdgpu_fence_count_emitted(&adev->uvd.inst[i].ring_enc[j]); + } + } + + if (fences == 0) { + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, false); + } else { + amdgpu_asic_set_uvd_clocks(adev, 0, 0); + /* shutdown the UVD block */ + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); + } + } else { + schedule_delayed_work(&adev->uvd.idle_work, UVD_IDLE_TIMEOUT); + } +} + +void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + bool set_clocks; + + if (amdgpu_sriov_vf(adev)) + return; + + set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work); + if (set_clocks) { + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_uvd(adev, true); + } else { + amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_UNGATE); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_UNGATE); + } + } +} + +void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring) +{ + if (!amdgpu_sriov_vf(ring->adev)) + schedule_delayed_work(&ring->adev->uvd.idle_work, UVD_IDLE_TIMEOUT); +} + +/** + * amdgpu_uvd_ring_test_ib - test ib execution + * + * @ring: amdgpu_ring pointer + * + * Test if we can successfully execute an IB + */ +int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct dma_fence *fence; + long r; + + r = amdgpu_uvd_get_create_msg(ring, 1, NULL); + if (r) + goto error; + + r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence); + if (r) + goto error; + + r = dma_fence_wait_timeout(fence, false, timeout); + if (r == 0) + r = -ETIMEDOUT; + else if (r > 0) + r = 0; + + dma_fence_put(fence); + +error: + return r; +} + +/** + * amdgpu_uvd_used_handles - returns used UVD handles + * + * @adev: amdgpu_device pointer + * + * Returns the number of UVD handles in use + */ +uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev) +{ + unsigned i; + uint32_t used_handles = 0; + + for (i = 0; i < adev->uvd.max_handles; ++i) { + /* + * Handles can be freed in any order, and not + * necessarily linear. So we need to count + * all non-zero handles. + */ + if (atomic_read(&adev->uvd.handles[i])) + used_handles++; + } + + return used_handles; +}
diff --git a/amdgpu/amdgpu_uvd.h b/amdgpu/amdgpu_uvd.h new file mode 100644 index 0000000..5eb6328 --- /dev/null +++ b/amdgpu/amdgpu_uvd.h
@@ -0,0 +1,89 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_UVD_H__ +#define __AMDGPU_UVD_H__ + +#define AMDGPU_DEFAULT_UVD_HANDLES 10 +#define AMDGPU_MAX_UVD_HANDLES 40 +#define AMDGPU_UVD_STACK_SIZE (200*1024) +#define AMDGPU_UVD_HEAP_SIZE (256*1024) +#define AMDGPU_UVD_SESSION_SIZE (50*1024) +#define AMDGPU_UVD_FIRMWARE_OFFSET 256 + +#define AMDGPU_MAX_UVD_INSTANCES 2 + +#define AMDGPU_UVD_FIRMWARE_SIZE(adev) \ + (AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(((const struct common_firmware_header *)(adev)->uvd.fw->data)->ucode_size_bytes) + \ + 8) - AMDGPU_UVD_FIRMWARE_OFFSET) + +struct amdgpu_uvd_inst { + struct amdgpu_bo *vcpu_bo; + void *cpu_addr; + uint64_t gpu_addr; + void *saved_bo; + struct amdgpu_ring ring; + struct amdgpu_ring ring_enc[AMDGPU_MAX_UVD_ENC_RINGS]; + struct amdgpu_irq_src irq; + uint32_t srbm_soft_reset; +}; + +#define AMDGPU_UVD_HARVEST_UVD0 (1 << 0) +#define AMDGPU_UVD_HARVEST_UVD1 (1 << 1) + +struct amdgpu_uvd { + const struct firmware *fw; /* UVD firmware */ + unsigned fw_version; + unsigned max_handles; + unsigned num_enc_rings; + uint8_t num_uvd_inst; + bool address_64_bit; + bool use_ctx_buf; + struct amdgpu_uvd_inst inst[AMDGPU_MAX_UVD_INSTANCES]; + struct drm_file *filp[AMDGPU_MAX_UVD_HANDLES]; + atomic_t handles[AMDGPU_MAX_UVD_HANDLES]; + struct drm_sched_entity entity; + struct delayed_work idle_work; + unsigned harvest_config; + /* store image width to adjust nb memory state */ + unsigned decode_image_width; +}; + +int amdgpu_uvd_sw_init(struct amdgpu_device *adev); +int amdgpu_uvd_sw_fini(struct amdgpu_device *adev); +int amdgpu_uvd_entity_init(struct amdgpu_device *adev); +int amdgpu_uvd_suspend(struct amdgpu_device *adev); +int amdgpu_uvd_resume(struct amdgpu_device *adev); +int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence); +int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + bool direct, struct dma_fence **fence); +void amdgpu_uvd_free_handles(struct amdgpu_device *adev, + struct drm_file *filp); +int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx); +void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring); +void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring); +int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout); +uint32_t amdgpu_uvd_used_handles(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_vce.c b/amdgpu/amdgpu_vce.c new file mode 100644 index 0000000..b70b3c4 --- /dev/null +++ b/amdgpu/amdgpu_vce.c
@@ -0,0 +1,1136 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * Authors: Christian König <christian.koenig@amd.com> + */ + +#include <linux/firmware.h> +#include <linux/module.h> + +#include <drm/drm.h> + +#include "amdgpu.h" +#include "amdgpu_pm.h" +#include "amdgpu_vce.h" +#include "cikd.h" + +/* 1 second timeout */ +#define VCE_IDLE_TIMEOUT msecs_to_jiffies(1000) + +/* Firmware Names */ +#ifdef CONFIG_DRM_AMDGPU_CIK +#define FIRMWARE_BONAIRE "amdgpu/bonaire_vce.bin" +#define FIRMWARE_KABINI "amdgpu/kabini_vce.bin" +#define FIRMWARE_KAVERI "amdgpu/kaveri_vce.bin" +#define FIRMWARE_HAWAII "amdgpu/hawaii_vce.bin" +#define FIRMWARE_MULLINS "amdgpu/mullins_vce.bin" +#endif +#define FIRMWARE_TONGA "amdgpu/tonga_vce.bin" +#define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin" +#define FIRMWARE_FIJI "amdgpu/fiji_vce.bin" +#define FIRMWARE_STONEY "amdgpu/stoney_vce.bin" +#define FIRMWARE_POLARIS10 "amdgpu/polaris10_vce.bin" +#define FIRMWARE_POLARIS11 "amdgpu/polaris11_vce.bin" +#define FIRMWARE_POLARIS12 "amdgpu/polaris12_vce.bin" +#define FIRMWARE_VEGAM "amdgpu/vegam_vce.bin" + +#define FIRMWARE_VEGA10 "amdgpu/vega10_vce.bin" +#define FIRMWARE_VEGA12 "amdgpu/vega12_vce.bin" +#define FIRMWARE_VEGA20 "amdgpu/vega20_vce.bin" + +#ifdef CONFIG_DRM_AMDGPU_CIK +MODULE_FIRMWARE(FIRMWARE_BONAIRE); +MODULE_FIRMWARE(FIRMWARE_KABINI); +MODULE_FIRMWARE(FIRMWARE_KAVERI); +MODULE_FIRMWARE(FIRMWARE_HAWAII); +MODULE_FIRMWARE(FIRMWARE_MULLINS); +#endif +MODULE_FIRMWARE(FIRMWARE_TONGA); +MODULE_FIRMWARE(FIRMWARE_CARRIZO); +MODULE_FIRMWARE(FIRMWARE_FIJI); +MODULE_FIRMWARE(FIRMWARE_STONEY); +MODULE_FIRMWARE(FIRMWARE_POLARIS10); +MODULE_FIRMWARE(FIRMWARE_POLARIS11); +MODULE_FIRMWARE(FIRMWARE_POLARIS12); +MODULE_FIRMWARE(FIRMWARE_VEGAM); + +MODULE_FIRMWARE(FIRMWARE_VEGA10); +MODULE_FIRMWARE(FIRMWARE_VEGA12); +MODULE_FIRMWARE(FIRMWARE_VEGA20); + +static void amdgpu_vce_idle_work_handler(struct work_struct *work); + +/** + * amdgpu_vce_init - allocate memory, load vce firmware + * + * @adev: amdgpu_device pointer + * + * First step to get VCE online, allocate memory and load the firmware + */ +int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) +{ + const char *fw_name; + const struct common_firmware_header *hdr; + unsigned ucode_version, version_major, version_minor, binary_id; + int i, r; + + switch (adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_BONAIRE: + fw_name = FIRMWARE_BONAIRE; + break; + case CHIP_KAVERI: + fw_name = FIRMWARE_KAVERI; + break; + case CHIP_KABINI: + fw_name = FIRMWARE_KABINI; + break; + case CHIP_HAWAII: + fw_name = FIRMWARE_HAWAII; + break; + case CHIP_MULLINS: + fw_name = FIRMWARE_MULLINS; + break; +#endif + case CHIP_TONGA: + fw_name = FIRMWARE_TONGA; + break; + case CHIP_CARRIZO: + fw_name = FIRMWARE_CARRIZO; + break; + case CHIP_FIJI: + fw_name = FIRMWARE_FIJI; + break; + case CHIP_STONEY: + fw_name = FIRMWARE_STONEY; + break; + case CHIP_POLARIS10: + fw_name = FIRMWARE_POLARIS10; + break; + case CHIP_POLARIS11: + fw_name = FIRMWARE_POLARIS11; + break; + case CHIP_POLARIS12: + fw_name = FIRMWARE_POLARIS12; + break; + case CHIP_VEGAM: + fw_name = FIRMWARE_VEGAM; + break; + case CHIP_VEGA10: + fw_name = FIRMWARE_VEGA10; + break; + case CHIP_VEGA12: + fw_name = FIRMWARE_VEGA12; + break; + case CHIP_VEGA20: + fw_name = FIRMWARE_VEGA20; + break; + + default: + return -EINVAL; + } + + r = request_firmware(&adev->vce.fw, fw_name, adev->dev); + if (r) { + dev_err(adev->dev, "amdgpu_vce: Can't load firmware \"%s\"\n", + fw_name); + return r; + } + + r = amdgpu_ucode_validate(adev->vce.fw); + if (r) { + dev_err(adev->dev, "amdgpu_vce: Can't validate firmware \"%s\"\n", + fw_name); + release_firmware(adev->vce.fw); + adev->vce.fw = NULL; + return r; + } + + hdr = (const struct common_firmware_header *)adev->vce.fw->data; + + ucode_version = le32_to_cpu(hdr->ucode_version); + version_major = (ucode_version >> 20) & 0xfff; + version_minor = (ucode_version >> 8) & 0xfff; + binary_id = ucode_version & 0xff; + DRM_INFO("Found VCE firmware Version: %hhd.%hhd Binary ID: %hhd\n", + version_major, version_minor, binary_id); + adev->vce.fw_version = ((version_major << 24) | (version_minor << 16) | + (binary_id << 8)); + + r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->vce.vcpu_bo, + &adev->vce.gpu_addr, &adev->vce.cpu_addr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r); + return r; + } + + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { + atomic_set(&adev->vce.handles[i], 0); + adev->vce.filp[i] = NULL; + } + + INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler); + mutex_init(&adev->vce.idle_mutex); + + return 0; +} + +/** + * amdgpu_vce_fini - free memory + * + * @adev: amdgpu_device pointer + * + * Last step on VCE teardown, free firmware memory + */ +int amdgpu_vce_sw_fini(struct amdgpu_device *adev) +{ + unsigned i; + + if (adev->vce.vcpu_bo == NULL) + return 0; + + drm_sched_entity_destroy(&adev->vce.entity); + + amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr, + (void **)&adev->vce.cpu_addr); + + for (i = 0; i < adev->vce.num_rings; i++) + amdgpu_ring_fini(&adev->vce.ring[i]); + + release_firmware(adev->vce.fw); + mutex_destroy(&adev->vce.idle_mutex); + + return 0; +} + +/** + * amdgpu_vce_entity_init - init entity + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_vce_entity_init(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring; + struct drm_sched_rq *rq; + int r; + + ring = &adev->vce.ring[0]; + rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; + r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL); + if (r != 0) { + DRM_ERROR("Failed setting up VCE run queue.\n"); + return r; + } + + return 0; +} + +/** + * amdgpu_vce_suspend - unpin VCE fw memory + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_vce_suspend(struct amdgpu_device *adev) +{ + int i; + + cancel_delayed_work_sync(&adev->vce.idle_work); + + if (adev->vce.vcpu_bo == NULL) + return 0; + + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) + if (atomic_read(&adev->vce.handles[i])) + break; + + if (i == AMDGPU_MAX_VCE_HANDLES) + return 0; + + /* TODO: suspending running encoding sessions isn't supported */ + return -EINVAL; +} + +/** + * amdgpu_vce_resume - pin VCE fw memory + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_vce_resume(struct amdgpu_device *adev) +{ + void *cpu_addr; + const struct common_firmware_header *hdr; + unsigned offset; + int r; + + if (adev->vce.vcpu_bo == NULL) + return -EINVAL; + + r = amdgpu_bo_reserve(adev->vce.vcpu_bo, false); + if (r) { + dev_err(adev->dev, "(%d) failed to reserve VCE bo\n", r); + return r; + } + + r = amdgpu_bo_kmap(adev->vce.vcpu_bo, &cpu_addr); + if (r) { + amdgpu_bo_unreserve(adev->vce.vcpu_bo); + dev_err(adev->dev, "(%d) VCE map failed\n", r); + return r; + } + + hdr = (const struct common_firmware_header *)adev->vce.fw->data; + offset = le32_to_cpu(hdr->ucode_array_offset_bytes); + memcpy_toio(cpu_addr, adev->vce.fw->data + offset, + adev->vce.fw->size - offset); + + amdgpu_bo_kunmap(adev->vce.vcpu_bo); + + amdgpu_bo_unreserve(adev->vce.vcpu_bo); + + return 0; +} + +/** + * amdgpu_vce_idle_work_handler - power off VCE + * + * @work: pointer to work structure + * + * power of VCE when it's not used any more + */ +static void amdgpu_vce_idle_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, vce.idle_work.work); + unsigned i, count = 0; + + for (i = 0; i < adev->vce.num_rings; i++) + count += amdgpu_fence_count_emitted(&adev->vce.ring[i]); + + if (count == 0) { + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_vce(adev, false); + } else { + amdgpu_asic_set_vce_clocks(adev, 0, 0); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_GATE); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_GATE); + } + } else { + schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT); + } +} + +/** + * amdgpu_vce_ring_begin_use - power up VCE + * + * @ring: amdgpu ring + * + * Make sure VCE is powerd up when we want to use it + */ +void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + bool set_clocks; + + if (amdgpu_sriov_vf(adev)) + return; + + mutex_lock(&adev->vce.idle_mutex); + set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work); + if (set_clocks) { + if (adev->pm.dpm_enabled) { + amdgpu_dpm_enable_vce(adev, true); + } else { + amdgpu_asic_set_vce_clocks(adev, 53300, 40000); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_UNGATE); + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_UNGATE); + + } + } + mutex_unlock(&adev->vce.idle_mutex); +} + +/** + * amdgpu_vce_ring_end_use - power VCE down + * + * @ring: amdgpu ring + * + * Schedule work to power VCE down again + */ +void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring) +{ + if (!amdgpu_sriov_vf(ring->adev)) + schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT); +} + +/** + * amdgpu_vce_free_handles - free still open VCE handles + * + * @adev: amdgpu_device pointer + * @filp: drm file pointer + * + * Close all VCE handles still open by this file pointer + */ +void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp) +{ + struct amdgpu_ring *ring = &adev->vce.ring[0]; + int i, r; + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { + uint32_t handle = atomic_read(&adev->vce.handles[i]); + + if (!handle || adev->vce.filp[i] != filp) + continue; + + r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL); + if (r) + DRM_ERROR("Error destroying VCE handle (%d)!\n", r); + + adev->vce.filp[i] = NULL; + atomic_set(&adev->vce.handles[i], 0); + } +} + +/** + * amdgpu_vce_get_create_msg - generate a VCE create msg + * + * @adev: amdgpu_device pointer + * @ring: ring we should submit the msg to + * @handle: VCE session handle to use + * @fence: optional fence to return + * + * Open up a stream for HW test + */ +int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + const unsigned ib_size_dw = 1024; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct dma_fence *f = NULL; + uint64_t dummy; + int i, r; + + r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); + if (r) + return r; + + ib = &job->ibs[0]; + + dummy = ib->gpu_addr + 1024; + + /* stitch together an VCE create msg */ + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x0000000c; /* len */ + ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */ + ib->ptr[ib->length_dw++] = handle; + + if ((ring->adev->vce.fw_version >> 24) >= 52) + ib->ptr[ib->length_dw++] = 0x00000040; /* len */ + else + ib->ptr[ib->length_dw++] = 0x00000030; /* len */ + ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */ + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000042; + ib->ptr[ib->length_dw++] = 0x0000000a; + ib->ptr[ib->length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = 0x00000080; + ib->ptr[ib->length_dw++] = 0x00000060; + ib->ptr[ib->length_dw++] = 0x00000100; + ib->ptr[ib->length_dw++] = 0x00000100; + ib->ptr[ib->length_dw++] = 0x0000000c; + ib->ptr[ib->length_dw++] = 0x00000000; + if ((ring->adev->vce.fw_version >> 24) >= 52) { + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + } + + ib->ptr[ib->length_dw++] = 0x00000014; /* len */ + ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */ + ib->ptr[ib->length_dw++] = upper_32_bits(dummy); + ib->ptr[ib->length_dw++] = dummy; + ib->ptr[ib->length_dw++] = 0x00000001; + + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err; + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + return 0; + +err: + amdgpu_job_free(job); + return r; +} + +/** + * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg + * + * @adev: amdgpu_device pointer + * @ring: ring we should submit the msg to + * @handle: VCE session handle to use + * @fence: optional fence to return + * + * Close up a stream for HW test or if userspace failed to do so + */ +int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + bool direct, struct dma_fence **fence) +{ + const unsigned ib_size_dw = 1024; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct dma_fence *f = NULL; + int i, r; + + r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); + if (r) + return r; + + ib = &job->ibs[0]; + + /* stitch together an VCE destroy msg */ + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x0000000c; /* len */ + ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */ + ib->ptr[ib->length_dw++] = handle; + + ib->ptr[ib->length_dw++] = 0x00000020; /* len */ + ib->ptr[ib->length_dw++] = 0x00000002; /* task info */ + ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */ + ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */ + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */ + ib->ptr[ib->length_dw++] = 0x00000000; + + ib->ptr[ib->length_dw++] = 0x00000008; /* len */ + ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */ + + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + + if (direct) + r = amdgpu_job_submit_direct(job, ring, &f); + else + r = amdgpu_job_submit(job, &ring->adev->vce.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, &f); + if (r) + goto err; + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + return 0; + +err: + amdgpu_job_free(job); + return r; +} + +/** + * amdgpu_vce_cs_validate_bo - make sure not to cross 4GB boundary + * + * @p: parser context + * @lo: address of lower dword + * @hi: address of higher dword + * @size: minimum size + * @index: bs/fb index + * + * Make sure that no BO cross a 4GB boundary. + */ +static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p, uint32_t ib_idx, + int lo, int hi, unsigned size, int32_t index) +{ + int64_t offset = ((uint64_t)size) * ((int64_t)index); + struct ttm_operation_ctx ctx = { false, false }; + struct amdgpu_bo_va_mapping *mapping; + unsigned i, fpfn, lpfn; + struct amdgpu_bo *bo; + uint64_t addr; + int r; + + addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) | + ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32; + if (index >= 0) { + addr += offset; + fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT; + lpfn = 0x100000000ULL >> PAGE_SHIFT; + } else { + fpfn = 0; + lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT; + } + + r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping); + if (r) { + DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n", + addr, lo, hi, size, index); + return r; + } + + for (i = 0; i < bo->placement.num_placement; ++i) { + bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn); + bo->placements[i].lpfn = bo->placements[i].lpfn ? + min(bo->placements[i].lpfn, lpfn) : lpfn; + } + return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); +} + + +/** + * amdgpu_vce_cs_reloc - command submission relocation + * + * @p: parser context + * @lo: address of lower dword + * @hi: address of higher dword + * @size: minimum size + * + * Patch relocation inside command stream with real buffer address + */ +static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx, + int lo, int hi, unsigned size, uint32_t index) +{ + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *bo; + uint64_t addr; + int r; + + if (index == 0xffffffff) + index = 0; + + addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) | + ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32; + addr += ((uint64_t)size) * ((uint64_t)index); + + r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping); + if (r) { + DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n", + addr, lo, hi, size, index); + return r; + } + + if ((addr + (uint64_t)size) > + (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) { + DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n", + addr, lo, hi); + return -EINVAL; + } + + addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE; + addr += amdgpu_bo_gpu_offset(bo); + addr -= ((uint64_t)size) * ((uint64_t)index); + + amdgpu_set_ib_value(p, ib_idx, lo, lower_32_bits(addr)); + amdgpu_set_ib_value(p, ib_idx, hi, upper_32_bits(addr)); + + return 0; +} + +/** + * amdgpu_vce_validate_handle - validate stream handle + * + * @p: parser context + * @handle: handle to validate + * @allocated: allocated a new handle? + * + * Validates the handle and return the found session index or -EINVAL + * we we don't have another free session index. + */ +static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p, + uint32_t handle, uint32_t *allocated) +{ + unsigned i; + + /* validate the handle */ + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { + if (atomic_read(&p->adev->vce.handles[i]) == handle) { + if (p->adev->vce.filp[i] != p->filp) { + DRM_ERROR("VCE handle collision detected!\n"); + return -EINVAL; + } + return i; + } + } + + /* handle not found try to alloc a new one */ + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { + if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) { + p->adev->vce.filp[i] = p->filp; + p->adev->vce.img_size[i] = 0; + *allocated |= 1 << i; + return i; + } + } + + DRM_ERROR("No more free VCE handles!\n"); + return -EINVAL; +} + +/** + * amdgpu_vce_cs_parse - parse and validate the command stream + * + * @p: parser context + * + */ +int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) +{ + struct amdgpu_ib *ib = &p->job->ibs[ib_idx]; + unsigned fb_idx = 0, bs_idx = 0; + int session_idx = -1; + uint32_t destroyed = 0; + uint32_t created = 0; + uint32_t allocated = 0; + uint32_t tmp, handle = 0; + uint32_t *size = &tmp; + unsigned idx; + int i, r = 0; + + p->job->vm = NULL; + ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); + + for (idx = 0; idx < ib->length_dw;) { + uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx); + uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1); + + if ((len < 8) || (len & 3)) { + DRM_ERROR("invalid VCE command length (%d)!\n", len); + r = -EINVAL; + goto out; + } + + switch (cmd) { + case 0x00000002: /* task info */ + fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6); + bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7); + break; + + case 0x03000001: /* encode */ + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 10, + idx + 9, 0, 0); + if (r) + goto out; + + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 12, + idx + 11, 0, 0); + if (r) + goto out; + break; + + case 0x05000001: /* context buffer */ + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, + idx + 2, 0, 0); + if (r) + goto out; + break; + + case 0x05000004: /* video bitstream buffer */ + tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4); + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2, + tmp, bs_idx); + if (r) + goto out; + break; + + case 0x05000005: /* feedback buffer */ + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2, + 4096, fb_idx); + if (r) + goto out; + break; + + case 0x0500000d: /* MV buffer */ + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, + idx + 2, 0, 0); + if (r) + goto out; + + r = amdgpu_vce_validate_bo(p, ib_idx, idx + 8, + idx + 7, 0, 0); + if (r) + goto out; + break; + } + + idx += len / 4; + } + + for (idx = 0; idx < ib->length_dw;) { + uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx); + uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1); + + switch (cmd) { + case 0x00000001: /* session */ + handle = amdgpu_get_ib_value(p, ib_idx, idx + 2); + session_idx = amdgpu_vce_validate_handle(p, handle, + &allocated); + if (session_idx < 0) { + r = session_idx; + goto out; + } + size = &p->adev->vce.img_size[session_idx]; + break; + + case 0x00000002: /* task info */ + fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6); + bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7); + break; + + case 0x01000001: /* create */ + created |= 1 << session_idx; + if (destroyed & (1 << session_idx)) { + destroyed &= ~(1 << session_idx); + allocated |= 1 << session_idx; + + } else if (!(allocated & (1 << session_idx))) { + DRM_ERROR("Handle already in use!\n"); + r = -EINVAL; + goto out; + } + + *size = amdgpu_get_ib_value(p, ib_idx, idx + 8) * + amdgpu_get_ib_value(p, ib_idx, idx + 10) * + 8 * 3 / 2; + break; + + case 0x04000001: /* config extension */ + case 0x04000002: /* pic control */ + case 0x04000005: /* rate control */ + case 0x04000007: /* motion estimation */ + case 0x04000008: /* rdo */ + case 0x04000009: /* vui */ + case 0x05000002: /* auxiliary buffer */ + case 0x05000009: /* clock table */ + break; + + case 0x0500000c: /* hw config */ + switch (p->adev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_KAVERI: + case CHIP_MULLINS: +#endif + case CHIP_CARRIZO: + break; + default: + r = -EINVAL; + goto out; + } + break; + + case 0x03000001: /* encode */ + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9, + *size, 0); + if (r) + goto out; + + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 12, idx + 11, + *size / 3, 0); + if (r) + goto out; + break; + + case 0x02000001: /* destroy */ + destroyed |= 1 << session_idx; + break; + + case 0x05000001: /* context buffer */ + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, + *size * 2, 0); + if (r) + goto out; + break; + + case 0x05000004: /* video bitstream buffer */ + tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4); + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, + tmp, bs_idx); + if (r) + goto out; + break; + + case 0x05000005: /* feedback buffer */ + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, + 4096, fb_idx); + if (r) + goto out; + break; + + case 0x0500000d: /* MV buffer */ + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, + idx + 2, *size, 0); + if (r) + goto out; + + r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 8, + idx + 7, *size / 12, 0); + if (r) + goto out; + break; + + default: + DRM_ERROR("invalid VCE command (0x%x)!\n", cmd); + r = -EINVAL; + goto out; + } + + if (session_idx == -1) { + DRM_ERROR("no session command at start of IB\n"); + r = -EINVAL; + goto out; + } + + idx += len / 4; + } + + if (allocated & ~created) { + DRM_ERROR("New session without create command!\n"); + r = -ENOENT; + } + +out: + if (!r) { + /* No error, free all destroyed handle slots */ + tmp = destroyed; + } else { + /* Error during parsing, free all allocated handle slots */ + tmp = allocated; + } + + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) + if (tmp & (1 << i)) + atomic_set(&p->adev->vce.handles[i], 0); + + return r; +} + +/** + * amdgpu_vce_cs_parse_vm - parse the command stream in VM mode + * + * @p: parser context + * + */ +int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx) +{ + struct amdgpu_ib *ib = &p->job->ibs[ib_idx]; + int session_idx = -1; + uint32_t destroyed = 0; + uint32_t created = 0; + uint32_t allocated = 0; + uint32_t tmp, handle = 0; + int i, r = 0, idx = 0; + + while (idx < ib->length_dw) { + uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx); + uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1); + + if ((len < 8) || (len & 3)) { + DRM_ERROR("invalid VCE command length (%d)!\n", len); + r = -EINVAL; + goto out; + } + + switch (cmd) { + case 0x00000001: /* session */ + handle = amdgpu_get_ib_value(p, ib_idx, idx + 2); + session_idx = amdgpu_vce_validate_handle(p, handle, + &allocated); + if (session_idx < 0) { + r = session_idx; + goto out; + } + break; + + case 0x01000001: /* create */ + created |= 1 << session_idx; + if (destroyed & (1 << session_idx)) { + destroyed &= ~(1 << session_idx); + allocated |= 1 << session_idx; + + } else if (!(allocated & (1 << session_idx))) { + DRM_ERROR("Handle already in use!\n"); + r = -EINVAL; + goto out; + } + + break; + + case 0x02000001: /* destroy */ + destroyed |= 1 << session_idx; + break; + + default: + break; + } + + if (session_idx == -1) { + DRM_ERROR("no session command at start of IB\n"); + r = -EINVAL; + goto out; + } + + idx += len / 4; + } + + if (allocated & ~created) { + DRM_ERROR("New session without create command!\n"); + r = -ENOENT; + } + +out: + if (!r) { + /* No error, free all destroyed handle slots */ + tmp = destroyed; + amdgpu_ib_free(p->adev, ib, NULL); + } else { + /* Error during parsing, free all allocated handle slots */ + tmp = allocated; + } + + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) + if (tmp & (1 << i)) + atomic_set(&p->adev->vce.handles[i], 0); + + return r; +} + +/** + * amdgpu_vce_ring_emit_ib - execute indirect buffer + * + * @ring: engine to use + * @ib: the IB to execute + * + */ +void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, + struct amdgpu_job *job, + struct amdgpu_ib *ib, + uint32_t flags) +{ + amdgpu_ring_write(ring, VCE_CMD_IB); + amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); + amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); + amdgpu_ring_write(ring, ib->length_dw); +} + +/** + * amdgpu_vce_ring_emit_fence - add a fence command to the ring + * + * @ring: engine to use + * @fence: the fence + * + */ +void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, + unsigned flags) +{ + WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT); + + amdgpu_ring_write(ring, VCE_CMD_FENCE); + amdgpu_ring_write(ring, addr); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, seq); + amdgpu_ring_write(ring, VCE_CMD_TRAP); + amdgpu_ring_write(ring, VCE_CMD_END); +} + +/** + * amdgpu_vce_ring_test_ring - test if VCE ring is working + * + * @ring: the engine to test on + * + */ +int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t rptr; + unsigned i; + int r, timeout = adev->usec_timeout; + + /* skip ring test for sriov*/ + if (amdgpu_sriov_vf(adev)) + return 0; + + r = amdgpu_ring_alloc(ring, 16); + if (r) + return r; + + rptr = amdgpu_ring_get_rptr(ring); + + amdgpu_ring_write(ring, VCE_CMD_END); + amdgpu_ring_commit(ring); + + for (i = 0; i < timeout; i++) { + if (amdgpu_ring_get_rptr(ring) != rptr) + break; + udelay(1); + } + + if (i >= timeout) + r = -ETIMEDOUT; + + return r; +} + +/** + * amdgpu_vce_ring_test_ib - test if VCE IBs are working + * + * @ring: the engine to test on + * + */ +int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct dma_fence *fence = NULL; + long r; + + /* skip vce ring1/2 ib test for now, since it's not reliable */ + if (ring != &ring->adev->vce.ring[0]) + return 0; + + r = amdgpu_vce_get_create_msg(ring, 1, NULL); + if (r) + goto error; + + r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence); + if (r) + goto error; + + r = dma_fence_wait_timeout(fence, false, timeout); + if (r == 0) + r = -ETIMEDOUT; + else if (r > 0) + r = 0; + +error: + dma_fence_put(fence); + return r; +}
diff --git a/amdgpu/amdgpu_vce.h b/amdgpu/amdgpu_vce.h new file mode 100644 index 0000000..30ea54d --- /dev/null +++ b/amdgpu/amdgpu_vce.h
@@ -0,0 +1,79 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_VCE_H__ +#define __AMDGPU_VCE_H__ + +#define AMDGPU_MAX_VCE_HANDLES 16 +#define AMDGPU_VCE_FIRMWARE_OFFSET 256 + +#define AMDGPU_VCE_HARVEST_VCE0 (1 << 0) +#define AMDGPU_VCE_HARVEST_VCE1 (1 << 1) + +#define AMDGPU_VCE_FW_53_45 ((53 << 24) | (45 << 16)) + +struct amdgpu_vce { + struct amdgpu_bo *vcpu_bo; + uint64_t gpu_addr; + void *cpu_addr; + void *saved_bo; + unsigned fw_version; + unsigned fb_version; + atomic_t handles[AMDGPU_MAX_VCE_HANDLES]; + struct drm_file *filp[AMDGPU_MAX_VCE_HANDLES]; + uint32_t img_size[AMDGPU_MAX_VCE_HANDLES]; + struct delayed_work idle_work; + struct mutex idle_mutex; + const struct firmware *fw; /* VCE firmware */ + struct amdgpu_ring ring[AMDGPU_MAX_VCE_RINGS]; + struct amdgpu_irq_src irq; + unsigned harvest_config; + struct drm_sched_entity entity; + uint32_t srbm_soft_reset; + unsigned num_rings; +}; + +int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size); +int amdgpu_vce_sw_fini(struct amdgpu_device *adev); +int amdgpu_vce_entity_init(struct amdgpu_device *adev); +int amdgpu_vce_suspend(struct amdgpu_device *adev); +int amdgpu_vce_resume(struct amdgpu_device *adev); +int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence); +int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + bool direct, struct dma_fence **fence); +void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp); +int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx); +int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx); +void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job, + struct amdgpu_ib *ib, uint32_t flags); +void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, + unsigned flags); +int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring); +int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout); +void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring); +void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring); +unsigned amdgpu_vce_ring_get_emit_ib_size(struct amdgpu_ring *ring); +unsigned amdgpu_vce_ring_get_dma_frame_size(struct amdgpu_ring *ring); + +#endif
diff --git a/amdgpu/amdgpu_vcn.c b/amdgpu/amdgpu_vcn.c new file mode 100644 index 0000000..2e12eeb --- /dev/null +++ b/amdgpu/amdgpu_vcn.c
@@ -0,0 +1,750 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ + +#include <linux/firmware.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include <drm/drm.h> + +#include "amdgpu.h" +#include "amdgpu_pm.h" +#include "amdgpu_vcn.h" +#include "soc15d.h" +#include "soc15_common.h" + +#include "vcn/vcn_1_0_offset.h" +#include "vcn/vcn_1_0_sh_mask.h" + +/* 1 second timeout */ +#define VCN_IDLE_TIMEOUT msecs_to_jiffies(1000) + +/* Firmware Names */ +#define FIRMWARE_RAVEN "amdgpu/raven_vcn.bin" +#define FIRMWARE_PICASSO "amdgpu/picasso_vcn.bin" +#define FIRMWARE_RAVEN2 "amdgpu/raven2_vcn.bin" +#define FIRMWARE_NAVI10 "amdgpu/navi10_vcn.bin" + +MODULE_FIRMWARE(FIRMWARE_RAVEN); +MODULE_FIRMWARE(FIRMWARE_PICASSO); +MODULE_FIRMWARE(FIRMWARE_RAVEN2); +MODULE_FIRMWARE(FIRMWARE_NAVI10); + +static void amdgpu_vcn_idle_work_handler(struct work_struct *work); + +int amdgpu_vcn_sw_init(struct amdgpu_device *adev) +{ + unsigned long bo_size; + const char *fw_name; + const struct common_firmware_header *hdr; + unsigned char fw_check; + int r; + + INIT_DELAYED_WORK(&adev->vcn.idle_work, amdgpu_vcn_idle_work_handler); + + switch (adev->asic_type) { + case CHIP_RAVEN: + if (adev->rev_id >= 8) + fw_name = FIRMWARE_RAVEN2; + else if (adev->pdev->device == 0x15d8) + fw_name = FIRMWARE_PICASSO; + else + fw_name = FIRMWARE_RAVEN; + break; + case CHIP_NAVI10: + fw_name = FIRMWARE_NAVI10; + if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) && + (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)) + adev->vcn.indirect_sram = true; + break; + default: + return -EINVAL; + } + + r = request_firmware(&adev->vcn.fw, fw_name, adev->dev); + if (r) { + dev_err(adev->dev, "amdgpu_vcn: Can't load firmware \"%s\"\n", + fw_name); + return r; + } + + r = amdgpu_ucode_validate(adev->vcn.fw); + if (r) { + dev_err(adev->dev, "amdgpu_vcn: Can't validate firmware \"%s\"\n", + fw_name); + release_firmware(adev->vcn.fw); + adev->vcn.fw = NULL; + return r; + } + + hdr = (const struct common_firmware_header *)adev->vcn.fw->data; + adev->vcn.fw_version = le32_to_cpu(hdr->ucode_version); + + /* Bit 20-23, it is encode major and non-zero for new naming convention. + * This field is part of version minor and DRM_DISABLED_FLAG in old naming + * convention. Since the l:wq!atest version minor is 0x5B and DRM_DISABLED_FLAG + * is zero in old naming convention, this field is always zero so far. + * These four bits are used to tell which naming convention is present. + */ + fw_check = (le32_to_cpu(hdr->ucode_version) >> 20) & 0xf; + if (fw_check) { + unsigned int dec_ver, enc_major, enc_minor, vep, fw_rev; + + fw_rev = le32_to_cpu(hdr->ucode_version) & 0xfff; + enc_minor = (le32_to_cpu(hdr->ucode_version) >> 12) & 0xff; + enc_major = fw_check; + dec_ver = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xf; + vep = (le32_to_cpu(hdr->ucode_version) >> 28) & 0xf; + DRM_INFO("Found VCN firmware Version ENC: %hu.%hu DEC: %hu VEP: %hu Revision: %hu\n", + enc_major, enc_minor, dec_ver, vep, fw_rev); + } else { + unsigned int version_major, version_minor, family_id; + + family_id = le32_to_cpu(hdr->ucode_version) & 0xff; + version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff; + version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff; + DRM_INFO("Found VCN firmware Version: %hu.%hu Family ID: %hu\n", + version_major, version_minor, family_id); + } + + bo_size = AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_CONTEXT_SIZE; + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) + bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); + r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.vcpu_bo, + &adev->vcn.gpu_addr, &adev->vcn.cpu_addr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate vcn bo\n", r); + return r; + } + + if (adev->vcn.indirect_sram) { + r = amdgpu_bo_create_kernel(adev, 64 * 2 * 4, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.dpg_sram_bo, + &adev->vcn.dpg_sram_gpu_addr, &adev->vcn.dpg_sram_cpu_addr); + if (r) { + dev_err(adev->dev, "(%d) failed to allocate DPG bo\n", r); + return r; + } + } + + return 0; +} + +int amdgpu_vcn_sw_fini(struct amdgpu_device *adev) +{ + int i; + + kvfree(adev->vcn.saved_bo); + + if (adev->vcn.indirect_sram) { + amdgpu_bo_free_kernel(&adev->vcn.dpg_sram_bo, + &adev->vcn.dpg_sram_gpu_addr, + (void **)&adev->vcn.dpg_sram_cpu_addr); + } + + amdgpu_bo_free_kernel(&adev->vcn.vcpu_bo, + &adev->vcn.gpu_addr, + (void **)&adev->vcn.cpu_addr); + + amdgpu_ring_fini(&adev->vcn.ring_dec); + + for (i = 0; i < adev->vcn.num_enc_rings; ++i) + amdgpu_ring_fini(&adev->vcn.ring_enc[i]); + + amdgpu_ring_fini(&adev->vcn.ring_jpeg); + + release_firmware(adev->vcn.fw); + + return 0; +} + +int amdgpu_vcn_suspend(struct amdgpu_device *adev) +{ + unsigned size; + void *ptr; + + cancel_delayed_work_sync(&adev->vcn.idle_work); + + if (adev->vcn.vcpu_bo == NULL) + return 0; + + size = amdgpu_bo_size(adev->vcn.vcpu_bo); + ptr = adev->vcn.cpu_addr; + + adev->vcn.saved_bo = kvmalloc(size, GFP_KERNEL); + if (!adev->vcn.saved_bo) + return -ENOMEM; + + memcpy_fromio(adev->vcn.saved_bo, ptr, size); + + return 0; +} + +int amdgpu_vcn_resume(struct amdgpu_device *adev) +{ + unsigned size; + void *ptr; + + if (adev->vcn.vcpu_bo == NULL) + return -EINVAL; + + size = amdgpu_bo_size(adev->vcn.vcpu_bo); + ptr = adev->vcn.cpu_addr; + + if (adev->vcn.saved_bo != NULL) { + memcpy_toio(ptr, adev->vcn.saved_bo, size); + kvfree(adev->vcn.saved_bo); + adev->vcn.saved_bo = NULL; + } else { + const struct common_firmware_header *hdr; + unsigned offset; + + hdr = (const struct common_firmware_header *)adev->vcn.fw->data; + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + offset = le32_to_cpu(hdr->ucode_array_offset_bytes); + memcpy_toio(adev->vcn.cpu_addr, adev->vcn.fw->data + offset, + le32_to_cpu(hdr->ucode_size_bytes)); + size -= le32_to_cpu(hdr->ucode_size_bytes); + ptr += le32_to_cpu(hdr->ucode_size_bytes); + } + memset_io(ptr, 0, size); + } + + return 0; +} + +static void amdgpu_vcn_idle_work_handler(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, vcn.idle_work.work); + unsigned int fences = 0; + unsigned int i; + + for (i = 0; i < adev->vcn.num_enc_rings; ++i) { + fences += amdgpu_fence_count_emitted(&adev->vcn.ring_enc[i]); + } + + if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { + struct dpg_pause_state new_state; + + if (fences) + new_state.fw_based = VCN_DPG_STATE__PAUSE; + else + new_state.fw_based = VCN_DPG_STATE__UNPAUSE; + + if (amdgpu_fence_count_emitted(&adev->vcn.ring_jpeg)) + new_state.jpeg = VCN_DPG_STATE__PAUSE; + else + new_state.jpeg = VCN_DPG_STATE__UNPAUSE; + + adev->vcn.pause_dpg_mode(adev, &new_state); + } + + fences += amdgpu_fence_count_emitted(&adev->vcn.ring_jpeg); + fences += amdgpu_fence_count_emitted(&adev->vcn.ring_dec); + + if (fences == 0) { + amdgpu_gfx_off_ctrl(adev, true); + if (adev->asic_type < CHIP_NAVI10 && adev->pm.dpm_enabled) + amdgpu_dpm_enable_uvd(adev, false); + else + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN, + AMD_PG_STATE_GATE); + } else { + schedule_delayed_work(&adev->vcn.idle_work, VCN_IDLE_TIMEOUT); + } +} + +void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work); + + if (set_clocks) { + amdgpu_gfx_off_ctrl(adev, false); + if (adev->asic_type < CHIP_NAVI10 && adev->pm.dpm_enabled) + amdgpu_dpm_enable_uvd(adev, true); + else + amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN, + AMD_PG_STATE_UNGATE); + } + + if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { + struct dpg_pause_state new_state; + unsigned int fences = 0; + unsigned int i; + + for (i = 0; i < adev->vcn.num_enc_rings; ++i) { + fences += amdgpu_fence_count_emitted(&adev->vcn.ring_enc[i]); + } + if (fences) + new_state.fw_based = VCN_DPG_STATE__PAUSE; + else + new_state.fw_based = VCN_DPG_STATE__UNPAUSE; + + if (amdgpu_fence_count_emitted(&adev->vcn.ring_jpeg)) + new_state.jpeg = VCN_DPG_STATE__PAUSE; + else + new_state.jpeg = VCN_DPG_STATE__UNPAUSE; + + if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC) + new_state.fw_based = VCN_DPG_STATE__PAUSE; + else if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) + new_state.jpeg = VCN_DPG_STATE__PAUSE; + + adev->vcn.pause_dpg_mode(adev, &new_state); + } +} + +void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring) +{ + schedule_delayed_work(&ring->adev->vcn.idle_work, VCN_IDLE_TIMEOUT); +} + +int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t tmp = 0; + unsigned i; + int r; + + WREG32(adev->vcn.external.scratch9, 0xCAFEDEAD); + r = amdgpu_ring_alloc(ring, 3); + if (r) + return r; + amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0)); + amdgpu_ring_write(ring, 0xDEADBEEF); + amdgpu_ring_commit(ring); + for (i = 0; i < adev->usec_timeout; i++) { + tmp = RREG32(adev->vcn.external.scratch9); + if (tmp == 0xDEADBEEF) + break; + udelay(1); + } + + if (i >= adev->usec_timeout) + r = -ETIMEDOUT; + + return r; +} + +static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, + struct amdgpu_bo *bo, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct dma_fence *f = NULL; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + uint64_t addr; + int i, r; + + r = amdgpu_job_alloc_with_ib(adev, 64, &job); + if (r) + goto err; + + ib = &job->ibs[0]; + addr = amdgpu_bo_gpu_offset(bo); + ib->ptr[0] = PACKET0(adev->vcn.internal.data0, 0); + ib->ptr[1] = addr; + ib->ptr[2] = PACKET0(adev->vcn.internal.data1, 0); + ib->ptr[3] = addr >> 32; + ib->ptr[4] = PACKET0(adev->vcn.internal.cmd, 0); + ib->ptr[5] = 0; + for (i = 6; i < 16; i += 2) { + ib->ptr[i] = PACKET0(adev->vcn.internal.nop, 0); + ib->ptr[i+1] = 0; + } + ib->length_dw = 16; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err_free; + + amdgpu_bo_fence(bo, f, false); + amdgpu_bo_unreserve(bo); + amdgpu_bo_unref(&bo); + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + + return 0; + +err_free: + amdgpu_job_free(job); + +err: + amdgpu_bo_unreserve(bo); + amdgpu_bo_unref(&bo); + return r; +} + +static int amdgpu_vcn_dec_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_bo *bo = NULL; + uint32_t *msg; + int r, i; + + r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL, (void **)&msg); + if (r) + return r; + + msg[0] = cpu_to_le32(0x00000028); + msg[1] = cpu_to_le32(0x00000038); + msg[2] = cpu_to_le32(0x00000001); + msg[3] = cpu_to_le32(0x00000000); + msg[4] = cpu_to_le32(handle); + msg[5] = cpu_to_le32(0x00000000); + msg[6] = cpu_to_le32(0x00000001); + msg[7] = cpu_to_le32(0x00000028); + msg[8] = cpu_to_le32(0x00000010); + msg[9] = cpu_to_le32(0x00000000); + msg[10] = cpu_to_le32(0x00000007); + msg[11] = cpu_to_le32(0x00000000); + msg[12] = cpu_to_le32(0x00000780); + msg[13] = cpu_to_le32(0x00000440); + for (i = 14; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + return amdgpu_vcn_dec_send_msg(ring, bo, fence); +} + +static int amdgpu_vcn_dec_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_bo *bo = NULL; + uint32_t *msg; + int r, i; + + r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL, (void **)&msg); + if (r) + return r; + + msg[0] = cpu_to_le32(0x00000028); + msg[1] = cpu_to_le32(0x00000018); + msg[2] = cpu_to_le32(0x00000000); + msg[3] = cpu_to_le32(0x00000002); + msg[4] = cpu_to_le32(handle); + msg[5] = cpu_to_le32(0x00000000); + for (i = 6; i < 1024; ++i) + msg[i] = cpu_to_le32(0x0); + + return amdgpu_vcn_dec_send_msg(ring, bo, fence); +} + +int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct dma_fence *fence; + long r; + + r = amdgpu_vcn_dec_get_create_msg(ring, 1, NULL); + if (r) + goto error; + + r = amdgpu_vcn_dec_get_destroy_msg(ring, 1, &fence); + if (r) + goto error; + + r = dma_fence_wait_timeout(fence, false, timeout); + if (r == 0) + r = -ETIMEDOUT; + else if (r > 0) + r = 0; + + dma_fence_put(fence); +error: + return r; +} + +int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t rptr; + unsigned i; + int r; + + r = amdgpu_ring_alloc(ring, 16); + if (r) + return r; + + rptr = amdgpu_ring_get_rptr(ring); + + amdgpu_ring_write(ring, VCN_ENC_CMD_END); + amdgpu_ring_commit(ring); + + for (i = 0; i < adev->usec_timeout; i++) { + if (amdgpu_ring_get_rptr(ring) != rptr) + break; + udelay(1); + } + + if (i >= adev->usec_timeout) + r = -ETIMEDOUT; + + return r; +} + +static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + const unsigned ib_size_dw = 16; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct dma_fence *f = NULL; + uint64_t dummy; + int i, r; + + r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); + if (r) + return r; + + ib = &job->ibs[0]; + dummy = ib->gpu_addr + 1024; + + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x00000018; + ib->ptr[ib->length_dw++] = 0x00000001; /* session info */ + ib->ptr[ib->length_dw++] = handle; + ib->ptr[ib->length_dw++] = upper_32_bits(dummy); + ib->ptr[ib->length_dw++] = dummy; + ib->ptr[ib->length_dw++] = 0x0000000b; + + ib->ptr[ib->length_dw++] = 0x00000014; + ib->ptr[ib->length_dw++] = 0x00000002; /* task info */ + ib->ptr[ib->length_dw++] = 0x0000001c; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + + ib->ptr[ib->length_dw++] = 0x00000008; + ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */ + + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err; + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + + return 0; + +err: + amdgpu_job_free(job); + return r; +} + +static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + const unsigned ib_size_dw = 16; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct dma_fence *f = NULL; + uint64_t dummy; + int i, r; + + r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); + if (r) + return r; + + ib = &job->ibs[0]; + dummy = ib->gpu_addr + 1024; + + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x00000018; + ib->ptr[ib->length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = handle; + ib->ptr[ib->length_dw++] = upper_32_bits(dummy); + ib->ptr[ib->length_dw++] = dummy; + ib->ptr[ib->length_dw++] = 0x0000000b; + + ib->ptr[ib->length_dw++] = 0x00000014; + ib->ptr[ib->length_dw++] = 0x00000002; + ib->ptr[ib->length_dw++] = 0x0000001c; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + + ib->ptr[ib->length_dw++] = 0x00000008; + ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */ + + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err; + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + + return 0; + +err: + amdgpu_job_free(job); + return r; +} + +int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct dma_fence *fence = NULL; + long r; + + r = amdgpu_vcn_enc_get_create_msg(ring, 1, NULL); + if (r) + goto error; + + r = amdgpu_vcn_enc_get_destroy_msg(ring, 1, &fence); + if (r) + goto error; + + r = dma_fence_wait_timeout(fence, false, timeout); + if (r == 0) + r = -ETIMEDOUT; + else if (r > 0) + r = 0; + +error: + dma_fence_put(fence); + return r; +} + +int amdgpu_vcn_jpeg_ring_test_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t tmp = 0; + unsigned i; + int r; + + WREG32(adev->vcn.external.jpeg_pitch, 0xCAFEDEAD); + r = amdgpu_ring_alloc(ring, 3); + if (r) + return r; + + amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.jpeg_pitch, 0)); + amdgpu_ring_write(ring, 0xDEADBEEF); + amdgpu_ring_commit(ring); + + for (i = 0; i < adev->usec_timeout; i++) { + tmp = RREG32(adev->vcn.external.jpeg_pitch); + if (tmp == 0xDEADBEEF) + break; + udelay(1); + } + + if (i >= adev->usec_timeout) + r = -ETIMEDOUT; + + return r; +} + +static int amdgpu_vcn_jpeg_set_reg(struct amdgpu_ring *ring, uint32_t handle, + struct dma_fence **fence) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + struct dma_fence *f = NULL; + const unsigned ib_size_dw = 16; + int i, r; + + r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); + if (r) + return r; + + ib = &job->ibs[0]; + + ib->ptr[0] = PACKETJ(adev->vcn.internal.jpeg_pitch, 0, 0, PACKETJ_TYPE0); + ib->ptr[1] = 0xDEADBEEF; + for (i = 2; i < 16; i += 2) { + ib->ptr[i] = PACKETJ(0, 0, 0, PACKETJ_TYPE6); + ib->ptr[i+1] = 0; + } + ib->length_dw = 16; + + r = amdgpu_job_submit_direct(job, ring, &f); + if (r) + goto err; + + if (fence) + *fence = dma_fence_get(f); + dma_fence_put(f); + + return 0; + +err: + amdgpu_job_free(job); + return r; +} + +int amdgpu_vcn_jpeg_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t tmp = 0; + unsigned i; + struct dma_fence *fence = NULL; + long r = 0; + + r = amdgpu_vcn_jpeg_set_reg(ring, 1, &fence); + if (r) + goto error; + + r = dma_fence_wait_timeout(fence, false, timeout); + if (r == 0) { + r = -ETIMEDOUT; + goto error; + } else if (r < 0) { + goto error; + } else { + r = 0; + } + + for (i = 0; i < adev->usec_timeout; i++) { + tmp = RREG32(adev->vcn.external.jpeg_pitch); + if (tmp == 0xDEADBEEF) + break; + udelay(1); + } + + if (i >= adev->usec_timeout) + r = -ETIMEDOUT; + + dma_fence_put(fence); +error: + return r; +}
diff --git a/amdgpu/amdgpu_vcn.h b/amdgpu/amdgpu_vcn.h new file mode 100644 index 0000000..19661c6 --- /dev/null +++ b/amdgpu/amdgpu_vcn.h
@@ -0,0 +1,195 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __AMDGPU_VCN_H__ +#define __AMDGPU_VCN_H__ + +#define AMDGPU_VCN_STACK_SIZE (128*1024) +#define AMDGPU_VCN_CONTEXT_SIZE (512*1024) + +#define AMDGPU_VCN_FIRMWARE_OFFSET 256 +#define AMDGPU_VCN_MAX_ENC_RINGS 3 + +#define VCN_DEC_KMD_CMD 0x80000000 +#define VCN_DEC_CMD_FENCE 0x00000000 +#define VCN_DEC_CMD_TRAP 0x00000001 +#define VCN_DEC_CMD_WRITE_REG 0x00000004 +#define VCN_DEC_CMD_REG_READ_COND_WAIT 0x00000006 +#define VCN_DEC_CMD_PACKET_START 0x0000000a +#define VCN_DEC_CMD_PACKET_END 0x0000000b + +#define VCN_ENC_CMD_NO_OP 0x00000000 +#define VCN_ENC_CMD_END 0x00000001 +#define VCN_ENC_CMD_IB 0x00000002 +#define VCN_ENC_CMD_FENCE 0x00000003 +#define VCN_ENC_CMD_TRAP 0x00000004 +#define VCN_ENC_CMD_REG_WRITE 0x0000000b +#define VCN_ENC_CMD_REG_WAIT 0x0000000c + +#define VCN_VID_SOC_ADDRESS_2_0 0x1fa00 +#define VCN_AON_SOC_ADDRESS_2_0 0x1f800 +#define VCN_VID_IP_ADDRESS_2_0 0x0 +#define VCN_AON_IP_ADDRESS_2_0 0x30000 + +#define RREG32_SOC15_DPG_MODE(ip, inst, reg, mask, sram_sel) \ + ({ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \ + WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \ + UVD_DPG_LMA_CTL__MASK_EN_MASK | \ + ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \ + << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \ + (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \ + RREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA); \ + }) + +#define WREG32_SOC15_DPG_MODE(ip, inst, reg, value, mask, sram_sel) \ + do { \ + WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA, value); \ + WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \ + WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \ + UVD_DPG_LMA_CTL__READ_WRITE_MASK | \ + ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \ + << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \ + (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \ + } while (0) + +#define SOC15_DPG_MODE_OFFSET_2_0(ip, inst, reg) \ + ({ \ + uint32_t internal_reg_offset, addr; \ + bool video_range, aon_range; \ + \ + addr = (adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg); \ + addr <<= 2; \ + video_range = ((((0xFFFFF & addr) >= (VCN_VID_SOC_ADDRESS_2_0)) && \ + ((0xFFFFF & addr) < ((VCN_VID_SOC_ADDRESS_2_0 + 0x2600))))); \ + aon_range = ((((0xFFFFF & addr) >= (VCN_AON_SOC_ADDRESS_2_0)) && \ + ((0xFFFFF & addr) < ((VCN_AON_SOC_ADDRESS_2_0 + 0x600))))); \ + if (video_range) \ + internal_reg_offset = ((0xFFFFF & addr) - (VCN_VID_SOC_ADDRESS_2_0) + \ + (VCN_VID_IP_ADDRESS_2_0)); \ + else if (aon_range) \ + internal_reg_offset = ((0xFFFFF & addr) - (VCN_AON_SOC_ADDRESS_2_0) + \ + (VCN_AON_IP_ADDRESS_2_0)); \ + else \ + internal_reg_offset = (0xFFFFF & addr); \ + \ + internal_reg_offset >>= 2; \ + }) + +#define RREG32_SOC15_DPG_MODE_2_0(offset, mask_en) \ + ({ \ + WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_CTL, \ + (0x0 << UVD_DPG_LMA_CTL__READ_WRITE__SHIFT | \ + mask_en << UVD_DPG_LMA_CTL__MASK_EN__SHIFT | \ + offset << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT)); \ + RREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_DATA); \ + }) + +#define WREG32_SOC15_DPG_MODE_2_0(offset, value, mask_en, indirect) \ + do { \ + if (!indirect) { \ + WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_DATA, value); \ + WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_CTL, \ + (0x1 << UVD_DPG_LMA_CTL__READ_WRITE__SHIFT | \ + mask_en << UVD_DPG_LMA_CTL__MASK_EN__SHIFT | \ + offset << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT)); \ + } else { \ + *adev->vcn.dpg_sram_curr_addr++ = offset; \ + *adev->vcn.dpg_sram_curr_addr++ = value; \ + } \ + } while (0) + +enum engine_status_constants { + UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON = 0x2AAAA0, + UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0 = 0xAAAA0, + UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON = 0x00000002, + UVD_STATUS__UVD_BUSY = 0x00000004, + GB_ADDR_CONFIG_DEFAULT = 0x26010011, + UVD_STATUS__IDLE = 0x2, + UVD_STATUS__BUSY = 0x5, + UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF = 0x1, + UVD_STATUS__RBC_BUSY = 0x1, + UVD_PGFSM_STATUS_UVDJ_PWR_ON = 0, +}; + +enum internal_dpg_state { + VCN_DPG_STATE__UNPAUSE = 0, + VCN_DPG_STATE__PAUSE, +}; + +struct dpg_pause_state { + enum internal_dpg_state fw_based; + enum internal_dpg_state jpeg; +}; + +struct amdgpu_vcn_reg{ + unsigned data0; + unsigned data1; + unsigned cmd; + unsigned nop; + unsigned scratch9; + unsigned jpeg_pitch; +}; + +struct amdgpu_vcn { + struct amdgpu_bo *vcpu_bo; + void *cpu_addr; + uint64_t gpu_addr; + unsigned fw_version; + void *saved_bo; + struct delayed_work idle_work; + const struct firmware *fw; /* VCN firmware */ + struct amdgpu_ring ring_dec; + struct amdgpu_ring ring_enc[AMDGPU_VCN_MAX_ENC_RINGS]; + struct amdgpu_ring ring_jpeg; + struct amdgpu_irq_src irq; + unsigned num_enc_rings; + enum amd_powergating_state cur_state; + struct dpg_pause_state pause_state; + struct amdgpu_vcn_reg internal, external; + int (*pause_dpg_mode)(struct amdgpu_device *adev, + struct dpg_pause_state *new_state); + + bool indirect_sram; + struct amdgpu_bo *dpg_sram_bo; + void *dpg_sram_cpu_addr; + uint64_t dpg_sram_gpu_addr; + uint32_t *dpg_sram_curr_addr; +}; + +int amdgpu_vcn_sw_init(struct amdgpu_device *adev); +int amdgpu_vcn_sw_fini(struct amdgpu_device *adev); +int amdgpu_vcn_suspend(struct amdgpu_device *adev); +int amdgpu_vcn_resume(struct amdgpu_device *adev); +void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring); +void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring); + +int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring); +int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout); + +int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring); +int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout); + +int amdgpu_vcn_jpeg_ring_test_ring(struct amdgpu_ring *ring); +int amdgpu_vcn_jpeg_ring_test_ib(struct amdgpu_ring *ring, long timeout); + +#endif
diff --git a/amdgpu/amdgpu_vf_error.c b/amdgpu/amdgpu_vf_error.c new file mode 100644 index 0000000..7f70979 --- /dev/null +++ b/amdgpu/amdgpu_vf_error.c
@@ -0,0 +1,86 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu.h" +#include "amdgpu_vf_error.h" +#include "mxgpu_ai.h" + +void amdgpu_vf_error_put(struct amdgpu_device *adev, + uint16_t sub_error_code, + uint16_t error_flags, + uint64_t error_data) +{ + int index; + uint16_t error_code; + + if (!amdgpu_sriov_vf(adev)) + return; + + error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code); + + mutex_lock(&adev->virt.vf_errors.lock); + index = adev->virt.vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE; + adev->virt.vf_errors.code [index] = error_code; + adev->virt.vf_errors.flags [index] = error_flags; + adev->virt.vf_errors.data [index] = error_data; + adev->virt.vf_errors.write_count ++; + mutex_unlock(&adev->virt.vf_errors.lock); +} + + +void amdgpu_vf_error_trans_all(struct amdgpu_device *adev) +{ + /* u32 pf2vf_flags = 0; */ + u32 data1, data2, data3; + int index; + + if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) || + (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) { + return; + } +/* + TODO: Enable these code when pv2vf_info is merged + AMDGPU_FW_VRAM_PF2VF_READ (adev, feature_flags, &pf2vf_flags); + if (!(pf2vf_flags & AMDGIM_FEATURE_ERROR_LOG_COLLECT)) { + return; + } +*/ + + mutex_lock(&adev->virt.vf_errors.lock); + /* The errors are overlay of array, correct read_count as full. */ + if (adev->virt.vf_errors.write_count - adev->virt.vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) { + adev->virt.vf_errors.read_count = adev->virt.vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE; + } + + while (adev->virt.vf_errors.read_count < adev->virt.vf_errors.write_count) { + index =adev->virt.vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE; + data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(adev->virt.vf_errors.code[index], + adev->virt.vf_errors.flags[index]); + data2 = adev->virt.vf_errors.data[index] & 0xFFFFFFFF; + data3 = (adev->virt.vf_errors.data[index] >> 32) & 0xFFFFFFFF; + + adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3); + adev->virt.vf_errors.read_count ++; + } + mutex_unlock(&adev->virt.vf_errors.lock); +}
diff --git a/amdgpu/amdgpu_vf_error.h b/amdgpu/amdgpu_vf_error.h new file mode 100644 index 0000000..6436bd0 --- /dev/null +++ b/amdgpu/amdgpu_vf_error.h
@@ -0,0 +1,65 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __VF_ERROR_H__ +#define __VF_ERROR_H__ + +#define AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(c,f) (((c & 0xFFFF) << 16) | (f & 0xFFFF)) +#define AMDGIM_ERROR_CODE(t,c) (((t&0xF)<<12)|(c&0xFFF)) + +/* Please keep enum same as AMD GIM driver */ +enum AMDGIM_ERROR_VF { + AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL = 0, + AMDGIM_ERROR_VF_NO_VBIOS, + AMDGIM_ERROR_VF_GPU_POST_ERROR, + AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, + AMDGIM_ERROR_VF_FENCE_INIT_FAIL, + + AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, + AMDGIM_ERROR_VF_IB_INIT_FAIL, + AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, + AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, + AMDGIM_ERROR_VF_GPU_RESET_FAIL, + + AMDGIM_ERROR_VF_TEST, + AMDGIM_ERROR_VF_MAX +}; + +enum AMDGIM_ERROR_CATEGORY { + AMDGIM_ERROR_CATEGORY_NON_USED = 0, + AMDGIM_ERROR_CATEGORY_GIM, + AMDGIM_ERROR_CATEGORY_PF, + AMDGIM_ERROR_CATEGORY_VF, + AMDGIM_ERROR_CATEGORY_VBIOS, + AMDGIM_ERROR_CATEGORY_MONITOR, + + AMDGIM_ERROR_CATEGORY_MAX +}; + +void amdgpu_vf_error_put(struct amdgpu_device *adev, + uint16_t sub_error_code, + uint16_t error_flags, + uint64_t error_data); +void amdgpu_vf_error_trans_all (struct amdgpu_device *adev); + +#endif /* __VF_ERROR_H__ */
diff --git a/amdgpu/amdgpu_virt.c b/amdgpu/amdgpu_virt.c new file mode 100644 index 0000000..59dd204 --- /dev/null +++ b/amdgpu/amdgpu_virt.c
@@ -0,0 +1,477 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/module.h> + +#include <drm/drm_drv.h> + +#include "amdgpu.h" + +bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) +{ + /* By now all MMIO pages except mailbox are blocked */ + /* if blocking is enabled in hypervisor. Choose the */ + /* SCRATCH_REG0 to test. */ + return RREG32_NO_KIQ(0xc040) == 0xffffffff; +} + +void amdgpu_virt_init_setting(struct amdgpu_device *adev) +{ + /* enable virtual display */ + adev->mode_info.num_crtc = 1; + adev->enable_virtual_display = true; + adev->ddev->driver->driver_features &= ~DRIVER_ATOMIC; + adev->cg_flags = 0; + adev->pg_flags = 0; +} + +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) +{ + signed long r, cnt = 0; + unsigned long flags; + uint32_t seq; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *ring = &kiq->ring; + + BUG_ON(!ring->funcs->emit_rreg); + + spin_lock_irqsave(&kiq->ring_lock, flags); + amdgpu_ring_alloc(ring, 32); + amdgpu_ring_emit_rreg(ring, reg); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + /* don't wait anymore for gpu reset case because this way may + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will + * never return if we keep waiting in virt_kiq_rreg, which cause + * gpu_recover() hang there. + * + * also don't wait anymore for IRQ context + * */ + if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + goto failed_kiq_read; + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) + goto failed_kiq_read; + + return adev->wb.wb[adev->virt.reg_val_offs]; + +failed_kiq_read: + pr_err("failed to read reg:%x\n", reg); + return ~0; +} + +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) +{ + signed long r, cnt = 0; + unsigned long flags; + uint32_t seq; + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *ring = &kiq->ring; + + BUG_ON(!ring->funcs->emit_wreg); + + spin_lock_irqsave(&kiq->ring_lock, flags); + amdgpu_ring_alloc(ring, 32); + amdgpu_ring_emit_wreg(ring, reg, v); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + /* don't wait anymore for gpu reset case because this way may + * block gpu_recover() routine forever, e.g. this virt_kiq_rreg + * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will + * never return if we keep waiting in virt_kiq_rreg, which cause + * gpu_recover() hang there. + * + * also don't wait anymore for IRQ context + * */ + if (r < 1 && (adev->in_gpu_reset || in_interrupt())) + goto failed_kiq_write; + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { + + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) + goto failed_kiq_write; + + return; + +failed_kiq_write: + pr_err("failed to write reg:%x\n", reg); +} + +void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, + uint32_t reg0, uint32_t reg1, + uint32_t ref, uint32_t mask) +{ + struct amdgpu_kiq *kiq = &adev->gfx.kiq; + struct amdgpu_ring *ring = &kiq->ring; + signed long r, cnt = 0; + unsigned long flags; + uint32_t seq; + + spin_lock_irqsave(&kiq->ring_lock, flags); + amdgpu_ring_alloc(ring, 32); + amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, + ref, mask); + amdgpu_fence_emit_polling(ring, &seq); + amdgpu_ring_commit(ring); + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + + /* don't wait anymore for IRQ context */ + if (r < 1 && in_interrupt()) + goto failed_kiq; + + might_sleep(); + while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { + + msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); + r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); + } + + if (cnt > MAX_KIQ_REG_TRY) + goto failed_kiq; + + return; + +failed_kiq: + pr_err("failed to write reg %x wait reg %x\n", reg0, reg1); +} + +/** + * amdgpu_virt_request_full_gpu() - request full gpu access + * @amdgpu: amdgpu device. + * @init: is driver init time. + * When start to init/fini driver, first need to request full gpu access. + * Return: Zero if request success, otherwise will return error. + */ +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) +{ + struct amdgpu_virt *virt = &adev->virt; + int r; + + if (virt->ops && virt->ops->req_full_gpu) { + r = virt->ops->req_full_gpu(adev, init); + if (r) + return r; + + adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; + } + + return 0; +} + +/** + * amdgpu_virt_release_full_gpu() - release full gpu access + * @amdgpu: amdgpu device. + * @init: is driver init time. + * When finishing driver init/fini, need to release full gpu access. + * Return: Zero if release success, otherwise will returen error. + */ +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) +{ + struct amdgpu_virt *virt = &adev->virt; + int r; + + if (virt->ops && virt->ops->rel_full_gpu) { + r = virt->ops->rel_full_gpu(adev, init); + if (r) + return r; + + adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; + } + return 0; +} + +/** + * amdgpu_virt_reset_gpu() - reset gpu + * @amdgpu: amdgpu device. + * Send reset command to GPU hypervisor to reset GPU that VM is using + * Return: Zero if reset success, otherwise will return error. + */ +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) +{ + struct amdgpu_virt *virt = &adev->virt; + int r; + + if (virt->ops && virt->ops->reset_gpu) { + r = virt->ops->reset_gpu(adev); + if (r) + return r; + + adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; + } + + return 0; +} + +/** + * amdgpu_virt_wait_reset() - wait for reset gpu completed + * @amdgpu: amdgpu device. + * Wait for GPU reset completed. + * Return: Zero if reset success, otherwise will return error. + */ +int amdgpu_virt_wait_reset(struct amdgpu_device *adev) +{ + struct amdgpu_virt *virt = &adev->virt; + + if (!virt->ops || !virt->ops->wait_reset) + return -EINVAL; + + return virt->ops->wait_reset(adev); +} + +/** + * amdgpu_virt_alloc_mm_table() - alloc memory for mm table + * @amdgpu: amdgpu device. + * MM table is used by UVD and VCE for its initialization + * Return: Zero if allocate success. + */ +int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) +{ + int r; + + if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) + return 0; + + r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->virt.mm_table.bo, + &adev->virt.mm_table.gpu_addr, + (void *)&adev->virt.mm_table.cpu_addr); + if (r) { + DRM_ERROR("failed to alloc mm table and error = %d.\n", r); + return r; + } + + memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); + DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n", + adev->virt.mm_table.gpu_addr, + adev->virt.mm_table.cpu_addr); + return 0; +} + +/** + * amdgpu_virt_free_mm_table() - free mm table memory + * @amdgpu: amdgpu device. + * Free MM table memory + */ +void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) +{ + if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) + return; + + amdgpu_bo_free_kernel(&adev->virt.mm_table.bo, + &adev->virt.mm_table.gpu_addr, + (void *)&adev->virt.mm_table.cpu_addr); + adev->virt.mm_table.gpu_addr = 0; +} + + +int amdgpu_virt_fw_reserve_get_checksum(void *obj, + unsigned long obj_size, + unsigned int key, + unsigned int chksum) +{ + unsigned int ret = key; + unsigned long i = 0; + unsigned char *pos; + + pos = (char *)obj; + /* calculate checksum */ + for (i = 0; i < obj_size; ++i) + ret += *(pos + i); + /* minus the chksum itself */ + pos = (char *)&chksum; + for (i = 0; i < sizeof(chksum); ++i) + ret -= *(pos + i); + return ret; +} + +void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) +{ + uint32_t pf2vf_size = 0; + uint32_t checksum = 0; + uint32_t checkval; + char *str; + + adev->virt.fw_reserve.p_pf2vf = NULL; + adev->virt.fw_reserve.p_vf2pf = NULL; + + if (adev->fw_vram_usage.va != NULL) { + adev->virt.fw_reserve.p_pf2vf = + (struct amd_sriov_msg_pf2vf_info_header *)( + adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET); + AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size); + AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum); + AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature); + + /* pf2vf message must be in 4K */ + if (pf2vf_size > 0 && pf2vf_size < 4096) { + checkval = amdgpu_virt_fw_reserve_get_checksum( + adev->virt.fw_reserve.p_pf2vf, pf2vf_size, + adev->virt.fw_reserve.checksum_key, checksum); + if (checkval == checksum) { + adev->virt.fw_reserve.p_vf2pf = + ((void *)adev->virt.fw_reserve.p_pf2vf + + pf2vf_size); + memset((void *)adev->virt.fw_reserve.p_vf2pf, 0, + sizeof(amdgim_vf2pf_info)); + AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.version, + AMDGPU_FW_VRAM_VF2PF_VER); + AMDGPU_FW_VRAM_VF2PF_WRITE(adev, header.size, + sizeof(amdgim_vf2pf_info)); + AMDGPU_FW_VRAM_VF2PF_READ(adev, driver_version, + &str); +#ifdef MODULE + if (THIS_MODULE->version != NULL) + strcpy(str, THIS_MODULE->version); + else +#endif + strcpy(str, "N/A"); + AMDGPU_FW_VRAM_VF2PF_WRITE(adev, driver_cert, + 0); + AMDGPU_FW_VRAM_VF2PF_WRITE(adev, checksum, + amdgpu_virt_fw_reserve_get_checksum( + adev->virt.fw_reserve.p_vf2pf, + pf2vf_size, + adev->virt.fw_reserve.checksum_key, 0)); + } + } + } +} + +static uint32_t parse_clk(char *buf, bool min) +{ + char *ptr = buf; + uint32_t clk = 0; + + do { + ptr = strchr(ptr, ':'); + if (!ptr) + break; + ptr+=2; + if (kstrtou32(ptr, 10, &clk)) + return 0; + } while (!min); + + return clk * 100; +} + +uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest) +{ + char *buf = NULL; + uint32_t clk = 0; + + buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf); + clk = parse_clk(buf, lowest); + + kfree(buf); + + return clk; +} + +uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest) +{ + char *buf = NULL; + uint32_t clk = 0; + + buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf); + clk = parse_clk(buf, lowest); + + kfree(buf); + + return clk; +} + +void amdgpu_virt_init_reg_access_mode(struct amdgpu_device *adev) +{ + struct amdgpu_virt *virt = &adev->virt; + + if (virt->ops && virt->ops->init_reg_access_mode) + virt->ops->init_reg_access_mode(adev); +} + +bool amdgpu_virt_support_psp_prg_ih_reg(struct amdgpu_device *adev) +{ + bool ret = false; + struct amdgpu_virt *virt = &adev->virt; + + if (amdgpu_sriov_vf(adev) + && (virt->reg_access_mode & AMDGPU_VIRT_REG_ACCESS_PSP_PRG_IH)) + ret = true; + + return ret; +} + +bool amdgpu_virt_support_rlc_prg_reg(struct amdgpu_device *adev) +{ + bool ret = false; + struct amdgpu_virt *virt = &adev->virt; + + if (amdgpu_sriov_vf(adev) + && (virt->reg_access_mode & AMDGPU_VIRT_REG_ACCESS_RLC) + && !(amdgpu_sriov_runtime(adev))) + ret = true; + + return ret; +} + +bool amdgpu_virt_support_skip_setting(struct amdgpu_device *adev) +{ + bool ret = false; + struct amdgpu_virt *virt = &adev->virt; + + if (amdgpu_sriov_vf(adev) + && (virt->reg_access_mode & AMDGPU_VIRT_REG_SKIP_SEETING)) + ret = true; + + return ret; +}
diff --git a/amdgpu/amdgpu_virt.h b/amdgpu/amdgpu_virt.h new file mode 100644 index 0000000..f510773 --- /dev/null +++ b/amdgpu/amdgpu_virt.h
@@ -0,0 +1,324 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Monk.liu@amd.com + */ +#ifndef AMDGPU_VIRT_H +#define AMDGPU_VIRT_H + +#define AMDGPU_SRIOV_CAPS_SRIOV_VBIOS (1 << 0) /* vBIOS is sr-iov ready */ +#define AMDGPU_SRIOV_CAPS_ENABLE_IOV (1 << 1) /* sr-iov is enabled on this GPU */ +#define AMDGPU_SRIOV_CAPS_IS_VF (1 << 2) /* this GPU is a virtual function */ +#define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU is pass through for VM */ +#define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full access mode */ + +struct amdgpu_mm_table { + struct amdgpu_bo *bo; + uint32_t *cpu_addr; + uint64_t gpu_addr; +}; + +#define AMDGPU_VF_ERROR_ENTRY_SIZE 16 + +/* struct error_entry - amdgpu VF error information. */ +struct amdgpu_vf_error_buffer { + struct mutex lock; + int read_count; + int write_count; + uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE]; + uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE]; + uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE]; +}; + +/* According to the fw feature, some new reg access modes are supported */ +#define AMDGPU_VIRT_REG_ACCESS_LEGACY (1 << 0) /* directly mmio */ +#define AMDGPU_VIRT_REG_ACCESS_PSP_PRG_IH (1 << 1) /* by PSP */ +#define AMDGPU_VIRT_REG_ACCESS_RLC (1 << 2) /* by RLC */ +#define AMDGPU_VIRT_REG_SKIP_SEETING (1 << 3) /* Skip setting reg */ + +/** + * struct amdgpu_virt_ops - amdgpu device virt operations + */ +struct amdgpu_virt_ops { + int (*req_full_gpu)(struct amdgpu_device *adev, bool init); + int (*rel_full_gpu)(struct amdgpu_device *adev, bool init); + int (*reset_gpu)(struct amdgpu_device *adev); + int (*wait_reset)(struct amdgpu_device *adev); + void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3); + int (*get_pp_clk)(struct amdgpu_device *adev, u32 type, char *buf); + int (*force_dpm_level)(struct amdgpu_device *adev, u32 level); + void (*init_reg_access_mode)(struct amdgpu_device *adev); +}; + +/* + * Firmware Reserve Frame buffer + */ +struct amdgpu_virt_fw_reserve { + struct amd_sriov_msg_pf2vf_info_header *p_pf2vf; + struct amd_sriov_msg_vf2pf_info_header *p_vf2pf; + unsigned int checksum_key; +}; +/* + * Defination between PF and VF + * Structures forcibly aligned to 4 to keep the same style as PF. + */ +#define AMDGIM_DATAEXCHANGE_OFFSET (64 * 1024) + +#define AMDGIM_GET_STRUCTURE_RESERVED_SIZE(total, u8, u16, u32, u64) \ + (total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2)) + +enum AMDGIM_FEATURE_FLAG { + /* GIM supports feature of Error log collecting */ + AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1, + /* GIM supports feature of loading uCodes */ + AMDGIM_FEATURE_GIM_LOAD_UCODES = 0x2, + /* VRAM LOST by GIM */ + AMDGIM_FEATURE_GIM_FLR_VRAMLOST = 0x4, + /* HW PERF SIM in GIM */ + AMDGIM_FEATURE_HW_PERF_SIMULATION = (1 << 3), +}; + +struct amd_sriov_msg_pf2vf_info_header { + /* the total structure size in byte. */ + uint32_t size; + /* version of this structure, written by the GIM */ + uint32_t version; + /* reserved */ + uint32_t reserved[2]; +} __aligned(4); +struct amdgim_pf2vf_info_v1 { + /* header contains size and version */ + struct amd_sriov_msg_pf2vf_info_header header; + /* max_width * max_height */ + unsigned int uvd_enc_max_pixels_count; + /* 16x16 pixels/sec, codec independent */ + unsigned int uvd_enc_max_bandwidth; + /* max_width * max_height */ + unsigned int vce_enc_max_pixels_count; + /* 16x16 pixels/sec, codec independent */ + unsigned int vce_enc_max_bandwidth; + /* MEC FW position in kb from the start of visible frame buffer */ + unsigned int mecfw_kboffset; + /* The features flags of the GIM driver supports. */ + unsigned int feature_flags; + /* use private key from mailbox 2 to create chueksum */ + unsigned int checksum; +} __aligned(4); + +struct amdgim_pf2vf_info_v2 { + /* header contains size and version */ + struct amd_sriov_msg_pf2vf_info_header header; + /* use private key from mailbox 2 to create chueksum */ + uint32_t checksum; + /* The features flags of the GIM driver supports. */ + uint32_t feature_flags; + /* max_width * max_height */ + uint32_t uvd_enc_max_pixels_count; + /* 16x16 pixels/sec, codec independent */ + uint32_t uvd_enc_max_bandwidth; + /* max_width * max_height */ + uint32_t vce_enc_max_pixels_count; + /* 16x16 pixels/sec, codec independent */ + uint32_t vce_enc_max_bandwidth; + /* MEC FW position in kb from the start of VF visible frame buffer */ + uint64_t mecfw_kboffset; + /* MEC FW size in KB */ + uint32_t mecfw_ksize; + /* UVD FW position in kb from the start of VF visible frame buffer */ + uint64_t uvdfw_kboffset; + /* UVD FW size in KB */ + uint32_t uvdfw_ksize; + /* VCE FW position in kb from the start of VF visible frame buffer */ + uint64_t vcefw_kboffset; + /* VCE FW size in KB */ + uint32_t vcefw_ksize; + uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 + sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)]; +} __aligned(4); + + +struct amd_sriov_msg_vf2pf_info_header { + /* the total structure size in byte. */ + uint32_t size; + /*version of this structure, written by the guest */ + uint32_t version; + /* reserved */ + uint32_t reserved[2]; +} __aligned(4); + +struct amdgim_vf2pf_info_v1 { + /* header contains size and version */ + struct amd_sriov_msg_vf2pf_info_header header; + /* driver version */ + char driver_version[64]; + /* driver certification, 1=WHQL, 0=None */ + unsigned int driver_cert; + /* guest OS type and version: need a define */ + unsigned int os_info; + /* in the unit of 1M */ + unsigned int fb_usage; + /* guest gfx engine usage percentage */ + unsigned int gfx_usage; + /* guest gfx engine health percentage */ + unsigned int gfx_health; + /* guest compute engine usage percentage */ + unsigned int compute_usage; + /* guest compute engine health percentage */ + unsigned int compute_health; + /* guest vce engine usage percentage. 0xffff means N/A. */ + unsigned int vce_enc_usage; + /* guest vce engine health percentage. 0xffff means N/A. */ + unsigned int vce_enc_health; + /* guest uvd engine usage percentage. 0xffff means N/A. */ + unsigned int uvd_enc_usage; + /* guest uvd engine usage percentage. 0xffff means N/A. */ + unsigned int uvd_enc_health; + unsigned int checksum; +} __aligned(4); + +struct amdgim_vf2pf_info_v2 { + /* header contains size and version */ + struct amd_sriov_msg_vf2pf_info_header header; + uint32_t checksum; + /* driver version */ + uint8_t driver_version[64]; + /* driver certification, 1=WHQL, 0=None */ + uint32_t driver_cert; + /* guest OS type and version: need a define */ + uint32_t os_info; + /* in the unit of 1M */ + uint32_t fb_usage; + /* guest gfx engine usage percentage */ + uint32_t gfx_usage; + /* guest gfx engine health percentage */ + uint32_t gfx_health; + /* guest compute engine usage percentage */ + uint32_t compute_usage; + /* guest compute engine health percentage */ + uint32_t compute_health; + /* guest vce engine usage percentage. 0xffff means N/A. */ + uint32_t vce_enc_usage; + /* guest vce engine health percentage. 0xffff means N/A. */ + uint32_t vce_enc_health; + /* guest uvd engine usage percentage. 0xffff means N/A. */ + uint32_t uvd_enc_usage; + /* guest uvd engine usage percentage. 0xffff means N/A. */ + uint32_t uvd_enc_health; + uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 64, 0, (12 + sizeof(struct amd_sriov_msg_vf2pf_info_header)/sizeof(uint32_t)), 0)]; +} __aligned(4); + +#define AMDGPU_FW_VRAM_VF2PF_VER 2 +typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; + +#define AMDGPU_FW_VRAM_VF2PF_WRITE(adev, field, val) \ + do { \ + ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field = (val); \ + } while (0) + +#define AMDGPU_FW_VRAM_VF2PF_READ(adev, field, val) \ + do { \ + (*val) = ((amdgim_vf2pf_info *)adev->virt.fw_reserve.p_vf2pf)->field; \ + } while (0) + +#define AMDGPU_FW_VRAM_PF2VF_READ(adev, field, val) \ + do { \ + if (!adev->virt.fw_reserve.p_pf2vf) \ + *(val) = 0; \ + else { \ + if (adev->virt.fw_reserve.p_pf2vf->version == 1) \ + *(val) = ((struct amdgim_pf2vf_info_v1 *)adev->virt.fw_reserve.p_pf2vf)->field; \ + if (adev->virt.fw_reserve.p_pf2vf->version == 2) \ + *(val) = ((struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf)->field; \ + } \ + } while (0) + +/* GPU virtualization */ +struct amdgpu_virt { + uint32_t caps; + struct amdgpu_bo *csa_obj; + void *csa_cpu_addr; + bool chained_ib_support; + uint32_t reg_val_offs; + struct amdgpu_irq_src ack_irq; + struct amdgpu_irq_src rcv_irq; + struct work_struct flr_work; + struct amdgpu_mm_table mm_table; + const struct amdgpu_virt_ops *ops; + struct amdgpu_vf_error_buffer vf_errors; + struct amdgpu_virt_fw_reserve fw_reserve; + uint32_t gim_feature; + /* protect DPM events to GIM */ + struct mutex dpm_mutex; + uint32_t reg_access_mode; +}; + +#define amdgpu_sriov_enabled(adev) \ +((adev)->virt.caps & AMDGPU_SRIOV_CAPS_ENABLE_IOV) + +#define amdgpu_sriov_vf(adev) \ +((adev)->virt.caps & AMDGPU_SRIOV_CAPS_IS_VF) + +#define amdgpu_sriov_bios(adev) \ +((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS) + +#define amdgpu_sriov_runtime(adev) \ +((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME) + +#define amdgpu_passthrough(adev) \ +((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE) + +static inline bool is_virtual_machine(void) +{ +#ifdef CONFIG_X86 + return boot_cpu_has(X86_FEATURE_HYPERVISOR); +#else + return false; +#endif +} + +#define amdgim_is_hwperf(adev) \ + ((adev)->virt.gim_feature & AMDGIM_FEATURE_HW_PERF_SIMULATION) + +bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev); +void amdgpu_virt_init_setting(struct amdgpu_device *adev); +uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); +void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v); +void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, + uint32_t reg0, uint32_t rreg1, + uint32_t ref, uint32_t mask); +int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init); +int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init); +int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); +int amdgpu_virt_wait_reset(struct amdgpu_device *adev); +int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev); +void amdgpu_virt_free_mm_table(struct amdgpu_device *adev); +int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size, + unsigned int key, + unsigned int chksum); +void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev); +uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest); +uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest); + +void amdgpu_virt_init_reg_access_mode(struct amdgpu_device *adev); +bool amdgpu_virt_support_psp_prg_ih_reg(struct amdgpu_device *adev); +bool amdgpu_virt_support_rlc_prg_reg(struct amdgpu_device *adev); +bool amdgpu_virt_support_skip_setting(struct amdgpu_device *adev); + +#endif
diff --git a/amdgpu/amdgpu_vm.c b/amdgpu/amdgpu_vm.c new file mode 100644 index 0000000..24c3c05 --- /dev/null +++ b/amdgpu/amdgpu_vm.c
@@ -0,0 +1,3115 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#include <linux/dma-fence-array.h> +#include <linux/interval_tree_generic.h> +#include <linux/idr.h> + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_gmc.h" +#include "amdgpu_xgmi.h" + +/** + * DOC: GPUVM + * + * GPUVM is similar to the legacy gart on older asics, however + * rather than there being a single global gart table + * for the entire GPU, there are multiple VM page tables active + * at any given time. The VM page tables can contain a mix + * vram pages and system memory pages and system memory pages + * can be mapped as snooped (cached system pages) or unsnooped + * (uncached system pages). + * Each VM has an ID associated with it and there is a page table + * associated with each VMID. When execting a command buffer, + * the kernel tells the the ring what VMID to use for that command + * buffer. VMIDs are allocated dynamically as commands are submitted. + * The userspace drivers maintain their own address space and the kernel + * sets up their pages tables accordingly when they submit their + * command buffers and a VMID is assigned. + * Cayman/Trinity support up to 8 active VMs at any given time; + * SI supports 16. + */ + +#define START(node) ((node)->start) +#define LAST(node) ((node)->last) + +INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, + START, LAST, static, amdgpu_vm_it) + +#undef START +#undef LAST + +/** + * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback + */ +struct amdgpu_prt_cb { + + /** + * @adev: amdgpu device + */ + struct amdgpu_device *adev; + + /** + * @cb: callback + */ + struct dma_fence_cb cb; +}; + +/** + * amdgpu_vm_level_shift - return the addr shift for each level + * + * @adev: amdgpu_device pointer + * @level: VMPT level + * + * Returns: + * The number of bits the pfn needs to be right shifted for a level. + */ +static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, + unsigned level) +{ + unsigned shift = 0xff; + + switch (level) { + case AMDGPU_VM_PDB2: + case AMDGPU_VM_PDB1: + case AMDGPU_VM_PDB0: + shift = 9 * (AMDGPU_VM_PDB0 - level) + + adev->vm_manager.block_size; + break; + case AMDGPU_VM_PTB: + shift = 0; + break; + default: + dev_err(adev->dev, "the level%d isn't supported.\n", level); + } + + return shift; +} + +/** + * amdgpu_vm_num_entries - return the number of entries in a PD/PT + * + * @adev: amdgpu_device pointer + * @level: VMPT level + * + * Returns: + * The number of entries in a page directory or page table. + */ +static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, + unsigned level) +{ + unsigned shift = amdgpu_vm_level_shift(adev, + adev->vm_manager.root_level); + + if (level == adev->vm_manager.root_level) + /* For the root directory */ + return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift; + else if (level != AMDGPU_VM_PTB) + /* Everything in between */ + return 512; + else + /* For the page tables on the leaves */ + return AMDGPU_VM_PTE_COUNT(adev); +} + +/** + * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD + * + * @adev: amdgpu_device pointer + * + * Returns: + * The number of entries in the root page directory which needs the ATS setting. + */ +static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev) +{ + unsigned shift; + + shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level); + return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); +} + +/** + * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT + * + * @adev: amdgpu_device pointer + * @level: VMPT level + * + * Returns: + * The mask to extract the entry number of a PD/PT from an address. + */ +static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev, + unsigned int level) +{ + if (level <= adev->vm_manager.root_level) + return 0xffffffff; + else if (level != AMDGPU_VM_PTB) + return 0x1ff; + else + return AMDGPU_VM_PTE_COUNT(adev) - 1; +} + +/** + * amdgpu_vm_bo_size - returns the size of the BOs in bytes + * + * @adev: amdgpu_device pointer + * @level: VMPT level + * + * Returns: + * The size of the BO for a page directory or page table in bytes. + */ +static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) +{ + return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); +} + +/** + * amdgpu_vm_bo_evicted - vm_bo is evicted + * + * @vm_bo: vm_bo which is evicted + * + * State for PDs/PTs and per VM BOs which are not at the location they should + * be. + */ +static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) +{ + struct amdgpu_vm *vm = vm_bo->vm; + struct amdgpu_bo *bo = vm_bo->bo; + + vm_bo->moved = true; + if (bo->tbo.type == ttm_bo_type_kernel) + list_move(&vm_bo->vm_status, &vm->evicted); + else + list_move_tail(&vm_bo->vm_status, &vm->evicted); +} + +/** + * amdgpu_vm_bo_relocated - vm_bo is reloacted + * + * @vm_bo: vm_bo which is relocated + * + * State for PDs/PTs which needs to update their parent PD. + */ +static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) +{ + list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); +} + +/** + * amdgpu_vm_bo_moved - vm_bo is moved + * + * @vm_bo: vm_bo which is moved + * + * State for per VM BOs which are moved, but that change is not yet reflected + * in the page tables. + */ +static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) +{ + list_move(&vm_bo->vm_status, &vm_bo->vm->moved); +} + +/** + * amdgpu_vm_bo_idle - vm_bo is idle + * + * @vm_bo: vm_bo which is now idle + * + * State for PDs/PTs and per VM BOs which have gone through the state machine + * and are now idle. + */ +static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) +{ + list_move(&vm_bo->vm_status, &vm_bo->vm->idle); + vm_bo->moved = false; +} + +/** + * amdgpu_vm_bo_invalidated - vm_bo is invalidated + * + * @vm_bo: vm_bo which is now invalidated + * + * State for normal BOs which are invalidated and that change not yet reflected + * in the PTs. + */ +static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) +{ + spin_lock(&vm_bo->vm->invalidated_lock); + list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); + spin_unlock(&vm_bo->vm->invalidated_lock); +} + +/** + * amdgpu_vm_bo_done - vm_bo is done + * + * @vm_bo: vm_bo which is now done + * + * State for normal BOs which are invalidated and that change has been updated + * in the PTs. + */ +static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) +{ + spin_lock(&vm_bo->vm->invalidated_lock); + list_del_init(&vm_bo->vm_status); + spin_unlock(&vm_bo->vm->invalidated_lock); +} + +/** + * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm + * + * @base: base structure for tracking BO usage in a VM + * @vm: vm to which bo is to be added + * @bo: amdgpu buffer object + * + * Initialize a bo_va_base structure and add it to the appropriate lists + * + */ +static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, + struct amdgpu_vm *vm, + struct amdgpu_bo *bo) +{ + base->vm = vm; + base->bo = bo; + base->next = NULL; + INIT_LIST_HEAD(&base->vm_status); + + if (!bo) + return; + base->next = bo->vm_bo; + bo->vm_bo = base; + + if (bo->tbo.resv != vm->root.base.bo->tbo.resv) + return; + + vm->bulk_moveable = false; + if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) + amdgpu_vm_bo_relocated(base); + else + amdgpu_vm_bo_idle(base); + + if (bo->preferred_domains & + amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)) + return; + + /* + * we checked all the prerequisites, but it looks like this per vm bo + * is currently evicted. add the bo to the evicted list to make sure it + * is validated on next vm use to avoid fault. + * */ + amdgpu_vm_bo_evicted(base); +} + +/** + * amdgpu_vm_pt_parent - get the parent page directory + * + * @pt: child page table + * + * Helper to get the parent entry for the child page table. NULL if we are at + * the root page directory. + */ +static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt) +{ + struct amdgpu_bo *parent = pt->base.bo->parent; + + if (!parent) + return NULL; + + return container_of(parent->vm_bo, struct amdgpu_vm_pt, base); +} + +/** + * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt + */ +struct amdgpu_vm_pt_cursor { + uint64_t pfn; + struct amdgpu_vm_pt *parent; + struct amdgpu_vm_pt *entry; + unsigned level; +}; + +/** + * amdgpu_vm_pt_start - start PD/PT walk + * + * @adev: amdgpu_device pointer + * @vm: amdgpu_vm structure + * @start: start address of the walk + * @cursor: state to initialize + * + * Initialize a amdgpu_vm_pt_cursor to start a walk. + */ +static void amdgpu_vm_pt_start(struct amdgpu_device *adev, + struct amdgpu_vm *vm, uint64_t start, + struct amdgpu_vm_pt_cursor *cursor) +{ + cursor->pfn = start; + cursor->parent = NULL; + cursor->entry = &vm->root; + cursor->level = adev->vm_manager.root_level; +} + +/** + * amdgpu_vm_pt_descendant - go to child node + * + * @adev: amdgpu_device pointer + * @cursor: current state + * + * Walk to the child node of the current node. + * Returns: + * True if the walk was possible, false otherwise. + */ +static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, + struct amdgpu_vm_pt_cursor *cursor) +{ + unsigned mask, shift, idx; + + if (!cursor->entry->entries) + return false; + + BUG_ON(!cursor->entry->base.bo); + mask = amdgpu_vm_entries_mask(adev, cursor->level); + shift = amdgpu_vm_level_shift(adev, cursor->level); + + ++cursor->level; + idx = (cursor->pfn >> shift) & mask; + cursor->parent = cursor->entry; + cursor->entry = &cursor->entry->entries[idx]; + return true; +} + +/** + * amdgpu_vm_pt_sibling - go to sibling node + * + * @adev: amdgpu_device pointer + * @cursor: current state + * + * Walk to the sibling node of the current node. + * Returns: + * True if the walk was possible, false otherwise. + */ +static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, + struct amdgpu_vm_pt_cursor *cursor) +{ + unsigned shift, num_entries; + + /* Root doesn't have a sibling */ + if (!cursor->parent) + return false; + + /* Go to our parents and see if we got a sibling */ + shift = amdgpu_vm_level_shift(adev, cursor->level - 1); + num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1); + + if (cursor->entry == &cursor->parent->entries[num_entries - 1]) + return false; + + cursor->pfn += 1ULL << shift; + cursor->pfn &= ~((1ULL << shift) - 1); + ++cursor->entry; + return true; +} + +/** + * amdgpu_vm_pt_ancestor - go to parent node + * + * @cursor: current state + * + * Walk to the parent node of the current node. + * Returns: + * True if the walk was possible, false otherwise. + */ +static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) +{ + if (!cursor->parent) + return false; + + --cursor->level; + cursor->entry = cursor->parent; + cursor->parent = amdgpu_vm_pt_parent(cursor->parent); + return true; +} + +/** + * amdgpu_vm_pt_next - get next PD/PT in hieratchy + * + * @adev: amdgpu_device pointer + * @cursor: current state + * + * Walk the PD/PT tree to the next node. + */ +static void amdgpu_vm_pt_next(struct amdgpu_device *adev, + struct amdgpu_vm_pt_cursor *cursor) +{ + /* First try a newborn child */ + if (amdgpu_vm_pt_descendant(adev, cursor)) + return; + + /* If that didn't worked try to find a sibling */ + while (!amdgpu_vm_pt_sibling(adev, cursor)) { + /* No sibling, go to our parents and grandparents */ + if (!amdgpu_vm_pt_ancestor(cursor)) { + cursor->pfn = ~0ll; + return; + } + } +} + +/** + * amdgpu_vm_pt_first_dfs - start a deep first search + * + * @adev: amdgpu_device structure + * @vm: amdgpu_vm structure + * @cursor: state to initialize + * + * Starts a deep first traversal of the PD/PT tree. + */ +static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_vm_pt_cursor *start, + struct amdgpu_vm_pt_cursor *cursor) +{ + if (start) + *cursor = *start; + else + amdgpu_vm_pt_start(adev, vm, 0, cursor); + while (amdgpu_vm_pt_descendant(adev, cursor)); +} + +/** + * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue + * + * @start: starting point for the search + * @entry: current entry + * + * Returns: + * True when the search should continue, false otherwise. + */ +static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, + struct amdgpu_vm_pt *entry) +{ + return entry && (!start || entry != start->entry); +} + +/** + * amdgpu_vm_pt_next_dfs - get the next node for a deep first search + * + * @adev: amdgpu_device structure + * @cursor: current state + * + * Move the cursor to the next node in a deep first search. + */ +static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, + struct amdgpu_vm_pt_cursor *cursor) +{ + if (!cursor->entry) + return; + + if (!cursor->parent) + cursor->entry = NULL; + else if (amdgpu_vm_pt_sibling(adev, cursor)) + while (amdgpu_vm_pt_descendant(adev, cursor)); + else + amdgpu_vm_pt_ancestor(cursor); +} + +/** + * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs + */ +#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ + for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ + (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ + amdgpu_vm_pt_continue_dfs((start), (entry)); \ + (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) + +/** + * amdgpu_vm_get_pd_bo - add the VM PD to a validation list + * + * @vm: vm providing the BOs + * @validated: head of validation list + * @entry: entry to add + * + * Add the page directory to the list of BOs to + * validate for command submission. + */ +void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, + struct list_head *validated, + struct amdgpu_bo_list_entry *entry) +{ + entry->priority = 0; + entry->tv.bo = &vm->root.base.bo->tbo; + /* One for the VM updates, one for TTM and one for the CS job */ + entry->tv.num_shared = 3; + entry->user_pages = NULL; + list_add(&entry->tv.head, validated); +} + +void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo) +{ + struct amdgpu_bo *abo; + struct amdgpu_vm_bo_base *bo_base; + + if (!amdgpu_bo_is_amdgpu_bo(bo)) + return; + + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) + return; + + abo = ttm_to_amdgpu_bo(bo); + if (!abo->parent) + return; + for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) { + struct amdgpu_vm *vm = bo_base->vm; + + if (abo->tbo.resv == vm->root.base.bo->tbo.resv) + vm->bulk_moveable = false; + } + +} +/** + * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU + * + * @adev: amdgpu device pointer + * @vm: vm providing the BOs + * + * Move all BOs to the end of LRU and remember their positions to put them + * together. + */ +void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct ttm_bo_global *glob = adev->mman.bdev.glob; + struct amdgpu_vm_bo_base *bo_base; + +#if 0 + if (vm->bulk_moveable) { + spin_lock(&glob->lru_lock); + ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); + spin_unlock(&glob->lru_lock); + return; + } +#endif + + memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); + + spin_lock(&glob->lru_lock); + list_for_each_entry(bo_base, &vm->idle, vm_status) { + struct amdgpu_bo *bo = bo_base->bo; + + if (!bo->parent) + continue; + + ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); + if (bo->shadow) + ttm_bo_move_to_lru_tail(&bo->shadow->tbo, + &vm->lru_bulk_move); + } + spin_unlock(&glob->lru_lock); + + vm->bulk_moveable = true; +} + +/** + * amdgpu_vm_validate_pt_bos - validate the page table BOs + * + * @adev: amdgpu device pointer + * @vm: vm providing the BOs + * @validate: callback to do the validation + * @param: parameter for the validation callback + * + * Validate the page table BOs on command submission if neccessary. + * + * Returns: + * Validation result. + */ +int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, + int (*validate)(void *p, struct amdgpu_bo *bo), + void *param) +{ + struct amdgpu_vm_bo_base *bo_base, *tmp; + int r = 0; + + vm->bulk_moveable &= list_empty(&vm->evicted); + + list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { + struct amdgpu_bo *bo = bo_base->bo; + + r = validate(param, bo); + if (r) + break; + + if (bo->tbo.type != ttm_bo_type_kernel) { + amdgpu_vm_bo_moved(bo_base); + } else { + vm->update_funcs->map_table(bo); + if (bo->parent) + amdgpu_vm_bo_relocated(bo_base); + else + amdgpu_vm_bo_idle(bo_base); + } + } + + return r; +} + +/** + * amdgpu_vm_ready - check VM is ready for updates + * + * @vm: VM to check + * + * Check if all VM PDs/PTs are ready for updates + * + * Returns: + * True if eviction list is empty. + */ +bool amdgpu_vm_ready(struct amdgpu_vm *vm) +{ + return list_empty(&vm->evicted); +} + +/** + * amdgpu_vm_clear_bo - initially clear the PDs/PTs + * + * @adev: amdgpu_device pointer + * @vm: VM to clear BO from + * @bo: BO to clear + * + * Root PD needs to be reserved when calling this. + * + * Returns: + * 0 on success, errno otherwise. + */ +static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo *bo) +{ + struct ttm_operation_ctx ctx = { true, false }; + unsigned level = adev->vm_manager.root_level; + struct amdgpu_vm_update_params params; + struct amdgpu_bo *ancestor = bo; + unsigned entries, ats_entries; + uint64_t addr; + int r; + + /* Figure out our place in the hierarchy */ + if (ancestor->parent) { + ++level; + while (ancestor->parent->parent) { + ++level; + ancestor = ancestor->parent; + } + } + + entries = amdgpu_bo_size(bo) / 8; + if (!vm->pte_support_ats) { + ats_entries = 0; + + } else if (!bo->parent) { + ats_entries = amdgpu_vm_num_ats_entries(adev); + ats_entries = min(ats_entries, entries); + entries -= ats_entries; + + } else { + struct amdgpu_vm_pt *pt; + + pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base); + ats_entries = amdgpu_vm_num_ats_entries(adev); + if ((pt - vm->root.entries) >= ats_entries) { + ats_entries = 0; + } else { + ats_entries = entries; + entries = 0; + } + } + + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (r) + return r; + + if (bo->shadow) { + r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement, + &ctx); + if (r) + return r; + } + + r = vm->update_funcs->map_table(bo); + if (r) + return r; + + memset(¶ms, 0, sizeof(params)); + params.adev = adev; + params.vm = vm; + + r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_KFD, NULL); + if (r) + return r; + + addr = 0; + if (ats_entries) { + uint64_t value = 0, flags; + + flags = AMDGPU_PTE_DEFAULT_ATC; + if (level != AMDGPU_VM_PTB) { + /* Handle leaf PDEs as PTEs */ + flags |= AMDGPU_PDE_PTE; + amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); + } + + r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries, + value, flags); + if (r) + return r; + + addr += ats_entries * 8; + } + + if (entries) { + uint64_t value = 0, flags = 0; + + if (adev->asic_type >= CHIP_VEGA10) { + if (level != AMDGPU_VM_PTB) { + /* Handle leaf PDEs as PTEs */ + flags |= AMDGPU_PDE_PTE; + amdgpu_gmc_get_vm_pde(adev, level, + &value, &flags); + } else { + /* Workaround for fault priority problem on GMC9 */ + flags = AMDGPU_PTE_EXECUTABLE; + } + } + + r = vm->update_funcs->update(¶ms, bo, addr, 0, entries, + value, flags); + if (r) + return r; + } + + return vm->update_funcs->commit(¶ms, NULL); +} + +/** + * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation + * + * @adev: amdgpu_device pointer + * @vm: requesting vm + * @bp: resulting BO allocation parameters + */ +static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm, + int level, struct amdgpu_bo_param *bp) +{ + memset(bp, 0, sizeof(*bp)); + + bp->size = amdgpu_vm_bo_size(adev, level); + bp->byte_align = AMDGPU_GPU_PAGE_SIZE; + bp->domain = AMDGPU_GEM_DOMAIN_VRAM; + bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain); + bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | + AMDGPU_GEM_CREATE_CPU_GTT_USWC; + if (vm->use_cpu_for_update) + bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + else if (!vm->root.base.bo || vm->root.base.bo->shadow) + bp->flags |= AMDGPU_GEM_CREATE_SHADOW; + bp->type = ttm_bo_type_kernel; + if (vm->root.base.bo) + bp->resv = vm->root.base.bo->tbo.resv; +} + +/** + * amdgpu_vm_alloc_pts - Allocate a specific page table + * + * @adev: amdgpu_device pointer + * @vm: VM to allocate page tables for + * @cursor: Which page table to allocate + * + * Make sure a specific page table or directory is allocated. + * + * Returns: + * 1 if page table needed to be allocated, 0 if page table was already + * allocated, negative errno if an error occurred. + */ +static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_vm_pt_cursor *cursor) +{ + struct amdgpu_vm_pt *entry = cursor->entry; + struct amdgpu_bo_param bp; + struct amdgpu_bo *pt; + int r; + + if (cursor->level < AMDGPU_VM_PTB && !entry->entries) { + unsigned num_entries; + + num_entries = amdgpu_vm_num_entries(adev, cursor->level); + entry->entries = kvmalloc_array(num_entries, + sizeof(*entry->entries), + GFP_KERNEL | __GFP_ZERO); + if (!entry->entries) + return -ENOMEM; + } + + if (entry->base.bo) + return 0; + + amdgpu_vm_bo_param(adev, vm, cursor->level, &bp); + + r = amdgpu_bo_create(adev, &bp, &pt); + if (r) + return r; + + /* Keep a reference to the root directory to avoid + * freeing them up in the wrong order. + */ + pt->parent = amdgpu_bo_ref(cursor->parent->base.bo); + amdgpu_vm_bo_base_init(&entry->base, vm, pt); + + r = amdgpu_vm_clear_bo(adev, vm, pt); + if (r) + goto error_free_pt; + + return 0; + +error_free_pt: + amdgpu_bo_unref(&pt->shadow); + amdgpu_bo_unref(&pt); + return r; +} + +/** + * amdgpu_vm_free_table - fre one PD/PT + * + * @entry: PDE to free + */ +static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry) +{ + if (entry->base.bo) { + entry->base.bo->vm_bo = NULL; + list_del(&entry->base.vm_status); + amdgpu_bo_unref(&entry->base.bo->shadow); + amdgpu_bo_unref(&entry->base.bo); + } + kvfree(entry->entries); + entry->entries = NULL; +} + +/** + * amdgpu_vm_free_pts - free PD/PT levels + * + * @adev: amdgpu device structure + * @vm: amdgpu vm structure + * @start: optional cursor where to start freeing PDs/PTs + * + * Free the page directory or page table level and all sub levels. + */ +static void amdgpu_vm_free_pts(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_vm_pt_cursor *start) +{ + struct amdgpu_vm_pt_cursor cursor; + struct amdgpu_vm_pt *entry; + + vm->bulk_moveable = false; + + for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) + amdgpu_vm_free_table(entry); + + if (start) + amdgpu_vm_free_table(start->entry); +} + +/** + * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug + * + * @adev: amdgpu_device pointer + */ +void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) +{ + const struct amdgpu_ip_block *ip_block; + bool has_compute_vm_bug; + struct amdgpu_ring *ring; + int i; + + has_compute_vm_bug = false; + + ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); + if (ip_block) { + /* Compute has a VM bug for GFX version < 7. + Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ + if (ip_block->version->major <= 7) + has_compute_vm_bug = true; + else if (ip_block->version->major == 8) + if (adev->gfx.mec_fw_version < 673) + has_compute_vm_bug = true; + } + + for (i = 0; i < adev->num_rings; i++) { + ring = adev->rings[i]; + if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) + /* only compute rings */ + ring->has_compute_vm_bug = has_compute_vm_bug; + else + ring->has_compute_vm_bug = false; + } +} + +/** + * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. + * + * @ring: ring on which the job will be submitted + * @job: job to submit + * + * Returns: + * True if sync is needed. + */ +bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, + struct amdgpu_job *job) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + struct amdgpu_vmid *id; + bool gds_switch_needed; + bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; + + if (job->vmid == 0) + return false; + id = &id_mgr->ids[job->vmid]; + gds_switch_needed = ring->funcs->emit_gds_switch && ( + id->gds_base != job->gds_base || + id->gds_size != job->gds_size || + id->gws_base != job->gws_base || + id->gws_size != job->gws_size || + id->oa_base != job->oa_base || + id->oa_size != job->oa_size); + + if (amdgpu_vmid_had_gpu_reset(adev, id)) + return true; + + return vm_flush_needed || gds_switch_needed; +} + +/** + * amdgpu_vm_flush - hardware flush the vm + * + * @ring: ring to use for flush + * @job: related job + * @need_pipe_sync: is pipe sync needed + * + * Emit a VM flush when it is necessary. + * + * Returns: + * 0 on success, errno otherwise. + */ +int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) +{ + struct amdgpu_device *adev = ring->adev; + unsigned vmhub = ring->funcs->vmhub; + struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; + struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; + bool gds_switch_needed = ring->funcs->emit_gds_switch && ( + id->gds_base != job->gds_base || + id->gds_size != job->gds_size || + id->gws_base != job->gws_base || + id->gws_size != job->gws_size || + id->oa_base != job->oa_base || + id->oa_size != job->oa_size); + bool vm_flush_needed = job->vm_needs_flush; + bool pasid_mapping_needed = id->pasid != job->pasid || + !id->pasid_mapping || + !dma_fence_is_signaled(id->pasid_mapping); + struct dma_fence *fence = NULL; + unsigned patch_offset = 0; + int r; + + if (amdgpu_vmid_had_gpu_reset(adev, id)) { + gds_switch_needed = true; + vm_flush_needed = true; + pasid_mapping_needed = true; + } + + gds_switch_needed &= !!ring->funcs->emit_gds_switch; + vm_flush_needed &= !!ring->funcs->emit_vm_flush && + job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; + pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && + ring->funcs->emit_wreg; + + if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) + return 0; + + if (ring->funcs->init_cond_exec) + patch_offset = amdgpu_ring_init_cond_exec(ring); + + if (need_pipe_sync) + amdgpu_ring_emit_pipeline_sync(ring); + + if (vm_flush_needed) { + trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); + amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); + } + + if (pasid_mapping_needed) + amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); + + if (vm_flush_needed || pasid_mapping_needed) { + r = amdgpu_fence_emit(ring, &fence, 0); + if (r) + return r; + } + + if (vm_flush_needed) { + mutex_lock(&id_mgr->lock); + dma_fence_put(id->last_flush); + id->last_flush = dma_fence_get(fence); + id->current_gpu_reset_count = + atomic_read(&adev->gpu_reset_counter); + mutex_unlock(&id_mgr->lock); + } + + if (pasid_mapping_needed) { + id->pasid = job->pasid; + dma_fence_put(id->pasid_mapping); + id->pasid_mapping = dma_fence_get(fence); + } + dma_fence_put(fence); + + if (ring->funcs->emit_gds_switch && gds_switch_needed) { + id->gds_base = job->gds_base; + id->gds_size = job->gds_size; + id->gws_base = job->gws_base; + id->gws_size = job->gws_size; + id->oa_base = job->oa_base; + id->oa_size = job->oa_size; + amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, + job->gds_size, job->gws_base, + job->gws_size, job->oa_base, + job->oa_size); + } + + if (ring->funcs->patch_cond_exec) + amdgpu_ring_patch_cond_exec(ring, patch_offset); + + /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ + if (ring->funcs->emit_switch_buffer) { + amdgpu_ring_emit_switch_buffer(ring); + amdgpu_ring_emit_switch_buffer(ring); + } + return 0; +} + +/** + * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo + * + * @vm: requested vm + * @bo: requested buffer object + * + * Find @bo inside the requested vm. + * Search inside the @bos vm list for the requested vm + * Returns the found bo_va or NULL if none is found + * + * Object has to be reserved! + * + * Returns: + * Found bo_va or NULL. + */ +struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, + struct amdgpu_bo *bo) +{ + struct amdgpu_vm_bo_base *base; + + for (base = bo->vm_bo; base; base = base->next) { + if (base->vm != vm) + continue; + + return container_of(base, struct amdgpu_bo_va, base); + } + return NULL; +} + +/** + * amdgpu_vm_map_gart - Resolve gart mapping of addr + * + * @pages_addr: optional DMA address to use for lookup + * @addr: the unmapped addr + * + * Look up the physical address of the page that the pte resolves + * to. + * + * Returns: + * The pointer for the page table entry. + */ +uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) +{ + uint64_t result; + + /* page table offset */ + result = pages_addr[addr >> PAGE_SHIFT]; + + /* in case cpu page size != gpu page size*/ + result |= addr & (~PAGE_MASK); + + result &= 0xFFFFFFFFFFFFF000ULL; + + return result; +} + +/* + * amdgpu_vm_update_pde - update a single level in the hierarchy + * + * @param: parameters for the update + * @vm: requested vm + * @entry: entry to update + * + * Makes sure the requested entry in parent is up to date. + */ +static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params, + struct amdgpu_vm *vm, + struct amdgpu_vm_pt *entry) +{ + struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry); + struct amdgpu_bo *bo = parent->base.bo, *pbo; + uint64_t pde, pt, flags; + unsigned level; + + for (level = 0, pbo = bo->parent; pbo; ++level) + pbo = pbo->parent; + + level += params->adev->vm_manager.root_level; + amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags); + pde = (entry - parent->entries) * 8; + return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags); +} + +/* + * amdgpu_vm_invalidate_pds - mark all PDs as invalid + * + * @adev: amdgpu_device pointer + * @vm: related vm + * + * Mark all PD level as invalid after an error. + */ +static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct amdgpu_vm_pt_cursor cursor; + struct amdgpu_vm_pt *entry; + + for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) + if (entry->base.bo && !entry->base.moved) + amdgpu_vm_bo_relocated(&entry->base); +} + +/* + * amdgpu_vm_update_directories - make sure that all directories are valid + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * + * Makes sure all directories are up to date. + * + * Returns: + * 0 for success, error for failure. + */ +int amdgpu_vm_update_directories(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct amdgpu_vm_update_params params; + int r; + + if (list_empty(&vm->relocated)) + return 0; + + memset(¶ms, 0, sizeof(params)); + params.adev = adev; + params.vm = vm; + + r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_VM, NULL); + if (r) + return r; + + while (!list_empty(&vm->relocated)) { + struct amdgpu_vm_pt *entry; + + entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt, + base.vm_status); + amdgpu_vm_bo_idle(&entry->base); + + r = amdgpu_vm_update_pde(¶ms, vm, entry); + if (r) + goto error; + } + + r = vm->update_funcs->commit(¶ms, &vm->last_update); + if (r) + goto error; + return 0; + +error: + amdgpu_vm_invalidate_pds(adev, vm); + return r; +} + +/** + * amdgpu_vm_update_flags - figure out flags for PTE updates + * + * Make sure to set the right flags for the PTEs at the desired level. + */ +static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params, + struct amdgpu_bo *bo, unsigned level, + uint64_t pe, uint64_t addr, + unsigned count, uint32_t incr, + uint64_t flags) + +{ + if (level != AMDGPU_VM_PTB) { + flags |= AMDGPU_PDE_PTE; + amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); + + } else if (params->adev->asic_type >= CHIP_VEGA10 && + !(flags & AMDGPU_PTE_VALID) && + !(flags & AMDGPU_PTE_PRT)) { + + /* Workaround for fault priority problem on GMC9 */ + flags |= AMDGPU_PTE_EXECUTABLE; + } + + params->vm->update_funcs->update(params, bo, pe, addr, count, incr, + flags); +} + +/** + * amdgpu_vm_fragment - get fragment for PTEs + * + * @params: see amdgpu_vm_update_params definition + * @start: first PTE to handle + * @end: last PTE to handle + * @flags: hw mapping flags + * @frag: resulting fragment size + * @frag_end: end of this fragment + * + * Returns the first possible fragment for the start and end address. + */ +static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params, + uint64_t start, uint64_t end, uint64_t flags, + unsigned int *frag, uint64_t *frag_end) +{ + /** + * The MC L1 TLB supports variable sized pages, based on a fragment + * field in the PTE. When this field is set to a non-zero value, page + * granularity is increased from 4KB to (1 << (12 + frag)). The PTE + * flags are considered valid for all PTEs within the fragment range + * and corresponding mappings are assumed to be physically contiguous. + * + * The L1 TLB can store a single PTE for the whole fragment, + * significantly increasing the space available for translation + * caching. This leads to large improvements in throughput when the + * TLB is under pressure. + * + * The L2 TLB distributes small and large fragments into two + * asymmetric partitions. The large fragment cache is significantly + * larger. Thus, we try to use large fragments wherever possible. + * Userspace can support this by aligning virtual base address and + * allocation size to the fragment size. + * + * Starting with Vega10 the fragment size only controls the L1. The L2 + * is now directly feed with small/huge/giant pages from the walker. + */ + unsigned max_frag; + + if (params->adev->asic_type < CHIP_VEGA10) + max_frag = params->adev->vm_manager.fragment_size; + else + max_frag = 31; + + /* system pages are non continuously */ + if (params->pages_addr) { + *frag = 0; + *frag_end = end; + return; + } + + /* This intentionally wraps around if no bit is set */ + *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1); + if (*frag >= max_frag) { + *frag = max_frag; + *frag_end = end & ~((1ULL << max_frag) - 1); + } else { + *frag_end = start + (1 << *frag); + } +} + +/** + * amdgpu_vm_update_ptes - make sure that page tables are valid + * + * @params: see amdgpu_vm_update_params definition + * @start: start of GPU address range + * @end: end of GPU address range + * @dst: destination address to map to, the next dst inside the function + * @flags: mapping flags + * + * Update the page tables in the range @start - @end. + * + * Returns: + * 0 for success, -EINVAL for failure. + */ +static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, + uint64_t start, uint64_t end, + uint64_t dst, uint64_t flags) +{ + struct amdgpu_device *adev = params->adev; + struct amdgpu_vm_pt_cursor cursor; + uint64_t frag_start = start, frag_end; + unsigned int frag; + int r; + + /* figure out the initial fragment */ + amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end); + + /* walk over the address space and update the PTs */ + amdgpu_vm_pt_start(adev, params->vm, start, &cursor); + while (cursor.pfn < end) { + unsigned shift, parent_shift, mask; + uint64_t incr, entry_end, pe_start; + struct amdgpu_bo *pt; + + r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor); + if (r) + return r; + + pt = cursor.entry->base.bo; + + /* The root level can't be a huge page */ + if (cursor.level == adev->vm_manager.root_level) { + if (!amdgpu_vm_pt_descendant(adev, &cursor)) + return -ENOENT; + continue; + } + + shift = amdgpu_vm_level_shift(adev, cursor.level); + parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1); + if (adev->asic_type < CHIP_VEGA10 && + (flags & AMDGPU_PTE_VALID)) { + /* No huge page support before GMC v9 */ + if (cursor.level != AMDGPU_VM_PTB) { + if (!amdgpu_vm_pt_descendant(adev, &cursor)) + return -ENOENT; + continue; + } + } else if (frag < shift) { + /* We can't use this level when the fragment size is + * smaller than the address shift. Go to the next + * child entry and try again. + */ + if (!amdgpu_vm_pt_descendant(adev, &cursor)) + return -ENOENT; + continue; + } else if (frag >= parent_shift && + cursor.level - 1 != adev->vm_manager.root_level) { + /* If the fragment size is even larger than the parent + * shift we should go up one level and check it again + * unless one level up is the root level. + */ + if (!amdgpu_vm_pt_ancestor(&cursor)) + return -ENOENT; + continue; + } + + /* Looks good so far, calculate parameters for the update */ + incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; + mask = amdgpu_vm_entries_mask(adev, cursor.level); + pe_start = ((cursor.pfn >> shift) & mask) * 8; + entry_end = (uint64_t)(mask + 1) << shift; + entry_end += cursor.pfn & ~(entry_end - 1); + entry_end = min(entry_end, end); + + do { + uint64_t upd_end = min(entry_end, frag_end); + unsigned nptes = (upd_end - frag_start) >> shift; + + amdgpu_vm_update_flags(params, pt, cursor.level, + pe_start, dst, nptes, incr, + flags | AMDGPU_PTE_FRAG(frag)); + + pe_start += nptes * 8; + dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift; + + frag_start = upd_end; + if (frag_start >= frag_end) { + /* figure out the next fragment */ + amdgpu_vm_fragment(params, frag_start, end, + flags, &frag, &frag_end); + if (frag < shift) + break; + } + } while (frag_start < entry_end); + + if (amdgpu_vm_pt_descendant(adev, &cursor)) { + /* Free all child entries */ + while (cursor.pfn < frag_start) { + amdgpu_vm_free_pts(adev, params->vm, &cursor); + amdgpu_vm_pt_next(adev, &cursor); + } + + } else if (frag >= shift) { + /* or just move on to the next on the same level. */ + amdgpu_vm_pt_next(adev, &cursor); + } + } + + return 0; +} + +/** + * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table + * + * @adev: amdgpu_device pointer + * @exclusive: fence we need to sync to + * @pages_addr: DMA addresses to use for mapping + * @vm: requested vm + * @start: start of mapped range + * @last: last mapped entry + * @flags: flags for the entries + * @addr: addr to set the area to + * @fence: optional resulting fence + * + * Fill in the page table entries between @start and @last. + * + * Returns: + * 0 for success, -EINVAL for failure. + */ +static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, + struct dma_fence *exclusive, + dma_addr_t *pages_addr, + struct amdgpu_vm *vm, + uint64_t start, uint64_t last, + uint64_t flags, uint64_t addr, + struct dma_fence **fence) +{ + struct amdgpu_vm_update_params params; + void *owner = AMDGPU_FENCE_OWNER_VM; + int r; + + memset(¶ms, 0, sizeof(params)); + params.adev = adev; + params.vm = vm; + params.pages_addr = pages_addr; + + /* sync to everything except eviction fences on unmapping */ + if (!(flags & AMDGPU_PTE_VALID)) + owner = AMDGPU_FENCE_OWNER_KFD; + + r = vm->update_funcs->prepare(¶ms, owner, exclusive); + if (r) + return r; + + r = amdgpu_vm_update_ptes(¶ms, start, last + 1, addr, flags); + if (r) + return r; + + return vm->update_funcs->commit(¶ms, fence); +} + +/** + * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks + * + * @adev: amdgpu_device pointer + * @exclusive: fence we need to sync to + * @pages_addr: DMA addresses to use for mapping + * @vm: requested vm + * @mapping: mapped range and flags to use for the update + * @flags: HW flags for the mapping + * @bo_adev: amdgpu_device pointer that bo actually been allocated + * @nodes: array of drm_mm_nodes with the MC addresses + * @fence: optional resulting fence + * + * Split the mapping into smaller chunks so that each update fits + * into a SDMA IB. + * + * Returns: + * 0 for success, -EINVAL for failure. + */ +static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, + struct dma_fence *exclusive, + dma_addr_t *pages_addr, + struct amdgpu_vm *vm, + struct amdgpu_bo_va_mapping *mapping, + uint64_t flags, + struct amdgpu_device *bo_adev, + struct drm_mm_node *nodes, + struct dma_fence **fence) +{ + unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size; + uint64_t pfn, start = mapping->start; + int r; + + /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here + * but in case of something, we filter the flags in first place + */ + if (!(mapping->flags & AMDGPU_PTE_READABLE)) + flags &= ~AMDGPU_PTE_READABLE; + if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) + flags &= ~AMDGPU_PTE_WRITEABLE; + + flags &= ~AMDGPU_PTE_EXECUTABLE; + flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; + + if (adev->asic_type == CHIP_NAVI10) { + flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; + flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); + } else { + flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; + flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK); + } + + if ((mapping->flags & AMDGPU_PTE_PRT) && + (adev->asic_type >= CHIP_VEGA10)) { + flags |= AMDGPU_PTE_PRT; + if (adev->asic_type >= CHIP_NAVI10) { + flags |= AMDGPU_PTE_SNOOPED; + flags |= AMDGPU_PTE_LOG; + flags |= AMDGPU_PTE_SYSTEM; + } + flags &= ~AMDGPU_PTE_VALID; + } + + trace_amdgpu_vm_bo_update(mapping); + + pfn = mapping->offset >> PAGE_SHIFT; + if (nodes) { + while (pfn >= nodes->size) { + pfn -= nodes->size; + ++nodes; + } + } + + do { + dma_addr_t *dma_addr = NULL; + uint64_t max_entries; + uint64_t addr, last; + + if (nodes) { + addr = nodes->start << PAGE_SHIFT; + max_entries = (nodes->size - pfn) * + AMDGPU_GPU_PAGES_IN_CPU_PAGE; + } else { + addr = 0; + max_entries = S64_MAX; + } + + if (pages_addr) { + uint64_t count; + + for (count = 1; + count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE; + ++count) { + uint64_t idx = pfn + count; + + if (pages_addr[idx] != + (pages_addr[idx - 1] + PAGE_SIZE)) + break; + } + + if (count < min_linear_pages) { + addr = pfn << PAGE_SHIFT; + dma_addr = pages_addr; + } else { + addr = pages_addr[pfn]; + max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE; + } + + } else if (flags & AMDGPU_PTE_VALID) { + addr += bo_adev->vm_manager.vram_base_offset; + addr += pfn << PAGE_SHIFT; + } + + last = min((uint64_t)mapping->last, start + max_entries - 1); + r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm, + start, last, flags, addr, + fence); + if (r) + return r; + + pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE; + if (nodes && nodes->size == pfn) { + pfn = 0; + ++nodes; + } + start = last + 1; + + } while (unlikely(start != mapping->last + 1)); + + return 0; +} + +/** + * amdgpu_vm_bo_update - update all BO mappings in the vm page table + * + * @adev: amdgpu_device pointer + * @bo_va: requested BO and VM object + * @clear: if true clear the entries + * + * Fill in the page table entries for @bo_va. + * + * Returns: + * 0 for success, -EINVAL for failure. + */ +int amdgpu_vm_bo_update(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + bool clear) +{ + struct amdgpu_bo *bo = bo_va->base.bo; + struct amdgpu_vm *vm = bo_va->base.vm; + struct amdgpu_bo_va_mapping *mapping; + dma_addr_t *pages_addr = NULL; + struct ttm_mem_reg *mem; + struct drm_mm_node *nodes; + struct dma_fence *exclusive, **last_update; + uint64_t flags; + struct amdgpu_device *bo_adev = adev; + int r; + + if (clear || !bo) { + mem = NULL; + nodes = NULL; + exclusive = NULL; + } else { + struct ttm_dma_tt *ttm; + + mem = &bo->tbo.mem; + nodes = mem->mm_node; + if (mem->mem_type == TTM_PL_TT) { + ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); + pages_addr = ttm->dma_address; + } + exclusive = reservation_object_get_excl(bo->tbo.resv); + } + + if (bo) { + flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); + bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); + } else { + flags = 0x0; + } + + if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv)) + last_update = &vm->last_update; + else + last_update = &bo_va->last_pt_update; + + if (!clear && bo_va->base.moved) { + bo_va->base.moved = false; + list_splice_init(&bo_va->valids, &bo_va->invalids); + + } else if (bo_va->cleared != clear) { + list_splice_init(&bo_va->valids, &bo_va->invalids); + } + + list_for_each_entry(mapping, &bo_va->invalids, list) { + r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm, + mapping, flags, bo_adev, nodes, + last_update); + if (r) + return r; + } + + if (vm->use_cpu_for_update) { + /* Flush HDP */ + mb(); + amdgpu_asic_flush_hdp(adev, NULL); + } + + /* If the BO is not in its preferred location add it back to + * the evicted list so that it gets validated again on the + * next command submission. + */ + if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) { + uint32_t mem_type = bo->tbo.mem.mem_type; + + if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type))) + amdgpu_vm_bo_evicted(&bo_va->base); + else + amdgpu_vm_bo_idle(&bo_va->base); + } else { + amdgpu_vm_bo_done(&bo_va->base); + } + + list_splice_init(&bo_va->invalids, &bo_va->valids); + bo_va->cleared = clear; + + if (trace_amdgpu_vm_bo_mapping_enabled()) { + list_for_each_entry(mapping, &bo_va->valids, list) + trace_amdgpu_vm_bo_mapping(mapping); + } + + return 0; +} + +/** + * amdgpu_vm_update_prt_state - update the global PRT state + * + * @adev: amdgpu_device pointer + */ +static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) +{ + unsigned long flags; + bool enable; + + spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); + enable = !!atomic_read(&adev->vm_manager.num_prt_users); + adev->gmc.gmc_funcs->set_prt(adev, enable); + spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); +} + +/** + * amdgpu_vm_prt_get - add a PRT user + * + * @adev: amdgpu_device pointer + */ +static void amdgpu_vm_prt_get(struct amdgpu_device *adev) +{ + if (!adev->gmc.gmc_funcs->set_prt) + return; + + if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) + amdgpu_vm_update_prt_state(adev); +} + +/** + * amdgpu_vm_prt_put - drop a PRT user + * + * @adev: amdgpu_device pointer + */ +static void amdgpu_vm_prt_put(struct amdgpu_device *adev) +{ + if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) + amdgpu_vm_update_prt_state(adev); +} + +/** + * amdgpu_vm_prt_cb - callback for updating the PRT status + * + * @fence: fence for the callback + * @_cb: the callback function + */ +static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) +{ + struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); + + amdgpu_vm_prt_put(cb->adev); + kfree(cb); +} + +/** + * amdgpu_vm_add_prt_cb - add callback for updating the PRT status + * + * @adev: amdgpu_device pointer + * @fence: fence for the callback + */ +static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, + struct dma_fence *fence) +{ + struct amdgpu_prt_cb *cb; + + if (!adev->gmc.gmc_funcs->set_prt) + return; + + cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); + if (!cb) { + /* Last resort when we are OOM */ + if (fence) + dma_fence_wait(fence, false); + + amdgpu_vm_prt_put(adev); + } else { + cb->adev = adev; + if (!fence || dma_fence_add_callback(fence, &cb->cb, + amdgpu_vm_prt_cb)) + amdgpu_vm_prt_cb(fence, &cb->cb); + } +} + +/** + * amdgpu_vm_free_mapping - free a mapping + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * @mapping: mapping to be freed + * @fence: fence of the unmap operation + * + * Free a mapping and make sure we decrease the PRT usage count if applicable. + */ +static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo_va_mapping *mapping, + struct dma_fence *fence) +{ + if (mapping->flags & AMDGPU_PTE_PRT) + amdgpu_vm_add_prt_cb(adev, fence); + kfree(mapping); +} + +/** + * amdgpu_vm_prt_fini - finish all prt mappings + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * + * Register a cleanup callback to disable PRT support after VM dies. + */ +static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) +{ + struct reservation_object *resv = vm->root.base.bo->tbo.resv; + struct dma_fence *excl, **shared; + unsigned i, shared_count; + int r; + + r = reservation_object_get_fences_rcu(resv, &excl, + &shared_count, &shared); + if (r) { + /* Not enough memory to grab the fence list, as last resort + * block for all the fences to complete. + */ + reservation_object_wait_timeout_rcu(resv, true, false, + MAX_SCHEDULE_TIMEOUT); + return; + } + + /* Add a callback for each fence in the reservation object */ + amdgpu_vm_prt_get(adev); + amdgpu_vm_add_prt_cb(adev, excl); + + for (i = 0; i < shared_count; ++i) { + amdgpu_vm_prt_get(adev); + amdgpu_vm_add_prt_cb(adev, shared[i]); + } + + kfree(shared); +} + +/** + * amdgpu_vm_clear_freed - clear freed BOs in the PT + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * @fence: optional resulting fence (unchanged if no work needed to be done + * or if an error occurred) + * + * Make sure all freed BOs are cleared in the PT. + * PTs have to be reserved and mutex must be locked! + * + * Returns: + * 0 for success. + * + */ +int amdgpu_vm_clear_freed(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct dma_fence **fence) +{ + struct amdgpu_bo_va_mapping *mapping; + uint64_t init_pte_value = 0; + struct dma_fence *f = NULL; + int r; + + while (!list_empty(&vm->freed)) { + mapping = list_first_entry(&vm->freed, + struct amdgpu_bo_va_mapping, list); + list_del(&mapping->list); + + if (vm->pte_support_ats && + mapping->start < AMDGPU_GMC_HOLE_START) + init_pte_value = AMDGPU_PTE_DEFAULT_ATC; + + r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm, + mapping->start, mapping->last, + init_pte_value, 0, &f); + amdgpu_vm_free_mapping(adev, vm, mapping, f); + if (r) { + dma_fence_put(f); + return r; + } + } + + if (fence && f) { + dma_fence_put(*fence); + *fence = f; + } else { + dma_fence_put(f); + } + + return 0; + +} + +/** + * amdgpu_vm_handle_moved - handle moved BOs in the PT + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * + * Make sure all BOs which are moved are updated in the PTs. + * + * Returns: + * 0 for success. + * + * PTs have to be reserved! + */ +int amdgpu_vm_handle_moved(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct amdgpu_bo_va *bo_va, *tmp; + struct reservation_object *resv; + bool clear; + int r; + + list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { + /* Per VM BOs never need to bo cleared in the page tables */ + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + return r; + } + + spin_lock(&vm->invalidated_lock); + while (!list_empty(&vm->invalidated)) { + bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, + base.vm_status); + resv = bo_va->base.bo->tbo.resv; + spin_unlock(&vm->invalidated_lock); + + /* Try to reserve the BO to avoid clearing its ptes */ + if (!amdgpu_vm_debug && reservation_object_trylock(resv)) + clear = false; + /* Somebody else is using the BO right now */ + else + clear = true; + + r = amdgpu_vm_bo_update(adev, bo_va, clear); + if (r) + return r; + + if (!clear) + reservation_object_unlock(resv); + spin_lock(&vm->invalidated_lock); + } + spin_unlock(&vm->invalidated_lock); + + return 0; +} + +/** + * amdgpu_vm_bo_add - add a bo to a specific vm + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * @bo: amdgpu buffer object + * + * Add @bo into the requested vm. + * Add @bo to the list of bos associated with the vm + * + * Returns: + * Newly added bo_va or NULL for failure + * + * Object has to be reserved! + */ +struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo *bo) +{ + struct amdgpu_bo_va *bo_va; + + bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); + if (bo_va == NULL) { + return NULL; + } + amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); + + bo_va->ref_count = 1; + INIT_LIST_HEAD(&bo_va->valids); + INIT_LIST_HEAD(&bo_va->invalids); + + if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) && + (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { + bo_va->is_xgmi = true; + mutex_lock(&adev->vm_manager.lock_pstate); + /* Power up XGMI if it can be potentially used */ + if (++adev->vm_manager.xgmi_map_counter == 1) + amdgpu_xgmi_set_pstate(adev, 1); + mutex_unlock(&adev->vm_manager.lock_pstate); + } + + return bo_va; +} + + +/** + * amdgpu_vm_bo_insert_mapping - insert a new mapping + * + * @adev: amdgpu_device pointer + * @bo_va: bo_va to store the address + * @mapping: the mapping to insert + * + * Insert a new mapping into all structures. + */ +static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + struct amdgpu_bo_va_mapping *mapping) +{ + struct amdgpu_vm *vm = bo_va->base.vm; + struct amdgpu_bo *bo = bo_va->base.bo; + + mapping->bo_va = bo_va; + list_add(&mapping->list, &bo_va->invalids); + amdgpu_vm_it_insert(mapping, &vm->va); + + if (mapping->flags & AMDGPU_PTE_PRT) + amdgpu_vm_prt_get(adev); + + if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv && + !bo_va->base.moved) { + list_move(&bo_va->base.vm_status, &vm->moved); + } + trace_amdgpu_vm_bo_map(bo_va, mapping); +} + +/** + * amdgpu_vm_bo_map - map bo inside a vm + * + * @adev: amdgpu_device pointer + * @bo_va: bo_va to store the address + * @saddr: where to map the BO + * @offset: requested offset in the BO + * @size: BO size in bytes + * @flags: attributes of pages (read/write/valid/etc.) + * + * Add a mapping of the BO at the specefied addr into the VM. + * + * Returns: + * 0 for success, error for failure. + * + * Object has to be reserved and unreserved outside! + */ +int amdgpu_vm_bo_map(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t saddr, uint64_t offset, + uint64_t size, uint64_t flags) +{ + struct amdgpu_bo_va_mapping *mapping, *tmp; + struct amdgpu_bo *bo = bo_va->base.bo; + struct amdgpu_vm *vm = bo_va->base.vm; + uint64_t eaddr; + + /* validate the parameters */ + if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || + size == 0 || size & AMDGPU_GPU_PAGE_MASK) + return -EINVAL; + + /* make sure object fit at this offset */ + eaddr = saddr + size - 1; + if (saddr >= eaddr || + (bo && offset + size > amdgpu_bo_size(bo))) + return -EINVAL; + + saddr /= AMDGPU_GPU_PAGE_SIZE; + eaddr /= AMDGPU_GPU_PAGE_SIZE; + + tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); + if (tmp) { + /* bo and tmp overlap, invalid addr */ + dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " + "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, + tmp->start, tmp->last + 1); + return -EINVAL; + } + + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + mapping->start = saddr; + mapping->last = eaddr; + mapping->offset = offset; + mapping->flags = flags; + + amdgpu_vm_bo_insert_map(adev, bo_va, mapping); + + return 0; +} + +/** + * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings + * + * @adev: amdgpu_device pointer + * @bo_va: bo_va to store the address + * @saddr: where to map the BO + * @offset: requested offset in the BO + * @size: BO size in bytes + * @flags: attributes of pages (read/write/valid/etc.) + * + * Add a mapping of the BO at the specefied addr into the VM. Replace existing + * mappings as we do so. + * + * Returns: + * 0 for success, error for failure. + * + * Object has to be reserved and unreserved outside! + */ +int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t saddr, uint64_t offset, + uint64_t size, uint64_t flags) +{ + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *bo = bo_va->base.bo; + uint64_t eaddr; + int r; + + /* validate the parameters */ + if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || + size == 0 || size & AMDGPU_GPU_PAGE_MASK) + return -EINVAL; + + /* make sure object fit at this offset */ + eaddr = saddr + size - 1; + if (saddr >= eaddr || + (bo && offset + size > amdgpu_bo_size(bo))) + return -EINVAL; + + /* Allocate all the needed memory */ + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); + if (r) { + kfree(mapping); + return r; + } + + saddr /= AMDGPU_GPU_PAGE_SIZE; + eaddr /= AMDGPU_GPU_PAGE_SIZE; + + mapping->start = saddr; + mapping->last = eaddr; + mapping->offset = offset; + mapping->flags = flags; + + amdgpu_vm_bo_insert_map(adev, bo_va, mapping); + + return 0; +} + +/** + * amdgpu_vm_bo_unmap - remove bo mapping from vm + * + * @adev: amdgpu_device pointer + * @bo_va: bo_va to remove the address from + * @saddr: where to the BO is mapped + * + * Remove a mapping of the BO at the specefied addr from the VM. + * + * Returns: + * 0 for success, error for failure. + * + * Object has to be reserved and unreserved outside! + */ +int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t saddr) +{ + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_vm *vm = bo_va->base.vm; + bool valid = true; + + saddr /= AMDGPU_GPU_PAGE_SIZE; + + list_for_each_entry(mapping, &bo_va->valids, list) { + if (mapping->start == saddr) + break; + } + + if (&mapping->list == &bo_va->valids) { + valid = false; + + list_for_each_entry(mapping, &bo_va->invalids, list) { + if (mapping->start == saddr) + break; + } + + if (&mapping->list == &bo_va->invalids) + return -ENOENT; + } + + list_del(&mapping->list); + amdgpu_vm_it_remove(mapping, &vm->va); + mapping->bo_va = NULL; + trace_amdgpu_vm_bo_unmap(bo_va, mapping); + + if (valid) + list_add(&mapping->list, &vm->freed); + else + amdgpu_vm_free_mapping(adev, vm, mapping, + bo_va->last_pt_update); + + return 0; +} + +/** + * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range + * + * @adev: amdgpu_device pointer + * @vm: VM structure to use + * @saddr: start of the range + * @size: size of the range + * + * Remove all mappings in a range, split them as appropriate. + * + * Returns: + * 0 for success, error for failure. + */ +int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + uint64_t saddr, uint64_t size) +{ + struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; + LIST_HEAD(removed); + uint64_t eaddr; + + eaddr = saddr + size - 1; + saddr /= AMDGPU_GPU_PAGE_SIZE; + eaddr /= AMDGPU_GPU_PAGE_SIZE; + + /* Allocate all the needed memory */ + before = kzalloc(sizeof(*before), GFP_KERNEL); + if (!before) + return -ENOMEM; + INIT_LIST_HEAD(&before->list); + + after = kzalloc(sizeof(*after), GFP_KERNEL); + if (!after) { + kfree(before); + return -ENOMEM; + } + INIT_LIST_HEAD(&after->list); + + /* Now gather all removed mappings */ + tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); + while (tmp) { + /* Remember mapping split at the start */ + if (tmp->start < saddr) { + before->start = tmp->start; + before->last = saddr - 1; + before->offset = tmp->offset; + before->flags = tmp->flags; + before->bo_va = tmp->bo_va; + list_add(&before->list, &tmp->bo_va->invalids); + } + + /* Remember mapping split at the end */ + if (tmp->last > eaddr) { + after->start = eaddr + 1; + after->last = tmp->last; + after->offset = tmp->offset; + after->offset += after->start - tmp->start; + after->flags = tmp->flags; + after->bo_va = tmp->bo_va; + list_add(&after->list, &tmp->bo_va->invalids); + } + + list_del(&tmp->list); + list_add(&tmp->list, &removed); + + tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); + } + + /* And free them up */ + list_for_each_entry_safe(tmp, next, &removed, list) { + amdgpu_vm_it_remove(tmp, &vm->va); + list_del(&tmp->list); + + if (tmp->start < saddr) + tmp->start = saddr; + if (tmp->last > eaddr) + tmp->last = eaddr; + + tmp->bo_va = NULL; + list_add(&tmp->list, &vm->freed); + trace_amdgpu_vm_bo_unmap(NULL, tmp); + } + + /* Insert partial mapping before the range */ + if (!list_empty(&before->list)) { + amdgpu_vm_it_insert(before, &vm->va); + if (before->flags & AMDGPU_PTE_PRT) + amdgpu_vm_prt_get(adev); + } else { + kfree(before); + } + + /* Insert partial mapping after the range */ + if (!list_empty(&after->list)) { + amdgpu_vm_it_insert(after, &vm->va); + if (after->flags & AMDGPU_PTE_PRT) + amdgpu_vm_prt_get(adev); + } else { + kfree(after); + } + + return 0; +} + +/** + * amdgpu_vm_bo_lookup_mapping - find mapping by address + * + * @vm: the requested VM + * @addr: the address + * + * Find a mapping by it's address. + * + * Returns: + * The amdgpu_bo_va_mapping matching for addr or NULL + * + */ +struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, + uint64_t addr) +{ + return amdgpu_vm_it_iter_first(&vm->va, addr, addr); +} + +/** + * amdgpu_vm_bo_trace_cs - trace all reserved mappings + * + * @vm: the requested vm + * @ticket: CS ticket + * + * Trace all mappings of BOs reserved during a command submission. + */ +void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) +{ + struct amdgpu_bo_va_mapping *mapping; + + if (!trace_amdgpu_vm_bo_cs_enabled()) + return; + + for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; + mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { + if (mapping->bo_va && mapping->bo_va->base.bo) { + struct amdgpu_bo *bo; + + bo = mapping->bo_va->base.bo; + if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket) + continue; + } + + trace_amdgpu_vm_bo_cs(mapping); + } +} + +/** + * amdgpu_vm_bo_rmv - remove a bo to a specific vm + * + * @adev: amdgpu_device pointer + * @bo_va: requested bo_va + * + * Remove @bo_va->bo from the requested vm. + * + * Object have to be reserved! + */ +void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va) +{ + struct amdgpu_bo_va_mapping *mapping, *next; + struct amdgpu_bo *bo = bo_va->base.bo; + struct amdgpu_vm *vm = bo_va->base.vm; + struct amdgpu_vm_bo_base **base; + + if (bo) { + if (bo->tbo.resv == vm->root.base.bo->tbo.resv) + vm->bulk_moveable = false; + + for (base = &bo_va->base.bo->vm_bo; *base; + base = &(*base)->next) { + if (*base != &bo_va->base) + continue; + + *base = bo_va->base.next; + break; + } + } + + spin_lock(&vm->invalidated_lock); + list_del(&bo_va->base.vm_status); + spin_unlock(&vm->invalidated_lock); + + list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { + list_del(&mapping->list); + amdgpu_vm_it_remove(mapping, &vm->va); + mapping->bo_va = NULL; + trace_amdgpu_vm_bo_unmap(bo_va, mapping); + list_add(&mapping->list, &vm->freed); + } + list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { + list_del(&mapping->list); + amdgpu_vm_it_remove(mapping, &vm->va); + amdgpu_vm_free_mapping(adev, vm, mapping, + bo_va->last_pt_update); + } + + dma_fence_put(bo_va->last_pt_update); + + if (bo && bo_va->is_xgmi) { + mutex_lock(&adev->vm_manager.lock_pstate); + if (--adev->vm_manager.xgmi_map_counter == 0) + amdgpu_xgmi_set_pstate(adev, 0); + mutex_unlock(&adev->vm_manager.lock_pstate); + } + + kfree(bo_va); +} + +/** + * amdgpu_vm_bo_invalidate - mark the bo as invalid + * + * @adev: amdgpu_device pointer + * @bo: amdgpu buffer object + * @evicted: is the BO evicted + * + * Mark @bo as invalid. + */ +void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, + struct amdgpu_bo *bo, bool evicted) +{ + struct amdgpu_vm_bo_base *bo_base; + + /* shadow bo doesn't have bo base, its validation needs its parent */ + if (bo->parent && bo->parent->shadow == bo) + bo = bo->parent; + + for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { + struct amdgpu_vm *vm = bo_base->vm; + + if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) { + amdgpu_vm_bo_evicted(bo_base); + continue; + } + + if (bo_base->moved) + continue; + bo_base->moved = true; + + if (bo->tbo.type == ttm_bo_type_kernel) + amdgpu_vm_bo_relocated(bo_base); + else if (bo->tbo.resv == vm->root.base.bo->tbo.resv) + amdgpu_vm_bo_moved(bo_base); + else + amdgpu_vm_bo_invalidated(bo_base); + } +} + +/** + * amdgpu_vm_get_block_size - calculate VM page table size as power of two + * + * @vm_size: VM size + * + * Returns: + * VM page table as power of two + */ +static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) +{ + /* Total bits covered by PD + PTs */ + unsigned bits = ilog2(vm_size) + 18; + + /* Make sure the PD is 4K in size up to 8GB address space. + Above that split equal between PD and PTs */ + if (vm_size <= 8) + return (bits - 9); + else + return ((bits + 3) / 2); +} + +/** + * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size + * + * @adev: amdgpu_device pointer + * @min_vm_size: the minimum vm size in GB if it's set auto + * @fragment_size_default: Default PTE fragment size + * @max_level: max VMPT level + * @max_bits: max address space size in bits + * + */ +void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, + uint32_t fragment_size_default, unsigned max_level, + unsigned max_bits) +{ + unsigned int max_size = 1 << (max_bits - 30); + unsigned int vm_size; + uint64_t tmp; + + /* adjust vm size first */ + if (amdgpu_vm_size != -1) { + vm_size = amdgpu_vm_size; + if (vm_size > max_size) { + dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", + amdgpu_vm_size, max_size); + vm_size = max_size; + } + } else { + struct sysinfo si; + unsigned int phys_ram_gb; + + /* Optimal VM size depends on the amount of physical + * RAM available. Underlying requirements and + * assumptions: + * + * - Need to map system memory and VRAM from all GPUs + * - VRAM from other GPUs not known here + * - Assume VRAM <= system memory + * - On GFX8 and older, VM space can be segmented for + * different MTYPEs + * - Need to allow room for fragmentation, guard pages etc. + * + * This adds up to a rough guess of system memory x3. + * Round up to power of two to maximize the available + * VM size with the given page table size. + */ + si_meminfo(&si); + phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + + (1 << 30) - 1) >> 30; + vm_size = roundup_pow_of_two( + min(max(phys_ram_gb * 3, min_vm_size), max_size)); + } + + adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; + + tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); + if (amdgpu_vm_block_size != -1) + tmp >>= amdgpu_vm_block_size - 9; + tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; + adev->vm_manager.num_level = min(max_level, (unsigned)tmp); + switch (adev->vm_manager.num_level) { + case 3: + adev->vm_manager.root_level = AMDGPU_VM_PDB2; + break; + case 2: + adev->vm_manager.root_level = AMDGPU_VM_PDB1; + break; + case 1: + adev->vm_manager.root_level = AMDGPU_VM_PDB0; + break; + default: + dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); + } + /* block size depends on vm size and hw setup*/ + if (amdgpu_vm_block_size != -1) + adev->vm_manager.block_size = + min((unsigned)amdgpu_vm_block_size, max_bits + - AMDGPU_GPU_PAGE_SHIFT + - 9 * adev->vm_manager.num_level); + else if (adev->vm_manager.num_level > 1) + adev->vm_manager.block_size = 9; + else + adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); + + if (amdgpu_vm_fragment_size == -1) + adev->vm_manager.fragment_size = fragment_size_default; + else + adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; + + DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", + vm_size, adev->vm_manager.num_level + 1, + adev->vm_manager.block_size, + adev->vm_manager.fragment_size); +} + +/** + * amdgpu_vm_wait_idle - wait for the VM to become idle + * + * @vm: VM object to wait for + * @timeout: timeout to wait for VM to become idle + */ +long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) +{ + return reservation_object_wait_timeout_rcu(vm->root.base.bo->tbo.resv, + true, true, timeout); +} + +/** + * amdgpu_vm_init - initialize a vm instance + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * @vm_context: Indicates if it GFX or Compute context + * @pasid: Process address space identifier + * + * Init @vm fields. + * + * Returns: + * 0 for success, error for failure. + */ +int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, + int vm_context, unsigned int pasid) +{ + struct amdgpu_bo_param bp; + struct amdgpu_bo *root; + int r, i; + + vm->va = RB_ROOT_CACHED; + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) + vm->reserved_vmid[i] = NULL; + INIT_LIST_HEAD(&vm->evicted); + INIT_LIST_HEAD(&vm->relocated); + INIT_LIST_HEAD(&vm->moved); + INIT_LIST_HEAD(&vm->idle); + INIT_LIST_HEAD(&vm->invalidated); + spin_lock_init(&vm->invalidated_lock); + INIT_LIST_HEAD(&vm->freed); + + /* create scheduler entity for page table updates */ + r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs, + adev->vm_manager.vm_pte_num_rqs, NULL); + if (r) + return r; + + vm->pte_support_ats = false; + + if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) { + vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & + AMDGPU_VM_USE_CPU_FOR_COMPUTE); + + if (adev->asic_type == CHIP_RAVEN) + vm->pte_support_ats = true; + } else { + vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & + AMDGPU_VM_USE_CPU_FOR_GFX); + } + DRM_DEBUG_DRIVER("VM update mode is %s\n", + vm->use_cpu_for_update ? "CPU" : "SDMA"); + WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), + "CPU update of VM recommended only for large BAR system\n"); + + if (vm->use_cpu_for_update) + vm->update_funcs = &amdgpu_vm_cpu_funcs; + else + vm->update_funcs = &amdgpu_vm_sdma_funcs; + vm->last_update = NULL; + + amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp); + if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) + bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW; + r = amdgpu_bo_create(adev, &bp, &root); + if (r) + goto error_free_sched_entity; + + r = amdgpu_bo_reserve(root, true); + if (r) + goto error_free_root; + + r = reservation_object_reserve_shared(root->tbo.resv, 1); + if (r) + goto error_unreserve; + + amdgpu_vm_bo_base_init(&vm->root.base, vm, root); + + r = amdgpu_vm_clear_bo(adev, vm, root); + if (r) + goto error_unreserve; + + amdgpu_bo_unreserve(vm->root.base.bo); + + if (pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, + GFP_ATOMIC); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + if (r < 0) + goto error_free_root; + + vm->pasid = pasid; + } + + INIT_KFIFO(vm->faults); + + return 0; + +error_unreserve: + amdgpu_bo_unreserve(vm->root.base.bo); + +error_free_root: + amdgpu_bo_unref(&vm->root.base.bo->shadow); + amdgpu_bo_unref(&vm->root.base.bo); + vm->root.base.bo = NULL; + +error_free_sched_entity: + drm_sched_entity_destroy(&vm->entity); + + return r; +} + +/** + * amdgpu_vm_check_clean_reserved - check if a VM is clean + * + * @adev: amdgpu_device pointer + * @vm: the VM to check + * + * check all entries of the root PD, if any subsequent PDs are allocated, + * it means there are page table creating and filling, and is no a clean + * VM + * + * Returns: + * 0 if this VM is clean + */ +static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + enum amdgpu_vm_level root = adev->vm_manager.root_level; + unsigned int entries = amdgpu_vm_num_entries(adev, root); + unsigned int i = 0; + + if (!(vm->root.entries)) + return 0; + + for (i = 0; i < entries; i++) { + if (vm->root.entries[i].base.bo) + return -EINVAL; + } + + return 0; +} + +/** + * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * + * This only works on GFX VMs that don't have any BOs added and no + * page tables allocated yet. + * + * Changes the following VM parameters: + * - use_cpu_for_update + * - pte_supports_ats + * - pasid (old PASID is released, because compute manages its own PASIDs) + * + * Reinitializes the page directory to reflect the changed ATS + * setting. + * + * Returns: + * 0 for success, -errno for errors. + */ +int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid) +{ + bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); + int r; + + r = amdgpu_bo_reserve(vm->root.base.bo, true); + if (r) + return r; + + /* Sanity checks */ + r = amdgpu_vm_check_clean_reserved(adev, vm); + if (r) + goto unreserve_bo; + + if (pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, + GFP_ATOMIC); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + + if (r == -ENOSPC) + goto unreserve_bo; + r = 0; + } + + /* Check if PD needs to be reinitialized and do it before + * changing any other state, in case it fails. + */ + if (pte_support_ats != vm->pte_support_ats) { + vm->pte_support_ats = pte_support_ats; + r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo); + if (r) + goto free_idr; + } + + /* Update VM state */ + vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & + AMDGPU_VM_USE_CPU_FOR_COMPUTE); + DRM_DEBUG_DRIVER("VM update mode is %s\n", + vm->use_cpu_for_update ? "CPU" : "SDMA"); + WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), + "CPU update of VM recommended only for large BAR system\n"); + + if (vm->pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + + /* Free the original amdgpu allocated pasid + * Will be replaced with kfd allocated pasid + */ + amdgpu_pasid_free(vm->pasid); + vm->pasid = 0; + } + + /* Free the shadow bo for compute VM */ + amdgpu_bo_unref(&vm->root.base.bo->shadow); + + if (pasid) + vm->pasid = pasid; + + goto unreserve_bo; + +free_idr: + if (pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + idr_remove(&adev->vm_manager.pasid_idr, pasid); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + } +unreserve_bo: + amdgpu_bo_unreserve(vm->root.base.bo); + return r; +} + +/** + * amdgpu_vm_release_compute - release a compute vm + * @adev: amdgpu_device pointer + * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute + * + * This is a correspondant of amdgpu_vm_make_compute. It decouples compute + * pasid from vm. Compute should stop use of vm after this call. + */ +void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) +{ + if (vm->pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + } + vm->pasid = 0; +} + +/** + * amdgpu_vm_fini - tear down a vm instance + * + * @adev: amdgpu_device pointer + * @vm: requested vm + * + * Tear down @vm. + * Unbind the VM and remove all bos from the vm bo list + */ +void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) +{ + struct amdgpu_bo_va_mapping *mapping, *tmp; + bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; + struct amdgpu_bo *root; + int i, r; + + amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); + + if (vm->pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + } + + drm_sched_entity_destroy(&vm->entity); + + if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { + dev_err(adev->dev, "still active bo inside vm\n"); + } + rbtree_postorder_for_each_entry_safe(mapping, tmp, + &vm->va.rb_root, rb) { + /* Don't remove the mapping here, we don't want to trigger a + * rebalance and the tree is about to be destroyed anyway. + */ + list_del(&mapping->list); + kfree(mapping); + } + list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { + if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { + amdgpu_vm_prt_fini(adev, vm); + prt_fini_needed = false; + } + + list_del(&mapping->list); + amdgpu_vm_free_mapping(adev, vm, mapping, NULL); + } + + root = amdgpu_bo_ref(vm->root.base.bo); + r = amdgpu_bo_reserve(root, true); + if (r) { + dev_err(adev->dev, "Leaking page tables because BO reservation failed\n"); + } else { + amdgpu_vm_free_pts(adev, vm, NULL); + amdgpu_bo_unreserve(root); + } + amdgpu_bo_unref(&root); + WARN_ON(vm->root.base.bo); + dma_fence_put(vm->last_update); + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) + amdgpu_vmid_free_reserved(adev, vm, i); +} + +/** + * amdgpu_vm_manager_init - init the VM manager + * + * @adev: amdgpu_device pointer + * + * Initialize the VM manager structures + */ +void amdgpu_vm_manager_init(struct amdgpu_device *adev) +{ + unsigned i; + + amdgpu_vmid_mgr_init(adev); + + adev->vm_manager.fence_context = + dma_fence_context_alloc(AMDGPU_MAX_RINGS); + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) + adev->vm_manager.seqno[i] = 0; + + spin_lock_init(&adev->vm_manager.prt_lock); + atomic_set(&adev->vm_manager.num_prt_users, 0); + + /* If not overridden by the user, by default, only in large BAR systems + * Compute VM tables will be updated by CPU + */ +#ifdef CONFIG_X86_64 + if (amdgpu_vm_update_mode == -1) { + if (amdgpu_gmc_vram_full_visible(&adev->gmc)) + adev->vm_manager.vm_update_mode = + AMDGPU_VM_USE_CPU_FOR_COMPUTE; + else + adev->vm_manager.vm_update_mode = 0; + } else + adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; +#else + adev->vm_manager.vm_update_mode = 0; +#endif + + idr_init(&adev->vm_manager.pasid_idr); + spin_lock_init(&adev->vm_manager.pasid_lock); + + adev->vm_manager.xgmi_map_counter = 0; + mutex_init(&adev->vm_manager.lock_pstate); +} + +/** + * amdgpu_vm_manager_fini - cleanup VM manager + * + * @adev: amdgpu_device pointer + * + * Cleanup the VM manager and free resources. + */ +void amdgpu_vm_manager_fini(struct amdgpu_device *adev) +{ + WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr)); + idr_destroy(&adev->vm_manager.pasid_idr); + + amdgpu_vmid_mgr_fini(adev); +} + +/** + * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. + * + * @dev: drm device pointer + * @data: drm_amdgpu_vm + * @filp: drm file pointer + * + * Returns: + * 0 for success, -errno for errors. + */ +int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) +{ + union drm_amdgpu_vm *args = data; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + int r; + + switch (args->in.op) { + case AMDGPU_VM_OP_RESERVE_VMID: + /* current, we only have requirement to reserve vmid from gfxhub */ + r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB); + if (r) + return r; + break; + case AMDGPU_VM_OP_UNRESERVE_VMID: + amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB); + break; + default: + return -EINVAL; + } + + return 0; +} + +/** + * amdgpu_vm_get_task_info - Extracts task info for a PASID. + * + * @adev: drm device pointer + * @pasid: PASID identifier for VM + * @task_info: task_info to fill. + */ +void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, + struct amdgpu_task_info *task_info) +{ + struct amdgpu_vm *vm; + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + + vm = idr_find(&adev->vm_manager.pasid_idr, pasid); + if (vm) + *task_info = vm->task_info; + + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); +} + +/** + * amdgpu_vm_set_task_info - Sets VMs task info. + * + * @vm: vm for which to set the info + */ +void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) +{ + if (!vm->task_info.pid) { + vm->task_info.pid = current->pid; + get_task_comm(vm->task_info.task_name, current); + + if (current->group_leader->mm == current->mm) { + vm->task_info.tgid = current->group_leader->pid; + get_task_comm(vm->task_info.process_name, current->group_leader); + } + } +}
diff --git a/amdgpu/amdgpu_vm.h b/amdgpu/amdgpu_vm.h new file mode 100644 index 0000000..489a162 --- /dev/null +++ b/amdgpu/amdgpu_vm.h
@@ -0,0 +1,413 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ +#ifndef __AMDGPU_VM_H__ +#define __AMDGPU_VM_H__ + +#include <linux/idr.h> +#include <linux/kfifo.h> +#include <linux/rbtree.h> +#include <drm/gpu_scheduler.h> +#include <drm/drm_file.h> +#include <drm/ttm/ttm_bo_driver.h> + +#include "amdgpu_sync.h" +#include "amdgpu_ring.h" +#include "amdgpu_ids.h" + +struct amdgpu_bo_va; +struct amdgpu_job; +struct amdgpu_bo_list_entry; + +/* + * GPUVM handling + */ + +/* Maximum number of PTEs the hardware can write with one command */ +#define AMDGPU_VM_MAX_UPDATE_SIZE 0x3FFFF + +/* number of entries in page table */ +#define AMDGPU_VM_PTE_COUNT(adev) (1 << (adev)->vm_manager.block_size) + +#define AMDGPU_PTE_VALID (1ULL << 0) +#define AMDGPU_PTE_SYSTEM (1ULL << 1) +#define AMDGPU_PTE_SNOOPED (1ULL << 2) + +/* VI only */ +#define AMDGPU_PTE_EXECUTABLE (1ULL << 4) + +#define AMDGPU_PTE_READABLE (1ULL << 5) +#define AMDGPU_PTE_WRITEABLE (1ULL << 6) + +#define AMDGPU_PTE_FRAG(x) ((x & 0x1fULL) << 7) + +/* TILED for VEGA10, reserved for older ASICs */ +#define AMDGPU_PTE_PRT (1ULL << 51) + +/* PDE is handled as PTE for VEGA10 */ +#define AMDGPU_PDE_PTE (1ULL << 54) + +#define AMDGPU_PTE_LOG (1ULL << 55) + +/* PTE is handled as PDE for VEGA10 (Translate Further) */ +#define AMDGPU_PTE_TF (1ULL << 56) + +/* PDE Block Fragment Size for VEGA10 */ +#define AMDGPU_PDE_BFS(a) ((uint64_t)a << 59) + + +/* For GFX9 */ +#define AMDGPU_PTE_MTYPE_VG10(a) ((uint64_t)(a) << 57) +#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL) + +#define AMDGPU_MTYPE_NC 0 +#define AMDGPU_MTYPE_CC 2 + +#define AMDGPU_PTE_DEFAULT_ATC (AMDGPU_PTE_SYSTEM \ + | AMDGPU_PTE_SNOOPED \ + | AMDGPU_PTE_EXECUTABLE \ + | AMDGPU_PTE_READABLE \ + | AMDGPU_PTE_WRITEABLE \ + | AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_CC)) + +/* NAVI10 only */ +#define AMDGPU_PTE_MTYPE_NV10(a) ((uint64_t)(a) << 48) +#define AMDGPU_PTE_MTYPE_NV10_MASK AMDGPU_PTE_MTYPE_NV10(7ULL) + +/* How to programm VM fault handling */ +#define AMDGPU_VM_FAULT_STOP_NEVER 0 +#define AMDGPU_VM_FAULT_STOP_FIRST 1 +#define AMDGPU_VM_FAULT_STOP_ALWAYS 2 + +/* max number of VMHUB */ +#define AMDGPU_MAX_VMHUBS 2 +#define AMDGPU_GFXHUB 0 +#define AMDGPU_MMHUB 1 + +/* hardcode that limit for now */ +#define AMDGPU_VA_RESERVED_SIZE (1ULL << 20) + +/* max vmids dedicated for process */ +#define AMDGPU_VM_MAX_RESERVED_VMID 1 + +#define AMDGPU_VM_CONTEXT_GFX 0 +#define AMDGPU_VM_CONTEXT_COMPUTE 1 + +/* See vm_update_mode */ +#define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0) +#define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1) + +/* VMPT level enumerate, and the hiberachy is: + * PDB2->PDB1->PDB0->PTB + */ +enum amdgpu_vm_level { + AMDGPU_VM_PDB2, + AMDGPU_VM_PDB1, + AMDGPU_VM_PDB0, + AMDGPU_VM_PTB +}; + +/* base structure for tracking BO usage in a VM */ +struct amdgpu_vm_bo_base { + /* constant after initialization */ + struct amdgpu_vm *vm; + struct amdgpu_bo *bo; + + /* protected by bo being reserved */ + struct amdgpu_vm_bo_base *next; + + /* protected by spinlock */ + struct list_head vm_status; + + /* protected by the BO being reserved */ + bool moved; +}; + +struct amdgpu_vm_pt { + struct amdgpu_vm_bo_base base; + + /* array of page tables, one for each directory entry */ + struct amdgpu_vm_pt *entries; +}; + +/* provided by hw blocks that can write ptes, e.g., sdma */ +struct amdgpu_vm_pte_funcs { + /* number of dw to reserve per operation */ + unsigned copy_pte_num_dw; + + /* copy pte entries from GART */ + void (*copy_pte)(struct amdgpu_ib *ib, + uint64_t pe, uint64_t src, + unsigned count); + + /* write pte one entry at a time with addr mapping */ + void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe, + uint64_t value, unsigned count, + uint32_t incr); + /* for linear pte/pde updates without addr mapping */ + void (*set_pte_pde)(struct amdgpu_ib *ib, + uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint64_t flags); +}; + +struct amdgpu_task_info { + char process_name[TASK_COMM_LEN]; + char task_name[TASK_COMM_LEN]; + pid_t pid; + pid_t tgid; +}; + +/** + * struct amdgpu_vm_update_params + * + * Encapsulate some VM table update parameters to reduce + * the number of function parameters + * + */ +struct amdgpu_vm_update_params { + + /** + * @adev: amdgpu device we do this update for + */ + struct amdgpu_device *adev; + + /** + * @vm: optional amdgpu_vm we do this update for + */ + struct amdgpu_vm *vm; + + /** + * @pages_addr: + * + * DMA addresses to use for mapping + */ + dma_addr_t *pages_addr; + + /** + * @job: job to used for hw submission + */ + struct amdgpu_job *job; + + /** + * @num_dw_left: number of dw left for the IB + */ + unsigned int num_dw_left; +}; + +struct amdgpu_vm_update_funcs { + int (*map_table)(struct amdgpu_bo *bo); + int (*prepare)(struct amdgpu_vm_update_params *p, void * owner, + struct dma_fence *exclusive); + int (*update)(struct amdgpu_vm_update_params *p, + struct amdgpu_bo *bo, uint64_t pe, uint64_t addr, + unsigned count, uint32_t incr, uint64_t flags); + int (*commit)(struct amdgpu_vm_update_params *p, + struct dma_fence **fence); +}; + +struct amdgpu_vm { + /* tree of virtual addresses mapped */ + struct rb_root_cached va; + + /* BOs who needs a validation */ + struct list_head evicted; + + /* PT BOs which relocated and their parent need an update */ + struct list_head relocated; + + /* per VM BOs moved, but not yet updated in the PT */ + struct list_head moved; + + /* All BOs of this VM not currently in the state machine */ + struct list_head idle; + + /* regular invalidated BOs, but not yet updated in the PT */ + struct list_head invalidated; + spinlock_t invalidated_lock; + + /* BO mappings freed, but not yet updated in the PT */ + struct list_head freed; + + /* contains the page directory */ + struct amdgpu_vm_pt root; + struct dma_fence *last_update; + + /* Scheduler entity for page table updates */ + struct drm_sched_entity entity; + + unsigned int pasid; + /* dedicated to vm */ + struct amdgpu_vmid *reserved_vmid[AMDGPU_MAX_VMHUBS]; + + /* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */ + bool use_cpu_for_update; + + /* Functions to use for VM table updates */ + const struct amdgpu_vm_update_funcs *update_funcs; + + /* Flag to indicate ATS support from PTE for GFX9 */ + bool pte_support_ats; + + /* Up to 128 pending retry page faults */ + DECLARE_KFIFO(faults, u64, 128); + + /* Points to the KFD process VM info */ + struct amdkfd_process_info *process_info; + + /* List node in amdkfd_process_info.vm_list_head */ + struct list_head vm_list_node; + + /* Valid while the PD is reserved or fenced */ + uint64_t pd_phys_addr; + + /* Some basic info about the task */ + struct amdgpu_task_info task_info; + + /* Store positions of group of BOs */ + struct ttm_lru_bulk_move lru_bulk_move; + /* mark whether can do the bulk move */ + bool bulk_moveable; +}; + +struct amdgpu_vm_manager { + /* Handling of VMIDs */ + struct amdgpu_vmid_mgr id_mgr[AMDGPU_MAX_VMHUBS]; + + /* Handling of VM fences */ + u64 fence_context; + unsigned seqno[AMDGPU_MAX_RINGS]; + + uint64_t max_pfn; + uint32_t num_level; + uint32_t block_size; + uint32_t fragment_size; + enum amdgpu_vm_level root_level; + /* vram base address for page table entry */ + u64 vram_base_offset; + /* vm pte handling */ + const struct amdgpu_vm_pte_funcs *vm_pte_funcs; + struct drm_sched_rq *vm_pte_rqs[AMDGPU_MAX_RINGS]; + unsigned vm_pte_num_rqs; + struct amdgpu_ring *page_fault; + + /* partial resident texture handling */ + spinlock_t prt_lock; + atomic_t num_prt_users; + + /* controls how VM page tables are updated for Graphics and Compute. + * BIT0[= 0] Graphics updated by SDMA [= 1] by CPU + * BIT1[= 0] Compute updated by SDMA [= 1] by CPU + */ + int vm_update_mode; + + /* PASID to VM mapping, will be used in interrupt context to + * look up VM of a page fault + */ + struct idr pasid_idr; + spinlock_t pasid_lock; + + /* counter of mapped memory through xgmi */ + uint32_t xgmi_map_counter; + struct mutex lock_pstate; +}; + +#define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count))) +#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr))) +#define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags))) + +extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs; +extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs; + +void amdgpu_vm_manager_init(struct amdgpu_device *adev); +void amdgpu_vm_manager_fini(struct amdgpu_device *adev); + +long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout); +int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, + int vm_context, unsigned int pasid); +int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid); +void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm); +void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); +void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, + struct list_head *validated, + struct amdgpu_bo_list_entry *entry); +bool amdgpu_vm_ready(struct amdgpu_vm *vm); +int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, + int (*callback)(void *p, struct amdgpu_bo *bo), + void *param); +int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync); +int amdgpu_vm_update_directories(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +int amdgpu_vm_clear_freed(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct dma_fence **fence); +int amdgpu_vm_handle_moved(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +int amdgpu_vm_bo_update(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + bool clear); +void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, + struct amdgpu_bo *bo, bool evicted); +uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr); +struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, + struct amdgpu_bo *bo); +struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo *bo); +int amdgpu_vm_bo_map(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t addr, uint64_t offset, + uint64_t size, uint64_t flags); +int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t addr, uint64_t offset, + uint64_t size, uint64_t flags); +int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va, + uint64_t addr); +int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + uint64_t saddr, uint64_t size); +struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, + uint64_t addr); +void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket); +void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, + struct amdgpu_bo_va *bo_va); +void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, + uint32_t fragment_size_default, unsigned max_level, + unsigned max_bits); +int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); +bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, + struct amdgpu_job *job); +void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev); + +void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, + struct amdgpu_task_info *task_info); + +void amdgpu_vm_set_task_info(struct amdgpu_vm *vm); + +void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo); + +#endif
diff --git a/amdgpu/amdgpu_vm_cpu.c b/amdgpu/amdgpu_vm_cpu.c new file mode 100644 index 0000000..5222d16 --- /dev/null +++ b/amdgpu/amdgpu_vm_cpu.c
@@ -0,0 +1,127 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "amdgpu_vm.h" +#include "amdgpu_object.h" +#include "amdgpu_trace.h" + +/** + * amdgpu_vm_cpu_map_table - make sure new PDs/PTs are kmapped + * + * @table: newly allocated or validated PD/PT + */ +static int amdgpu_vm_cpu_map_table(struct amdgpu_bo *table) +{ + return amdgpu_bo_kmap(table, NULL); +} + +/** + * amdgpu_vm_cpu_prepare - prepare page table update with the CPU + * + * @p: see amdgpu_vm_update_params definition + * @owner: owner we need to sync to + * @exclusive: exclusive move fence we need to sync to + * + * Returns: + * Negativ errno, 0 for success. + */ +static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p, void *owner, + struct dma_fence *exclusive) +{ + int r; + + /* Wait for PT BOs to be idle. PTs share the same resv. object + * as the root PD BO + */ + r = amdgpu_bo_sync_wait(p->vm->root.base.bo, owner, true); + if (unlikely(r)) + return r; + + /* Wait for any BO move to be completed */ + if (exclusive) { + r = dma_fence_wait(exclusive, true); + if (unlikely(r)) + return r; + } + + return 0; +} + +/** + * amdgpu_vm_cpu_update - helper to update page tables via CPU + * + * @p: see amdgpu_vm_update_params definition + * @bo: PD/PT to update + * @pe: kmap addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Write count number of PT/PD entries directly. + */ +static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p, + struct amdgpu_bo *bo, uint64_t pe, + uint64_t addr, unsigned count, uint32_t incr, + uint64_t flags) +{ + unsigned int i; + uint64_t value; + + pe += (unsigned long)amdgpu_bo_kptr(bo); + + trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags); + + for (i = 0; i < count; i++) { + value = p->pages_addr ? + amdgpu_vm_map_gart(p->pages_addr, addr) : + addr; + amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe, + i, value, flags); + addr += incr; + } + return 0; +} + +/** + * amdgpu_vm_cpu_commit - commit page table update to the HW + * + * @p: see amdgpu_vm_update_params definition + * @fence: unused + * + * Make sure that the hardware sees the page table updates. + */ +static int amdgpu_vm_cpu_commit(struct amdgpu_vm_update_params *p, + struct dma_fence **fence) +{ + /* Flush HDP */ + mb(); + amdgpu_asic_flush_hdp(p->adev, NULL); + return 0; +} + +const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs = { + .map_table = amdgpu_vm_cpu_map_table, + .prepare = amdgpu_vm_cpu_prepare, + .update = amdgpu_vm_cpu_update, + .commit = amdgpu_vm_cpu_commit +};
diff --git a/amdgpu/amdgpu_vm_sdma.c b/amdgpu/amdgpu_vm_sdma.c new file mode 100644 index 0000000..ddd181f --- /dev/null +++ b/amdgpu/amdgpu_vm_sdma.c
@@ -0,0 +1,270 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "amdgpu_vm.h" +#include "amdgpu_job.h" +#include "amdgpu_object.h" +#include "amdgpu_trace.h" + +#define AMDGPU_VM_SDMA_MIN_NUM_DW 256u +#define AMDGPU_VM_SDMA_MAX_NUM_DW (16u * 1024u) + +/** + * amdgpu_vm_sdma_map_table - make sure new PDs/PTs are GTT mapped + * + * @table: newly allocated or validated PD/PT + */ +static int amdgpu_vm_sdma_map_table(struct amdgpu_bo *table) +{ + int r; + + r = amdgpu_ttm_alloc_gart(&table->tbo); + if (r) + return r; + + if (table->shadow) + r = amdgpu_ttm_alloc_gart(&table->shadow->tbo); + + return r; +} + +/** + * amdgpu_vm_sdma_prepare - prepare SDMA command submission + * + * @p: see amdgpu_vm_update_params definition + * @owner: owner we need to sync to + * @exclusive: exclusive move fence we need to sync to + * + * Returns: + * Negativ errno, 0 for success. + */ +static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, + void *owner, struct dma_fence *exclusive) +{ + struct amdgpu_bo *root = p->vm->root.base.bo; + unsigned int ndw = AMDGPU_VM_SDMA_MIN_NUM_DW; + int r; + + r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job); + if (r) + return r; + + r = amdgpu_sync_fence(p->adev, &p->job->sync, exclusive, false); + if (r) + return r; + + r = amdgpu_sync_resv(p->adev, &p->job->sync, root->tbo.resv, + owner, false); + if (r) + return r; + + p->num_dw_left = ndw; + return 0; +} + +/** + * amdgpu_vm_sdma_commit - commit SDMA command submission + * + * @p: see amdgpu_vm_update_params definition + * @fence: resulting fence + * + * Returns: + * Negativ errno, 0 for success. + */ +static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, + struct dma_fence **fence) +{ + struct amdgpu_bo *root = p->vm->root.base.bo; + struct amdgpu_ib *ib = p->job->ibs; + struct amdgpu_ring *ring; + struct dma_fence *f; + int r; + + ring = container_of(p->vm->entity.rq->sched, struct amdgpu_ring, sched); + + WARN_ON(ib->length_dw == 0); + amdgpu_ring_pad_ib(ring, ib); + WARN_ON(ib->length_dw > p->num_dw_left); + r = amdgpu_job_submit(p->job, &p->vm->entity, + AMDGPU_FENCE_OWNER_VM, &f); + if (r) + goto error; + + amdgpu_bo_fence(root, f, true); + if (fence) + swap(*fence, f); + dma_fence_put(f); + return 0; + +error: + amdgpu_job_free(p->job); + return r; +} + + +/** + * amdgpu_vm_sdma_copy_ptes - copy the PTEs from mapping + * + * @p: see amdgpu_vm_update_params definition + * @bo: PD/PT to update + * @pe: addr of the page entry + * @count: number of page entries to copy + * + * Traces the parameters and calls the DMA function to copy the PTEs. + */ +static void amdgpu_vm_sdma_copy_ptes(struct amdgpu_vm_update_params *p, + struct amdgpu_bo *bo, uint64_t pe, + unsigned count) +{ + struct amdgpu_ib *ib = p->job->ibs; + uint64_t src = ib->gpu_addr; + + src += p->num_dw_left * 4; + + pe += amdgpu_bo_gpu_offset(bo); + trace_amdgpu_vm_copy_ptes(pe, src, count); + + amdgpu_vm_copy_pte(p->adev, ib, pe, src, count); +} + +/** + * amdgpu_vm_sdma_set_ptes - helper to call the right asic function + * + * @p: see amdgpu_vm_update_params definition + * @bo: PD/PT to update + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Traces the parameters and calls the right asic functions + * to setup the page table using the DMA. + */ +static void amdgpu_vm_sdma_set_ptes(struct amdgpu_vm_update_params *p, + struct amdgpu_bo *bo, uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint64_t flags) +{ + struct amdgpu_ib *ib = p->job->ibs; + + pe += amdgpu_bo_gpu_offset(bo); + trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags); + if (count < 3) { + amdgpu_vm_write_pte(p->adev, ib, pe, addr | flags, + count, incr); + } else { + amdgpu_vm_set_pte_pde(p->adev, ib, pe, addr, + count, incr, flags); + } +} + +/** + * amdgpu_vm_sdma_update - execute VM update + * + * @p: see amdgpu_vm_update_params definition + * @bo: PD/PT to update + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: hw access flags + * + * Reserve space in the IB, setup mapping buffer on demand and write commands to + * the IB. + */ +static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p, + struct amdgpu_bo *bo, uint64_t pe, + uint64_t addr, unsigned count, uint32_t incr, + uint64_t flags) +{ + unsigned int i, ndw, nptes; + uint64_t *pte; + int r; + + do { + ndw = p->num_dw_left; + ndw -= p->job->ibs->length_dw; + + if (ndw < 32) { + r = amdgpu_vm_sdma_commit(p, NULL); + if (r) + return r; + + /* estimate how many dw we need */ + ndw = 32; + if (p->pages_addr) + ndw += count * 2; + ndw = max(ndw, AMDGPU_VM_SDMA_MIN_NUM_DW); + ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW); + + r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job); + if (r) + return r; + + p->num_dw_left = ndw; + } + + if (!p->pages_addr) { + /* set page commands needed */ + if (bo->shadow) + amdgpu_vm_sdma_set_ptes(p, bo->shadow, pe, addr, + count, incr, flags); + amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count, + incr, flags); + return 0; + } + + /* copy commands needed */ + ndw -= p->adev->vm_manager.vm_pte_funcs->copy_pte_num_dw * + (bo->shadow ? 2 : 1); + + /* for padding */ + ndw -= 7; + + nptes = min(count, ndw / 2); + + /* Put the PTEs at the end of the IB. */ + p->num_dw_left -= nptes * 2; + pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]); + for (i = 0; i < nptes; ++i, addr += incr) { + pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr); + pte[i] |= flags; + } + + if (bo->shadow) + amdgpu_vm_sdma_copy_ptes(p, bo->shadow, pe, nptes); + amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes); + + pe += nptes * 8; + count -= nptes; + } while (count); + + return 0; +} + +const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs = { + .map_table = amdgpu_vm_sdma_map_table, + .prepare = amdgpu_vm_sdma_prepare, + .update = amdgpu_vm_sdma_update, + .commit = amdgpu_vm_sdma_commit +};
diff --git a/amdgpu/amdgpu_vram_mgr.c b/amdgpu/amdgpu_vram_mgr.c new file mode 100644 index 0000000..3a9d8c1 --- /dev/null +++ b/amdgpu/amdgpu_vram_mgr.c
@@ -0,0 +1,469 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include "amdgpu.h" + +struct amdgpu_vram_mgr { + struct drm_mm mm; + spinlock_t lock; + atomic64_t usage; + atomic64_t vis_usage; +}; + +/** + * DOC: mem_info_vram_total + * + * The amdgpu driver provides a sysfs API for reporting current total VRAM + * available on the device + * The file mem_info_vram_total is used for this and returns the total + * amount of VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.real_vram_size); +} + +/** + * DOC: mem_info_vis_vram_total + * + * The amdgpu driver provides a sysfs API for reporting current total + * visible VRAM available on the device + * The file mem_info_vis_vram_total is used for this and returns the total + * amount of visible VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.visible_vram_size); +} + +/** + * DOC: mem_info_vram_used + * + * The amdgpu driver provides a sysfs API for reporting current total VRAM + * available on the device + * The file mem_info_vram_used is used for this and returns the total + * amount of currently used VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", + amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM])); +} + +/** + * DOC: mem_info_vis_vram_used + * + * The amdgpu driver provides a sysfs API for reporting current total of + * used visible VRAM + * The file mem_info_vis_vram_used is used for this and returns the total + * amount of currently used visible VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", + amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM])); +} + +static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, + amdgpu_mem_info_vram_total_show, NULL); +static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, + amdgpu_mem_info_vis_vram_total_show,NULL); +static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, + amdgpu_mem_info_vram_used_show, NULL); +static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, + amdgpu_mem_info_vis_vram_used_show, NULL); + +/** + * amdgpu_vram_mgr_init - init VRAM manager and DRM MM + * + * @man: TTM memory type manager + * @p_size: maximum size of VRAM + * + * Allocate and initialize the VRAM manager. + */ +static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man, + unsigned long p_size) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_vram_mgr *mgr; + int ret; + + mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); + if (!mgr) + return -ENOMEM; + + drm_mm_init(&mgr->mm, 0, p_size); + spin_lock_init(&mgr->lock); + man->priv = mgr; + + /* Add the two VRAM-related sysfs files */ + ret = device_create_file(adev->dev, &dev_attr_mem_info_vram_total); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_vram_total\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_mem_info_vis_vram_total); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_vis_vram_total\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_mem_info_vram_used); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_vram_used\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_mem_info_vis_vram_used); + if (ret) { + DRM_ERROR("Failed to create device file mem_info_vis_vram_used\n"); + return ret; + } + + return 0; +} + +/** + * amdgpu_vram_mgr_fini - free and destroy VRAM manager + * + * @man: TTM memory type manager + * + * Destroy and free the VRAM manager, returns -EBUSY if ranges are still + * allocated inside it. + */ +static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_vram_mgr *mgr = man->priv; + + spin_lock(&mgr->lock); + drm_mm_takedown(&mgr->mm); + spin_unlock(&mgr->lock); + kfree(mgr); + man->priv = NULL; + device_remove_file(adev->dev, &dev_attr_mem_info_vram_total); + device_remove_file(adev->dev, &dev_attr_mem_info_vis_vram_total); + device_remove_file(adev->dev, &dev_attr_mem_info_vram_used); + device_remove_file(adev->dev, &dev_attr_mem_info_vis_vram_used); + return 0; +} + +/** + * amdgpu_vram_mgr_vis_size - Calculate visible node size + * + * @adev: amdgpu device structure + * @node: MM node structure + * + * Calculate how many bytes of the MM node are inside visible VRAM + */ +static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, + struct drm_mm_node *node) +{ + uint64_t start = node->start << PAGE_SHIFT; + uint64_t end = (node->size + node->start) << PAGE_SHIFT; + + if (start >= adev->gmc.visible_vram_size) + return 0; + + return (end > adev->gmc.visible_vram_size ? + adev->gmc.visible_vram_size : end) - start; +} + +/** + * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size + * + * @bo: &amdgpu_bo buffer object (must be in VRAM) + * + * Returns: + * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. + */ +u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_mem_reg *mem = &bo->tbo.mem; + struct drm_mm_node *nodes = mem->mm_node; + unsigned pages = mem->num_pages; + u64 usage; + + if (amdgpu_gmc_vram_full_visible(&adev->gmc)) + return amdgpu_bo_size(bo); + + if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) + return 0; + + for (usage = 0; nodes && pages; pages -= nodes->size, nodes++) + usage += amdgpu_vram_mgr_vis_size(adev, nodes); + + return usage; +} + +/** + * amdgpu_vram_mgr_virt_start - update virtual start address + * + * @mem: ttm_mem_reg to update + * @node: just allocated node + * + * Calculate a virtual BO start address to easily check if everything is CPU + * accessible. + */ +static void amdgpu_vram_mgr_virt_start(struct ttm_mem_reg *mem, + struct drm_mm_node *node) +{ + unsigned long start; + + start = node->start + node->size; + if (start > mem->num_pages) + start -= mem->num_pages; + else + start = 0; + mem->start = max(mem->start, start); +} + +/** + * amdgpu_vram_mgr_new - allocate new ranges + * + * @man: TTM memory type manager + * @tbo: TTM BO we need this range for + * @place: placement flags and restrictions + * @mem: the resulting mem object + * + * Allocate VRAM for the given BO. + */ +static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_mem_reg *mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_vram_mgr *mgr = man->priv; + struct drm_mm *mm = &mgr->mm; + struct drm_mm_node *nodes; + enum drm_mm_insert_mode mode; + unsigned long lpfn, num_nodes, pages_per_node, pages_left; + uint64_t vis_usage = 0, mem_bytes; + unsigned i; + int r; + + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size; + + /* bail out quickly if there's likely not enough VRAM for this BO */ + mem_bytes = (u64)mem->num_pages << PAGE_SHIFT; + if (atomic64_add_return(mem_bytes, &mgr->usage) > adev->gmc.mc_vram_size) { + atomic64_sub(mem_bytes, &mgr->usage); + mem->mm_node = NULL; + return 0; + } + + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + pages_per_node = ~0ul; + num_nodes = 1; + } else { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pages_per_node = HPAGE_PMD_NR; +#else + /* default to 2MB */ + pages_per_node = (2UL << (20UL - PAGE_SHIFT)); +#endif + pages_per_node = max((uint32_t)pages_per_node, mem->page_alignment); + num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); + } + + nodes = kvmalloc_array((uint32_t)num_nodes, sizeof(*nodes), + GFP_KERNEL | __GFP_ZERO); + if (!nodes) { + atomic64_sub(mem_bytes, &mgr->usage); + return -ENOMEM; + } + + mode = DRM_MM_INSERT_BEST; + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mode = DRM_MM_INSERT_HIGH; + + mem->start = 0; + pages_left = mem->num_pages; + + spin_lock(&mgr->lock); + for (i = 0; pages_left >= pages_per_node; ++i) { + unsigned long pages = rounddown_pow_of_two(pages_left); + + r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, + pages_per_node, 0, + place->fpfn, lpfn, + mode); + if (unlikely(r)) + break; + + vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); + amdgpu_vram_mgr_virt_start(mem, &nodes[i]); + pages_left -= pages; + } + + for (; pages_left; ++i) { + unsigned long pages = min(pages_left, pages_per_node); + uint32_t alignment = mem->page_alignment; + + if (pages == pages_per_node) + alignment = pages_per_node; + + r = drm_mm_insert_node_in_range(mm, &nodes[i], + pages, alignment, 0, + place->fpfn, lpfn, + mode); + if (unlikely(r)) + goto error; + + vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); + amdgpu_vram_mgr_virt_start(mem, &nodes[i]); + pages_left -= pages; + } + spin_unlock(&mgr->lock); + + atomic64_add(vis_usage, &mgr->vis_usage); + + mem->mm_node = nodes; + + return 0; + +error: + while (i--) + drm_mm_remove_node(&nodes[i]); + spin_unlock(&mgr->lock); + atomic64_sub(mem->num_pages << PAGE_SHIFT, &mgr->usage); + + kvfree(nodes); + return r == -ENOSPC ? 0 : r; +} + +/** + * amdgpu_vram_mgr_del - free ranges + * + * @man: TTM memory type manager + * @tbo: TTM BO we need this range for + * @place: placement flags and restrictions + * @mem: TTM memory object + * + * Free the allocated VRAM again. + */ +static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); + struct amdgpu_vram_mgr *mgr = man->priv; + struct drm_mm_node *nodes = mem->mm_node; + uint64_t usage = 0, vis_usage = 0; + unsigned pages = mem->num_pages; + + if (!mem->mm_node) + return; + + spin_lock(&mgr->lock); + while (pages) { + pages -= nodes->size; + drm_mm_remove_node(nodes); + usage += nodes->size << PAGE_SHIFT; + vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); + ++nodes; + } + spin_unlock(&mgr->lock); + + atomic64_sub(usage, &mgr->usage); + atomic64_sub(vis_usage, &mgr->vis_usage); + + kvfree(mem->mm_node); + mem->mm_node = NULL; +} + +/** + * amdgpu_vram_mgr_usage - how many bytes are used in this domain + * + * @man: TTM memory type manager + * + * Returns how many bytes are used in this domain. + */ +uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man) +{ + struct amdgpu_vram_mgr *mgr = man->priv; + + return atomic64_read(&mgr->usage); +} + +/** + * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part + * + * @man: TTM memory type manager + * + * Returns how many bytes are used in the visible part of VRAM + */ +uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man) +{ + struct amdgpu_vram_mgr *mgr = man->priv; + + return atomic64_read(&mgr->vis_usage); +} + +/** + * amdgpu_vram_mgr_debug - dump VRAM table + * + * @man: TTM memory type manager + * @printer: DRM printer to use + * + * Dump the table content using printk. + */ +static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man, + struct drm_printer *printer) +{ + struct amdgpu_vram_mgr *mgr = man->priv; + + spin_lock(&mgr->lock); + drm_mm_print(&mgr->mm, printer); + spin_unlock(&mgr->lock); + + drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", + man->size, amdgpu_vram_mgr_usage(man) >> 20, + amdgpu_vram_mgr_vis_usage(man) >> 20); +} + +const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = { + .init = amdgpu_vram_mgr_init, + .takedown = amdgpu_vram_mgr_fini, + .get_node = amdgpu_vram_mgr_new, + .put_node = amdgpu_vram_mgr_del, + .debug = amdgpu_vram_mgr_debug +};
diff --git a/amdgpu/amdgpu_xgmi.c b/amdgpu/amdgpu_xgmi.c new file mode 100644 index 0000000..d11eba0 --- /dev/null +++ b/amdgpu/amdgpu_xgmi.c
@@ -0,0 +1,397 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include <linux/list.h> +#include "amdgpu.h" +#include "amdgpu_xgmi.h" +#include "amdgpu_smu.h" + + +static DEFINE_MUTEX(xgmi_mutex); + +#define AMDGPU_MAX_XGMI_HIVE 8 +#define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4 + +static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE]; +static unsigned hive_count = 0; + +void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive) +{ + return &hive->device_list; +} + +/** + * DOC: AMDGPU XGMI Support + * + * XGMI is a high speed interconnect that joins multiple GPU cards + * into a homogeneous memory space that is organized by a collective + * hive ID and individual node IDs, both of which are 64-bit numbers. + * + * The file xgmi_device_id contains the unique per GPU device ID and + * is stored in the /sys/class/drm/card${cardno}/device/ directory. + * + * Inside the device directory a sub-directory 'xgmi_hive_info' is + * created which contains the hive ID and the list of nodes. + * + * The hive ID is stored in: + * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id + * + * The node information is stored in numbered directories: + * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id + * + * Each device has their own xgmi_hive_info direction with a mirror + * set of node sub-directories. + * + * The XGMI memory space is built by contiguously adding the power of + * two padded VRAM space from each node to each other. + * + */ + + +static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct amdgpu_hive_info *hive = + container_of(attr, struct amdgpu_hive_info, dev_attr); + + return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id); +} + +static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev, + struct amdgpu_hive_info *hive) +{ + int ret = 0; + + if (WARN_ON(hive->kobj)) + return -EINVAL; + + hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj); + if (!hive->kobj) { + dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n"); + return -EINVAL; + } + + hive->dev_attr = (struct device_attribute) { + .attr = { + .name = "xgmi_hive_id", + .mode = S_IRUGO, + + }, + .show = amdgpu_xgmi_show_hive_id, + }; + + ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr); + if (ret) { + dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n"); + kobject_del(hive->kobj); + kobject_put(hive->kobj); + hive->kobj = NULL; + } + + return ret; +} + +static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev, + struct amdgpu_hive_info *hive) +{ + sysfs_remove_file(hive->kobj, &hive->dev_attr.attr); + kobject_del(hive->kobj); + kobject_put(hive->kobj); + hive->kobj = NULL; +} + +static ssize_t amdgpu_xgmi_show_device_id(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + + return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id); + +} + + +static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL); + + +static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, + struct amdgpu_hive_info *hive) +{ + int ret = 0; + char node[10] = { 0 }; + + /* Create xgmi device id file */ + ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id); + if (ret) { + dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n"); + return ret; + } + + /* Create sysfs link to hive info folder on the first device */ + if (adev != hive->adev) { + ret = sysfs_create_link(&adev->dev->kobj, hive->kobj, + "xgmi_hive_info"); + if (ret) { + dev_err(adev->dev, "XGMI: Failed to create link to hive info"); + goto remove_file; + } + } + + sprintf(node, "node%d", hive->number_devices); + /* Create sysfs link form the hive folder to yourself */ + ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node); + if (ret) { + dev_err(adev->dev, "XGMI: Failed to create link from hive info"); + goto remove_link; + } + + goto success; + + +remove_link: + sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); + +remove_file: + device_remove_file(adev->dev, &dev_attr_xgmi_device_id); + +success: + return ret; +} + +static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, + struct amdgpu_hive_info *hive) +{ + device_remove_file(adev->dev, &dev_attr_xgmi_device_id); + sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); + sysfs_remove_link(hive->kobj, adev->ddev->unique); +} + + + +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock) +{ + int i; + struct amdgpu_hive_info *tmp; + + if (!adev->gmc.xgmi.hive_id) + return NULL; + + mutex_lock(&xgmi_mutex); + + for (i = 0 ; i < hive_count; ++i) { + tmp = &xgmi_hives[i]; + if (tmp->hive_id == adev->gmc.xgmi.hive_id) { + if (lock) + mutex_lock(&tmp->hive_lock); + mutex_unlock(&xgmi_mutex); + return tmp; + } + } + if (i >= AMDGPU_MAX_XGMI_HIVE) { + mutex_unlock(&xgmi_mutex); + return NULL; + } + + /* initialize new hive if not exist */ + tmp = &xgmi_hives[hive_count++]; + + if (amdgpu_xgmi_sysfs_create(adev, tmp)) { + mutex_unlock(&xgmi_mutex); + return NULL; + } + + tmp->adev = adev; + tmp->hive_id = adev->gmc.xgmi.hive_id; + INIT_LIST_HEAD(&tmp->device_list); + mutex_init(&tmp->hive_lock); + mutex_init(&tmp->reset_lock); + + if (lock) + mutex_lock(&tmp->hive_lock); + tmp->pstate = -1; + mutex_unlock(&xgmi_mutex); + + return tmp; +} + +int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) +{ + int ret = 0; + struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); + + if (!hive) + return 0; + + if (hive->pstate == pstate) + return 0; + + dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate); + + if (is_support_sw_smu(adev)) + ret = smu_set_xgmi_pstate(&adev->smu, pstate); + if (ret) + dev_err(adev->dev, + "XGMI: Set pstate failure on device %llx, hive %llx, ret %d", + adev->gmc.xgmi.node_id, + adev->gmc.xgmi.hive_id, ret); + + return ret; +} + +int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev) +{ + int ret = -EINVAL; + + /* Each psp need to set the latest topology */ + ret = psp_xgmi_set_topology_info(&adev->psp, + hive->number_devices, + &adev->psp.xgmi_context.top_info); + if (ret) + dev_err(adev->dev, + "XGMI: Set topology failure on device %llx, hive %llx, ret %d", + adev->gmc.xgmi.node_id, + adev->gmc.xgmi.hive_id, ret); + + return ret; +} + + +int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev, + struct amdgpu_device *peer_adev) +{ + struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info; + int i; + + for (i = 0 ; i < top->num_nodes; ++i) + if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id) + return top->nodes[i].num_hops; + return -EINVAL; +} + +int amdgpu_xgmi_add_device(struct amdgpu_device *adev) +{ + struct psp_xgmi_topology_info *top_info; + struct amdgpu_hive_info *hive; + struct amdgpu_xgmi *entry; + struct amdgpu_device *tmp_adev = NULL; + + int count = 0, ret = -EINVAL; + + if (!adev->gmc.xgmi.supported) + return 0; + + ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id); + if (ret) { + dev_err(adev->dev, + "XGMI: Failed to get node id\n"); + return ret; + } + + ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id); + if (ret) { + dev_err(adev->dev, + "XGMI: Failed to get hive id\n"); + return ret; + } + + hive = amdgpu_get_xgmi_hive(adev, 1); + if (!hive) { + ret = -EINVAL; + dev_err(adev->dev, + "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n", + adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id); + goto exit; + } + + top_info = &adev->psp.xgmi_context.top_info; + + list_add_tail(&adev->gmc.xgmi.head, &hive->device_list); + list_for_each_entry(entry, &hive->device_list, head) + top_info->nodes[count++].node_id = entry->node_id; + top_info->num_nodes = count; + hive->number_devices = count; + + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + /* update node list for other device in the hive */ + if (tmp_adev != adev) { + top_info = &tmp_adev->psp.xgmi_context.top_info; + top_info->nodes[count - 1].node_id = adev->gmc.xgmi.node_id; + top_info->num_nodes = count; + } + ret = amdgpu_xgmi_update_topology(hive, tmp_adev); + if (ret) + goto exit; + } + + /* get latest topology info for each device from psp */ + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, + &tmp_adev->psp.xgmi_context.top_info); + if (ret) { + dev_err(tmp_adev->dev, + "XGMI: Get topology failure on device %llx, hive %llx, ret %d", + tmp_adev->gmc.xgmi.node_id, + tmp_adev->gmc.xgmi.hive_id, ret); + /* To do : continue with some node failed or disable the whole hive */ + goto exit; + } + } + + if (!ret) + ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive); + + + mutex_unlock(&hive->hive_lock); +exit: + if (!ret) + dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n", + adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id); + else + dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n", + adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id, + ret); + + return ret; +} + +void amdgpu_xgmi_remove_device(struct amdgpu_device *adev) +{ + struct amdgpu_hive_info *hive; + + if (!adev->gmc.xgmi.supported) + return; + + hive = amdgpu_get_xgmi_hive(adev, 1); + if (!hive) + return; + + if (!(hive->number_devices--)) { + amdgpu_xgmi_sysfs_destroy(adev, hive); + mutex_destroy(&hive->hive_lock); + mutex_destroy(&hive->reset_lock); + } else { + amdgpu_xgmi_sysfs_rem_dev_info(adev, hive); + mutex_unlock(&hive->hive_lock); + } +}
diff --git a/amdgpu/amdgpu_xgmi.h b/amdgpu/amdgpu_xgmi.h new file mode 100644 index 0000000..fbcee31 --- /dev/null +++ b/amdgpu/amdgpu_xgmi.h
@@ -0,0 +1,54 @@ +/* + * Copyright 2016 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __AMDGPU_XGMI_H__ +#define __AMDGPU_XGMI_H__ + +#include "amdgpu_psp.h" + +struct amdgpu_hive_info { + uint64_t hive_id; + struct list_head device_list; + int number_devices; + struct mutex hive_lock, reset_lock; + struct kobject *kobj; + struct device_attribute dev_attr; + struct amdgpu_device *adev; + int pstate; /*0 -- low , 1 -- high , -1 unknown*/ +}; + +struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock); +int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev); +int amdgpu_xgmi_add_device(struct amdgpu_device *adev); +void amdgpu_xgmi_remove_device(struct amdgpu_device *adev); +int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate); +int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev, + struct amdgpu_device *peer_adev); + +static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev, + struct amdgpu_device *bo_adev) +{ + return (adev != bo_adev && + adev->gmc.xgmi.hive_id && + adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id); +} + +#endif
diff --git a/amdgpu/athub_v2_0.c b/amdgpu/athub_v2_0.c new file mode 100644 index 0000000..89b32b6 --- /dev/null +++ b/amdgpu/athub_v2_0.c
@@ -0,0 +1,101 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "amdgpu.h" +#include "athub_v2_0.h" + +#include "athub/athub_2_0_0_offset.h" +#include "athub/athub_2_0_0_sh_mask.h" +#include "athub/athub_2_0_0_default.h" +#include "navi10_enum.h" + +#include "soc15_common.h" + +static void +athub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t def, data; + + def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) + data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK; + else + data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK; + + if (def != data) + WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data); +} + +static void +athub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *adev, + bool enable) +{ + uint32_t def, data; + + def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) && + (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) + data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK; + else + data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK; + + if (def != data) + WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data); +} + +int athub_v2_0_set_clockgating(struct amdgpu_device *adev, + enum amd_clockgating_state state) +{ + if (amdgpu_sriov_vf(adev)) + return 0; + + switch (adev->asic_type) { + case CHIP_NAVI10: + athub_v2_0_update_medium_grain_clock_gating(adev, + state == AMD_CG_STATE_GATE ? true : false); + athub_v2_0_update_medium_grain_light_sleep(adev, + state == AMD_CG_STATE_GATE ? true : false); + break; + default: + break; + } + + return 0; +} + +void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) +{ + int data; + + /* AMD_CG_SUPPORT_ATHUB_MGCG */ + data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); + if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK) + *flags |= AMD_CG_SUPPORT_ATHUB_MGCG; + + /* AMD_CG_SUPPORT_ATHUB_LS */ + if (data & ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK) + *flags |= AMD_CG_SUPPORT_ATHUB_LS; +}
diff --git a/amdgpu/athub_v2_0.h b/amdgpu/athub_v2_0.h new file mode 100644 index 0000000..02932c1 --- /dev/null +++ b/amdgpu/athub_v2_0.h
@@ -0,0 +1,30 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __ATHUB_V2_0_H__ +#define __ATHUB_V2_0_H__ + +int athub_v2_0_set_clockgating(struct amdgpu_device *adev, + enum amd_clockgating_state state); +void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags); + +#endif
diff --git a/amdgpu/atom.c b/amdgpu/atom.c new file mode 100644 index 0000000..dd30f4e --- /dev/null +++ b/amdgpu/atom.c
@@ -0,0 +1,1424 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Stanislaw Skowronek + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <asm/unaligned.h> + +#include <drm/drm_util.h> + +#define ATOM_DEBUG + +#include "atom.h" +#include "atom-names.h" +#include "atom-bits.h" +#include "amdgpu.h" + +#define ATOM_COND_ABOVE 0 +#define ATOM_COND_ABOVEOREQUAL 1 +#define ATOM_COND_ALWAYS 2 +#define ATOM_COND_BELOW 3 +#define ATOM_COND_BELOWOREQUAL 4 +#define ATOM_COND_EQUAL 5 +#define ATOM_COND_NOTEQUAL 6 + +#define ATOM_PORT_ATI 0 +#define ATOM_PORT_PCI 1 +#define ATOM_PORT_SYSIO 2 + +#define ATOM_UNIT_MICROSEC 0 +#define ATOM_UNIT_MILLISEC 1 + +#define PLL_INDEX 2 +#define PLL_DATA 3 + +typedef struct { + struct atom_context *ctx; + uint32_t *ps, *ws; + int ps_shift; + uint16_t start; + unsigned last_jump; + unsigned long last_jump_jiffies; + bool abort; +} atom_exec_context; + +int amdgpu_atom_debug = 0; +static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, uint32_t * params); +int amdgpu_atom_execute_table(struct atom_context *ctx, int index, uint32_t * params); + +static uint32_t atom_arg_mask[8] = + { 0xFFFFFFFF, 0xFFFF, 0xFFFF00, 0xFFFF0000, 0xFF, 0xFF00, 0xFF0000, +0xFF000000 }; +static int atom_arg_shift[8] = { 0, 0, 8, 16, 0, 8, 16, 24 }; + +static int atom_dst_to_src[8][4] = { + /* translate destination alignment field to the source alignment encoding */ + {0, 0, 0, 0}, + {1, 2, 3, 0}, + {1, 2, 3, 0}, + {1, 2, 3, 0}, + {4, 5, 6, 7}, + {4, 5, 6, 7}, + {4, 5, 6, 7}, + {4, 5, 6, 7}, +}; +static int atom_def_dst[8] = { 0, 0, 1, 2, 0, 1, 2, 3 }; + +static int debug_depth = 0; +#ifdef ATOM_DEBUG +static void debug_print_spaces(int n) +{ + while (n--) + printk(" "); +} + +#define DEBUG(...) do if (amdgpu_atom_debug) { printk(KERN_DEBUG __VA_ARGS__); } while (0) +#define SDEBUG(...) do if (amdgpu_atom_debug) { printk(KERN_DEBUG); debug_print_spaces(debug_depth); printk(__VA_ARGS__); } while (0) +#else +#define DEBUG(...) do { } while (0) +#define SDEBUG(...) do { } while (0) +#endif + +static uint32_t atom_iio_execute(struct atom_context *ctx, int base, + uint32_t index, uint32_t data) +{ + uint32_t temp = 0xCDCDCDCD; + + while (1) + switch (CU8(base)) { + case ATOM_IIO_NOP: + base++; + break; + case ATOM_IIO_READ: + temp = ctx->card->ioreg_read(ctx->card, CU16(base + 1)); + base += 3; + break; + case ATOM_IIO_WRITE: + ctx->card->ioreg_write(ctx->card, CU16(base + 1), temp); + base += 3; + break; + case ATOM_IIO_CLEAR: + temp &= + ~((0xFFFFFFFF >> (32 - CU8(base + 1))) << + CU8(base + 2)); + base += 3; + break; + case ATOM_IIO_SET: + temp |= + (0xFFFFFFFF >> (32 - CU8(base + 1))) << CU8(base + + 2); + base += 3; + break; + case ATOM_IIO_MOVE_INDEX: + temp &= + ~((0xFFFFFFFF >> (32 - CU8(base + 1))) << + CU8(base + 3)); + temp |= + ((index >> CU8(base + 2)) & + (0xFFFFFFFF >> (32 - CU8(base + 1)))) << CU8(base + + 3); + base += 4; + break; + case ATOM_IIO_MOVE_DATA: + temp &= + ~((0xFFFFFFFF >> (32 - CU8(base + 1))) << + CU8(base + 3)); + temp |= + ((data >> CU8(base + 2)) & + (0xFFFFFFFF >> (32 - CU8(base + 1)))) << CU8(base + + 3); + base += 4; + break; + case ATOM_IIO_MOVE_ATTR: + temp &= + ~((0xFFFFFFFF >> (32 - CU8(base + 1))) << + CU8(base + 3)); + temp |= + ((ctx-> + io_attr >> CU8(base + 2)) & (0xFFFFFFFF >> (32 - + CU8 + (base + + + 1)))) + << CU8(base + 3); + base += 4; + break; + case ATOM_IIO_END: + return temp; + default: + pr_info("Unknown IIO opcode\n"); + return 0; + } +} + +static uint32_t atom_get_src_int(atom_exec_context *ctx, uint8_t attr, + int *ptr, uint32_t *saved, int print) +{ + uint32_t idx, val = 0xCDCDCDCD, align, arg; + struct atom_context *gctx = ctx->ctx; + arg = attr & 7; + align = (attr >> 3) & 7; + switch (arg) { + case ATOM_ARG_REG: + idx = U16(*ptr); + (*ptr) += 2; + if (print) + DEBUG("REG[0x%04X]", idx); + idx += gctx->reg_block; + switch (gctx->io_mode) { + case ATOM_IO_MM: + val = gctx->card->reg_read(gctx->card, idx); + break; + case ATOM_IO_PCI: + pr_info("PCI registers are not implemented\n"); + return 0; + case ATOM_IO_SYSIO: + pr_info("SYSIO registers are not implemented\n"); + return 0; + default: + if (!(gctx->io_mode & 0x80)) { + pr_info("Bad IO mode\n"); + return 0; + } + if (!gctx->iio[gctx->io_mode & 0x7F]) { + pr_info("Undefined indirect IO read method %d\n", + gctx->io_mode & 0x7F); + return 0; + } + val = + atom_iio_execute(gctx, + gctx->iio[gctx->io_mode & 0x7F], + idx, 0); + } + break; + case ATOM_ARG_PS: + idx = U8(*ptr); + (*ptr)++; + /* get_unaligned_le32 avoids unaligned accesses from atombios + * tables, noticed on a DEC Alpha. */ + val = get_unaligned_le32((u32 *)&ctx->ps[idx]); + if (print) + DEBUG("PS[0x%02X,0x%04X]", idx, val); + break; + case ATOM_ARG_WS: + idx = U8(*ptr); + (*ptr)++; + if (print) + DEBUG("WS[0x%02X]", idx); + switch (idx) { + case ATOM_WS_QUOTIENT: + val = gctx->divmul[0]; + break; + case ATOM_WS_REMAINDER: + val = gctx->divmul[1]; + break; + case ATOM_WS_DATAPTR: + val = gctx->data_block; + break; + case ATOM_WS_SHIFT: + val = gctx->shift; + break; + case ATOM_WS_OR_MASK: + val = 1 << gctx->shift; + break; + case ATOM_WS_AND_MASK: + val = ~(1 << gctx->shift); + break; + case ATOM_WS_FB_WINDOW: + val = gctx->fb_base; + break; + case ATOM_WS_ATTRIBUTES: + val = gctx->io_attr; + break; + case ATOM_WS_REGPTR: + val = gctx->reg_block; + break; + default: + val = ctx->ws[idx]; + } + break; + case ATOM_ARG_ID: + idx = U16(*ptr); + (*ptr) += 2; + if (print) { + if (gctx->data_block) + DEBUG("ID[0x%04X+%04X]", idx, gctx->data_block); + else + DEBUG("ID[0x%04X]", idx); + } + val = U32(idx + gctx->data_block); + break; + case ATOM_ARG_FB: + idx = U8(*ptr); + (*ptr)++; + if ((gctx->fb_base + (idx * 4)) > gctx->scratch_size_bytes) { + DRM_ERROR("ATOM: fb read beyond scratch region: %d vs. %d\n", + gctx->fb_base + (idx * 4), gctx->scratch_size_bytes); + val = 0; + } else + val = gctx->scratch[(gctx->fb_base / 4) + idx]; + if (print) + DEBUG("FB[0x%02X]", idx); + break; + case ATOM_ARG_IMM: + switch (align) { + case ATOM_SRC_DWORD: + val = U32(*ptr); + (*ptr) += 4; + if (print) + DEBUG("IMM 0x%08X\n", val); + return val; + case ATOM_SRC_WORD0: + case ATOM_SRC_WORD8: + case ATOM_SRC_WORD16: + val = U16(*ptr); + (*ptr) += 2; + if (print) + DEBUG("IMM 0x%04X\n", val); + return val; + case ATOM_SRC_BYTE0: + case ATOM_SRC_BYTE8: + case ATOM_SRC_BYTE16: + case ATOM_SRC_BYTE24: + val = U8(*ptr); + (*ptr)++; + if (print) + DEBUG("IMM 0x%02X\n", val); + return val; + } + return 0; + case ATOM_ARG_PLL: + idx = U8(*ptr); + (*ptr)++; + if (print) + DEBUG("PLL[0x%02X]", idx); + val = gctx->card->pll_read(gctx->card, idx); + break; + case ATOM_ARG_MC: + idx = U8(*ptr); + (*ptr)++; + if (print) + DEBUG("MC[0x%02X]", idx); + val = gctx->card->mc_read(gctx->card, idx); + break; + } + if (saved) + *saved = val; + val &= atom_arg_mask[align]; + val >>= atom_arg_shift[align]; + if (print) + switch (align) { + case ATOM_SRC_DWORD: + DEBUG(".[31:0] -> 0x%08X\n", val); + break; + case ATOM_SRC_WORD0: + DEBUG(".[15:0] -> 0x%04X\n", val); + break; + case ATOM_SRC_WORD8: + DEBUG(".[23:8] -> 0x%04X\n", val); + break; + case ATOM_SRC_WORD16: + DEBUG(".[31:16] -> 0x%04X\n", val); + break; + case ATOM_SRC_BYTE0: + DEBUG(".[7:0] -> 0x%02X\n", val); + break; + case ATOM_SRC_BYTE8: + DEBUG(".[15:8] -> 0x%02X\n", val); + break; + case ATOM_SRC_BYTE16: + DEBUG(".[23:16] -> 0x%02X\n", val); + break; + case ATOM_SRC_BYTE24: + DEBUG(".[31:24] -> 0x%02X\n", val); + break; + } + return val; +} + +static void atom_skip_src_int(atom_exec_context *ctx, uint8_t attr, int *ptr) +{ + uint32_t align = (attr >> 3) & 7, arg = attr & 7; + switch (arg) { + case ATOM_ARG_REG: + case ATOM_ARG_ID: + (*ptr) += 2; + break; + case ATOM_ARG_PLL: + case ATOM_ARG_MC: + case ATOM_ARG_PS: + case ATOM_ARG_WS: + case ATOM_ARG_FB: + (*ptr)++; + break; + case ATOM_ARG_IMM: + switch (align) { + case ATOM_SRC_DWORD: + (*ptr) += 4; + return; + case ATOM_SRC_WORD0: + case ATOM_SRC_WORD8: + case ATOM_SRC_WORD16: + (*ptr) += 2; + return; + case ATOM_SRC_BYTE0: + case ATOM_SRC_BYTE8: + case ATOM_SRC_BYTE16: + case ATOM_SRC_BYTE24: + (*ptr)++; + return; + } + return; + } +} + +static uint32_t atom_get_src(atom_exec_context *ctx, uint8_t attr, int *ptr) +{ + return atom_get_src_int(ctx, attr, ptr, NULL, 1); +} + +static uint32_t atom_get_src_direct(atom_exec_context *ctx, uint8_t align, int *ptr) +{ + uint32_t val = 0xCDCDCDCD; + + switch (align) { + case ATOM_SRC_DWORD: + val = U32(*ptr); + (*ptr) += 4; + break; + case ATOM_SRC_WORD0: + case ATOM_SRC_WORD8: + case ATOM_SRC_WORD16: + val = U16(*ptr); + (*ptr) += 2; + break; + case ATOM_SRC_BYTE0: + case ATOM_SRC_BYTE8: + case ATOM_SRC_BYTE16: + case ATOM_SRC_BYTE24: + val = U8(*ptr); + (*ptr)++; + break; + } + return val; +} + +static uint32_t atom_get_dst(atom_exec_context *ctx, int arg, uint8_t attr, + int *ptr, uint32_t *saved, int print) +{ + return atom_get_src_int(ctx, + arg | atom_dst_to_src[(attr >> 3) & + 7][(attr >> 6) & 3] << 3, + ptr, saved, print); +} + +static void atom_skip_dst(atom_exec_context *ctx, int arg, uint8_t attr, int *ptr) +{ + atom_skip_src_int(ctx, + arg | atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & + 3] << 3, ptr); +} + +static void atom_put_dst(atom_exec_context *ctx, int arg, uint8_t attr, + int *ptr, uint32_t val, uint32_t saved) +{ + uint32_t align = + atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & 3], old_val = + val, idx; + struct atom_context *gctx = ctx->ctx; + old_val &= atom_arg_mask[align] >> atom_arg_shift[align]; + val <<= atom_arg_shift[align]; + val &= atom_arg_mask[align]; + saved &= ~atom_arg_mask[align]; + val |= saved; + switch (arg) { + case ATOM_ARG_REG: + idx = U16(*ptr); + (*ptr) += 2; + DEBUG("REG[0x%04X]", idx); + idx += gctx->reg_block; + switch (gctx->io_mode) { + case ATOM_IO_MM: + if (idx == 0) + gctx->card->reg_write(gctx->card, idx, + val << 2); + else + gctx->card->reg_write(gctx->card, idx, val); + break; + case ATOM_IO_PCI: + pr_info("PCI registers are not implemented\n"); + return; + case ATOM_IO_SYSIO: + pr_info("SYSIO registers are not implemented\n"); + return; + default: + if (!(gctx->io_mode & 0x80)) { + pr_info("Bad IO mode\n"); + return; + } + if (!gctx->iio[gctx->io_mode & 0xFF]) { + pr_info("Undefined indirect IO write method %d\n", + gctx->io_mode & 0x7F); + return; + } + atom_iio_execute(gctx, gctx->iio[gctx->io_mode & 0xFF], + idx, val); + } + break; + case ATOM_ARG_PS: + idx = U8(*ptr); + (*ptr)++; + DEBUG("PS[0x%02X]", idx); + ctx->ps[idx] = cpu_to_le32(val); + break; + case ATOM_ARG_WS: + idx = U8(*ptr); + (*ptr)++; + DEBUG("WS[0x%02X]", idx); + switch (idx) { + case ATOM_WS_QUOTIENT: + gctx->divmul[0] = val; + break; + case ATOM_WS_REMAINDER: + gctx->divmul[1] = val; + break; + case ATOM_WS_DATAPTR: + gctx->data_block = val; + break; + case ATOM_WS_SHIFT: + gctx->shift = val; + break; + case ATOM_WS_OR_MASK: + case ATOM_WS_AND_MASK: + break; + case ATOM_WS_FB_WINDOW: + gctx->fb_base = val; + break; + case ATOM_WS_ATTRIBUTES: + gctx->io_attr = val; + break; + case ATOM_WS_REGPTR: + gctx->reg_block = val; + break; + default: + ctx->ws[idx] = val; + } + break; + case ATOM_ARG_FB: + idx = U8(*ptr); + (*ptr)++; + if ((gctx->fb_base + (idx * 4)) > gctx->scratch_size_bytes) { + DRM_ERROR("ATOM: fb write beyond scratch region: %d vs. %d\n", + gctx->fb_base + (idx * 4), gctx->scratch_size_bytes); + } else + gctx->scratch[(gctx->fb_base / 4) + idx] = val; + DEBUG("FB[0x%02X]", idx); + break; + case ATOM_ARG_PLL: + idx = U8(*ptr); + (*ptr)++; + DEBUG("PLL[0x%02X]", idx); + gctx->card->pll_write(gctx->card, idx, val); + break; + case ATOM_ARG_MC: + idx = U8(*ptr); + (*ptr)++; + DEBUG("MC[0x%02X]", idx); + gctx->card->mc_write(gctx->card, idx, val); + return; + } + switch (align) { + case ATOM_SRC_DWORD: + DEBUG(".[31:0] <- 0x%08X\n", old_val); + break; + case ATOM_SRC_WORD0: + DEBUG(".[15:0] <- 0x%04X\n", old_val); + break; + case ATOM_SRC_WORD8: + DEBUG(".[23:8] <- 0x%04X\n", old_val); + break; + case ATOM_SRC_WORD16: + DEBUG(".[31:16] <- 0x%04X\n", old_val); + break; + case ATOM_SRC_BYTE0: + DEBUG(".[7:0] <- 0x%02X\n", old_val); + break; + case ATOM_SRC_BYTE8: + DEBUG(".[15:8] <- 0x%02X\n", old_val); + break; + case ATOM_SRC_BYTE16: + DEBUG(".[23:16] <- 0x%02X\n", old_val); + break; + case ATOM_SRC_BYTE24: + DEBUG(".[31:24] <- 0x%02X\n", old_val); + break; + } +} + +static void atom_op_add(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst += src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_and(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst &= src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_beep(atom_exec_context *ctx, int *ptr, int arg) +{ + printk("ATOM BIOS beeped!\n"); +} + +static void atom_op_calltable(atom_exec_context *ctx, int *ptr, int arg) +{ + int idx = U8((*ptr)++); + int r = 0; + + if (idx < ATOM_TABLE_NAMES_CNT) + SDEBUG(" table: %d (%s)\n", idx, atom_table_names[idx]); + else + SDEBUG(" table: %d\n", idx); + if (U16(ctx->ctx->cmd_table + 4 + 2 * idx)) + r = amdgpu_atom_execute_table_locked(ctx->ctx, idx, ctx->ps + ctx->ps_shift); + if (r) { + ctx->abort = true; + } +} + +static void atom_op_clear(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t saved; + int dptr = *ptr; + attr &= 0x38; + attr |= atom_def_dst[attr >> 3] << 6; + atom_get_dst(ctx, arg, attr, ptr, &saved, 0); + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, 0, saved); +} + +static void atom_op_compare(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + ctx->ctx->cs_equal = (dst == src); + ctx->ctx->cs_above = (dst > src); + SDEBUG(" result: %s %s\n", ctx->ctx->cs_equal ? "EQ" : "NE", + ctx->ctx->cs_above ? "GT" : "LE"); +} + +static void atom_op_delay(atom_exec_context *ctx, int *ptr, int arg) +{ + unsigned count = U8((*ptr)++); + SDEBUG(" count: %d\n", count); + if (arg == ATOM_UNIT_MICROSEC) + udelay(count); + else if (!drm_can_sleep()) + mdelay(count); + else + msleep(count); +} + +static void atom_op_div(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + if (src != 0) { + ctx->ctx->divmul[0] = dst / src; + ctx->ctx->divmul[1] = dst % src; + } else { + ctx->ctx->divmul[0] = 0; + ctx->ctx->divmul[1] = 0; + } +} + +static void atom_op_div32(atom_exec_context *ctx, int *ptr, int arg) +{ + uint64_t val64; + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + if (src != 0) { + val64 = dst; + val64 |= ((uint64_t)ctx->ctx->divmul[1]) << 32; + do_div(val64, src); + ctx->ctx->divmul[0] = lower_32_bits(val64); + ctx->ctx->divmul[1] = upper_32_bits(val64); + } else { + ctx->ctx->divmul[0] = 0; + ctx->ctx->divmul[1] = 0; + } +} + +static void atom_op_eot(atom_exec_context *ctx, int *ptr, int arg) +{ + /* functionally, a nop */ +} + +static void atom_op_jump(atom_exec_context *ctx, int *ptr, int arg) +{ + int execute = 0, target = U16(*ptr); + unsigned long cjiffies; + + (*ptr) += 2; + switch (arg) { + case ATOM_COND_ABOVE: + execute = ctx->ctx->cs_above; + break; + case ATOM_COND_ABOVEOREQUAL: + execute = ctx->ctx->cs_above || ctx->ctx->cs_equal; + break; + case ATOM_COND_ALWAYS: + execute = 1; + break; + case ATOM_COND_BELOW: + execute = !(ctx->ctx->cs_above || ctx->ctx->cs_equal); + break; + case ATOM_COND_BELOWOREQUAL: + execute = !ctx->ctx->cs_above; + break; + case ATOM_COND_EQUAL: + execute = ctx->ctx->cs_equal; + break; + case ATOM_COND_NOTEQUAL: + execute = !ctx->ctx->cs_equal; + break; + } + if (arg != ATOM_COND_ALWAYS) + SDEBUG(" taken: %s\n", execute ? "yes" : "no"); + SDEBUG(" target: 0x%04X\n", target); + if (execute) { + if (ctx->last_jump == (ctx->start + target)) { + cjiffies = jiffies; + if (time_after(cjiffies, ctx->last_jump_jiffies)) { + cjiffies -= ctx->last_jump_jiffies; + if ((jiffies_to_msecs(cjiffies) > 5000)) { + DRM_ERROR("atombios stuck in loop for more than 5secs aborting\n"); + ctx->abort = true; + } + } else { + /* jiffies wrap around we will just wait a little longer */ + ctx->last_jump_jiffies = jiffies; + } + } else { + ctx->last_jump = ctx->start + target; + ctx->last_jump_jiffies = jiffies; + } + *ptr = ctx->start + target; + } +} + +static void atom_op_mask(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, mask, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + mask = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr); + SDEBUG(" mask: 0x%08x", mask); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst &= mask; + dst |= src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_move(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t src, saved; + int dptr = *ptr; + if (((attr >> 3) & 7) != ATOM_SRC_DWORD) + atom_get_dst(ctx, arg, attr, ptr, &saved, 0); + else { + atom_skip_dst(ctx, arg, attr, ptr); + saved = 0xCDCDCDCD; + } + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, src, saved); +} + +static void atom_op_mul(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + ctx->ctx->divmul[0] = dst * src; +} + +static void atom_op_mul32(atom_exec_context *ctx, int *ptr, int arg) +{ + uint64_t val64; + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + val64 = (uint64_t)dst * (uint64_t)src; + ctx->ctx->divmul[0] = lower_32_bits(val64); + ctx->ctx->divmul[1] = upper_32_bits(val64); +} + +static void atom_op_nop(atom_exec_context *ctx, int *ptr, int arg) +{ + /* nothing */ +} + +static void atom_op_or(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst |= src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_postcard(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t val = U8((*ptr)++); + SDEBUG("POST card output: 0x%02X\n", val); +} + +static void atom_op_repeat(atom_exec_context *ctx, int *ptr, int arg) +{ + pr_info("unimplemented!\n"); +} + +static void atom_op_restorereg(atom_exec_context *ctx, int *ptr, int arg) +{ + pr_info("unimplemented!\n"); +} + +static void atom_op_savereg(atom_exec_context *ctx, int *ptr, int arg) +{ + pr_info("unimplemented!\n"); +} + +static void atom_op_setdatablock(atom_exec_context *ctx, int *ptr, int arg) +{ + int idx = U8(*ptr); + (*ptr)++; + SDEBUG(" block: %d\n", idx); + if (!idx) + ctx->ctx->data_block = 0; + else if (idx == 255) + ctx->ctx->data_block = ctx->start; + else + ctx->ctx->data_block = U16(ctx->ctx->data_table + 4 + 2 * idx); + SDEBUG(" base: 0x%04X\n", ctx->ctx->data_block); +} + +static void atom_op_setfbbase(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + SDEBUG(" fb_base: "); + ctx->ctx->fb_base = atom_get_src(ctx, attr, ptr); +} + +static void atom_op_setport(atom_exec_context *ctx, int *ptr, int arg) +{ + int port; + switch (arg) { + case ATOM_PORT_ATI: + port = U16(*ptr); + if (port < ATOM_IO_NAMES_CNT) + SDEBUG(" port: %d (%s)\n", port, atom_io_names[port]); + else + SDEBUG(" port: %d\n", port); + if (!port) + ctx->ctx->io_mode = ATOM_IO_MM; + else + ctx->ctx->io_mode = ATOM_IO_IIO | port; + (*ptr) += 2; + break; + case ATOM_PORT_PCI: + ctx->ctx->io_mode = ATOM_IO_PCI; + (*ptr)++; + break; + case ATOM_PORT_SYSIO: + ctx->ctx->io_mode = ATOM_IO_SYSIO; + (*ptr)++; + break; + } +} + +static void atom_op_setregblock(atom_exec_context *ctx, int *ptr, int arg) +{ + ctx->ctx->reg_block = U16(*ptr); + (*ptr) += 2; + SDEBUG(" base: 0x%04X\n", ctx->ctx->reg_block); +} + +static void atom_op_shift_left(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++), shift; + uint32_t saved, dst; + int dptr = *ptr; + attr &= 0x38; + attr |= atom_def_dst[attr >> 3] << 6; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + shift = atom_get_src_direct(ctx, ATOM_SRC_BYTE0, ptr); + SDEBUG(" shift: %d\n", shift); + dst <<= shift; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_shift_right(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++), shift; + uint32_t saved, dst; + int dptr = *ptr; + attr &= 0x38; + attr |= atom_def_dst[attr >> 3] << 6; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + shift = atom_get_src_direct(ctx, ATOM_SRC_BYTE0, ptr); + SDEBUG(" shift: %d\n", shift); + dst >>= shift; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_shl(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++), shift; + uint32_t saved, dst; + int dptr = *ptr; + uint32_t dst_align = atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & 3]; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + /* op needs to full dst value */ + dst = saved; + shift = atom_get_src(ctx, attr, ptr); + SDEBUG(" shift: %d\n", shift); + dst <<= shift; + dst &= atom_arg_mask[dst_align]; + dst >>= atom_arg_shift[dst_align]; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_shr(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++), shift; + uint32_t saved, dst; + int dptr = *ptr; + uint32_t dst_align = atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & 3]; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + /* op needs to full dst value */ + dst = saved; + shift = atom_get_src(ctx, attr, ptr); + SDEBUG(" shift: %d\n", shift); + dst >>= shift; + dst &= atom_arg_mask[dst_align]; + dst >>= atom_arg_shift[dst_align]; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_sub(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst -= src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_switch(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t src, val, target; + SDEBUG(" switch: "); + src = atom_get_src(ctx, attr, ptr); + while (U16(*ptr) != ATOM_CASE_END) + if (U8(*ptr) == ATOM_CASE_MAGIC) { + (*ptr)++; + SDEBUG(" case: "); + val = + atom_get_src(ctx, (attr & 0x38) | ATOM_ARG_IMM, + ptr); + target = U16(*ptr); + if (val == src) { + SDEBUG(" target: %04X\n", target); + *ptr = ctx->start + target; + return; + } + (*ptr) += 2; + } else { + pr_info("Bad case\n"); + return; + } + (*ptr) += 2; +} + +static void atom_op_test(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src; + SDEBUG(" src1: "); + dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); + SDEBUG(" src2: "); + src = atom_get_src(ctx, attr, ptr); + ctx->ctx->cs_equal = ((dst & src) == 0); + SDEBUG(" result: %s\n", ctx->ctx->cs_equal ? "EQ" : "NE"); +} + +static void atom_op_xor(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t attr = U8((*ptr)++); + uint32_t dst, src, saved; + int dptr = *ptr; + SDEBUG(" dst: "); + dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); + SDEBUG(" src: "); + src = atom_get_src(ctx, attr, ptr); + dst ^= src; + SDEBUG(" dst: "); + atom_put_dst(ctx, arg, attr, &dptr, dst, saved); +} + +static void atom_op_debug(atom_exec_context *ctx, int *ptr, int arg) +{ + uint8_t val = U8((*ptr)++); + SDEBUG("DEBUG output: 0x%02X\n", val); +} + +static void atom_op_processds(atom_exec_context *ctx, int *ptr, int arg) +{ + uint16_t val = U16(*ptr); + (*ptr) += val + 2; + SDEBUG("PROCESSDS output: 0x%02X\n", val); +} + +static struct { + void (*func) (atom_exec_context *, int *, int); + int arg; +} opcode_table[ATOM_OP_CNT] = { + { + NULL, 0}, { + atom_op_move, ATOM_ARG_REG}, { + atom_op_move, ATOM_ARG_PS}, { + atom_op_move, ATOM_ARG_WS}, { + atom_op_move, ATOM_ARG_FB}, { + atom_op_move, ATOM_ARG_PLL}, { + atom_op_move, ATOM_ARG_MC}, { + atom_op_and, ATOM_ARG_REG}, { + atom_op_and, ATOM_ARG_PS}, { + atom_op_and, ATOM_ARG_WS}, { + atom_op_and, ATOM_ARG_FB}, { + atom_op_and, ATOM_ARG_PLL}, { + atom_op_and, ATOM_ARG_MC}, { + atom_op_or, ATOM_ARG_REG}, { + atom_op_or, ATOM_ARG_PS}, { + atom_op_or, ATOM_ARG_WS}, { + atom_op_or, ATOM_ARG_FB}, { + atom_op_or, ATOM_ARG_PLL}, { + atom_op_or, ATOM_ARG_MC}, { + atom_op_shift_left, ATOM_ARG_REG}, { + atom_op_shift_left, ATOM_ARG_PS}, { + atom_op_shift_left, ATOM_ARG_WS}, { + atom_op_shift_left, ATOM_ARG_FB}, { + atom_op_shift_left, ATOM_ARG_PLL}, { + atom_op_shift_left, ATOM_ARG_MC}, { + atom_op_shift_right, ATOM_ARG_REG}, { + atom_op_shift_right, ATOM_ARG_PS}, { + atom_op_shift_right, ATOM_ARG_WS}, { + atom_op_shift_right, ATOM_ARG_FB}, { + atom_op_shift_right, ATOM_ARG_PLL}, { + atom_op_shift_right, ATOM_ARG_MC}, { + atom_op_mul, ATOM_ARG_REG}, { + atom_op_mul, ATOM_ARG_PS}, { + atom_op_mul, ATOM_ARG_WS}, { + atom_op_mul, ATOM_ARG_FB}, { + atom_op_mul, ATOM_ARG_PLL}, { + atom_op_mul, ATOM_ARG_MC}, { + atom_op_div, ATOM_ARG_REG}, { + atom_op_div, ATOM_ARG_PS}, { + atom_op_div, ATOM_ARG_WS}, { + atom_op_div, ATOM_ARG_FB}, { + atom_op_div, ATOM_ARG_PLL}, { + atom_op_div, ATOM_ARG_MC}, { + atom_op_add, ATOM_ARG_REG}, { + atom_op_add, ATOM_ARG_PS}, { + atom_op_add, ATOM_ARG_WS}, { + atom_op_add, ATOM_ARG_FB}, { + atom_op_add, ATOM_ARG_PLL}, { + atom_op_add, ATOM_ARG_MC}, { + atom_op_sub, ATOM_ARG_REG}, { + atom_op_sub, ATOM_ARG_PS}, { + atom_op_sub, ATOM_ARG_WS}, { + atom_op_sub, ATOM_ARG_FB}, { + atom_op_sub, ATOM_ARG_PLL}, { + atom_op_sub, ATOM_ARG_MC}, { + atom_op_setport, ATOM_PORT_ATI}, { + atom_op_setport, ATOM_PORT_PCI}, { + atom_op_setport, ATOM_PORT_SYSIO}, { + atom_op_setregblock, 0}, { + atom_op_setfbbase, 0}, { + atom_op_compare, ATOM_ARG_REG}, { + atom_op_compare, ATOM_ARG_PS}, { + atom_op_compare, ATOM_ARG_WS}, { + atom_op_compare, ATOM_ARG_FB}, { + atom_op_compare, ATOM_ARG_PLL}, { + atom_op_compare, ATOM_ARG_MC}, { + atom_op_switch, 0}, { + atom_op_jump, ATOM_COND_ALWAYS}, { + atom_op_jump, ATOM_COND_EQUAL}, { + atom_op_jump, ATOM_COND_BELOW}, { + atom_op_jump, ATOM_COND_ABOVE}, { + atom_op_jump, ATOM_COND_BELOWOREQUAL}, { + atom_op_jump, ATOM_COND_ABOVEOREQUAL}, { + atom_op_jump, ATOM_COND_NOTEQUAL}, { + atom_op_test, ATOM_ARG_REG}, { + atom_op_test, ATOM_ARG_PS}, { + atom_op_test, ATOM_ARG_WS}, { + atom_op_test, ATOM_ARG_FB}, { + atom_op_test, ATOM_ARG_PLL}, { + atom_op_test, ATOM_ARG_MC}, { + atom_op_delay, ATOM_UNIT_MILLISEC}, { + atom_op_delay, ATOM_UNIT_MICROSEC}, { + atom_op_calltable, 0}, { + atom_op_repeat, 0}, { + atom_op_clear, ATOM_ARG_REG}, { + atom_op_clear, ATOM_ARG_PS}, { + atom_op_clear, ATOM_ARG_WS}, { + atom_op_clear, ATOM_ARG_FB}, { + atom_op_clear, ATOM_ARG_PLL}, { + atom_op_clear, ATOM_ARG_MC}, { + atom_op_nop, 0}, { + atom_op_eot, 0}, { + atom_op_mask, ATOM_ARG_REG}, { + atom_op_mask, ATOM_ARG_PS}, { + atom_op_mask, ATOM_ARG_WS}, { + atom_op_mask, ATOM_ARG_FB}, { + atom_op_mask, ATOM_ARG_PLL}, { + atom_op_mask, ATOM_ARG_MC}, { + atom_op_postcard, 0}, { + atom_op_beep, 0}, { + atom_op_savereg, 0}, { + atom_op_restorereg, 0}, { + atom_op_setdatablock, 0}, { + atom_op_xor, ATOM_ARG_REG}, { + atom_op_xor, ATOM_ARG_PS}, { + atom_op_xor, ATOM_ARG_WS}, { + atom_op_xor, ATOM_ARG_FB}, { + atom_op_xor, ATOM_ARG_PLL}, { + atom_op_xor, ATOM_ARG_MC}, { + atom_op_shl, ATOM_ARG_REG}, { + atom_op_shl, ATOM_ARG_PS}, { + atom_op_shl, ATOM_ARG_WS}, { + atom_op_shl, ATOM_ARG_FB}, { + atom_op_shl, ATOM_ARG_PLL}, { + atom_op_shl, ATOM_ARG_MC}, { + atom_op_shr, ATOM_ARG_REG}, { + atom_op_shr, ATOM_ARG_PS}, { + atom_op_shr, ATOM_ARG_WS}, { + atom_op_shr, ATOM_ARG_FB}, { + atom_op_shr, ATOM_ARG_PLL}, { + atom_op_shr, ATOM_ARG_MC}, { + atom_op_debug, 0}, { + atom_op_processds, 0}, { + atom_op_mul32, ATOM_ARG_PS}, { + atom_op_mul32, ATOM_ARG_WS}, { + atom_op_div32, ATOM_ARG_PS}, { + atom_op_div32, ATOM_ARG_WS}, +}; + +static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, uint32_t * params) +{ + int base = CU16(ctx->cmd_table + 4 + 2 * index); + int len, ws, ps, ptr; + unsigned char op; + atom_exec_context ectx; + int ret = 0; + + if (!base) + return -EINVAL; + + len = CU16(base + ATOM_CT_SIZE_PTR); + ws = CU8(base + ATOM_CT_WS_PTR); + ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK; + ptr = base + ATOM_CT_CODE_PTR; + + SDEBUG(">> execute %04X (len %d, WS %d, PS %d)\n", base, len, ws, ps); + + ectx.ctx = ctx; + ectx.ps_shift = ps / 4; + ectx.start = base; + ectx.ps = params; + ectx.abort = false; + ectx.last_jump = 0; + if (ws) + ectx.ws = kcalloc(4, ws, GFP_KERNEL); + else + ectx.ws = NULL; + + debug_depth++; + while (1) { + op = CU8(ptr++); + if (op < ATOM_OP_NAMES_CNT) + SDEBUG("%s @ 0x%04X\n", atom_op_names[op], ptr - 1); + else + SDEBUG("[%d] @ 0x%04X\n", op, ptr - 1); + if (ectx.abort) { + DRM_ERROR("atombios stuck executing %04X (len %d, WS %d, PS %d) @ 0x%04X\n", + base, len, ws, ps, ptr - 1); + ret = -EINVAL; + goto free; + } + + if (op < ATOM_OP_CNT && op > 0) + opcode_table[op].func(&ectx, &ptr, + opcode_table[op].arg); + else + break; + + if (op == ATOM_OP_EOT) + break; + } + debug_depth--; + SDEBUG("<<\n"); + +free: + if (ws) + kfree(ectx.ws); + return ret; +} + +int amdgpu_atom_execute_table(struct atom_context *ctx, int index, uint32_t * params) +{ + int r; + + mutex_lock(&ctx->mutex); + /* reset data block */ + ctx->data_block = 0; + /* reset reg block */ + ctx->reg_block = 0; + /* reset fb window */ + ctx->fb_base = 0; + /* reset io mode */ + ctx->io_mode = ATOM_IO_MM; + /* reset divmul */ + ctx->divmul[0] = 0; + ctx->divmul[1] = 0; + r = amdgpu_atom_execute_table_locked(ctx, index, params); + mutex_unlock(&ctx->mutex); + return r; +} + +static int atom_iio_len[] = { 1, 2, 3, 3, 3, 3, 4, 4, 4, 3 }; + +static void atom_index_iio(struct atom_context *ctx, int base) +{ + ctx->iio = kzalloc(2 * 256, GFP_KERNEL); + if (!ctx->iio) + return; + while (CU8(base) == ATOM_IIO_START) { + ctx->iio[CU8(base + 1)] = base + 2; + base += 2; + while (CU8(base) != ATOM_IIO_END) + base += atom_iio_len[CU8(base)]; + base += 3; + } +} + +struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios) +{ + int base; + struct atom_context *ctx = + kzalloc(sizeof(struct atom_context), GFP_KERNEL); + char *str; + u16 idx; + + if (!ctx) + return NULL; + + ctx->card = card; + ctx->bios = bios; + + if (CU16(0) != ATOM_BIOS_MAGIC) { + pr_info("Invalid BIOS magic\n"); + kfree(ctx); + return NULL; + } + if (strncmp + (CSTR(ATOM_ATI_MAGIC_PTR), ATOM_ATI_MAGIC, + strlen(ATOM_ATI_MAGIC))) { + pr_info("Invalid ATI magic\n"); + kfree(ctx); + return NULL; + } + + base = CU16(ATOM_ROM_TABLE_PTR); + if (strncmp + (CSTR(base + ATOM_ROM_MAGIC_PTR), ATOM_ROM_MAGIC, + strlen(ATOM_ROM_MAGIC))) { + pr_info("Invalid ATOM magic\n"); + kfree(ctx); + return NULL; + } + + ctx->cmd_table = CU16(base + ATOM_ROM_CMD_PTR); + ctx->data_table = CU16(base + ATOM_ROM_DATA_PTR); + atom_index_iio(ctx, CU16(ctx->data_table + ATOM_DATA_IIO_PTR) + 4); + if (!ctx->iio) { + amdgpu_atom_destroy(ctx); + return NULL; + } + + idx = CU16(ATOM_ROM_PART_NUMBER_PTR); + if (idx == 0) + idx = 0x80; + + str = CSTR(idx); + if (*str != '\0') { + pr_info("ATOM BIOS: %s\n", str); + strlcpy(ctx->vbios_version, str, sizeof(ctx->vbios_version)); + } + + + return ctx; +} + +int amdgpu_atom_asic_init(struct atom_context *ctx) +{ + int hwi = CU16(ctx->data_table + ATOM_DATA_FWI_PTR); + uint32_t ps[16]; + int ret; + + memset(ps, 0, 64); + + ps[0] = cpu_to_le32(CU32(hwi + ATOM_FWI_DEFSCLK_PTR)); + ps[1] = cpu_to_le32(CU32(hwi + ATOM_FWI_DEFMCLK_PTR)); + if (!ps[0] || !ps[1]) + return 1; + + if (!CU16(ctx->cmd_table + 4 + 2 * ATOM_CMD_INIT)) + return 1; + ret = amdgpu_atom_execute_table(ctx, ATOM_CMD_INIT, ps); + if (ret) + return ret; + + memset(ps, 0, 64); + + return ret; +} + +void amdgpu_atom_destroy(struct atom_context *ctx) +{ + kfree(ctx->iio); + kfree(ctx); +} + +bool amdgpu_atom_parse_data_header(struct atom_context *ctx, int index, + uint16_t * size, uint8_t * frev, uint8_t * crev, + uint16_t * data_start) +{ + int offset = index * 2 + 4; + int idx = CU16(ctx->data_table + offset); + u16 *mdt = (u16 *)(ctx->bios + ctx->data_table + 4); + + if (!mdt[index]) + return false; + + if (size) + *size = CU16(idx); + if (frev) + *frev = CU8(idx + 2); + if (crev) + *crev = CU8(idx + 3); + *data_start = idx; + return true; +} + +bool amdgpu_atom_parse_cmd_header(struct atom_context *ctx, int index, uint8_t * frev, + uint8_t * crev) +{ + int offset = index * 2 + 4; + int idx = CU16(ctx->cmd_table + offset); + u16 *mct = (u16 *)(ctx->bios + ctx->cmd_table + 4); + + if (!mct[index]) + return false; + + if (frev) + *frev = CU8(idx + 2); + if (crev) + *crev = CU8(idx + 3); + return true; +} +
diff --git a/amdgpu/atom.h b/amdgpu/atom.h new file mode 100644 index 0000000..4205bbe --- /dev/null +++ b/amdgpu/atom.h
@@ -0,0 +1,161 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Author: Stanislaw Skowronek + */ + +#ifndef ATOM_H +#define ATOM_H + +#include <linux/types.h> + +struct drm_device; + +#define ATOM_BIOS_MAGIC 0xAA55 +#define ATOM_ATI_MAGIC_PTR 0x30 +#define ATOM_ATI_MAGIC " 761295520" +#define ATOM_ROM_TABLE_PTR 0x48 +#define ATOM_ROM_PART_NUMBER_PTR 0x6E + +#define ATOM_ROM_MAGIC "ATOM" +#define ATOM_ROM_MAGIC_PTR 4 + +#define ATOM_ROM_MSG_PTR 0x10 +#define ATOM_ROM_CMD_PTR 0x1E +#define ATOM_ROM_DATA_PTR 0x20 + +#define ATOM_CMD_INIT 0 +#define ATOM_CMD_SETSCLK 0x0A +#define ATOM_CMD_SETMCLK 0x0B +#define ATOM_CMD_SETPCLK 0x0C +#define ATOM_CMD_SPDFANCNTL 0x39 + +#define ATOM_DATA_FWI_PTR 0xC +#define ATOM_DATA_IIO_PTR 0x32 + +#define ATOM_FWI_DEFSCLK_PTR 8 +#define ATOM_FWI_DEFMCLK_PTR 0xC +#define ATOM_FWI_MAXSCLK_PTR 0x24 +#define ATOM_FWI_MAXMCLK_PTR 0x28 + +#define ATOM_CT_SIZE_PTR 0 +#define ATOM_CT_WS_PTR 4 +#define ATOM_CT_PS_PTR 5 +#define ATOM_CT_PS_MASK 0x7F +#define ATOM_CT_CODE_PTR 6 + +#define ATOM_OP_CNT 127 +#define ATOM_OP_EOT 91 + +#define ATOM_CASE_MAGIC 0x63 +#define ATOM_CASE_END 0x5A5A + +#define ATOM_ARG_REG 0 +#define ATOM_ARG_PS 1 +#define ATOM_ARG_WS 2 +#define ATOM_ARG_FB 3 +#define ATOM_ARG_ID 4 +#define ATOM_ARG_IMM 5 +#define ATOM_ARG_PLL 6 +#define ATOM_ARG_MC 7 + +#define ATOM_SRC_DWORD 0 +#define ATOM_SRC_WORD0 1 +#define ATOM_SRC_WORD8 2 +#define ATOM_SRC_WORD16 3 +#define ATOM_SRC_BYTE0 4 +#define ATOM_SRC_BYTE8 5 +#define ATOM_SRC_BYTE16 6 +#define ATOM_SRC_BYTE24 7 + +#define ATOM_WS_QUOTIENT 0x40 +#define ATOM_WS_REMAINDER 0x41 +#define ATOM_WS_DATAPTR 0x42 +#define ATOM_WS_SHIFT 0x43 +#define ATOM_WS_OR_MASK 0x44 +#define ATOM_WS_AND_MASK 0x45 +#define ATOM_WS_FB_WINDOW 0x46 +#define ATOM_WS_ATTRIBUTES 0x47 +#define ATOM_WS_REGPTR 0x48 + +#define ATOM_IIO_NOP 0 +#define ATOM_IIO_START 1 +#define ATOM_IIO_READ 2 +#define ATOM_IIO_WRITE 3 +#define ATOM_IIO_CLEAR 4 +#define ATOM_IIO_SET 5 +#define ATOM_IIO_MOVE_INDEX 6 +#define ATOM_IIO_MOVE_ATTR 7 +#define ATOM_IIO_MOVE_DATA 8 +#define ATOM_IIO_END 9 + +#define ATOM_IO_MM 0 +#define ATOM_IO_PCI 1 +#define ATOM_IO_SYSIO 2 +#define ATOM_IO_IIO 0x80 + +struct card_info { + struct drm_device *dev; + void (* reg_write)(struct card_info *, uint32_t, uint32_t); /* filled by driver */ + uint32_t (* reg_read)(struct card_info *, uint32_t); /* filled by driver */ + void (* ioreg_write)(struct card_info *, uint32_t, uint32_t); /* filled by driver */ + uint32_t (* ioreg_read)(struct card_info *, uint32_t); /* filled by driver */ + void (* mc_write)(struct card_info *, uint32_t, uint32_t); /* filled by driver */ + uint32_t (* mc_read)(struct card_info *, uint32_t); /* filled by driver */ + void (* pll_write)(struct card_info *, uint32_t, uint32_t); /* filled by driver */ + uint32_t (* pll_read)(struct card_info *, uint32_t); /* filled by driver */ +}; + +struct atom_context { + struct card_info *card; + struct mutex mutex; + void *bios; + uint32_t cmd_table, data_table; + uint16_t *iio; + + uint16_t data_block; + uint32_t fb_base; + uint32_t divmul[2]; + uint16_t io_attr; + uint16_t reg_block; + uint8_t shift; + int cs_equal, cs_above; + int io_mode; + uint32_t *scratch; + int scratch_size_bytes; + char vbios_version[20]; +}; + +extern int amdgpu_atom_debug; + +struct atom_context *amdgpu_atom_parse(struct card_info *, void *); +int amdgpu_atom_execute_table(struct atom_context *, int, uint32_t *); +int amdgpu_atom_asic_init(struct atom_context *); +void amdgpu_atom_destroy(struct atom_context *); +bool amdgpu_atom_parse_data_header(struct atom_context *ctx, int index, uint16_t *size, + uint8_t *frev, uint8_t *crev, uint16_t *data_start); +bool amdgpu_atom_parse_cmd_header(struct atom_context *ctx, int index, + uint8_t *frev, uint8_t *crev); +#include "atom-types.h" +#include "atombios.h" +#include "ObjectID.h" + +#endif
diff --git a/amdgpu/atombios_crtc.c b/amdgpu/atombios_crtc.c new file mode 100644 index 0000000..213e62a --- /dev/null +++ b/amdgpu/atombios_crtc.c
@@ -0,0 +1,886 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <drm/drm_crtc_helper.h> +#include <drm/amdgpu_drm.h> +#include <drm/drm_fixed.h> +#include "amdgpu.h" +#include "atom.h" +#include "atom-bits.h" +#include "atombios_encoders.h" +#include "atombios_crtc.h" +#include "amdgpu_atombios.h" +#include "amdgpu_pll.h" +#include "amdgpu_connectors.h" + +void amdgpu_atombios_crtc_overscan_setup(struct drm_crtc *crtc, + struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + SET_CRTC_OVERSCAN_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, SetCRTC_OverScan); + int a1, a2; + + memset(&args, 0, sizeof(args)); + + args.ucCRTC = amdgpu_crtc->crtc_id; + + switch (amdgpu_crtc->rmx_type) { + case RMX_CENTER: + args.usOverscanTop = cpu_to_le16((adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2); + args.usOverscanBottom = cpu_to_le16((adjusted_mode->crtc_vdisplay - mode->crtc_vdisplay) / 2); + args.usOverscanLeft = cpu_to_le16((adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2); + args.usOverscanRight = cpu_to_le16((adjusted_mode->crtc_hdisplay - mode->crtc_hdisplay) / 2); + break; + case RMX_ASPECT: + a1 = mode->crtc_vdisplay * adjusted_mode->crtc_hdisplay; + a2 = adjusted_mode->crtc_vdisplay * mode->crtc_hdisplay; + + if (a1 > a2) { + args.usOverscanLeft = cpu_to_le16((adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2); + args.usOverscanRight = cpu_to_le16((adjusted_mode->crtc_hdisplay - (a2 / mode->crtc_vdisplay)) / 2); + } else if (a2 > a1) { + args.usOverscanTop = cpu_to_le16((adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2); + args.usOverscanBottom = cpu_to_le16((adjusted_mode->crtc_vdisplay - (a1 / mode->crtc_hdisplay)) / 2); + } + break; + case RMX_FULL: + default: + args.usOverscanRight = cpu_to_le16(amdgpu_crtc->h_border); + args.usOverscanLeft = cpu_to_le16(amdgpu_crtc->h_border); + args.usOverscanBottom = cpu_to_le16(amdgpu_crtc->v_border); + args.usOverscanTop = cpu_to_le16(amdgpu_crtc->v_border); + break; + } + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_scaler_setup(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + ENABLE_SCALER_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, EnableScaler); + + memset(&args, 0, sizeof(args)); + + args.ucScaler = amdgpu_crtc->crtc_id; + + switch (amdgpu_crtc->rmx_type) { + case RMX_FULL: + args.ucEnable = ATOM_SCALER_EXPANSION; + break; + case RMX_CENTER: + args.ucEnable = ATOM_SCALER_CENTER; + break; + case RMX_ASPECT: + args.ucEnable = ATOM_SCALER_EXPANSION; + break; + default: + args.ucEnable = ATOM_SCALER_DISABLE; + break; + } + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_lock(struct drm_crtc *crtc, int lock) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + int index = + GetIndexIntoMasterTable(COMMAND, UpdateCRTC_DoubleBufferRegisters); + ENABLE_CRTC_PS_ALLOCATION args; + + memset(&args, 0, sizeof(args)); + + args.ucCRTC = amdgpu_crtc->crtc_id; + args.ucEnable = lock; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_enable(struct drm_crtc *crtc, int state) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + int index = GetIndexIntoMasterTable(COMMAND, EnableCRTC); + ENABLE_CRTC_PS_ALLOCATION args; + + memset(&args, 0, sizeof(args)); + + args.ucCRTC = amdgpu_crtc->crtc_id; + args.ucEnable = state; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_blank(struct drm_crtc *crtc, int state) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + int index = GetIndexIntoMasterTable(COMMAND, BlankCRTC); + BLANK_CRTC_PS_ALLOCATION args; + + memset(&args, 0, sizeof(args)); + + args.ucCRTC = amdgpu_crtc->crtc_id; + args.ucBlanking = state; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_powergate(struct drm_crtc *crtc, int state) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + int index = GetIndexIntoMasterTable(COMMAND, EnableDispPowerGating); + ENABLE_DISP_POWER_GATING_PS_ALLOCATION args; + + memset(&args, 0, sizeof(args)); + + args.ucDispPipeId = amdgpu_crtc->crtc_id; + args.ucEnable = state; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_powergate_init(struct amdgpu_device *adev) +{ + int index = GetIndexIntoMasterTable(COMMAND, EnableDispPowerGating); + ENABLE_DISP_POWER_GATING_PS_ALLOCATION args; + + memset(&args, 0, sizeof(args)); + + args.ucEnable = ATOM_INIT; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +void amdgpu_atombios_crtc_set_dtd_timing(struct drm_crtc *crtc, + struct drm_display_mode *mode) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + SET_CRTC_USING_DTD_TIMING_PARAMETERS args; + int index = GetIndexIntoMasterTable(COMMAND, SetCRTC_UsingDTDTiming); + u16 misc = 0; + + memset(&args, 0, sizeof(args)); + args.usH_Size = cpu_to_le16(mode->crtc_hdisplay - (amdgpu_crtc->h_border * 2)); + args.usH_Blanking_Time = + cpu_to_le16(mode->crtc_hblank_end - mode->crtc_hdisplay + (amdgpu_crtc->h_border * 2)); + args.usV_Size = cpu_to_le16(mode->crtc_vdisplay - (amdgpu_crtc->v_border * 2)); + args.usV_Blanking_Time = + cpu_to_le16(mode->crtc_vblank_end - mode->crtc_vdisplay + (amdgpu_crtc->v_border * 2)); + args.usH_SyncOffset = + cpu_to_le16(mode->crtc_hsync_start - mode->crtc_hdisplay + amdgpu_crtc->h_border); + args.usH_SyncWidth = + cpu_to_le16(mode->crtc_hsync_end - mode->crtc_hsync_start); + args.usV_SyncOffset = + cpu_to_le16(mode->crtc_vsync_start - mode->crtc_vdisplay + amdgpu_crtc->v_border); + args.usV_SyncWidth = + cpu_to_le16(mode->crtc_vsync_end - mode->crtc_vsync_start); + args.ucH_Border = amdgpu_crtc->h_border; + args.ucV_Border = amdgpu_crtc->v_border; + + if (mode->flags & DRM_MODE_FLAG_NVSYNC) + misc |= ATOM_VSYNC_POLARITY; + if (mode->flags & DRM_MODE_FLAG_NHSYNC) + misc |= ATOM_HSYNC_POLARITY; + if (mode->flags & DRM_MODE_FLAG_CSYNC) + misc |= ATOM_COMPOSITESYNC; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + misc |= ATOM_INTERLACE; + if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + misc |= ATOM_DOUBLE_CLOCK_MODE; + + args.susModeMiscInfo.usAccess = cpu_to_le16(misc); + args.ucCRTC = amdgpu_crtc->crtc_id; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +union atom_enable_ss { + ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION v1; + ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2 v2; + ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3 v3; +}; + +static void amdgpu_atombios_crtc_program_ss(struct amdgpu_device *adev, + int enable, + int pll_id, + int crtc_id, + struct amdgpu_atom_ss *ss) +{ + unsigned i; + int index = GetIndexIntoMasterTable(COMMAND, EnableSpreadSpectrumOnPPLL); + union atom_enable_ss args; + + if (enable) { + /* Don't mess with SS if percentage is 0 or external ss. + * SS is already disabled previously, and disabling it + * again can cause display problems if the pll is already + * programmed. + */ + if (ss->percentage == 0) + return; + if (ss->type & ATOM_EXTERNAL_SS_MASK) + return; + } else { + for (i = 0; i < adev->mode_info.num_crtc; i++) { + if (adev->mode_info.crtcs[i] && + adev->mode_info.crtcs[i]->enabled && + i != crtc_id && + pll_id == adev->mode_info.crtcs[i]->pll_id) { + /* one other crtc is using this pll don't turn + * off spread spectrum as it might turn off + * display on active crtc + */ + return; + } + } + } + + memset(&args, 0, sizeof(args)); + + args.v3.usSpreadSpectrumAmountFrac = cpu_to_le16(0); + args.v3.ucSpreadSpectrumType = ss->type & ATOM_SS_CENTRE_SPREAD_MODE_MASK; + switch (pll_id) { + case ATOM_PPLL1: + args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P1PLL; + break; + case ATOM_PPLL2: + args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P2PLL; + break; + case ATOM_DCPLL: + args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_DCPLL; + break; + case ATOM_PPLL_INVALID: + return; + } + args.v3.usSpreadSpectrumAmount = cpu_to_le16(ss->amount); + args.v3.usSpreadSpectrumStep = cpu_to_le16(ss->step); + args.v3.ucEnable = enable; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +union adjust_pixel_clock { + ADJUST_DISPLAY_PLL_PS_ALLOCATION v1; + ADJUST_DISPLAY_PLL_PS_ALLOCATION_V3 v3; +}; + +static u32 amdgpu_atombios_crtc_adjust_pll(struct drm_crtc *crtc, + struct drm_display_mode *mode) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct drm_encoder *encoder = amdgpu_crtc->encoder; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + u32 adjusted_clock = mode->clock; + int encoder_mode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + u32 dp_clock = mode->clock; + u32 clock = mode->clock; + int bpc = amdgpu_crtc->bpc; + bool is_duallink = amdgpu_dig_monitor_is_duallink(encoder, mode->clock); + union adjust_pixel_clock args; + u8 frev, crev; + int index; + + amdgpu_crtc->pll_flags = AMDGPU_PLL_USE_FRAC_FB_DIV; + + if ((amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) || + (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) { + if (connector) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + + dp_clock = dig_connector->dp_clock; + } + } + + /* use recommended ref_div for ss */ + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + if (amdgpu_crtc->ss_enabled) { + if (amdgpu_crtc->ss.refdiv) { + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_REF_DIV; + amdgpu_crtc->pll_reference_div = amdgpu_crtc->ss.refdiv; + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_FRAC_FB_DIV; + } + } + } + + /* DVO wants 2x pixel clock if the DVO chip is in 12 bit mode */ + if (amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1) + adjusted_clock = mode->clock * 2; + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + amdgpu_crtc->pll_flags |= AMDGPU_PLL_PREFER_CLOSEST_LOWER; + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) + amdgpu_crtc->pll_flags |= AMDGPU_PLL_IS_LCD; + + + /* adjust pll for deep color modes */ + if (encoder_mode == ATOM_ENCODER_MODE_HDMI) { + switch (bpc) { + case 8: + default: + break; + case 10: + clock = (clock * 5) / 4; + break; + case 12: + clock = (clock * 3) / 2; + break; + case 16: + clock = clock * 2; + break; + } + } + + /* DCE3+ has an AdjustDisplayPll that will adjust the pixel clock + * accordingly based on the encoder/transmitter to work around + * special hw requirements. + */ + index = GetIndexIntoMasterTable(COMMAND, AdjustDisplayPll); + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, + &crev)) + return adjusted_clock; + + memset(&args, 0, sizeof(args)); + + switch (frev) { + case 1: + switch (crev) { + case 1: + case 2: + args.v1.usPixelClock = cpu_to_le16(clock / 10); + args.v1.ucTransmitterID = amdgpu_encoder->encoder_id; + args.v1.ucEncodeMode = encoder_mode; + if (amdgpu_crtc->ss_enabled && amdgpu_crtc->ss.percentage) + args.v1.ucConfig |= + ADJUST_DISPLAY_CONFIG_SS_ENABLE; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, + index, (uint32_t *)&args); + adjusted_clock = le16_to_cpu(args.v1.usPixelClock) * 10; + break; + case 3: + args.v3.sInput.usPixelClock = cpu_to_le16(clock / 10); + args.v3.sInput.ucTransmitterID = amdgpu_encoder->encoder_id; + args.v3.sInput.ucEncodeMode = encoder_mode; + args.v3.sInput.ucDispPllConfig = 0; + if (amdgpu_crtc->ss_enabled && amdgpu_crtc->ss.percentage) + args.v3.sInput.ucDispPllConfig |= + DISPPLL_CONFIG_SS_ENABLE; + if (ENCODER_MODE_IS_DP(encoder_mode)) { + args.v3.sInput.ucDispPllConfig |= + DISPPLL_CONFIG_COHERENT_MODE; + /* 16200 or 27000 */ + args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10); + } else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + if (dig->coherent_mode) + args.v3.sInput.ucDispPllConfig |= + DISPPLL_CONFIG_COHERENT_MODE; + if (is_duallink) + args.v3.sInput.ucDispPllConfig |= + DISPPLL_CONFIG_DUAL_LINK; + } + if (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != + ENCODER_OBJECT_ID_NONE) + args.v3.sInput.ucExtTransmitterID = + amdgpu_encoder_get_dp_bridge_encoder_id(encoder); + else + args.v3.sInput.ucExtTransmitterID = 0; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, + index, (uint32_t *)&args); + adjusted_clock = le32_to_cpu(args.v3.sOutput.ulDispPllFreq) * 10; + if (args.v3.sOutput.ucRefDiv) { + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_FRAC_FB_DIV; + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_REF_DIV; + amdgpu_crtc->pll_reference_div = args.v3.sOutput.ucRefDiv; + } + if (args.v3.sOutput.ucPostDiv) { + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_FRAC_FB_DIV; + amdgpu_crtc->pll_flags |= AMDGPU_PLL_USE_POST_DIV; + amdgpu_crtc->pll_post_div = args.v3.sOutput.ucPostDiv; + } + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return adjusted_clock; + } + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return adjusted_clock; + } + + return adjusted_clock; +} + +union set_pixel_clock { + SET_PIXEL_CLOCK_PS_ALLOCATION base; + PIXEL_CLOCK_PARAMETERS v1; + PIXEL_CLOCK_PARAMETERS_V2 v2; + PIXEL_CLOCK_PARAMETERS_V3 v3; + PIXEL_CLOCK_PARAMETERS_V5 v5; + PIXEL_CLOCK_PARAMETERS_V6 v6; + PIXEL_CLOCK_PARAMETERS_V7 v7; +}; + +/* on DCE5, make sure the voltage is high enough to support the + * required disp clk. + */ +void amdgpu_atombios_crtc_set_disp_eng_pll(struct amdgpu_device *adev, + u32 dispclk) +{ + u8 frev, crev; + int index; + union set_pixel_clock args; + + memset(&args, 0, sizeof(args)); + + index = GetIndexIntoMasterTable(COMMAND, SetPixelClock); + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, + &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 5: + /* if the default dcpll clock is specified, + * SetPixelClock provides the dividers + */ + args.v5.ucCRTC = ATOM_CRTC_INVALID; + args.v5.usPixelClock = cpu_to_le16(dispclk); + args.v5.ucPpll = ATOM_DCPLL; + break; + case 6: + /* if the default dcpll clock is specified, + * SetPixelClock provides the dividers + */ + args.v6.ulDispEngClkFreq = cpu_to_le32(dispclk); + if (adev->asic_type == CHIP_TAHITI || + adev->asic_type == CHIP_PITCAIRN || + adev->asic_type == CHIP_VERDE || + adev->asic_type == CHIP_OLAND) + args.v6.ucPpll = ATOM_PPLL0; + else + args.v6.ucPpll = ATOM_EXT_PLL1; + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return; + } + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return; + } + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +union set_dce_clock { + SET_DCE_CLOCK_PS_ALLOCATION_V1_1 v1_1; + SET_DCE_CLOCK_PS_ALLOCATION_V2_1 v2_1; +}; + +u32 amdgpu_atombios_crtc_set_dce_clock(struct amdgpu_device *adev, + u32 freq, u8 clk_type, u8 clk_src) +{ + u8 frev, crev; + int index; + union set_dce_clock args; + u32 ret_freq = 0; + + memset(&args, 0, sizeof(args)); + + index = GetIndexIntoMasterTable(COMMAND, SetDCEClock); + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, + &crev)) + return 0; + + switch (frev) { + case 2: + switch (crev) { + case 1: + args.v2_1.asParam.ulDCEClkFreq = cpu_to_le32(freq); /* 10kHz units */ + args.v2_1.asParam.ucDCEClkType = clk_type; + args.v2_1.asParam.ucDCEClkSrc = clk_src; + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + ret_freq = le32_to_cpu(args.v2_1.asParam.ulDCEClkFreq) * 10; + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return 0; + } + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return 0; + } + + return ret_freq; +} + +static bool is_pixel_clock_source_from_pll(u32 encoder_mode, int pll_id) +{ + if (ENCODER_MODE_IS_DP(encoder_mode)) { + if (pll_id < ATOM_EXT_PLL1) + return true; + else + return false; + } else { + return true; + } +} + +void amdgpu_atombios_crtc_program_pll(struct drm_crtc *crtc, + u32 crtc_id, + int pll_id, + u32 encoder_mode, + u32 encoder_id, + u32 clock, + u32 ref_div, + u32 fb_div, + u32 frac_fb_div, + u32 post_div, + int bpc, + bool ss_enabled, + struct amdgpu_atom_ss *ss) +{ + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + u8 frev, crev; + int index = GetIndexIntoMasterTable(COMMAND, SetPixelClock); + union set_pixel_clock args; + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, + &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 1: + if (clock == ATOM_DISABLE) + return; + args.v1.usPixelClock = cpu_to_le16(clock / 10); + args.v1.usRefDiv = cpu_to_le16(ref_div); + args.v1.usFbDiv = cpu_to_le16(fb_div); + args.v1.ucFracFbDiv = frac_fb_div; + args.v1.ucPostDiv = post_div; + args.v1.ucPpll = pll_id; + args.v1.ucCRTC = crtc_id; + args.v1.ucRefDivSrc = 1; + break; + case 2: + args.v2.usPixelClock = cpu_to_le16(clock / 10); + args.v2.usRefDiv = cpu_to_le16(ref_div); + args.v2.usFbDiv = cpu_to_le16(fb_div); + args.v2.ucFracFbDiv = frac_fb_div; + args.v2.ucPostDiv = post_div; + args.v2.ucPpll = pll_id; + args.v2.ucCRTC = crtc_id; + args.v2.ucRefDivSrc = 1; + break; + case 3: + args.v3.usPixelClock = cpu_to_le16(clock / 10); + args.v3.usRefDiv = cpu_to_le16(ref_div); + args.v3.usFbDiv = cpu_to_le16(fb_div); + args.v3.ucFracFbDiv = frac_fb_div; + args.v3.ucPostDiv = post_div; + args.v3.ucPpll = pll_id; + if (crtc_id == ATOM_CRTC2) + args.v3.ucMiscInfo = PIXEL_CLOCK_MISC_CRTC_SEL_CRTC2; + else + args.v3.ucMiscInfo = PIXEL_CLOCK_MISC_CRTC_SEL_CRTC1; + if (ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK)) + args.v3.ucMiscInfo |= PIXEL_CLOCK_MISC_REF_DIV_SRC; + args.v3.ucTransmitterId = encoder_id; + args.v3.ucEncoderMode = encoder_mode; + break; + case 5: + args.v5.ucCRTC = crtc_id; + args.v5.usPixelClock = cpu_to_le16(clock / 10); + args.v5.ucRefDiv = ref_div; + args.v5.usFbDiv = cpu_to_le16(fb_div); + args.v5.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000); + args.v5.ucPostDiv = post_div; + args.v5.ucMiscInfo = 0; /* HDMI depth, etc. */ + if ((ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK)) && + (pll_id < ATOM_EXT_PLL1)) + args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_REF_DIV_SRC; + if (encoder_mode == ATOM_ENCODER_MODE_HDMI) { + switch (bpc) { + case 8: + default: + args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_24BPP; + break; + case 10: + /* yes this is correct, the atom define is wrong */ + args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_32BPP; + break; + case 12: + /* yes this is correct, the atom define is wrong */ + args.v5.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_30BPP; + break; + } + } + args.v5.ucTransmitterID = encoder_id; + args.v5.ucEncoderMode = encoder_mode; + args.v5.ucPpll = pll_id; + break; + case 6: + args.v6.ulDispEngClkFreq = cpu_to_le32(crtc_id << 24 | clock / 10); + args.v6.ucRefDiv = ref_div; + args.v6.usFbDiv = cpu_to_le16(fb_div); + args.v6.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000); + args.v6.ucPostDiv = post_div; + args.v6.ucMiscInfo = 0; /* HDMI depth, etc. */ + if ((ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK)) && + (pll_id < ATOM_EXT_PLL1) && + !is_pixel_clock_source_from_pll(encoder_mode, pll_id)) + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_REF_DIV_SRC; + if (encoder_mode == ATOM_ENCODER_MODE_HDMI) { + switch (bpc) { + case 8: + default: + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_24BPP; + break; + case 10: + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6; + break; + case 12: + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6; + break; + case 16: + args.v6.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_48BPP; + break; + } + } + args.v6.ucTransmitterID = encoder_id; + args.v6.ucEncoderMode = encoder_mode; + args.v6.ucPpll = pll_id; + break; + case 7: + args.v7.ulPixelClock = cpu_to_le32(clock * 10); /* 100 hz units */ + args.v7.ucMiscInfo = 0; + if ((encoder_mode == ATOM_ENCODER_MODE_DVI) && + (clock > 165000)) + args.v7.ucMiscInfo |= PIXEL_CLOCK_V7_MISC_DVI_DUALLINK_EN; + args.v7.ucCRTC = crtc_id; + if (encoder_mode == ATOM_ENCODER_MODE_HDMI) { + switch (bpc) { + case 8: + default: + args.v7.ucDeepColorRatio = PIXEL_CLOCK_V7_DEEPCOLOR_RATIO_DIS; + break; + case 10: + args.v7.ucDeepColorRatio = PIXEL_CLOCK_V7_DEEPCOLOR_RATIO_5_4; + break; + case 12: + args.v7.ucDeepColorRatio = PIXEL_CLOCK_V7_DEEPCOLOR_RATIO_3_2; + break; + case 16: + args.v7.ucDeepColorRatio = PIXEL_CLOCK_V7_DEEPCOLOR_RATIO_2_1; + break; + } + } + args.v7.ucTransmitterID = encoder_id; + args.v7.ucEncoderMode = encoder_mode; + args.v7.ucPpll = pll_id; + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return; + } + break; + default: + DRM_ERROR("Unknown table version %d %d\n", frev, crev); + return; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +int amdgpu_atombios_crtc_prepare_pll(struct drm_crtc *crtc, + struct drm_display_mode *mode) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = + to_amdgpu_encoder(amdgpu_crtc->encoder); + int encoder_mode = amdgpu_atombios_encoder_get_encoder_mode(amdgpu_crtc->encoder); + + amdgpu_crtc->bpc = 8; + amdgpu_crtc->ss_enabled = false; + + if ((amdgpu_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) || + (amdgpu_encoder_get_dp_bridge_encoder_id(amdgpu_crtc->encoder) != ENCODER_OBJECT_ID_NONE)) { + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + struct drm_connector *connector = + amdgpu_get_connector_for_encoder(amdgpu_crtc->encoder); + struct amdgpu_connector *amdgpu_connector = + to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + int dp_clock; + + /* Assign mode clock for hdmi deep color max clock limit check */ + amdgpu_connector->pixelclock_for_modeset = mode->clock; + amdgpu_crtc->bpc = amdgpu_connector_get_monitor_bpc(connector); + + switch (encoder_mode) { + case ATOM_ENCODER_MODE_DP_MST: + case ATOM_ENCODER_MODE_DP: + /* DP/eDP */ + dp_clock = dig_connector->dp_clock / 10; + amdgpu_crtc->ss_enabled = + amdgpu_atombios_get_asic_ss_info(adev, &amdgpu_crtc->ss, + ASIC_INTERNAL_SS_ON_DP, + dp_clock); + break; + case ATOM_ENCODER_MODE_LVDS: + amdgpu_crtc->ss_enabled = + amdgpu_atombios_get_asic_ss_info(adev, + &amdgpu_crtc->ss, + dig->lcd_ss_id, + mode->clock / 10); + break; + case ATOM_ENCODER_MODE_DVI: + amdgpu_crtc->ss_enabled = + amdgpu_atombios_get_asic_ss_info(adev, + &amdgpu_crtc->ss, + ASIC_INTERNAL_SS_ON_TMDS, + mode->clock / 10); + break; + case ATOM_ENCODER_MODE_HDMI: + amdgpu_crtc->ss_enabled = + amdgpu_atombios_get_asic_ss_info(adev, + &amdgpu_crtc->ss, + ASIC_INTERNAL_SS_ON_HDMI, + mode->clock / 10); + break; + default: + break; + } + } + + /* adjust pixel clock as needed */ + amdgpu_crtc->adjusted_clock = amdgpu_atombios_crtc_adjust_pll(crtc, mode); + + return 0; +} + +void amdgpu_atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode) +{ + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); + struct drm_device *dev = crtc->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = + to_amdgpu_encoder(amdgpu_crtc->encoder); + u32 pll_clock = mode->clock; + u32 clock = mode->clock; + u32 ref_div = 0, fb_div = 0, frac_fb_div = 0, post_div = 0; + struct amdgpu_pll *pll; + int encoder_mode = amdgpu_atombios_encoder_get_encoder_mode(amdgpu_crtc->encoder); + + /* pass the actual clock to amdgpu_atombios_crtc_program_pll for HDMI */ + if ((encoder_mode == ATOM_ENCODER_MODE_HDMI) && + (amdgpu_crtc->bpc > 8)) + clock = amdgpu_crtc->adjusted_clock; + + switch (amdgpu_crtc->pll_id) { + case ATOM_PPLL1: + pll = &adev->clock.ppll[0]; + break; + case ATOM_PPLL2: + pll = &adev->clock.ppll[1]; + break; + case ATOM_PPLL0: + case ATOM_PPLL_INVALID: + default: + pll = &adev->clock.ppll[2]; + break; + } + + /* update pll params */ + pll->flags = amdgpu_crtc->pll_flags; + pll->reference_div = amdgpu_crtc->pll_reference_div; + pll->post_div = amdgpu_crtc->pll_post_div; + + amdgpu_pll_compute(pll, amdgpu_crtc->adjusted_clock, &pll_clock, + &fb_div, &frac_fb_div, &ref_div, &post_div); + + amdgpu_atombios_crtc_program_ss(adev, ATOM_DISABLE, amdgpu_crtc->pll_id, + amdgpu_crtc->crtc_id, &amdgpu_crtc->ss); + + amdgpu_atombios_crtc_program_pll(crtc, amdgpu_crtc->crtc_id, amdgpu_crtc->pll_id, + encoder_mode, amdgpu_encoder->encoder_id, clock, + ref_div, fb_div, frac_fb_div, post_div, + amdgpu_crtc->bpc, amdgpu_crtc->ss_enabled, &amdgpu_crtc->ss); + + if (amdgpu_crtc->ss_enabled) { + /* calculate ss amount and step size */ + u32 step_size; + u32 amount = (((fb_div * 10) + frac_fb_div) * + (u32)amdgpu_crtc->ss.percentage) / + (100 * (u32)amdgpu_crtc->ss.percentage_divider); + amdgpu_crtc->ss.amount = (amount / 10) & ATOM_PPLL_SS_AMOUNT_V2_FBDIV_MASK; + amdgpu_crtc->ss.amount |= ((amount - (amount / 10)) << ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT) & + ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK; + if (amdgpu_crtc->ss.type & ATOM_PPLL_SS_TYPE_V2_CENTRE_SPREAD) + step_size = (4 * amount * ref_div * ((u32)amdgpu_crtc->ss.rate * 2048)) / + (125 * 25 * pll->reference_freq / 100); + else + step_size = (2 * amount * ref_div * ((u32)amdgpu_crtc->ss.rate * 2048)) / + (125 * 25 * pll->reference_freq / 100); + amdgpu_crtc->ss.step = step_size; + + amdgpu_atombios_crtc_program_ss(adev, ATOM_ENABLE, amdgpu_crtc->pll_id, + amdgpu_crtc->crtc_id, &amdgpu_crtc->ss); + } +} +
diff --git a/amdgpu/atombios_crtc.h b/amdgpu/atombios_crtc.h new file mode 100644 index 0000000..0eeda8e --- /dev/null +++ b/amdgpu/atombios_crtc.h
@@ -0,0 +1,60 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __ATOMBIOS_CRTC_H__ +#define __ATOMBIOS_CRTC_H__ + +void amdgpu_atombios_crtc_overscan_setup(struct drm_crtc *crtc, + struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); +void amdgpu_atombios_crtc_scaler_setup(struct drm_crtc *crtc); +void amdgpu_atombios_crtc_lock(struct drm_crtc *crtc, int lock); +void amdgpu_atombios_crtc_enable(struct drm_crtc *crtc, int state); +void amdgpu_atombios_crtc_blank(struct drm_crtc *crtc, int state); +void amdgpu_atombios_crtc_powergate(struct drm_crtc *crtc, int state); +void amdgpu_atombios_crtc_powergate_init(struct amdgpu_device *adev); +void amdgpu_atombios_crtc_set_dtd_timing(struct drm_crtc *crtc, + struct drm_display_mode *mode); +void amdgpu_atombios_crtc_set_disp_eng_pll(struct amdgpu_device *adev, + u32 dispclk); +u32 amdgpu_atombios_crtc_set_dce_clock(struct amdgpu_device *adev, + u32 freq, u8 clk_type, u8 clk_src); +void amdgpu_atombios_crtc_program_pll(struct drm_crtc *crtc, + u32 crtc_id, + int pll_id, + u32 encoder_mode, + u32 encoder_id, + u32 clock, + u32 ref_div, + u32 fb_div, + u32 frac_fb_div, + u32 post_div, + int bpc, + bool ss_enabled, + struct amdgpu_atom_ss *ss); +int amdgpu_atombios_crtc_prepare_pll(struct drm_crtc *crtc, + struct drm_display_mode *mode); +void amdgpu_atombios_crtc_set_pll(struct drm_crtc *crtc, + struct drm_display_mode *mode); + +#endif
diff --git a/amdgpu/atombios_dp.c b/amdgpu/atombios_dp.c new file mode 100644 index 0000000..6858cde --- /dev/null +++ b/amdgpu/atombios_dp.c
@@ -0,0 +1,762 @@ +/* + * Copyright 2007-8 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" + +#include "atom.h" +#include "atom-bits.h" +#include "atombios_encoders.h" +#include "atombios_dp.h" +#include "amdgpu_connectors.h" +#include "amdgpu_atombios.h" +#include <drm/drm_dp_helper.h> + +/* move these to drm_dp_helper.c/h */ +#define DP_LINK_CONFIGURATION_SIZE 9 +#define DP_DPCD_SIZE DP_RECEIVER_CAP_SIZE + +static char *voltage_names[] = { + "0.4V", "0.6V", "0.8V", "1.2V" +}; +static char *pre_emph_names[] = { + "0dB", "3.5dB", "6dB", "9.5dB" +}; + +/***** amdgpu AUX functions *****/ + +union aux_channel_transaction { + PROCESS_AUX_CHANNEL_TRANSACTION_PS_ALLOCATION v1; + PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 v2; +}; + +static int amdgpu_atombios_dp_process_aux_ch(struct amdgpu_i2c_chan *chan, + u8 *send, int send_bytes, + u8 *recv, int recv_size, + u8 delay, u8 *ack) +{ + struct drm_device *dev = chan->dev; + struct amdgpu_device *adev = dev->dev_private; + union aux_channel_transaction args; + int index = GetIndexIntoMasterTable(COMMAND, ProcessAuxChannelTransaction); + unsigned char *base; + int recv_bytes; + int r = 0; + + memset(&args, 0, sizeof(args)); + + mutex_lock(&chan->mutex); + + base = (unsigned char *)(adev->mode_info.atom_context->scratch + 1); + + amdgpu_atombios_copy_swap(base, send, send_bytes, true); + + args.v2.lpAuxRequest = cpu_to_le16((u16)(0 + 4)); + args.v2.lpDataOut = cpu_to_le16((u16)(16 + 4)); + args.v2.ucDataOutLen = 0; + args.v2.ucChannelID = chan->rec.i2c_id; + args.v2.ucDelay = delay / 10; + args.v2.ucHPD_ID = chan->rec.hpd; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + *ack = args.v2.ucReplyStatus; + + /* timeout */ + if (args.v2.ucReplyStatus == 1) { + r = -ETIMEDOUT; + goto done; + } + + /* flags not zero */ + if (args.v2.ucReplyStatus == 2) { + DRM_DEBUG_KMS("dp_aux_ch flags not zero\n"); + r = -EIO; + goto done; + } + + /* error */ + if (args.v2.ucReplyStatus == 3) { + DRM_DEBUG_KMS("dp_aux_ch error\n"); + r = -EIO; + goto done; + } + + recv_bytes = args.v1.ucDataOutLen; + if (recv_bytes > recv_size) + recv_bytes = recv_size; + + if (recv && recv_size) + amdgpu_atombios_copy_swap(recv, base + 16, recv_bytes, false); + + r = recv_bytes; +done: + mutex_unlock(&chan->mutex); + + return r; +} + +#define BARE_ADDRESS_SIZE 3 +#define HEADER_SIZE (BARE_ADDRESS_SIZE + 1) + +static ssize_t +amdgpu_atombios_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) +{ + struct amdgpu_i2c_chan *chan = + container_of(aux, struct amdgpu_i2c_chan, aux); + int ret; + u8 tx_buf[20]; + size_t tx_size; + u8 ack, delay = 0; + + if (WARN_ON(msg->size > 16)) + return -E2BIG; + + tx_buf[0] = msg->address & 0xff; + tx_buf[1] = msg->address >> 8; + tx_buf[2] = (msg->request << 4) | + ((msg->address >> 16) & 0xf); + tx_buf[3] = msg->size ? (msg->size - 1) : 0; + + switch (msg->request & ~DP_AUX_I2C_MOT) { + case DP_AUX_NATIVE_WRITE: + case DP_AUX_I2C_WRITE: + /* tx_size needs to be 4 even for bare address packets since the atom + * table needs the info in tx_buf[3]. + */ + tx_size = HEADER_SIZE + msg->size; + if (msg->size == 0) + tx_buf[3] |= BARE_ADDRESS_SIZE << 4; + else + tx_buf[3] |= tx_size << 4; + memcpy(tx_buf + HEADER_SIZE, msg->buffer, msg->size); + ret = amdgpu_atombios_dp_process_aux_ch(chan, + tx_buf, tx_size, NULL, 0, delay, &ack); + if (ret >= 0) + /* Return payload size. */ + ret = msg->size; + break; + case DP_AUX_NATIVE_READ: + case DP_AUX_I2C_READ: + /* tx_size needs to be 4 even for bare address packets since the atom + * table needs the info in tx_buf[3]. + */ + tx_size = HEADER_SIZE; + if (msg->size == 0) + tx_buf[3] |= BARE_ADDRESS_SIZE << 4; + else + tx_buf[3] |= tx_size << 4; + ret = amdgpu_atombios_dp_process_aux_ch(chan, + tx_buf, tx_size, msg->buffer, msg->size, delay, &ack); + break; + default: + ret = -EINVAL; + break; + } + + if (ret >= 0) + msg->reply = ack >> 4; + + return ret; +} + +void amdgpu_atombios_dp_aux_init(struct amdgpu_connector *amdgpu_connector) +{ + int ret; + + amdgpu_connector->ddc_bus->rec.hpd = amdgpu_connector->hpd.hpd; + amdgpu_connector->ddc_bus->aux.dev = amdgpu_connector->base.kdev; + amdgpu_connector->ddc_bus->aux.transfer = amdgpu_atombios_dp_aux_transfer; + ret = drm_dp_aux_register(&amdgpu_connector->ddc_bus->aux); + if (!ret) + amdgpu_connector->ddc_bus->has_aux = true; + + WARN(ret, "drm_dp_aux_register_i2c_bus() failed with error %d\n", ret); +} + +/***** general DP utility functions *****/ + +#define DP_VOLTAGE_MAX DP_TRAIN_VOLTAGE_SWING_LEVEL_3 +#define DP_PRE_EMPHASIS_MAX DP_TRAIN_PRE_EMPH_LEVEL_3 + +static void amdgpu_atombios_dp_get_adjust_train(const u8 link_status[DP_LINK_STATUS_SIZE], + int lane_count, + u8 train_set[4]) +{ + u8 v = 0; + u8 p = 0; + int lane; + + for (lane = 0; lane < lane_count; lane++) { + u8 this_v = drm_dp_get_adjust_request_voltage(link_status, lane); + u8 this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + + DRM_DEBUG_KMS("requested signal parameters: lane %d voltage %s pre_emph %s\n", + lane, + voltage_names[this_v >> DP_TRAIN_VOLTAGE_SWING_SHIFT], + pre_emph_names[this_p >> DP_TRAIN_PRE_EMPHASIS_SHIFT]); + + if (this_v > v) + v = this_v; + if (this_p > p) + p = this_p; + } + + if (v >= DP_VOLTAGE_MAX) + v |= DP_TRAIN_MAX_SWING_REACHED; + + if (p >= DP_PRE_EMPHASIS_MAX) + p |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED; + + DRM_DEBUG_KMS("using signal parameters: voltage %s pre_emph %s\n", + voltage_names[(v & DP_TRAIN_VOLTAGE_SWING_MASK) >> DP_TRAIN_VOLTAGE_SWING_SHIFT], + pre_emph_names[(p & DP_TRAIN_PRE_EMPHASIS_MASK) >> DP_TRAIN_PRE_EMPHASIS_SHIFT]); + + for (lane = 0; lane < 4; lane++) + train_set[lane] = v | p; +} + +/* convert bits per color to bits per pixel */ +/* get bpc from the EDID */ +static unsigned amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc) +{ + if (bpc == 0) + return 24; + else + return bpc * 3; +} + +/***** amdgpu specific DP functions *****/ + +static int amdgpu_atombios_dp_get_dp_link_config(struct drm_connector *connector, + const u8 dpcd[DP_DPCD_SIZE], + unsigned pix_clock, + unsigned *dp_lanes, unsigned *dp_rate) +{ + unsigned bpp = + amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector)); + static const unsigned link_rates[3] = { 162000, 270000, 540000 }; + unsigned max_link_rate = drm_dp_max_link_rate(dpcd); + unsigned max_lane_num = drm_dp_max_lane_count(dpcd); + unsigned lane_num, i, max_pix_clock; + + if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) == + ENCODER_OBJECT_ID_NUTMEG) { + for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) { + max_pix_clock = (lane_num * 270000 * 8) / bpp; + if (max_pix_clock >= pix_clock) { + *dp_lanes = lane_num; + *dp_rate = 270000; + return 0; + } + } + } else { + for (i = 0; i < ARRAY_SIZE(link_rates) && link_rates[i] <= max_link_rate; i++) { + for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) { + max_pix_clock = (lane_num * link_rates[i] * 8) / bpp; + if (max_pix_clock >= pix_clock) { + *dp_lanes = lane_num; + *dp_rate = link_rates[i]; + return 0; + } + } + } + } + + return -EINVAL; +} + +static u8 amdgpu_atombios_dp_encoder_service(struct amdgpu_device *adev, + int action, int dp_clock, + u8 ucconfig, u8 lane_num) +{ + DP_ENCODER_SERVICE_PARAMETERS args; + int index = GetIndexIntoMasterTable(COMMAND, DPEncoderService); + + memset(&args, 0, sizeof(args)); + args.ucLinkClock = dp_clock / 10; + args.ucConfig = ucconfig; + args.ucAction = action; + args.ucLaneNum = lane_num; + args.ucStatus = 0; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + return args.ucStatus; +} + +u8 amdgpu_atombios_dp_get_sinktype(struct amdgpu_connector *amdgpu_connector) +{ + struct drm_device *dev = amdgpu_connector->base.dev; + struct amdgpu_device *adev = dev->dev_private; + + return amdgpu_atombios_dp_encoder_service(adev, ATOM_DP_ACTION_GET_SINK_TYPE, 0, + amdgpu_connector->ddc_bus->rec.i2c_id, 0); +} + +static void amdgpu_atombios_dp_probe_oui(struct amdgpu_connector *amdgpu_connector) +{ + struct amdgpu_connector_atom_dig *dig_connector = amdgpu_connector->con_priv; + u8 buf[3]; + + if (!(dig_connector->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) + return; + + if (drm_dp_dpcd_read(&amdgpu_connector->ddc_bus->aux, DP_SINK_OUI, buf, 3) == 3) + DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n", + buf[0], buf[1], buf[2]); + + if (drm_dp_dpcd_read(&amdgpu_connector->ddc_bus->aux, DP_BRANCH_OUI, buf, 3) == 3) + DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n", + buf[0], buf[1], buf[2]); +} + +int amdgpu_atombios_dp_get_dpcd(struct amdgpu_connector *amdgpu_connector) +{ + struct amdgpu_connector_atom_dig *dig_connector = amdgpu_connector->con_priv; + u8 msg[DP_DPCD_SIZE]; + int ret; + + ret = drm_dp_dpcd_read(&amdgpu_connector->ddc_bus->aux, DP_DPCD_REV, + msg, DP_DPCD_SIZE); + if (ret == DP_DPCD_SIZE) { + memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE); + + DRM_DEBUG_KMS("DPCD: %*ph\n", (int)sizeof(dig_connector->dpcd), + dig_connector->dpcd); + + amdgpu_atombios_dp_probe_oui(amdgpu_connector); + + return 0; + } + + dig_connector->dpcd[0] = 0; + return -EINVAL; +} + +int amdgpu_atombios_dp_get_panel_mode(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector; + int panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE; + u16 dp_bridge = amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector); + u8 tmp; + + if (!amdgpu_connector->con_priv) + return panel_mode; + + dig_connector = amdgpu_connector->con_priv; + + if (dp_bridge != ENCODER_OBJECT_ID_NONE) { + /* DP bridge chips */ + if (drm_dp_dpcd_readb(&amdgpu_connector->ddc_bus->aux, + DP_EDP_CONFIGURATION_CAP, &tmp) == 1) { + if (tmp & 1) + panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE; + else if ((dp_bridge == ENCODER_OBJECT_ID_NUTMEG) || + (dp_bridge == ENCODER_OBJECT_ID_TRAVIS)) + panel_mode = DP_PANEL_MODE_INTERNAL_DP1_MODE; + else + panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE; + } + } else if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { + /* eDP */ + if (drm_dp_dpcd_readb(&amdgpu_connector->ddc_bus->aux, + DP_EDP_CONFIGURATION_CAP, &tmp) == 1) { + if (tmp & 1) + panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE; + } + } + + return panel_mode; +} + +void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector; + int ret; + + if (!amdgpu_connector->con_priv) + return; + dig_connector = amdgpu_connector->con_priv; + + if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || + (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { + ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd, + mode->clock, + &dig_connector->dp_lane_count, + &dig_connector->dp_clock); + if (ret) { + dig_connector->dp_clock = 0; + dig_connector->dp_lane_count = 0; + } + } +} + +int amdgpu_atombios_dp_mode_valid_helper(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector; + unsigned dp_lanes, dp_clock; + int ret; + + if (!amdgpu_connector->con_priv) + return MODE_CLOCK_HIGH; + dig_connector = amdgpu_connector->con_priv; + + ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd, + mode->clock, &dp_lanes, &dp_clock); + if (ret) + return MODE_CLOCK_HIGH; + + if ((dp_clock == 540000) && + (!amdgpu_connector_is_dp12_capable(connector))) + return MODE_CLOCK_HIGH; + + return MODE_OK; +} + +bool amdgpu_atombios_dp_needs_link_train(struct amdgpu_connector *amdgpu_connector) +{ + u8 link_status[DP_LINK_STATUS_SIZE]; + struct amdgpu_connector_atom_dig *dig = amdgpu_connector->con_priv; + + if (drm_dp_dpcd_read_link_status(&amdgpu_connector->ddc_bus->aux, link_status) + <= 0) + return false; + if (drm_dp_channel_eq_ok(link_status, dig->dp_lane_count)) + return false; + return true; +} + +void amdgpu_atombios_dp_set_rx_power_state(struct drm_connector *connector, + u8 power_state) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector; + + if (!amdgpu_connector->con_priv) + return; + + dig_connector = amdgpu_connector->con_priv; + + /* power up/down the sink */ + if (dig_connector->dpcd[0] >= 0x11) { + drm_dp_dpcd_writeb(&amdgpu_connector->ddc_bus->aux, + DP_SET_POWER, power_state); + usleep_range(1000, 2000); + } +} + +struct amdgpu_atombios_dp_link_train_info { + struct amdgpu_device *adev; + struct drm_encoder *encoder; + struct drm_connector *connector; + int dp_clock; + int dp_lane_count; + bool tp3_supported; + u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 train_set[4]; + u8 link_status[DP_LINK_STATUS_SIZE]; + u8 tries; + struct drm_dp_aux *aux; +}; + +static void +amdgpu_atombios_dp_update_vs_emph(struct amdgpu_atombios_dp_link_train_info *dp_info) +{ + /* set the initial vs/emph on the source */ + amdgpu_atombios_encoder_setup_dig_transmitter(dp_info->encoder, + ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH, + 0, dp_info->train_set[0]); /* sets all lanes at once */ + + /* set the vs/emph on the sink */ + drm_dp_dpcd_write(dp_info->aux, DP_TRAINING_LANE0_SET, + dp_info->train_set, dp_info->dp_lane_count); +} + +static void +amdgpu_atombios_dp_set_tp(struct amdgpu_atombios_dp_link_train_info *dp_info, int tp) +{ + int rtp = 0; + + /* set training pattern on the source */ + switch (tp) { + case DP_TRAINING_PATTERN_1: + rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1; + break; + case DP_TRAINING_PATTERN_2: + rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2; + break; + case DP_TRAINING_PATTERN_3: + rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN3; + break; + } + amdgpu_atombios_encoder_setup_dig_encoder(dp_info->encoder, rtp, 0); + + /* enable training pattern on the sink */ + drm_dp_dpcd_writeb(dp_info->aux, DP_TRAINING_PATTERN_SET, tp); +} + +static int +amdgpu_atombios_dp_link_train_init(struct amdgpu_atombios_dp_link_train_info *dp_info) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(dp_info->encoder); + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + u8 tmp; + + /* power up the sink */ + amdgpu_atombios_dp_set_rx_power_state(dp_info->connector, DP_SET_POWER_D0); + + /* possibly enable downspread on the sink */ + if (dp_info->dpcd[3] & 0x1) + drm_dp_dpcd_writeb(dp_info->aux, + DP_DOWNSPREAD_CTRL, DP_SPREAD_AMP_0_5); + else + drm_dp_dpcd_writeb(dp_info->aux, + DP_DOWNSPREAD_CTRL, 0); + + if (dig->panel_mode == DP_PANEL_MODE_INTERNAL_DP2_MODE) + drm_dp_dpcd_writeb(dp_info->aux, DP_EDP_CONFIGURATION_SET, 1); + + /* set the lane count on the sink */ + tmp = dp_info->dp_lane_count; + if (drm_dp_enhanced_frame_cap(dp_info->dpcd)) + tmp |= DP_LANE_COUNT_ENHANCED_FRAME_EN; + drm_dp_dpcd_writeb(dp_info->aux, DP_LANE_COUNT_SET, tmp); + + /* set the link rate on the sink */ + tmp = drm_dp_link_rate_to_bw_code(dp_info->dp_clock); + drm_dp_dpcd_writeb(dp_info->aux, DP_LINK_BW_SET, tmp); + + /* start training on the source */ + amdgpu_atombios_encoder_setup_dig_encoder(dp_info->encoder, + ATOM_ENCODER_CMD_DP_LINK_TRAINING_START, 0); + + /* disable the training pattern on the sink */ + drm_dp_dpcd_writeb(dp_info->aux, + DP_TRAINING_PATTERN_SET, + DP_TRAINING_PATTERN_DISABLE); + + return 0; +} + +static int +amdgpu_atombios_dp_link_train_finish(struct amdgpu_atombios_dp_link_train_info *dp_info) +{ + udelay(400); + + /* disable the training pattern on the sink */ + drm_dp_dpcd_writeb(dp_info->aux, + DP_TRAINING_PATTERN_SET, + DP_TRAINING_PATTERN_DISABLE); + + /* disable the training pattern on the source */ + amdgpu_atombios_encoder_setup_dig_encoder(dp_info->encoder, + ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE, 0); + + return 0; +} + +static int +amdgpu_atombios_dp_link_train_cr(struct amdgpu_atombios_dp_link_train_info *dp_info) +{ + bool clock_recovery; + u8 voltage; + int i; + + amdgpu_atombios_dp_set_tp(dp_info, DP_TRAINING_PATTERN_1); + memset(dp_info->train_set, 0, 4); + amdgpu_atombios_dp_update_vs_emph(dp_info); + + udelay(400); + + /* clock recovery loop */ + clock_recovery = false; + dp_info->tries = 0; + voltage = 0xff; + while (1) { + drm_dp_link_train_clock_recovery_delay(dp_info->dpcd); + + if (drm_dp_dpcd_read_link_status(dp_info->aux, + dp_info->link_status) <= 0) { + DRM_ERROR("displayport link status failed\n"); + break; + } + + if (drm_dp_clock_recovery_ok(dp_info->link_status, dp_info->dp_lane_count)) { + clock_recovery = true; + break; + } + + for (i = 0; i < dp_info->dp_lane_count; i++) { + if ((dp_info->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0) + break; + } + if (i == dp_info->dp_lane_count) { + DRM_ERROR("clock recovery reached max voltage\n"); + break; + } + + if ((dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) { + ++dp_info->tries; + if (dp_info->tries == 5) { + DRM_ERROR("clock recovery tried 5 times\n"); + break; + } + } else + dp_info->tries = 0; + + voltage = dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; + + /* Compute new train_set as requested by sink */ + amdgpu_atombios_dp_get_adjust_train(dp_info->link_status, dp_info->dp_lane_count, + dp_info->train_set); + + amdgpu_atombios_dp_update_vs_emph(dp_info); + } + if (!clock_recovery) { + DRM_ERROR("clock recovery failed\n"); + return -1; + } else { + DRM_DEBUG_KMS("clock recovery at voltage %d pre-emphasis %d\n", + dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK, + (dp_info->train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK) >> + DP_TRAIN_PRE_EMPHASIS_SHIFT); + return 0; + } +} + +static int +amdgpu_atombios_dp_link_train_ce(struct amdgpu_atombios_dp_link_train_info *dp_info) +{ + bool channel_eq; + + if (dp_info->tp3_supported) + amdgpu_atombios_dp_set_tp(dp_info, DP_TRAINING_PATTERN_3); + else + amdgpu_atombios_dp_set_tp(dp_info, DP_TRAINING_PATTERN_2); + + /* channel equalization loop */ + dp_info->tries = 0; + channel_eq = false; + while (1) { + drm_dp_link_train_channel_eq_delay(dp_info->dpcd); + + if (drm_dp_dpcd_read_link_status(dp_info->aux, + dp_info->link_status) <= 0) { + DRM_ERROR("displayport link status failed\n"); + break; + } + + if (drm_dp_channel_eq_ok(dp_info->link_status, dp_info->dp_lane_count)) { + channel_eq = true; + break; + } + + /* Try 5 times */ + if (dp_info->tries > 5) { + DRM_ERROR("channel eq failed: 5 tries\n"); + break; + } + + /* Compute new train_set as requested by sink */ + amdgpu_atombios_dp_get_adjust_train(dp_info->link_status, dp_info->dp_lane_count, + dp_info->train_set); + + amdgpu_atombios_dp_update_vs_emph(dp_info); + dp_info->tries++; + } + + if (!channel_eq) { + DRM_ERROR("channel eq failed\n"); + return -1; + } else { + DRM_DEBUG_KMS("channel eq at voltage %d pre-emphasis %d\n", + dp_info->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK, + (dp_info->train_set[0] & DP_TRAIN_PRE_EMPHASIS_MASK) + >> DP_TRAIN_PRE_EMPHASIS_SHIFT); + return 0; + } +} + +void amdgpu_atombios_dp_link_train(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_encoder_atom_dig *dig; + struct amdgpu_connector *amdgpu_connector; + struct amdgpu_connector_atom_dig *dig_connector; + struct amdgpu_atombios_dp_link_train_info dp_info; + u8 tmp; + + if (!amdgpu_encoder->enc_priv) + return; + dig = amdgpu_encoder->enc_priv; + + amdgpu_connector = to_amdgpu_connector(connector); + if (!amdgpu_connector->con_priv) + return; + dig_connector = amdgpu_connector->con_priv; + + if ((dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_DISPLAYPORT) && + (dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_eDP)) + return; + + if (drm_dp_dpcd_readb(&amdgpu_connector->ddc_bus->aux, DP_MAX_LANE_COUNT, &tmp) + == 1) { + if (tmp & DP_TPS3_SUPPORTED) + dp_info.tp3_supported = true; + else + dp_info.tp3_supported = false; + } else { + dp_info.tp3_supported = false; + } + + memcpy(dp_info.dpcd, dig_connector->dpcd, DP_RECEIVER_CAP_SIZE); + dp_info.adev = adev; + dp_info.encoder = encoder; + dp_info.connector = connector; + dp_info.dp_lane_count = dig_connector->dp_lane_count; + dp_info.dp_clock = dig_connector->dp_clock; + dp_info.aux = &amdgpu_connector->ddc_bus->aux; + + if (amdgpu_atombios_dp_link_train_init(&dp_info)) + goto done; + if (amdgpu_atombios_dp_link_train_cr(&dp_info)) + goto done; + if (amdgpu_atombios_dp_link_train_ce(&dp_info)) + goto done; +done: + if (amdgpu_atombios_dp_link_train_finish(&dp_info)) + return; +}
diff --git a/amdgpu/atombios_dp.h b/amdgpu/atombios_dp.h new file mode 100644 index 0000000..f59d85e --- /dev/null +++ b/amdgpu/atombios_dp.h
@@ -0,0 +1,42 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __ATOMBIOS_DP_H__ +#define __ATOMBIOS_DP_H__ + +void amdgpu_atombios_dp_aux_init(struct amdgpu_connector *amdgpu_connector); +u8 amdgpu_atombios_dp_get_sinktype(struct amdgpu_connector *amdgpu_connector); +int amdgpu_atombios_dp_get_dpcd(struct amdgpu_connector *amdgpu_connector); +int amdgpu_atombios_dp_get_panel_mode(struct drm_encoder *encoder, + struct drm_connector *connector); +void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector, + const struct drm_display_mode *mode); +int amdgpu_atombios_dp_mode_valid_helper(struct drm_connector *connector, + struct drm_display_mode *mode); +bool amdgpu_atombios_dp_needs_link_train(struct amdgpu_connector *amdgpu_connector); +void amdgpu_atombios_dp_set_rx_power_state(struct drm_connector *connector, + u8 power_state); +void amdgpu_atombios_dp_link_train(struct drm_encoder *encoder, + struct drm_connector *connector); + +#endif
diff --git a/amdgpu/atombios_encoders.c b/amdgpu/atombios_encoders.c new file mode 100644 index 0000000..1e94a9b --- /dev/null +++ b/amdgpu/atombios_encoders.c
@@ -0,0 +1,2158 @@ +/* + * Copyright 2007-11 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + */ + +#include <linux/pci.h> + +#include <drm/drm_crtc_helper.h> +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "amdgpu_connectors.h" +#include "amdgpu_display.h" +#include "atom.h" +#include "atombios_encoders.h" +#include "atombios_dp.h" +#include <linux/backlight.h> +#include "bif/bif_4_1_d.h" + +u8 +amdgpu_atombios_encoder_get_backlight_level_from_reg(struct amdgpu_device *adev) +{ + u8 backlight_level; + u32 bios_2_scratch; + + bios_2_scratch = RREG32(mmBIOS_SCRATCH_2); + + backlight_level = ((bios_2_scratch & ATOM_S2_CURRENT_BL_LEVEL_MASK) >> + ATOM_S2_CURRENT_BL_LEVEL_SHIFT); + + return backlight_level; +} + +void +amdgpu_atombios_encoder_set_backlight_level_to_reg(struct amdgpu_device *adev, + u8 backlight_level) +{ + u32 bios_2_scratch; + + bios_2_scratch = RREG32(mmBIOS_SCRATCH_2); + + bios_2_scratch &= ~ATOM_S2_CURRENT_BL_LEVEL_MASK; + bios_2_scratch |= ((backlight_level << ATOM_S2_CURRENT_BL_LEVEL_SHIFT) & + ATOM_S2_CURRENT_BL_LEVEL_MASK); + + WREG32(mmBIOS_SCRATCH_2, bios_2_scratch); +} + +u8 +amdgpu_atombios_encoder_get_backlight_level(struct amdgpu_encoder *amdgpu_encoder) +{ + struct drm_device *dev = amdgpu_encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) + return 0; + + return amdgpu_atombios_encoder_get_backlight_level_from_reg(adev); +} + +void +amdgpu_atombios_encoder_set_backlight_level(struct amdgpu_encoder *amdgpu_encoder, + u8 level) +{ + struct drm_encoder *encoder = &amdgpu_encoder->base; + struct drm_device *dev = amdgpu_encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder_atom_dig *dig; + + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) + return; + + if ((amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) && + amdgpu_encoder->enc_priv) { + dig = amdgpu_encoder->enc_priv; + dig->backlight_level = level; + amdgpu_atombios_encoder_set_backlight_level_to_reg(adev, dig->backlight_level); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + if (dig->backlight_level == 0) + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); + else { + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_BL_BRIGHTNESS_CONTROL, 0, 0); + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0); + } + break; + default: + break; + } + } +} + +#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) + +static u8 amdgpu_atombios_encoder_backlight_level(struct backlight_device *bd) +{ + u8 level; + + /* Convert brightness to hardware level */ + if (bd->props.brightness < 0) + level = 0; + else if (bd->props.brightness > AMDGPU_MAX_BL_LEVEL) + level = AMDGPU_MAX_BL_LEVEL; + else + level = bd->props.brightness; + + return level; +} + +static int amdgpu_atombios_encoder_update_backlight_status(struct backlight_device *bd) +{ + struct amdgpu_backlight_privdata *pdata = bl_get_data(bd); + struct amdgpu_encoder *amdgpu_encoder = pdata->encoder; + + amdgpu_atombios_encoder_set_backlight_level(amdgpu_encoder, + amdgpu_atombios_encoder_backlight_level(bd)); + + return 0; +} + +static int +amdgpu_atombios_encoder_get_backlight_brightness(struct backlight_device *bd) +{ + struct amdgpu_backlight_privdata *pdata = bl_get_data(bd); + struct amdgpu_encoder *amdgpu_encoder = pdata->encoder; + struct drm_device *dev = amdgpu_encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + + return amdgpu_atombios_encoder_get_backlight_level_from_reg(adev); +} + +static const struct backlight_ops amdgpu_atombios_encoder_backlight_ops = { + .get_brightness = amdgpu_atombios_encoder_get_backlight_brightness, + .update_status = amdgpu_atombios_encoder_update_backlight_status, +}; + +void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encoder, + struct drm_connector *drm_connector) +{ + struct drm_device *dev = amdgpu_encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + struct backlight_device *bd; + struct backlight_properties props; + struct amdgpu_backlight_privdata *pdata; + struct amdgpu_encoder_atom_dig *dig; + u8 backlight_level; + char bl_name[16]; + + /* Mac laptops with multiple GPUs use the gmux driver for backlight + * so don't register a backlight device + */ + if ((adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && + (adev->pdev->device == 0x6741)) + return; + + if (!amdgpu_encoder->enc_priv) + return; + + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) + return; + + pdata = kmalloc(sizeof(struct amdgpu_backlight_privdata), GFP_KERNEL); + if (!pdata) { + DRM_ERROR("Memory allocation failed\n"); + goto error; + } + + memset(&props, 0, sizeof(props)); + props.max_brightness = AMDGPU_MAX_BL_LEVEL; + props.type = BACKLIGHT_RAW; + snprintf(bl_name, sizeof(bl_name), + "amdgpu_bl%d", dev->primary->index); + bd = backlight_device_register(bl_name, drm_connector->kdev, + pdata, &amdgpu_atombios_encoder_backlight_ops, &props); + if (IS_ERR(bd)) { + DRM_ERROR("Backlight registration failed\n"); + goto error; + } + + pdata->encoder = amdgpu_encoder; + + backlight_level = amdgpu_atombios_encoder_get_backlight_level_from_reg(adev); + + dig = amdgpu_encoder->enc_priv; + dig->bl_dev = bd; + + bd->props.brightness = amdgpu_atombios_encoder_get_backlight_brightness(bd); + bd->props.power = FB_BLANK_UNBLANK; + backlight_update_status(bd); + + DRM_INFO("amdgpu atom DIG backlight initialized\n"); + + return; + +error: + kfree(pdata); + return; +} + +void +amdgpu_atombios_encoder_fini_backlight(struct amdgpu_encoder *amdgpu_encoder) +{ + struct drm_device *dev = amdgpu_encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + struct backlight_device *bd = NULL; + struct amdgpu_encoder_atom_dig *dig; + + if (!amdgpu_encoder->enc_priv) + return; + + if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU)) + return; + + dig = amdgpu_encoder->enc_priv; + bd = dig->bl_dev; + dig->bl_dev = NULL; + + if (bd) { + struct amdgpu_legacy_backlight_privdata *pdata; + + pdata = bl_get_data(bd); + backlight_device_unregister(bd); + kfree(pdata); + + DRM_INFO("amdgpu atom LVDS backlight unloaded\n"); + } +} + +#else /* !CONFIG_BACKLIGHT_CLASS_DEVICE */ + +void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *encoder) +{ +} + +void amdgpu_atombios_encoder_fini_backlight(struct amdgpu_encoder *encoder) +{ +} + +#endif + +bool amdgpu_atombios_encoder_is_digital(struct drm_encoder *encoder) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + return true; + default: + return false; + } +} + +bool amdgpu_atombios_encoder_mode_fixup(struct drm_encoder *encoder, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + + /* set the active encoder to connector routing */ + amdgpu_encoder_set_active_device(encoder); + drm_mode_set_crtcinfo(adjusted_mode, 0); + + /* hw bug */ + if ((mode->flags & DRM_MODE_FLAG_INTERLACE) + && (mode->crtc_vsync_start < (mode->crtc_vdisplay + 2))) + adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vdisplay + 2; + + /* vertical FP must be at least 1 */ + if (mode->crtc_vsync_start == mode->crtc_vdisplay) + adjusted_mode->crtc_vsync_start++; + + /* get the native mode for scaling */ + if (amdgpu_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT)) + amdgpu_panel_mode_fixup(encoder, adjusted_mode); + else if (amdgpu_encoder->rmx_type != RMX_OFF) + amdgpu_panel_mode_fixup(encoder, adjusted_mode); + + if ((amdgpu_encoder->active_device & (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) || + (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) { + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + amdgpu_atombios_dp_set_link_config(connector, adjusted_mode); + } + + return true; +} + +static void +amdgpu_atombios_encoder_setup_dac(struct drm_encoder *encoder, int action) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + DAC_ENCODER_CONTROL_PS_ALLOCATION args; + int index = 0; + + memset(&args, 0, sizeof(args)); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_DAC1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: + index = GetIndexIntoMasterTable(COMMAND, DAC1EncoderControl); + break; + case ENCODER_OBJECT_ID_INTERNAL_DAC2: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2: + index = GetIndexIntoMasterTable(COMMAND, DAC2EncoderControl); + break; + } + + args.ucAction = action; + args.ucDacStandard = ATOM_DAC1_PS2; + args.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + +} + +static u8 amdgpu_atombios_encoder_get_bpc(struct drm_encoder *encoder) +{ + int bpc = 8; + + if (encoder->crtc) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); + bpc = amdgpu_crtc->bpc; + } + + switch (bpc) { + case 0: + return PANEL_BPC_UNDEFINE; + case 6: + return PANEL_6BIT_PER_COLOR; + case 8: + default: + return PANEL_8BIT_PER_COLOR; + case 10: + return PANEL_10BIT_PER_COLOR; + case 12: + return PANEL_12BIT_PER_COLOR; + case 16: + return PANEL_16BIT_PER_COLOR; + } +} + +union dvo_encoder_control { + ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION ext_tmds; + DVO_ENCODER_CONTROL_PS_ALLOCATION dvo; + DVO_ENCODER_CONTROL_PS_ALLOCATION_V3 dvo_v3; + DVO_ENCODER_CONTROL_PS_ALLOCATION_V1_4 dvo_v4; +}; + +static void +amdgpu_atombios_encoder_setup_dvo(struct drm_encoder *encoder, int action) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + union dvo_encoder_control args; + int index = GetIndexIntoMasterTable(COMMAND, DVOEncoderControl); + uint8_t frev, crev; + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 1: + /* R4xx, R5xx */ + args.ext_tmds.sXTmdsEncoder.ucEnable = action; + + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.ext_tmds.sXTmdsEncoder.ucMisc |= PANEL_ENCODER_MISC_DUAL; + + args.ext_tmds.sXTmdsEncoder.ucMisc |= ATOM_PANEL_MISC_888RGB; + break; + case 2: + /* RS600/690/740 */ + args.dvo.sDVOEncoder.ucAction = action; + args.dvo.sDVOEncoder.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + /* DFP1, CRT1, TV1 depending on the type of port */ + args.dvo.sDVOEncoder.ucDeviceType = ATOM_DEVICE_DFP1_INDEX; + + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.dvo.sDVOEncoder.usDevAttr.sDigAttrib.ucAttribute |= PANEL_ENCODER_MISC_DUAL; + break; + case 3: + /* R6xx */ + args.dvo_v3.ucAction = action; + args.dvo_v3.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + args.dvo_v3.ucDVOConfig = 0; /* XXX */ + break; + case 4: + /* DCE8 */ + args.dvo_v4.ucAction = action; + args.dvo_v4.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + args.dvo_v4.ucDVOConfig = 0; /* XXX */ + args.dvo_v4.ucBitPerColor = amdgpu_atombios_encoder_get_bpc(encoder); + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +int amdgpu_atombios_encoder_get_encoder_mode(struct drm_encoder *encoder) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_connector *connector; + struct amdgpu_connector *amdgpu_connector; + struct amdgpu_connector_atom_dig *dig_connector; + + /* dp bridges are always DP */ + if (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE) + return ATOM_ENCODER_MODE_DP; + + /* DVO is always DVO */ + if ((amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_DVO1) || + (amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1)) + return ATOM_ENCODER_MODE_DVO; + + connector = amdgpu_get_connector_for_encoder(encoder); + /* if we don't have an active device yet, just use one of + * the connectors tied to the encoder. + */ + if (!connector) + connector = amdgpu_get_connector_for_encoder_init(encoder); + amdgpu_connector = to_amdgpu_connector(connector); + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */ + if (amdgpu_audio != 0) { + if (amdgpu_connector->use_digital && + (amdgpu_connector->audio == AMDGPU_AUDIO_ENABLE)) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector)) && + (amdgpu_connector->audio == AMDGPU_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else if (amdgpu_connector->use_digital) + return ATOM_ENCODER_MODE_DVI; + else + return ATOM_ENCODER_MODE_CRT; + } else if (amdgpu_connector->use_digital) { + return ATOM_ENCODER_MODE_DVI; + } else { + return ATOM_ENCODER_MODE_CRT; + } + break; + case DRM_MODE_CONNECTOR_DVID: + case DRM_MODE_CONNECTOR_HDMIA: + default: + if (amdgpu_audio != 0) { + if (amdgpu_connector->audio == AMDGPU_AUDIO_ENABLE) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector)) && + (amdgpu_connector->audio == AMDGPU_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else + return ATOM_ENCODER_MODE_DVI; + } else { + return ATOM_ENCODER_MODE_DVI; + } + break; + case DRM_MODE_CONNECTOR_LVDS: + return ATOM_ENCODER_MODE_LVDS; + break; + case DRM_MODE_CONNECTOR_DisplayPort: + dig_connector = amdgpu_connector->con_priv; + if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || + (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { + return ATOM_ENCODER_MODE_DP; + } else if (amdgpu_audio != 0) { + if (amdgpu_connector->audio == AMDGPU_AUDIO_ENABLE) + return ATOM_ENCODER_MODE_HDMI; + else if (drm_detect_hdmi_monitor(amdgpu_connector_edid(connector)) && + (amdgpu_connector->audio == AMDGPU_AUDIO_AUTO)) + return ATOM_ENCODER_MODE_HDMI; + else + return ATOM_ENCODER_MODE_DVI; + } else { + return ATOM_ENCODER_MODE_DVI; + } + break; + case DRM_MODE_CONNECTOR_eDP: + return ATOM_ENCODER_MODE_DP; + case DRM_MODE_CONNECTOR_DVIA: + case DRM_MODE_CONNECTOR_VGA: + return ATOM_ENCODER_MODE_CRT; + break; + case DRM_MODE_CONNECTOR_Composite: + case DRM_MODE_CONNECTOR_SVIDEO: + case DRM_MODE_CONNECTOR_9PinDIN: + /* fix me */ + return ATOM_ENCODER_MODE_TV; + /*return ATOM_ENCODER_MODE_CV;*/ + break; + } +} + +/* + * DIG Encoder/Transmitter Setup + * + * DCE 6.0 + * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B). + * Supports up to 6 digital outputs + * - 6 DIG encoder blocks. + * - DIG to PHY mapping is hardcoded + * DIG1 drives UNIPHY0 link A, A+B + * DIG2 drives UNIPHY0 link B + * DIG3 drives UNIPHY1 link A, A+B + * DIG4 drives UNIPHY1 link B + * DIG5 drives UNIPHY2 link A, A+B + * DIG6 drives UNIPHY2 link B + * + * Routing + * crtc -> dig encoder -> UNIPHY/LVTMA (1 or 2 links) + * Examples: + * crtc0 -> dig2 -> LVTMA links A+B -> TMDS/HDMI + * crtc1 -> dig1 -> UNIPHY0 link B -> DP + * crtc0 -> dig1 -> UNIPHY2 link A -> LVDS + * crtc1 -> dig2 -> UNIPHY1 link B+A -> TMDS/HDMI + */ + +union dig_encoder_control { + DIG_ENCODER_CONTROL_PS_ALLOCATION v1; + DIG_ENCODER_CONTROL_PARAMETERS_V2 v2; + DIG_ENCODER_CONTROL_PARAMETERS_V3 v3; + DIG_ENCODER_CONTROL_PARAMETERS_V4 v4; + DIG_ENCODER_CONTROL_PARAMETERS_V5 v5; +}; + +void +amdgpu_atombios_encoder_setup_dig_encoder(struct drm_encoder *encoder, + int action, int panel_mode) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + union dig_encoder_control args; + int index = GetIndexIntoMasterTable(COMMAND, DIGxEncoderControl); + uint8_t frev, crev; + int dp_clock = 0; + int dp_lane_count = 0; + int hpd_id = AMDGPU_HPD_NONE; + + if (connector) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + + dp_clock = dig_connector->dp_clock; + dp_lane_count = dig_connector->dp_lane_count; + hpd_id = amdgpu_connector->hpd.hpd; + } + + /* no dig encoder assigned */ + if (dig->dig_encoder == -1) + return; + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 1: + args.v1.ucAction = action; + args.v1.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + if (action == ATOM_ENCODER_CMD_SETUP_PANEL_MODE) + args.v3.ucPanelMode = panel_mode; + else + args.v1.ucEncoderMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode)) + args.v1.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v1.ucLaneNum = 8; + else + args.v1.ucLaneNum = 4; + + if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000)) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER2; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER3; + break; + } + if (dig->linkb) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKB; + else + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKA; + break; + case 2: + case 3: + args.v3.ucAction = action; + args.v3.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + if (action == ATOM_ENCODER_CMD_SETUP_PANEL_MODE) + args.v3.ucPanelMode = panel_mode; + else + args.v3.ucEncoderMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (ENCODER_MODE_IS_DP(args.v3.ucEncoderMode)) + args.v3.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v3.ucLaneNum = 8; + else + args.v3.ucLaneNum = 4; + + if (ENCODER_MODE_IS_DP(args.v3.ucEncoderMode) && (dp_clock == 270000)) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ; + args.v3.acConfig.ucDigSel = dig->dig_encoder; + args.v3.ucBitPerColor = amdgpu_atombios_encoder_get_bpc(encoder); + break; + case 4: + args.v4.ucAction = action; + args.v4.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + if (action == ATOM_ENCODER_CMD_SETUP_PANEL_MODE) + args.v4.ucPanelMode = panel_mode; + else + args.v4.ucEncoderMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode)) + args.v4.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v4.ucLaneNum = 8; + else + args.v4.ucLaneNum = 4; + + if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode)) { + if (dp_clock == 540000) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ; + else if (dp_clock == 324000) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_3_24GHZ; + else if (dp_clock == 270000) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ; + else + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_1_62GHZ; + } + args.v4.acConfig.ucDigSel = dig->dig_encoder; + args.v4.ucBitPerColor = amdgpu_atombios_encoder_get_bpc(encoder); + if (hpd_id == AMDGPU_HPD_NONE) + args.v4.ucHPD_ID = 0; + else + args.v4.ucHPD_ID = hpd_id + 1; + break; + case 5: + switch (action) { + case ATOM_ENCODER_CMD_SETUP_PANEL_MODE: + args.v5.asDPPanelModeParam.ucAction = action; + args.v5.asDPPanelModeParam.ucPanelMode = panel_mode; + args.v5.asDPPanelModeParam.ucDigId = dig->dig_encoder; + break; + case ATOM_ENCODER_CMD_STREAM_SETUP: + args.v5.asStreamParam.ucAction = action; + args.v5.asStreamParam.ucDigId = dig->dig_encoder; + args.v5.asStreamParam.ucDigMode = + amdgpu_atombios_encoder_get_encoder_mode(encoder); + if (ENCODER_MODE_IS_DP(args.v5.asStreamParam.ucDigMode)) + args.v5.asStreamParam.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, + amdgpu_encoder->pixel_clock)) + args.v5.asStreamParam.ucLaneNum = 8; + else + args.v5.asStreamParam.ucLaneNum = 4; + args.v5.asStreamParam.ulPixelClock = + cpu_to_le32(amdgpu_encoder->pixel_clock / 10); + args.v5.asStreamParam.ucBitPerColor = + amdgpu_atombios_encoder_get_bpc(encoder); + args.v5.asStreamParam.ucLinkRateIn270Mhz = dp_clock / 27000; + break; + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_START: + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1: + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2: + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN3: + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN4: + case ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE: + case ATOM_ENCODER_CMD_DP_VIDEO_OFF: + case ATOM_ENCODER_CMD_DP_VIDEO_ON: + args.v5.asCmdParam.ucAction = action; + args.v5.asCmdParam.ucDigId = dig->dig_encoder; + break; + default: + DRM_ERROR("Unsupported action 0x%x\n", action); + break; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + +} + +union dig_transmitter_control { + DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1; + DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2; + DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 v3; + DIG_TRANSMITTER_CONTROL_PARAMETERS_V4 v4; + DIG_TRANSMITTER_CONTROL_PARAMETERS_V1_5 v5; + DIG_TRANSMITTER_CONTROL_PARAMETERS_V1_6 v6; +}; + +void +amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int action, + uint8_t lane_num, uint8_t lane_set) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + struct drm_connector *connector; + union dig_transmitter_control args; + int index = 0; + uint8_t frev, crev; + bool is_dp = false; + int pll_id = 0; + int dp_clock = 0; + int dp_lane_count = 0; + int connector_object_id = 0; + int igp_lane_info = 0; + int dig_encoder = dig->dig_encoder; + int hpd_id = AMDGPU_HPD_NONE; + + if (action == ATOM_TRANSMITTER_ACTION_INIT) { + connector = amdgpu_get_connector_for_encoder_init(encoder); + /* just needed to avoid bailing in the encoder check. the encoder + * isn't used for init + */ + dig_encoder = 0; + } else + connector = amdgpu_get_connector_for_encoder(encoder); + + if (connector) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + + hpd_id = amdgpu_connector->hpd.hpd; + dp_clock = dig_connector->dp_clock; + dp_lane_count = dig_connector->dp_lane_count; + connector_object_id = + (amdgpu_connector->connector_object_id & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT; + } + + if (encoder->crtc) { + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); + pll_id = amdgpu_crtc->pll_id; + } + + /* no dig encoder assigned */ + if (dig_encoder == -1) + return; + + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder))) + is_dp = true; + + memset(&args, 0, sizeof(args)); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl); + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl); + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + index = GetIndexIntoMasterTable(COMMAND, LVTMATransmitterControl); + break; + } + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 1: + args.v1.ucAction = action; + if (action == ATOM_TRANSMITTER_ACTION_INIT) { + args.v1.usInitInfo = cpu_to_le16(connector_object_id); + } else if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) { + args.v1.asMode.ucLaneSel = lane_num; + args.v1.asMode.ucLaneSet = lane_set; + } else { + if (is_dp) + args.v1.usPixelClock = cpu_to_le16(dp_clock / 10); + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v1.usPixelClock = cpu_to_le16((amdgpu_encoder->pixel_clock / 2) / 10); + else + args.v1.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + } + + args.v1.ucConfig = ATOM_TRANSMITTER_CONFIG_CLKSRC_PPLL; + + if (dig_encoder) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG2_ENCODER; + else + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG1_ENCODER; + + if ((adev->flags & AMD_IS_APU) && + (amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_UNIPHY)) { + if (is_dp || + !amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) { + if (igp_lane_info & 0x1) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_0_3; + else if (igp_lane_info & 0x2) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_4_7; + else if (igp_lane_info & 0x4) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_8_11; + else if (igp_lane_info & 0x8) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_12_15; + } else { + if (igp_lane_info & 0x3) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_0_7; + else if (igp_lane_info & 0xc) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LANE_8_15; + } + } + + if (dig->linkb) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LINKB; + else + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_LINKA; + + if (is_dp) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_COHERENT; + else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + if (dig->coherent_mode) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_COHERENT; + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_8LANE_LINK; + } + break; + case 2: + args.v2.ucAction = action; + if (action == ATOM_TRANSMITTER_ACTION_INIT) { + args.v2.usInitInfo = cpu_to_le16(connector_object_id); + } else if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) { + args.v2.asMode.ucLaneSel = lane_num; + args.v2.asMode.ucLaneSet = lane_set; + } else { + if (is_dp) + args.v2.usPixelClock = cpu_to_le16(dp_clock / 10); + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v2.usPixelClock = cpu_to_le16((amdgpu_encoder->pixel_clock / 2) / 10); + else + args.v2.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + } + + args.v2.acConfig.ucEncoderSel = dig_encoder; + if (dig->linkb) + args.v2.acConfig.ucLinkSel = 1; + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + args.v2.acConfig.ucTransmitterSel = 0; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + args.v2.acConfig.ucTransmitterSel = 1; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + args.v2.acConfig.ucTransmitterSel = 2; + break; + } + + if (is_dp) { + args.v2.acConfig.fCoherentMode = 1; + args.v2.acConfig.fDPConnector = 1; + } else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + if (dig->coherent_mode) + args.v2.acConfig.fCoherentMode = 1; + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v2.acConfig.fDualLinkConnector = 1; + } + break; + case 3: + args.v3.ucAction = action; + if (action == ATOM_TRANSMITTER_ACTION_INIT) { + args.v3.usInitInfo = cpu_to_le16(connector_object_id); + } else if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) { + args.v3.asMode.ucLaneSel = lane_num; + args.v3.asMode.ucLaneSet = lane_set; + } else { + if (is_dp) + args.v3.usPixelClock = cpu_to_le16(dp_clock / 10); + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v3.usPixelClock = cpu_to_le16((amdgpu_encoder->pixel_clock / 2) / 10); + else + args.v3.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + } + + if (is_dp) + args.v3.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v3.ucLaneNum = 8; + else + args.v3.ucLaneNum = 4; + + if (dig->linkb) + args.v3.acConfig.ucLinkSel = 1; + if (dig_encoder & 1) + args.v3.acConfig.ucEncoderSel = 1; + + /* Select the PLL for the PHY + * DP PHY should be clocked from external src if there is + * one. + */ + /* On DCE4, if there is an external clock, it generates the DP ref clock */ + if (is_dp && adev->clock.dp_extclk) + args.v3.acConfig.ucRefClkSource = 2; /* external src */ + else + args.v3.acConfig.ucRefClkSource = pll_id; + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + args.v3.acConfig.ucTransmitterSel = 0; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + args.v3.acConfig.ucTransmitterSel = 1; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + args.v3.acConfig.ucTransmitterSel = 2; + break; + } + + if (is_dp) + args.v3.acConfig.fCoherentMode = 1; /* DP requires coherent */ + else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + if (dig->coherent_mode) + args.v3.acConfig.fCoherentMode = 1; + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v3.acConfig.fDualLinkConnector = 1; + } + break; + case 4: + args.v4.ucAction = action; + if (action == ATOM_TRANSMITTER_ACTION_INIT) { + args.v4.usInitInfo = cpu_to_le16(connector_object_id); + } else if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) { + args.v4.asMode.ucLaneSel = lane_num; + args.v4.asMode.ucLaneSet = lane_set; + } else { + if (is_dp) + args.v4.usPixelClock = cpu_to_le16(dp_clock / 10); + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v4.usPixelClock = cpu_to_le16((amdgpu_encoder->pixel_clock / 2) / 10); + else + args.v4.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + } + + if (is_dp) + args.v4.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v4.ucLaneNum = 8; + else + args.v4.ucLaneNum = 4; + + if (dig->linkb) + args.v4.acConfig.ucLinkSel = 1; + if (dig_encoder & 1) + args.v4.acConfig.ucEncoderSel = 1; + + /* Select the PLL for the PHY + * DP PHY should be clocked from external src if there is + * one. + */ + /* On DCE5 DCPLL usually generates the DP ref clock */ + if (is_dp) { + if (adev->clock.dp_extclk) + args.v4.acConfig.ucRefClkSource = ENCODER_REFCLK_SRC_EXTCLK; + else + args.v4.acConfig.ucRefClkSource = ENCODER_REFCLK_SRC_DCPLL; + } else + args.v4.acConfig.ucRefClkSource = pll_id; + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + args.v4.acConfig.ucTransmitterSel = 0; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + args.v4.acConfig.ucTransmitterSel = 1; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + args.v4.acConfig.ucTransmitterSel = 2; + break; + } + + if (is_dp) + args.v4.acConfig.fCoherentMode = 1; /* DP requires coherent */ + else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + if (dig->coherent_mode) + args.v4.acConfig.fCoherentMode = 1; + if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v4.acConfig.fDualLinkConnector = 1; + } + break; + case 5: + args.v5.ucAction = action; + if (is_dp) + args.v5.usSymClock = cpu_to_le16(dp_clock / 10); + else + args.v5.usSymClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + if (dig->linkb) + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYB; + else + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYA; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + if (dig->linkb) + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYD; + else + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYC; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + if (dig->linkb) + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYF; + else + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYE; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + args.v5.ucPhyId = ATOM_PHY_ID_UNIPHYG; + break; + } + if (is_dp) + args.v5.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v5.ucLaneNum = 8; + else + args.v5.ucLaneNum = 4; + args.v5.ucConnObjId = connector_object_id; + args.v5.ucDigMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (is_dp && adev->clock.dp_extclk) + args.v5.asConfig.ucPhyClkSrcId = ENCODER_REFCLK_SRC_EXTCLK; + else + args.v5.asConfig.ucPhyClkSrcId = pll_id; + + if (is_dp) + args.v5.asConfig.ucCoherentMode = 1; /* DP requires coherent */ + else if (amdgpu_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) { + if (dig->coherent_mode) + args.v5.asConfig.ucCoherentMode = 1; + } + if (hpd_id == AMDGPU_HPD_NONE) + args.v5.asConfig.ucHPDSel = 0; + else + args.v5.asConfig.ucHPDSel = hpd_id + 1; + args.v5.ucDigEncoderSel = 1 << dig_encoder; + args.v5.ucDPLaneSet = lane_set; + break; + case 6: + args.v6.ucAction = action; + if (is_dp) + args.v6.ulSymClock = cpu_to_le32(dp_clock / 10); + else + args.v6.ulSymClock = cpu_to_le32(amdgpu_encoder->pixel_clock / 10); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + if (dig->linkb) + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYB; + else + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYA; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + if (dig->linkb) + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYD; + else + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYC; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + if (dig->linkb) + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYF; + else + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYE; + break; + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + args.v6.ucPhyId = ATOM_PHY_ID_UNIPHYG; + break; + } + if (is_dp) + args.v6.ucLaneNum = dp_lane_count; + else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v6.ucLaneNum = 8; + else + args.v6.ucLaneNum = 4; + args.v6.ucConnObjId = connector_object_id; + if (action == ATOM_TRANSMITTER_ACTION_SETUP_VSEMPH) + args.v6.ucDPLaneSet = lane_set; + else + args.v6.ucDigMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (hpd_id == AMDGPU_HPD_NONE) + args.v6.ucHPDSel = 0; + else + args.v6.ucHPDSel = hpd_id + 1; + args.v6.ucDigEncoderSel = 1 << dig_encoder; + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + break; + default: + DRM_ERROR("Unknown table version %d, %d\n", frev, crev); + break; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +bool +amdgpu_atombios_encoder_set_edp_panel_power(struct drm_connector *connector, + int action) +{ + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_device *dev = amdgpu_connector->base.dev; + struct amdgpu_device *adev = dev->dev_private; + union dig_transmitter_control args; + int index = GetIndexIntoMasterTable(COMMAND, UNIPHYTransmitterControl); + uint8_t frev, crev; + + if (connector->connector_type != DRM_MODE_CONNECTOR_eDP) + goto done; + + if ((action != ATOM_TRANSMITTER_ACTION_POWER_ON) && + (action != ATOM_TRANSMITTER_ACTION_POWER_OFF)) + goto done; + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + goto done; + + memset(&args, 0, sizeof(args)); + + args.v1.ucAction = action; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + /* wait for the panel to power up */ + if (action == ATOM_TRANSMITTER_ACTION_POWER_ON) { + int i; + + for (i = 0; i < 300; i++) { + if (amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) + return true; + mdelay(1); + } + return false; + } +done: + return true; +} + +union external_encoder_control { + EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION v1; + EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3 v3; +}; + +static void +amdgpu_atombios_encoder_setup_external_encoder(struct drm_encoder *encoder, + struct drm_encoder *ext_encoder, + int action) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_encoder *ext_amdgpu_encoder = to_amdgpu_encoder(ext_encoder); + union external_encoder_control args; + struct drm_connector *connector; + int index = GetIndexIntoMasterTable(COMMAND, ExternalEncoderControl); + u8 frev, crev; + int dp_clock = 0; + int dp_lane_count = 0; + int connector_object_id = 0; + u32 ext_enum = (ext_amdgpu_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT; + + if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT) + connector = amdgpu_get_connector_for_encoder_init(encoder); + else + connector = amdgpu_get_connector_for_encoder(encoder); + + if (connector) { + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct amdgpu_connector_atom_dig *dig_connector = + amdgpu_connector->con_priv; + + dp_clock = dig_connector->dp_clock; + dp_lane_count = dig_connector->dp_lane_count; + connector_object_id = + (amdgpu_connector->connector_object_id & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT; + } + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return; + + switch (frev) { + case 1: + /* no params on frev 1 */ + break; + case 2: + switch (crev) { + case 1: + case 2: + args.v1.sDigEncoder.ucAction = action; + args.v1.sDigEncoder.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + args.v1.sDigEncoder.ucEncoderMode = + amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (ENCODER_MODE_IS_DP(args.v1.sDigEncoder.ucEncoderMode)) { + if (dp_clock == 270000) + args.v1.sDigEncoder.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; + args.v1.sDigEncoder.ucLaneNum = dp_lane_count; + } else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v1.sDigEncoder.ucLaneNum = 8; + else + args.v1.sDigEncoder.ucLaneNum = 4; + break; + case 3: + args.v3.sExtEncoder.ucAction = action; + if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT) + args.v3.sExtEncoder.usConnectorId = cpu_to_le16(connector_object_id); + else + args.v3.sExtEncoder.usPixelClock = cpu_to_le16(amdgpu_encoder->pixel_clock / 10); + args.v3.sExtEncoder.ucEncoderMode = + amdgpu_atombios_encoder_get_encoder_mode(encoder); + + if (ENCODER_MODE_IS_DP(args.v3.sExtEncoder.ucEncoderMode)) { + if (dp_clock == 270000) + args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ; + else if (dp_clock == 540000) + args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_5_40GHZ; + args.v3.sExtEncoder.ucLaneNum = dp_lane_count; + } else if (amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) + args.v3.sExtEncoder.ucLaneNum = 8; + else + args.v3.sExtEncoder.ucLaneNum = 4; + switch (ext_enum) { + case GRAPH_OBJECT_ENUM_ID1: + args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER1; + break; + case GRAPH_OBJECT_ENUM_ID2: + args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER2; + break; + case GRAPH_OBJECT_ENUM_ID3: + args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER3; + break; + } + args.v3.sExtEncoder.ucBitPerColor = amdgpu_atombios_encoder_get_bpc(encoder); + break; + default: + DRM_ERROR("Unknown table version: %d, %d\n", frev, crev); + return; + } + break; + default: + DRM_ERROR("Unknown table version: %d, %d\n", frev, crev); + return; + } + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +static void +amdgpu_atombios_encoder_setup_dig(struct drm_encoder *encoder, int action) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_encoder *ext_encoder = amdgpu_get_external_encoder(encoder); + struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv; + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + struct amdgpu_connector *amdgpu_connector = NULL; + struct amdgpu_connector_atom_dig *amdgpu_dig_connector = NULL; + + if (connector) { + amdgpu_connector = to_amdgpu_connector(connector); + amdgpu_dig_connector = amdgpu_connector->con_priv; + } + + if (action == ATOM_ENABLE) { + if (!connector) + dig->panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE; + else + dig->panel_mode = amdgpu_atombios_dp_get_panel_mode(encoder, connector); + + /* setup and enable the encoder */ + amdgpu_atombios_encoder_setup_dig_encoder(encoder, ATOM_ENCODER_CMD_SETUP, 0); + amdgpu_atombios_encoder_setup_dig_encoder(encoder, + ATOM_ENCODER_CMD_SETUP_PANEL_MODE, + dig->panel_mode); + if (ext_encoder) + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, + EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP); + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder)) && + connector) { + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_ON); + amdgpu_dig_connector->edp_on = true; + } + } + /* enable the transmitter */ + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_ENABLE, + 0, 0); + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder)) && + connector) { + /* DP_SET_POWER_D0 is set in amdgpu_atombios_dp_link_train */ + amdgpu_atombios_dp_link_train(encoder, connector); + amdgpu_atombios_encoder_setup_dig_encoder(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0); + } + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) + amdgpu_atombios_encoder_set_backlight_level(amdgpu_encoder, dig->backlight_level); + if (ext_encoder) + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, ATOM_ENABLE); + } else { + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder)) && + connector) + amdgpu_atombios_encoder_setup_dig_encoder(encoder, + ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0); + if (ext_encoder) + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, ATOM_DISABLE); + if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); + + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder)) && + connector) + amdgpu_atombios_dp_set_rx_power_state(connector, DP_SET_POWER_D3); + /* disable the transmitter */ + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, + ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0); + if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(encoder)) && + connector) { + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) { + amdgpu_atombios_encoder_set_edp_panel_power(connector, + ATOM_TRANSMITTER_ACTION_POWER_OFF); + amdgpu_dig_connector->edp_on = false; + } + } + } +} + +void +amdgpu_atombios_encoder_dpms(struct drm_encoder *encoder, int mode) +{ + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + + DRM_DEBUG_KMS("encoder dpms %d to mode %d, devices %08x, active_devices %08x\n", + amdgpu_encoder->encoder_id, mode, amdgpu_encoder->devices, + amdgpu_encoder->active_device); + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + switch (mode) { + case DRM_MODE_DPMS_ON: + amdgpu_atombios_encoder_setup_dig(encoder, ATOM_ENABLE); + break; + case DRM_MODE_DPMS_STANDBY: + case DRM_MODE_DPMS_SUSPEND: + case DRM_MODE_DPMS_OFF: + amdgpu_atombios_encoder_setup_dig(encoder, ATOM_DISABLE); + break; + } + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + switch (mode) { + case DRM_MODE_DPMS_ON: + amdgpu_atombios_encoder_setup_dvo(encoder, ATOM_ENABLE); + break; + case DRM_MODE_DPMS_STANDBY: + case DRM_MODE_DPMS_SUSPEND: + case DRM_MODE_DPMS_OFF: + amdgpu_atombios_encoder_setup_dvo(encoder, ATOM_DISABLE); + break; + } + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: + switch (mode) { + case DRM_MODE_DPMS_ON: + amdgpu_atombios_encoder_setup_dac(encoder, ATOM_ENABLE); + break; + case DRM_MODE_DPMS_STANDBY: + case DRM_MODE_DPMS_SUSPEND: + case DRM_MODE_DPMS_OFF: + amdgpu_atombios_encoder_setup_dac(encoder, ATOM_DISABLE); + break; + } + break; + default: + return; + } +} + +union crtc_source_param { + SELECT_CRTC_SOURCE_PS_ALLOCATION v1; + SELECT_CRTC_SOURCE_PARAMETERS_V2 v2; + SELECT_CRTC_SOURCE_PARAMETERS_V3 v3; +}; + +void +amdgpu_atombios_encoder_set_crtc_source(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc); + union crtc_source_param args; + int index = GetIndexIntoMasterTable(COMMAND, SelectCRTC_Source); + uint8_t frev, crev; + struct amdgpu_encoder_atom_dig *dig; + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return; + + switch (frev) { + case 1: + switch (crev) { + case 1: + default: + args.v1.ucCRTC = amdgpu_crtc->crtc_id; + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_TMDS1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1: + args.v1.ucDevice = ATOM_DEVICE_DFP1_INDEX; + break; + case ENCODER_OBJECT_ID_INTERNAL_LVDS: + case ENCODER_OBJECT_ID_INTERNAL_LVTM1: + if (amdgpu_encoder->devices & ATOM_DEVICE_LCD1_SUPPORT) + args.v1.ucDevice = ATOM_DEVICE_LCD1_INDEX; + else + args.v1.ucDevice = ATOM_DEVICE_DFP3_INDEX; + break; + case ENCODER_OBJECT_ID_INTERNAL_DVO1: + case ENCODER_OBJECT_ID_INTERNAL_DDI: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + args.v1.ucDevice = ATOM_DEVICE_DFP2_INDEX; + break; + case ENCODER_OBJECT_ID_INTERNAL_DAC1: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v1.ucDevice = ATOM_DEVICE_TV1_INDEX; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v1.ucDevice = ATOM_DEVICE_CV_INDEX; + else + args.v1.ucDevice = ATOM_DEVICE_CRT1_INDEX; + break; + case ENCODER_OBJECT_ID_INTERNAL_DAC2: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v1.ucDevice = ATOM_DEVICE_TV1_INDEX; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v1.ucDevice = ATOM_DEVICE_CV_INDEX; + else + args.v1.ucDevice = ATOM_DEVICE_CRT2_INDEX; + break; + } + break; + case 2: + args.v2.ucCRTC = amdgpu_crtc->crtc_id; + if (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE) { + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + + if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS) + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_LVDS; + else if (connector->connector_type == DRM_MODE_CONNECTOR_VGA) + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_CRT; + else + args.v2.ucEncodeMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + } else if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_LVDS; + } else { + args.v2.ucEncodeMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + } + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + dig = amdgpu_encoder->enc_priv; + switch (dig->dig_encoder) { + case 0: + args.v2.ucEncoderID = ASIC_INT_DIG1_ENCODER_ID; + break; + case 1: + args.v2.ucEncoderID = ASIC_INT_DIG2_ENCODER_ID; + break; + case 2: + args.v2.ucEncoderID = ASIC_INT_DIG3_ENCODER_ID; + break; + case 3: + args.v2.ucEncoderID = ASIC_INT_DIG4_ENCODER_ID; + break; + case 4: + args.v2.ucEncoderID = ASIC_INT_DIG5_ENCODER_ID; + break; + case 5: + args.v2.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID; + break; + case 6: + args.v2.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID; + break; + } + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + args.v2.ucEncoderID = ASIC_INT_DVO_ENCODER_ID; + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v2.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v2.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else + args.v2.ucEncoderID = ASIC_INT_DAC1_ENCODER_ID; + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v2.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v2.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else + args.v2.ucEncoderID = ASIC_INT_DAC2_ENCODER_ID; + break; + } + break; + case 3: + args.v3.ucCRTC = amdgpu_crtc->crtc_id; + if (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE) { + struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder); + + if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS) + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_LVDS; + else if (connector->connector_type == DRM_MODE_CONNECTOR_VGA) + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_CRT; + else + args.v2.ucEncodeMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + } else if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) { + args.v2.ucEncodeMode = ATOM_ENCODER_MODE_LVDS; + } else { + args.v2.ucEncodeMode = amdgpu_atombios_encoder_get_encoder_mode(encoder); + } + args.v3.ucDstBpc = amdgpu_atombios_encoder_get_bpc(encoder); + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: + dig = amdgpu_encoder->enc_priv; + switch (dig->dig_encoder) { + case 0: + args.v3.ucEncoderID = ASIC_INT_DIG1_ENCODER_ID; + break; + case 1: + args.v3.ucEncoderID = ASIC_INT_DIG2_ENCODER_ID; + break; + case 2: + args.v3.ucEncoderID = ASIC_INT_DIG3_ENCODER_ID; + break; + case 3: + args.v3.ucEncoderID = ASIC_INT_DIG4_ENCODER_ID; + break; + case 4: + args.v3.ucEncoderID = ASIC_INT_DIG5_ENCODER_ID; + break; + case 5: + args.v3.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID; + break; + case 6: + args.v3.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID; + break; + } + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: + args.v3.ucEncoderID = ASIC_INT_DVO_ENCODER_ID; + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v3.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v3.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else + args.v3.ucEncoderID = ASIC_INT_DAC1_ENCODER_ID; + break; + case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2: + if (amdgpu_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) + args.v3.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else if (amdgpu_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) + args.v3.ucEncoderID = ASIC_INT_TV_ENCODER_ID; + else + args.v3.ucEncoderID = ASIC_INT_DAC2_ENCODER_ID; + break; + } + break; + } + break; + default: + DRM_ERROR("Unknown table version: %d, %d\n", frev, crev); + return; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +} + +/* This only needs to be called once at startup */ +void +amdgpu_atombios_encoder_init_dig(struct amdgpu_device *adev) +{ + struct drm_device *dev = adev->ddev; + struct drm_encoder *encoder; + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct drm_encoder *ext_encoder = amdgpu_get_external_encoder(encoder); + + switch (amdgpu_encoder->encoder_id) { + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: + amdgpu_atombios_encoder_setup_dig_transmitter(encoder, ATOM_TRANSMITTER_ACTION_INIT, + 0, 0); + break; + } + + if (ext_encoder) + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, + EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT); + } +} + +static bool +amdgpu_atombios_encoder_dac_load_detect(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + + if (amdgpu_encoder->devices & (ATOM_DEVICE_TV_SUPPORT | + ATOM_DEVICE_CV_SUPPORT | + ATOM_DEVICE_CRT_SUPPORT)) { + DAC_LOAD_DETECTION_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, DAC_LoadDetection); + uint8_t frev, crev; + + memset(&args, 0, sizeof(args)); + + if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev)) + return false; + + args.sDacload.ucMisc = 0; + + if ((amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_DAC1) || + (amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1)) + args.sDacload.ucDacType = ATOM_DAC_A; + else + args.sDacload.ucDacType = ATOM_DAC_B; + + if (amdgpu_connector->devices & ATOM_DEVICE_CRT1_SUPPORT) + args.sDacload.usDeviceID = cpu_to_le16(ATOM_DEVICE_CRT1_SUPPORT); + else if (amdgpu_connector->devices & ATOM_DEVICE_CRT2_SUPPORT) + args.sDacload.usDeviceID = cpu_to_le16(ATOM_DEVICE_CRT2_SUPPORT); + else if (amdgpu_connector->devices & ATOM_DEVICE_CV_SUPPORT) { + args.sDacload.usDeviceID = cpu_to_le16(ATOM_DEVICE_CV_SUPPORT); + if (crev >= 3) + args.sDacload.ucMisc = DAC_LOAD_MISC_YPrPb; + } else if (amdgpu_connector->devices & ATOM_DEVICE_TV1_SUPPORT) { + args.sDacload.usDeviceID = cpu_to_le16(ATOM_DEVICE_TV1_SUPPORT); + if (crev >= 3) + args.sDacload.ucMisc = DAC_LOAD_MISC_YPrPb; + } + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + return true; + } else + return false; +} + +enum drm_connector_status +amdgpu_atombios_encoder_dac_detect(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + uint32_t bios_0_scratch; + + if (!amdgpu_atombios_encoder_dac_load_detect(encoder, connector)) { + DRM_DEBUG_KMS("detect returned false \n"); + return connector_status_unknown; + } + + bios_0_scratch = RREG32(mmBIOS_SCRATCH_0); + + DRM_DEBUG_KMS("Bios 0 scratch %x %08x\n", bios_0_scratch, amdgpu_encoder->devices); + if (amdgpu_connector->devices & ATOM_DEVICE_CRT1_SUPPORT) { + if (bios_0_scratch & ATOM_S0_CRT1_MASK) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_CRT2_SUPPORT) { + if (bios_0_scratch & ATOM_S0_CRT2_MASK) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_CV_SUPPORT) { + if (bios_0_scratch & (ATOM_S0_CV_MASK|ATOM_S0_CV_MASK_A)) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_TV1_SUPPORT) { + if (bios_0_scratch & (ATOM_S0_TV1_COMPOSITE | ATOM_S0_TV1_COMPOSITE_A)) + return connector_status_connected; /* CTV */ + else if (bios_0_scratch & (ATOM_S0_TV1_SVIDEO | ATOM_S0_TV1_SVIDEO_A)) + return connector_status_connected; /* STV */ + } + return connector_status_disconnected; +} + +enum drm_connector_status +amdgpu_atombios_encoder_dig_detect(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + struct drm_device *dev = encoder->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); + struct drm_encoder *ext_encoder = amdgpu_get_external_encoder(encoder); + u32 bios_0_scratch; + + if (!ext_encoder) + return connector_status_unknown; + + if ((amdgpu_connector->devices & ATOM_DEVICE_CRT_SUPPORT) == 0) + return connector_status_unknown; + + /* load detect on the dp bridge */ + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, + EXTERNAL_ENCODER_ACTION_V3_DACLOAD_DETECTION); + + bios_0_scratch = RREG32(mmBIOS_SCRATCH_0); + + DRM_DEBUG_KMS("Bios 0 scratch %x %08x\n", bios_0_scratch, amdgpu_encoder->devices); + if (amdgpu_connector->devices & ATOM_DEVICE_CRT1_SUPPORT) { + if (bios_0_scratch & ATOM_S0_CRT1_MASK) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_CRT2_SUPPORT) { + if (bios_0_scratch & ATOM_S0_CRT2_MASK) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_CV_SUPPORT) { + if (bios_0_scratch & (ATOM_S0_CV_MASK|ATOM_S0_CV_MASK_A)) + return connector_status_connected; + } + if (amdgpu_connector->devices & ATOM_DEVICE_TV1_SUPPORT) { + if (bios_0_scratch & (ATOM_S0_TV1_COMPOSITE | ATOM_S0_TV1_COMPOSITE_A)) + return connector_status_connected; /* CTV */ + else if (bios_0_scratch & (ATOM_S0_TV1_SVIDEO | ATOM_S0_TV1_SVIDEO_A)) + return connector_status_connected; /* STV */ + } + return connector_status_disconnected; +} + +void +amdgpu_atombios_encoder_setup_ext_encoder_ddc(struct drm_encoder *encoder) +{ + struct drm_encoder *ext_encoder = amdgpu_get_external_encoder(encoder); + + if (ext_encoder) + /* ddc_setup on the dp bridge */ + amdgpu_atombios_encoder_setup_external_encoder(encoder, ext_encoder, + EXTERNAL_ENCODER_ACTION_V3_DDC_SETUP); + +} + +void +amdgpu_atombios_encoder_set_bios_scratch_regs(struct drm_connector *connector, + struct drm_encoder *encoder, + bool connected) +{ + struct drm_device *dev = connector->dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_connector *amdgpu_connector = + to_amdgpu_connector(connector); + struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder); + uint32_t bios_0_scratch, bios_3_scratch, bios_6_scratch; + + bios_0_scratch = RREG32(mmBIOS_SCRATCH_0); + bios_3_scratch = RREG32(mmBIOS_SCRATCH_3); + bios_6_scratch = RREG32(mmBIOS_SCRATCH_6); + + if ((amdgpu_encoder->devices & ATOM_DEVICE_LCD1_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_LCD1_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("LCD1 connected\n"); + bios_0_scratch |= ATOM_S0_LCD1; + bios_3_scratch |= ATOM_S3_LCD1_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_LCD1; + } else { + DRM_DEBUG_KMS("LCD1 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_LCD1; + bios_3_scratch &= ~ATOM_S3_LCD1_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_LCD1; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_CRT1_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_CRT1_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("CRT1 connected\n"); + bios_0_scratch |= ATOM_S0_CRT1_COLOR; + bios_3_scratch |= ATOM_S3_CRT1_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_CRT1; + } else { + DRM_DEBUG_KMS("CRT1 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_CRT1_MASK; + bios_3_scratch &= ~ATOM_S3_CRT1_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_CRT1; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_CRT2_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_CRT2_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("CRT2 connected\n"); + bios_0_scratch |= ATOM_S0_CRT2_COLOR; + bios_3_scratch |= ATOM_S3_CRT2_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_CRT2; + } else { + DRM_DEBUG_KMS("CRT2 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_CRT2_MASK; + bios_3_scratch &= ~ATOM_S3_CRT2_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_CRT2; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP1_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP1_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP1 connected\n"); + bios_0_scratch |= ATOM_S0_DFP1; + bios_3_scratch |= ATOM_S3_DFP1_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP1; + } else { + DRM_DEBUG_KMS("DFP1 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP1; + bios_3_scratch &= ~ATOM_S3_DFP1_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP1; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP2_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP2_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP2 connected\n"); + bios_0_scratch |= ATOM_S0_DFP2; + bios_3_scratch |= ATOM_S3_DFP2_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP2; + } else { + DRM_DEBUG_KMS("DFP2 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP2; + bios_3_scratch &= ~ATOM_S3_DFP2_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP2; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP3_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP3_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP3 connected\n"); + bios_0_scratch |= ATOM_S0_DFP3; + bios_3_scratch |= ATOM_S3_DFP3_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP3; + } else { + DRM_DEBUG_KMS("DFP3 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP3; + bios_3_scratch &= ~ATOM_S3_DFP3_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP3; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP4_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP4_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP4 connected\n"); + bios_0_scratch |= ATOM_S0_DFP4; + bios_3_scratch |= ATOM_S3_DFP4_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP4; + } else { + DRM_DEBUG_KMS("DFP4 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP4; + bios_3_scratch &= ~ATOM_S3_DFP4_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP4; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP5_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP5_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP5 connected\n"); + bios_0_scratch |= ATOM_S0_DFP5; + bios_3_scratch |= ATOM_S3_DFP5_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP5; + } else { + DRM_DEBUG_KMS("DFP5 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP5; + bios_3_scratch &= ~ATOM_S3_DFP5_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP5; + } + } + if ((amdgpu_encoder->devices & ATOM_DEVICE_DFP6_SUPPORT) && + (amdgpu_connector->devices & ATOM_DEVICE_DFP6_SUPPORT)) { + if (connected) { + DRM_DEBUG_KMS("DFP6 connected\n"); + bios_0_scratch |= ATOM_S0_DFP6; + bios_3_scratch |= ATOM_S3_DFP6_ACTIVE; + bios_6_scratch |= ATOM_S6_ACC_REQ_DFP6; + } else { + DRM_DEBUG_KMS("DFP6 disconnected\n"); + bios_0_scratch &= ~ATOM_S0_DFP6; + bios_3_scratch &= ~ATOM_S3_DFP6_ACTIVE; + bios_6_scratch &= ~ATOM_S6_ACC_REQ_DFP6; + } + } + + WREG32(mmBIOS_SCRATCH_0, bios_0_scratch); + WREG32(mmBIOS_SCRATCH_3, bios_3_scratch); + WREG32(mmBIOS_SCRATCH_6, bios_6_scratch); +} + +union lvds_info { + struct _ATOM_LVDS_INFO info; + struct _ATOM_LVDS_INFO_V12 info_12; +}; + +struct amdgpu_encoder_atom_dig * +amdgpu_atombios_encoder_get_lcd_info(struct amdgpu_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_mode_info *mode_info = &adev->mode_info; + int index = GetIndexIntoMasterTable(DATA, LVDS_Info); + uint16_t data_offset, misc; + union lvds_info *lvds_info; + uint8_t frev, crev; + struct amdgpu_encoder_atom_dig *lvds = NULL; + int encoder_enum = (encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT; + + if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL, + &frev, &crev, &data_offset)) { + lvds_info = + (union lvds_info *)(mode_info->atom_context->bios + data_offset); + lvds = + kzalloc(sizeof(struct amdgpu_encoder_atom_dig), GFP_KERNEL); + + if (!lvds) + return NULL; + + lvds->native_mode.clock = + le16_to_cpu(lvds_info->info.sLCDTiming.usPixClk) * 10; + lvds->native_mode.hdisplay = + le16_to_cpu(lvds_info->info.sLCDTiming.usHActive); + lvds->native_mode.vdisplay = + le16_to_cpu(lvds_info->info.sLCDTiming.usVActive); + lvds->native_mode.htotal = lvds->native_mode.hdisplay + + le16_to_cpu(lvds_info->info.sLCDTiming.usHBlanking_Time); + lvds->native_mode.hsync_start = lvds->native_mode.hdisplay + + le16_to_cpu(lvds_info->info.sLCDTiming.usHSyncOffset); + lvds->native_mode.hsync_end = lvds->native_mode.hsync_start + + le16_to_cpu(lvds_info->info.sLCDTiming.usHSyncWidth); + lvds->native_mode.vtotal = lvds->native_mode.vdisplay + + le16_to_cpu(lvds_info->info.sLCDTiming.usVBlanking_Time); + lvds->native_mode.vsync_start = lvds->native_mode.vdisplay + + le16_to_cpu(lvds_info->info.sLCDTiming.usVSyncOffset); + lvds->native_mode.vsync_end = lvds->native_mode.vsync_start + + le16_to_cpu(lvds_info->info.sLCDTiming.usVSyncWidth); + lvds->panel_pwr_delay = + le16_to_cpu(lvds_info->info.usOffDelayInMs); + lvds->lcd_misc = lvds_info->info.ucLVDS_Misc; + + misc = le16_to_cpu(lvds_info->info.sLCDTiming.susModeMiscInfo.usAccess); + if (misc & ATOM_VSYNC_POLARITY) + lvds->native_mode.flags |= DRM_MODE_FLAG_NVSYNC; + if (misc & ATOM_HSYNC_POLARITY) + lvds->native_mode.flags |= DRM_MODE_FLAG_NHSYNC; + if (misc & ATOM_COMPOSITESYNC) + lvds->native_mode.flags |= DRM_MODE_FLAG_CSYNC; + if (misc & ATOM_INTERLACE) + lvds->native_mode.flags |= DRM_MODE_FLAG_INTERLACE; + if (misc & ATOM_DOUBLE_CLOCK_MODE) + lvds->native_mode.flags |= DRM_MODE_FLAG_DBLSCAN; + + lvds->native_mode.width_mm = le16_to_cpu(lvds_info->info.sLCDTiming.usImageHSize); + lvds->native_mode.height_mm = le16_to_cpu(lvds_info->info.sLCDTiming.usImageVSize); + + /* set crtc values */ + drm_mode_set_crtcinfo(&lvds->native_mode, CRTC_INTERLACE_HALVE_V); + + lvds->lcd_ss_id = lvds_info->info.ucSS_Id; + + encoder->native_mode = lvds->native_mode; + + if (encoder_enum == 2) + lvds->linkb = true; + else + lvds->linkb = false; + + /* parse the lcd record table */ + if (le16_to_cpu(lvds_info->info.usModePatchTableOffset)) { + ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record; + ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record; + bool bad_record = false; + u8 *record; + + if ((frev == 1) && (crev < 2)) + /* absolute */ + record = (u8 *)(mode_info->atom_context->bios + + le16_to_cpu(lvds_info->info.usModePatchTableOffset)); + else + /* relative */ + record = (u8 *)(mode_info->atom_context->bios + + data_offset + + le16_to_cpu(lvds_info->info.usModePatchTableOffset)); + while (*record != ATOM_RECORD_END_TYPE) { + switch (*record) { + case LCD_MODE_PATCH_RECORD_MODE_TYPE: + record += sizeof(ATOM_PATCH_RECORD_MODE); + break; + case LCD_RTS_RECORD_TYPE: + record += sizeof(ATOM_LCD_RTS_RECORD); + break; + case LCD_CAP_RECORD_TYPE: + record += sizeof(ATOM_LCD_MODE_CONTROL_CAP); + break; + case LCD_FAKE_EDID_PATCH_RECORD_TYPE: + fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record; + if (fake_edid_record->ucFakeEDIDLength) { + struct edid *edid; + int edid_size = + max((int)EDID_LENGTH, (int)fake_edid_record->ucFakeEDIDLength); + edid = kmalloc(edid_size, GFP_KERNEL); + if (edid) { + memcpy((u8 *)edid, (u8 *)&fake_edid_record->ucFakeEDIDString[0], + fake_edid_record->ucFakeEDIDLength); + + if (drm_edid_is_valid(edid)) { + adev->mode_info.bios_hardcoded_edid = edid; + adev->mode_info.bios_hardcoded_edid_size = edid_size; + } else + kfree(edid); + } + } + record += fake_edid_record->ucFakeEDIDLength ? + fake_edid_record->ucFakeEDIDLength + 2 : + sizeof(ATOM_FAKE_EDID_PATCH_RECORD); + break; + case LCD_PANEL_RESOLUTION_RECORD_TYPE: + panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record; + lvds->native_mode.width_mm = panel_res_record->usHSize; + lvds->native_mode.height_mm = panel_res_record->usVSize; + record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD); + break; + default: + DRM_ERROR("Bad LCD record %d\n", *record); + bad_record = true; + break; + } + if (bad_record) + break; + } + } + } + return lvds; +} + +struct amdgpu_encoder_atom_dig * +amdgpu_atombios_encoder_get_dig_info(struct amdgpu_encoder *amdgpu_encoder) +{ + int encoder_enum = (amdgpu_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT; + struct amdgpu_encoder_atom_dig *dig = kzalloc(sizeof(struct amdgpu_encoder_atom_dig), GFP_KERNEL); + + if (!dig) + return NULL; + + /* coherent mode by default */ + dig->coherent_mode = true; + dig->dig_encoder = -1; + + if (encoder_enum == 2) + dig->linkb = true; + else + dig->linkb = false; + + return dig; +} +
diff --git a/amdgpu/atombios_encoders.h b/amdgpu/atombios_encoders.h new file mode 100644 index 0000000..f77cbde --- /dev/null +++ b/amdgpu/atombios_encoders.h
@@ -0,0 +1,78 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __ATOMBIOS_ENCODER_H__ +#define __ATOMBIOS_ENCODER_H__ + +u8 +amdgpu_atombios_encoder_get_backlight_level_from_reg(struct amdgpu_device *adev); +void +amdgpu_atombios_encoder_set_backlight_level_to_reg(struct amdgpu_device *adev, + u8 backlight_level); +u8 +amdgpu_atombios_encoder_get_backlight_level(struct amdgpu_encoder *amdgpu_encoder); +void +amdgpu_atombios_encoder_set_backlight_level(struct amdgpu_encoder *amdgpu_encoder, + u8 level); +void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encoder, + struct drm_connector *drm_connector); +void +amdgpu_atombios_encoder_fini_backlight(struct amdgpu_encoder *amdgpu_encoder); +bool amdgpu_atombios_encoder_is_digital(struct drm_encoder *encoder); +bool amdgpu_atombios_encoder_mode_fixup(struct drm_encoder *encoder, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); +int amdgpu_atombios_encoder_get_encoder_mode(struct drm_encoder *encoder); +void +amdgpu_atombios_encoder_setup_dig_encoder(struct drm_encoder *encoder, + int action, int panel_mode); +void +amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int action, + uint8_t lane_num, uint8_t lane_set); +bool +amdgpu_atombios_encoder_set_edp_panel_power(struct drm_connector *connector, + int action); +void +amdgpu_atombios_encoder_dpms(struct drm_encoder *encoder, int mode); +void +amdgpu_atombios_encoder_set_crtc_source(struct drm_encoder *encoder); +void +amdgpu_atombios_encoder_init_dig(struct amdgpu_device *adev); +enum drm_connector_status +amdgpu_atombios_encoder_dac_detect(struct drm_encoder *encoder, + struct drm_connector *connector); +enum drm_connector_status +amdgpu_atombios_encoder_dig_detect(struct drm_encoder *encoder, + struct drm_connector *connector); +void +amdgpu_atombios_encoder_setup_ext_encoder_ddc(struct drm_encoder *encoder); +void +amdgpu_atombios_encoder_set_bios_scratch_regs(struct drm_connector *connector, + struct drm_encoder *encoder, + bool connected); +struct amdgpu_encoder_atom_dig * +amdgpu_atombios_encoder_get_lcd_info(struct amdgpu_encoder *encoder); +struct amdgpu_encoder_atom_dig * +amdgpu_atombios_encoder_get_dig_info(struct amdgpu_encoder *amdgpu_encoder); + +#endif
diff --git a/amdgpu/atombios_i2c.c b/amdgpu/atombios_i2c.c new file mode 100644 index 0000000..980c363 --- /dev/null +++ b/amdgpu/atombios_i2c.c
@@ -0,0 +1,181 @@ +/* + * Copyright 2011 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + * + */ + +#include <drm/amdgpu_drm.h> +#include "amdgpu.h" +#include "atom.h" +#include "amdgpu_atombios.h" +#include "atombios_i2c.h" + +#define TARGET_HW_I2C_CLOCK 50 + +/* these are a limitation of ProcessI2cChannelTransaction not the hw */ +#define ATOM_MAX_HW_I2C_WRITE 3 +#define ATOM_MAX_HW_I2C_READ 255 + +static int amdgpu_atombios_i2c_process_i2c_ch(struct amdgpu_i2c_chan *chan, + u8 slave_addr, u8 flags, + u8 *buf, u8 num) +{ + struct drm_device *dev = chan->dev; + struct amdgpu_device *adev = dev->dev_private; + PROCESS_I2C_CHANNEL_TRANSACTION_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, ProcessI2cChannelTransaction); + unsigned char *base; + u16 out = cpu_to_le16(0); + int r = 0; + + memset(&args, 0, sizeof(args)); + + mutex_lock(&chan->mutex); + + base = (unsigned char *)adev->mode_info.atom_context->scratch; + + if (flags & HW_I2C_WRITE) { + if (num > ATOM_MAX_HW_I2C_WRITE) { + DRM_ERROR("hw i2c: tried to write too many bytes (%d vs 3)\n", num); + r = -EINVAL; + goto done; + } + if (buf == NULL) + args.ucRegIndex = 0; + else + args.ucRegIndex = buf[0]; + if (num) + num--; + if (num) { + if (buf) { + memcpy(&out, &buf[1], num); + } else { + DRM_ERROR("hw i2c: missing buf with num > 1\n"); + r = -EINVAL; + goto done; + } + } + args.lpI2CDataOut = cpu_to_le16(out); + } else { + if (num > ATOM_MAX_HW_I2C_READ) { + DRM_ERROR("hw i2c: tried to read too many bytes (%d vs 255)\n", num); + r = -EINVAL; + goto done; + } + args.ucRegIndex = 0; + args.lpI2CDataOut = 0; + } + + args.ucFlag = flags; + args.ucI2CSpeed = TARGET_HW_I2C_CLOCK; + args.ucTransBytes = num; + args.ucSlaveAddr = slave_addr << 1; + args.ucLineNumber = chan->rec.i2c_id; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); + + /* error */ + if (args.ucStatus != HW_ASSISTED_I2C_STATUS_SUCCESS) { + DRM_DEBUG_KMS("hw_i2c error\n"); + r = -EIO; + goto done; + } + + if (!(flags & HW_I2C_WRITE)) + amdgpu_atombios_copy_swap(buf, base, num, false); + +done: + mutex_unlock(&chan->mutex); + + return r; +} + +int amdgpu_atombios_i2c_xfer(struct i2c_adapter *i2c_adap, + struct i2c_msg *msgs, int num) +{ + struct amdgpu_i2c_chan *i2c = i2c_get_adapdata(i2c_adap); + struct i2c_msg *p; + int i, remaining, current_count, buffer_offset, max_bytes, ret; + u8 flags; + + /* check for bus probe */ + p = &msgs[0]; + if ((num == 1) && (p->len == 0)) { + ret = amdgpu_atombios_i2c_process_i2c_ch(i2c, + p->addr, HW_I2C_WRITE, + NULL, 0); + if (ret) + return ret; + else + return num; + } + + for (i = 0; i < num; i++) { + p = &msgs[i]; + remaining = p->len; + buffer_offset = 0; + /* max_bytes are a limitation of ProcessI2cChannelTransaction not the hw */ + if (p->flags & I2C_M_RD) { + max_bytes = ATOM_MAX_HW_I2C_READ; + flags = HW_I2C_READ; + } else { + max_bytes = ATOM_MAX_HW_I2C_WRITE; + flags = HW_I2C_WRITE; + } + while (remaining) { + if (remaining > max_bytes) + current_count = max_bytes; + else + current_count = remaining; + ret = amdgpu_atombios_i2c_process_i2c_ch(i2c, + p->addr, flags, + &p->buf[buffer_offset], current_count); + if (ret) + return ret; + remaining -= current_count; + buffer_offset += current_count; + } + } + + return num; +} + +u32 amdgpu_atombios_i2c_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +void amdgpu_atombios_i2c_channel_trans(struct amdgpu_device* adev, u8 slave_addr, u8 line_number, u8 offset, u8 data) +{ + PROCESS_I2C_CHANNEL_TRANSACTION_PS_ALLOCATION args; + int index = GetIndexIntoMasterTable(COMMAND, ProcessI2cChannelTransaction); + + args.ucRegIndex = offset; + args.lpI2CDataOut = data; + args.ucFlag = 1; + args.ucI2CSpeed = TARGET_HW_I2C_CLOCK; + args.ucTransBytes = 1; + args.ucSlaveAddr = slave_addr; + args.ucLineNumber = line_number; + + amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args); +}
diff --git a/amdgpu/atombios_i2c.h b/amdgpu/atombios_i2c.h new file mode 100644 index 0000000..251aaf4 --- /dev/null +++ b/amdgpu/atombios_i2c.h
@@ -0,0 +1,33 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __ATOMBIOS_I2C_H__ +#define __ATOMBIOS_I2C_H__ + +int amdgpu_atombios_i2c_xfer(struct i2c_adapter *i2c_adap, + struct i2c_msg *msgs, int num); +u32 amdgpu_atombios_i2c_func(struct i2c_adapter *adap); +void amdgpu_atombios_i2c_channel_trans(struct amdgpu_device* adev, + u8 slave_addr, u8 line_number, u8 offset, u8 data); + +#endif
diff --git a/amdgpu/cik.c b/amdgpu/cik.c new file mode 100644 index 0000000..1ffbc0d --- /dev/null +++ b/amdgpu/cik.c
@@ -0,0 +1,2160 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#include <linux/firmware.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "amdgpu_ih.h" +#include "amdgpu_uvd.h" +#include "amdgpu_vce.h" +#include "cikd.h" +#include "atom.h" +#include "amd_pcie.h" + +#include "cik.h" +#include "gmc_v7_0.h" +#include "cik_ih.h" +#include "dce_v8_0.h" +#include "gfx_v7_0.h" +#include "cik_sdma.h" +#include "uvd_v4_2.h" +#include "vce_v2_0.h" +#include "cik_dpm.h" + +#include "uvd/uvd_4_2_d.h" + +#include "smu/smu_7_0_1_d.h" +#include "smu/smu_7_0_1_sh_mask.h" + +#include "dce/dce_8_0_d.h" +#include "dce/dce_8_0_sh_mask.h" + +#include "bif/bif_4_1_d.h" +#include "bif/bif_4_1_sh_mask.h" + +#include "gca/gfx_7_2_d.h" +#include "gca/gfx_7_2_enum.h" +#include "gca/gfx_7_2_sh_mask.h" + +#include "gmc/gmc_7_1_d.h" +#include "gmc/gmc_7_1_sh_mask.h" + +#include "oss/oss_2_0_d.h" +#include "oss/oss_2_0_sh_mask.h" + +#include "amdgpu_dm.h" +#include "amdgpu_amdkfd.h" +#include "dce_virtual.h" + +/* + * Indirect registers accessor + */ +static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&adev->pcie_idx_lock, flags); + WREG32(mmPCIE_INDEX, reg); + (void)RREG32(mmPCIE_INDEX); + r = RREG32(mmPCIE_DATA); + spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); + return r; +} + +static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&adev->pcie_idx_lock, flags); + WREG32(mmPCIE_INDEX, reg); + (void)RREG32(mmPCIE_INDEX); + WREG32(mmPCIE_DATA, v); + (void)RREG32(mmPCIE_DATA); + spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); +} + +static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + WREG32(mmSMC_IND_INDEX_0, (reg)); + r = RREG32(mmSMC_IND_DATA_0); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + return r; +} + +static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + WREG32(mmSMC_IND_INDEX_0, (reg)); + WREG32(mmSMC_IND_DATA_0, (v)); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); +} + +static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags); + WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff)); + r = RREG32(mmUVD_CTX_DATA); + spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags); + return r; +} + +static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags); + WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff)); + WREG32(mmUVD_CTX_DATA, (v)); + spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags); +} + +static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&adev->didt_idx_lock, flags); + WREG32(mmDIDT_IND_INDEX, (reg)); + r = RREG32(mmDIDT_IND_DATA); + spin_unlock_irqrestore(&adev->didt_idx_lock, flags); + return r; +} + +static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&adev->didt_idx_lock, flags); + WREG32(mmDIDT_IND_INDEX, (reg)); + WREG32(mmDIDT_IND_DATA, (v)); + spin_unlock_irqrestore(&adev->didt_idx_lock, flags); +} + +static const u32 bonaire_golden_spm_registers[] = +{ + 0xc200, 0xe0ffffff, 0xe0000000 +}; + +static const u32 bonaire_golden_common_registers[] = +{ + 0x31dc, 0xffffffff, 0x00000800, + 0x31dd, 0xffffffff, 0x00000800, + 0x31e6, 0xffffffff, 0x00007fbf, + 0x31e7, 0xffffffff, 0x00007faf +}; + +static const u32 bonaire_golden_registers[] = +{ + 0xcd5, 0x00000333, 0x00000333, + 0xcd4, 0x000c0fc0, 0x00040200, + 0x2684, 0x00010000, 0x00058208, + 0xf000, 0xffff1fff, 0x00140000, + 0xf080, 0xfdfc0fff, 0x00000100, + 0xf08d, 0x40000000, 0x40000200, + 0x260c, 0xffffffff, 0x00000000, + 0x260d, 0xf00fffff, 0x00000400, + 0x260e, 0x0002021c, 0x00020200, + 0x31e, 0x00000080, 0x00000000, + 0x16ec, 0x000000f0, 0x00000070, + 0x16f0, 0xf0311fff, 0x80300000, + 0x263e, 0x73773777, 0x12010001, + 0xd43, 0x00810000, 0x408af000, + 0x1c0c, 0x31000111, 0x00000011, + 0xbd2, 0x73773777, 0x12010001, + 0x883, 0x00007fb6, 0x0021a1b1, + 0x884, 0x00007fb6, 0x002021b1, + 0x860, 0x00007fb6, 0x00002191, + 0x886, 0x00007fb6, 0x002121b1, + 0x887, 0x00007fb6, 0x002021b1, + 0x877, 0x00007fb6, 0x00002191, + 0x878, 0x00007fb6, 0x00002191, + 0xd8a, 0x0000003f, 0x0000000a, + 0xd8b, 0x0000003f, 0x0000000a, + 0xab9, 0x00073ffe, 0x000022a2, + 0x903, 0x000007ff, 0x00000000, + 0x2285, 0xf000003f, 0x00000007, + 0x22fc, 0x00002001, 0x00000001, + 0x22c9, 0xffffffff, 0x00ffffff, + 0xc281, 0x0000ff0f, 0x00000000, + 0xa293, 0x07ffffff, 0x06000000, + 0x136, 0x00000fff, 0x00000100, + 0xf9e, 0x00000001, 0x00000002, + 0x2440, 0x03000000, 0x0362c688, + 0x2300, 0x000000ff, 0x00000001, + 0x390, 0x00001fff, 0x00001fff, + 0x2418, 0x0000007f, 0x00000020, + 0x2542, 0x00010000, 0x00010000, + 0x2b05, 0x000003ff, 0x000000f3, + 0x2b03, 0xffffffff, 0x00001032 +}; + +static const u32 bonaire_mgcg_cgcg_init[] = +{ + 0x3108, 0xffffffff, 0xfffffffc, + 0xc200, 0xffffffff, 0xe0000000, + 0xf0a8, 0xffffffff, 0x00000100, + 0xf082, 0xffffffff, 0x00000100, + 0xf0b0, 0xffffffff, 0xc0000100, + 0xf0b2, 0xffffffff, 0xc0000100, + 0xf0b1, 0xffffffff, 0xc0000100, + 0x1579, 0xffffffff, 0x00600100, + 0xf0a0, 0xffffffff, 0x00000100, + 0xf085, 0xffffffff, 0x06000100, + 0xf088, 0xffffffff, 0x00000100, + 0xf086, 0xffffffff, 0x06000100, + 0xf081, 0xffffffff, 0x00000100, + 0xf0b8, 0xffffffff, 0x00000100, + 0xf089, 0xffffffff, 0x00000100, + 0xf080, 0xffffffff, 0x00000100, + 0xf08c, 0xffffffff, 0x00000100, + 0xf08d, 0xffffffff, 0x00000100, + 0xf094, 0xffffffff, 0x00000100, + 0xf095, 0xffffffff, 0x00000100, + 0xf096, 0xffffffff, 0x00000100, + 0xf097, 0xffffffff, 0x00000100, + 0xf098, 0xffffffff, 0x00000100, + 0xf09f, 0xffffffff, 0x00000100, + 0xf09e, 0xffffffff, 0x00000100, + 0xf084, 0xffffffff, 0x06000100, + 0xf0a4, 0xffffffff, 0x00000100, + 0xf09d, 0xffffffff, 0x00000100, + 0xf0ad, 0xffffffff, 0x00000100, + 0xf0ac, 0xffffffff, 0x00000100, + 0xf09c, 0xffffffff, 0x00000100, + 0xc200, 0xffffffff, 0xe0000000, + 0xf008, 0xffffffff, 0x00010000, + 0xf009, 0xffffffff, 0x00030002, + 0xf00a, 0xffffffff, 0x00040007, + 0xf00b, 0xffffffff, 0x00060005, + 0xf00c, 0xffffffff, 0x00090008, + 0xf00d, 0xffffffff, 0x00010000, + 0xf00e, 0xffffffff, 0x00030002, + 0xf00f, 0xffffffff, 0x00040007, + 0xf010, 0xffffffff, 0x00060005, + 0xf011, 0xffffffff, 0x00090008, + 0xf012, 0xffffffff, 0x00010000, + 0xf013, 0xffffffff, 0x00030002, + 0xf014, 0xffffffff, 0x00040007, + 0xf015, 0xffffffff, 0x00060005, + 0xf016, 0xffffffff, 0x00090008, + 0xf017, 0xffffffff, 0x00010000, + 0xf018, 0xffffffff, 0x00030002, + 0xf019, 0xffffffff, 0x00040007, + 0xf01a, 0xffffffff, 0x00060005, + 0xf01b, 0xffffffff, 0x00090008, + 0xf01c, 0xffffffff, 0x00010000, + 0xf01d, 0xffffffff, 0x00030002, + 0xf01e, 0xffffffff, 0x00040007, + 0xf01f, 0xffffffff, 0x00060005, + 0xf020, 0xffffffff, 0x00090008, + 0xf021, 0xffffffff, 0x00010000, + 0xf022, 0xffffffff, 0x00030002, + 0xf023, 0xffffffff, 0x00040007, + 0xf024, 0xffffffff, 0x00060005, + 0xf025, 0xffffffff, 0x00090008, + 0xf026, 0xffffffff, 0x00010000, + 0xf027, 0xffffffff, 0x00030002, + 0xf028, 0xffffffff, 0x00040007, + 0xf029, 0xffffffff, 0x00060005, + 0xf02a, 0xffffffff, 0x00090008, + 0xf000, 0xffffffff, 0x96e00200, + 0x21c2, 0xffffffff, 0x00900100, + 0x3109, 0xffffffff, 0x0020003f, + 0xe, 0xffffffff, 0x0140001c, + 0xf, 0x000f0000, 0x000f0000, + 0x88, 0xffffffff, 0xc060000c, + 0x89, 0xc0000fff, 0x00000100, + 0x3e4, 0xffffffff, 0x00000100, + 0x3e6, 0x00000101, 0x00000000, + 0x82a, 0xffffffff, 0x00000104, + 0x1579, 0xff000fff, 0x00000100, + 0xc33, 0xc0000fff, 0x00000104, + 0x3079, 0x00000001, 0x00000001, + 0x3403, 0xff000ff0, 0x00000100, + 0x3603, 0xff000ff0, 0x00000100 +}; + +static const u32 spectre_golden_spm_registers[] = +{ + 0xc200, 0xe0ffffff, 0xe0000000 +}; + +static const u32 spectre_golden_common_registers[] = +{ + 0x31dc, 0xffffffff, 0x00000800, + 0x31dd, 0xffffffff, 0x00000800, + 0x31e6, 0xffffffff, 0x00007fbf, + 0x31e7, 0xffffffff, 0x00007faf +}; + +static const u32 spectre_golden_registers[] = +{ + 0xf000, 0xffff1fff, 0x96940200, + 0xf003, 0xffff0001, 0xff000000, + 0xf080, 0xfffc0fff, 0x00000100, + 0x1bb6, 0x00010101, 0x00010000, + 0x260d, 0xf00fffff, 0x00000400, + 0x260e, 0xfffffffc, 0x00020200, + 0x16ec, 0x000000f0, 0x00000070, + 0x16f0, 0xf0311fff, 0x80300000, + 0x263e, 0x73773777, 0x12010001, + 0x26df, 0x00ff0000, 0x00fc0000, + 0xbd2, 0x73773777, 0x12010001, + 0x2285, 0xf000003f, 0x00000007, + 0x22c9, 0xffffffff, 0x00ffffff, + 0xa0d4, 0x3f3f3fff, 0x00000082, + 0xa0d5, 0x0000003f, 0x00000000, + 0xf9e, 0x00000001, 0x00000002, + 0x244f, 0xffff03df, 0x00000004, + 0x31da, 0x00000008, 0x00000008, + 0x2300, 0x000008ff, 0x00000800, + 0x2542, 0x00010000, 0x00010000, + 0x2b03, 0xffffffff, 0x54763210, + 0x853e, 0x01ff01ff, 0x00000002, + 0x8526, 0x007ff800, 0x00200000, + 0x8057, 0xffffffff, 0x00000f40, + 0xc24d, 0xffffffff, 0x00000001 +}; + +static const u32 spectre_mgcg_cgcg_init[] = +{ + 0x3108, 0xffffffff, 0xfffffffc, + 0xc200, 0xffffffff, 0xe0000000, + 0xf0a8, 0xffffffff, 0x00000100, + 0xf082, 0xffffffff, 0x00000100, + 0xf0b0, 0xffffffff, 0x00000100, + 0xf0b2, 0xffffffff, 0x00000100, + 0xf0b1, 0xffffffff, 0x00000100, + 0x1579, 0xffffffff, 0x00600100, + 0xf0a0, 0xffffffff, 0x00000100, + 0xf085, 0xffffffff, 0x06000100, + 0xf088, 0xffffffff, 0x00000100, + 0xf086, 0xffffffff, 0x06000100, + 0xf081, 0xffffffff, 0x00000100, + 0xf0b8, 0xffffffff, 0x00000100, + 0xf089, 0xffffffff, 0x00000100, + 0xf080, 0xffffffff, 0x00000100, + 0xf08c, 0xffffffff, 0x00000100, + 0xf08d, 0xffffffff, 0x00000100, + 0xf094, 0xffffffff, 0x00000100, + 0xf095, 0xffffffff, 0x00000100, + 0xf096, 0xffffffff, 0x00000100, + 0xf097, 0xffffffff, 0x00000100, + 0xf098, 0xffffffff, 0x00000100, + 0xf09f, 0xffffffff, 0x00000100, + 0xf09e, 0xffffffff, 0x00000100, + 0xf084, 0xffffffff, 0x06000100, + 0xf0a4, 0xffffffff, 0x00000100, + 0xf09d, 0xffffffff, 0x00000100, + 0xf0ad, 0xffffffff, 0x00000100, + 0xf0ac, 0xffffffff, 0x00000100, + 0xf09c, 0xffffffff, 0x00000100, + 0xc200, 0xffffffff, 0xe0000000, + 0xf008, 0xffffffff, 0x00010000, + 0xf009, 0xffffffff, 0x00030002, + 0xf00a, 0xffffffff, 0x00040007, + 0xf00b, 0xffffffff, 0x00060005, + 0xf00c, 0xffffffff, 0x00090008, + 0xf00d, 0xffffffff, 0x00010000, + 0xf00e, 0xffffffff, 0x00030002, + 0xf00f, 0xffffffff, 0x00040007, + 0xf010, 0xffffffff, 0x00060005, + 0xf011, 0xffffffff, 0x00090008, + 0xf012, 0xffffffff, 0x00010000, + 0xf013, 0xffffffff, 0x00030002, + 0xf014, 0xffffffff, 0x00040007, + 0xf015, 0xffffffff, 0x00060005, + 0xf016, 0xffffffff, 0x00090008, + 0xf017, 0xffffffff, 0x00010000, + 0xf018, 0xffffffff, 0x00030002, + 0xf019, 0xffffffff, 0x00040007, + 0xf01a, 0xffffffff, 0x00060005, + 0xf01b, 0xffffffff, 0x00090008, + 0xf01c, 0xffffffff, 0x00010000, + 0xf01d, 0xffffffff, 0x00030002, + 0xf01e, 0xffffffff, 0x00040007, + 0xf01f, 0xffffffff, 0x00060005, + 0xf020, 0xffffffff, 0x00090008, + 0xf021, 0xffffffff, 0x00010000, + 0xf022, 0xffffffff, 0x00030002, + 0xf023, 0xffffffff, 0x00040007, + 0xf024, 0xffffffff, 0x00060005, + 0xf025, 0xffffffff, 0x00090008, + 0xf026, 0xffffffff, 0x00010000, + 0xf027, 0xffffffff, 0x00030002, + 0xf028, 0xffffffff, 0x00040007, + 0xf029, 0xffffffff, 0x00060005, + 0xf02a, 0xffffffff, 0x00090008, + 0xf02b, 0xffffffff, 0x00010000, + 0xf02c, 0xffffffff, 0x00030002, + 0xf02d, 0xffffffff, 0x00040007, + 0xf02e, 0xffffffff, 0x00060005, + 0xf02f, 0xffffffff, 0x00090008, + 0xf000, 0xffffffff, 0x96e00200, + 0x21c2, 0xffffffff, 0x00900100, + 0x3109, 0xffffffff, 0x0020003f, + 0xe, 0xffffffff, 0x0140001c, + 0xf, 0x000f0000, 0x000f0000, + 0x88, 0xffffffff, 0xc060000c, + 0x89, 0xc0000fff, 0x00000100, + 0x3e4, 0xffffffff, 0x00000100, + 0x3e6, 0x00000101, 0x00000000, + 0x82a, 0xffffffff, 0x00000104, + 0x1579, 0xff000fff, 0x00000100, + 0xc33, 0xc0000fff, 0x00000104, + 0x3079, 0x00000001, 0x00000001, + 0x3403, 0xff000ff0, 0x00000100, + 0x3603, 0xff000ff0, 0x00000100 +}; + +static const u32 kalindi_golden_spm_registers[] = +{ + 0xc200, 0xe0ffffff, 0xe0000000 +}; + +static const u32 kalindi_golden_common_registers[] = +{ + 0x31dc, 0xffffffff, 0x00000800, + 0x31dd, 0xffffffff, 0x00000800, + 0x31e6, 0xffffffff, 0x00007fbf, + 0x31e7, 0xffffffff, 0x00007faf +}; + +static const u32 kalindi_golden_registers[] = +{ + 0xf000, 0xffffdfff, 0x6e944040, + 0x1579, 0xff607fff, 0xfc000100, + 0xf088, 0xff000fff, 0x00000100, + 0xf089, 0xff000fff, 0x00000100, + 0xf080, 0xfffc0fff, 0x00000100, + 0x1bb6, 0x00010101, 0x00010000, + 0x260c, 0xffffffff, 0x00000000, + 0x260d, 0xf00fffff, 0x00000400, + 0x16ec, 0x000000f0, 0x00000070, + 0x16f0, 0xf0311fff, 0x80300000, + 0x263e, 0x73773777, 0x12010001, + 0x263f, 0xffffffff, 0x00000010, + 0x26df, 0x00ff0000, 0x00fc0000, + 0x200c, 0x00001f0f, 0x0000100a, + 0xbd2, 0x73773777, 0x12010001, + 0x902, 0x000fffff, 0x000c007f, + 0x2285, 0xf000003f, 0x00000007, + 0x22c9, 0x3fff3fff, 0x00ffcfff, + 0xc281, 0x0000ff0f, 0x00000000, + 0xa293, 0x07ffffff, 0x06000000, + 0x136, 0x00000fff, 0x00000100, + 0xf9e, 0x00000001, 0x00000002, + 0x31da, 0x00000008, 0x00000008, + 0x2300, 0x000000ff, 0x00000003, + 0x853e, 0x01ff01ff, 0x00000002, + 0x8526, 0x007ff800, 0x00200000, + 0x8057, 0xffffffff, 0x00000f40, + 0x2231, 0x001f3ae3, 0x00000082, + 0x2235, 0x0000001f, 0x00000010, + 0xc24d, 0xffffffff, 0x00000000 +}; + +static const u32 kalindi_mgcg_cgcg_init[] = +{ + 0x3108, 0xffffffff, 0xfffffffc, + 0xc200, 0xffffffff, 0xe0000000, + 0xf0a8, 0xffffffff, 0x00000100, + 0xf082, 0xffffffff, 0x00000100, + 0xf0b0, 0xffffffff, 0x00000100, + 0xf0b2, 0xffffffff, 0x00000100, + 0xf0b1, 0xffffffff, 0x00000100, + 0x1579, 0xffffffff, 0x00600100, + 0xf0a0, 0xffffffff, 0x00000100, + 0xf085, 0xffffffff, 0x06000100, + 0xf088, 0xffffffff, 0x00000100, + 0xf086, 0xffffffff, 0x06000100, + 0xf081, 0xffffffff, 0x00000100, + 0xf0b8, 0xffffffff, 0x00000100, + 0xf089, 0xffffffff, 0x00000100, + 0xf080, 0xffffffff, 0x00000100, + 0xf08c, 0xffffffff, 0x00000100, + 0xf08d, 0xffffffff, 0x00000100, + 0xf094, 0xffffffff, 0x00000100, + 0xf095, 0xffffffff, 0x00000100, + 0xf096, 0xffffffff, 0x00000100, + 0xf097, 0xffffffff, 0x00000100, + 0xf098, 0xffffffff, 0x00000100, + 0xf09f, 0xffffffff, 0x00000100, + 0xf09e, 0xffffffff, 0x00000100, + 0xf084, 0xffffffff, 0x06000100, + 0xf0a4, 0xffffffff, 0x00000100, + 0xf09d, 0xffffffff, 0x00000100, + 0xf0ad, 0xffffffff, 0x00000100, + 0xf0ac, 0xffffffff, 0x00000100, + 0xf09c, 0xffffffff, 0x00000100, + 0xc200, 0xffffffff, 0xe0000000, + 0xf008, 0xffffffff, 0x00010000, + 0xf009, 0xffffffff, 0x00030002, + 0xf00a, 0xffffffff, 0x00040007, + 0xf00b, 0xffffffff, 0x00060005, + 0xf00c, 0xffffffff, 0x00090008, + 0xf00d, 0xffffffff, 0x00010000, + 0xf00e, 0xffffffff, 0x00030002, + 0xf00f, 0xffffffff, 0x00040007, + 0xf010, 0xffffffff, 0x00060005, + 0xf011, 0xffffffff, 0x00090008, + 0xf000, 0xffffffff, 0x96e00200, + 0x21c2, 0xffffffff, 0x00900100, + 0x3109, 0xffffffff, 0x0020003f, + 0xe, 0xffffffff, 0x0140001c, + 0xf, 0x000f0000, 0x000f0000, + 0x88, 0xffffffff, 0xc060000c, + 0x89, 0xc0000fff, 0x00000100, + 0x82a, 0xffffffff, 0x00000104, + 0x1579, 0xff000fff, 0x00000100, + 0xc33, 0xc0000fff, 0x00000104, + 0x3079, 0x00000001, 0x00000001, + 0x3403, 0xff000ff0, 0x00000100, + 0x3603, 0xff000ff0, 0x00000100 +}; + +static const u32 hawaii_golden_spm_registers[] = +{ + 0xc200, 0xe0ffffff, 0xe0000000 +}; + +static const u32 hawaii_golden_common_registers[] = +{ + 0xc200, 0xffffffff, 0xe0000000, + 0xa0d4, 0xffffffff, 0x3a00161a, + 0xa0d5, 0xffffffff, 0x0000002e, + 0x2684, 0xffffffff, 0x00018208, + 0x263e, 0xffffffff, 0x12011003 +}; + +static const u32 hawaii_golden_registers[] = +{ + 0xcd5, 0x00000333, 0x00000333, + 0x2684, 0x00010000, 0x00058208, + 0x260c, 0xffffffff, 0x00000000, + 0x260d, 0xf00fffff, 0x00000400, + 0x260e, 0x0002021c, 0x00020200, + 0x31e, 0x00000080, 0x00000000, + 0x16ec, 0x000000f0, 0x00000070, + 0x16f0, 0xf0311fff, 0x80300000, + 0xd43, 0x00810000, 0x408af000, + 0x1c0c, 0x31000111, 0x00000011, + 0xbd2, 0x73773777, 0x12010001, + 0x848, 0x0000007f, 0x0000001b, + 0x877, 0x00007fb6, 0x00002191, + 0xd8a, 0x0000003f, 0x0000000a, + 0xd8b, 0x0000003f, 0x0000000a, + 0xab9, 0x00073ffe, 0x000022a2, + 0x903, 0x000007ff, 0x00000000, + 0x22fc, 0x00002001, 0x00000001, + 0x22c9, 0xffffffff, 0x00ffffff, + 0xc281, 0x0000ff0f, 0x00000000, + 0xa293, 0x07ffffff, 0x06000000, + 0xf9e, 0x00000001, 0x00000002, + 0x31da, 0x00000008, 0x00000008, + 0x31dc, 0x00000f00, 0x00000800, + 0x31dd, 0x00000f00, 0x00000800, + 0x31e6, 0x00ffffff, 0x00ff7fbf, + 0x31e7, 0x00ffffff, 0x00ff7faf, + 0x2300, 0x000000ff, 0x00000800, + 0x390, 0x00001fff, 0x00001fff, + 0x2418, 0x0000007f, 0x00000020, + 0x2542, 0x00010000, 0x00010000, + 0x2b80, 0x00100000, 0x000ff07c, + 0x2b05, 0x000003ff, 0x0000000f, + 0x2b04, 0xffffffff, 0x7564fdec, + 0x2b03, 0xffffffff, 0x3120b9a8, + 0x2b02, 0x20000000, 0x0f9c0000 +}; + +static const u32 hawaii_mgcg_cgcg_init[] = +{ + 0x3108, 0xffffffff, 0xfffffffd, + 0xc200, 0xffffffff, 0xe0000000, + 0xf0a8, 0xffffffff, 0x00000100, + 0xf082, 0xffffffff, 0x00000100, + 0xf0b0, 0xffffffff, 0x00000100, + 0xf0b2, 0xffffffff, 0x00000100, + 0xf0b1, 0xffffffff, 0x00000100, + 0x1579, 0xffffffff, 0x00200100, + 0xf0a0, 0xffffffff, 0x00000100, + 0xf085, 0xffffffff, 0x06000100, + 0xf088, 0xffffffff, 0x00000100, + 0xf086, 0xffffffff, 0x06000100, + 0xf081, 0xffffffff, 0x00000100, + 0xf0b8, 0xffffffff, 0x00000100, + 0xf089, 0xffffffff, 0x00000100, + 0xf080, 0xffffffff, 0x00000100, + 0xf08c, 0xffffffff, 0x00000100, + 0xf08d, 0xffffffff, 0x00000100, + 0xf094, 0xffffffff, 0x00000100, + 0xf095, 0xffffffff, 0x00000100, + 0xf096, 0xffffffff, 0x00000100, + 0xf097, 0xffffffff, 0x00000100, + 0xf098, 0xffffffff, 0x00000100, + 0xf09f, 0xffffffff, 0x00000100, + 0xf09e, 0xffffffff, 0x00000100, + 0xf084, 0xffffffff, 0x06000100, + 0xf0a4, 0xffffffff, 0x00000100, + 0xf09d, 0xffffffff, 0x00000100, + 0xf0ad, 0xffffffff, 0x00000100, + 0xf0ac, 0xffffffff, 0x00000100, + 0xf09c, 0xffffffff, 0x00000100, + 0xc200, 0xffffffff, 0xe0000000, + 0xf008, 0xffffffff, 0x00010000, + 0xf009, 0xffffffff, 0x00030002, + 0xf00a, 0xffffffff, 0x00040007, + 0xf00b, 0xffffffff, 0x00060005, + 0xf00c, 0xffffffff, 0x00090008, + 0xf00d, 0xffffffff, 0x00010000, + 0xf00e, 0xffffffff, 0x00030002, + 0xf00f, 0xffffffff, 0x00040007, + 0xf010, 0xffffffff, 0x00060005, + 0xf011, 0xffffffff, 0x00090008, + 0xf012, 0xffffffff, 0x00010000, + 0xf013, 0xffffffff, 0x00030002, + 0xf014, 0xffffffff, 0x00040007, + 0xf015, 0xffffffff, 0x00060005, + 0xf016, 0xffffffff, 0x00090008, + 0xf017, 0xffffffff, 0x00010000, + 0xf018, 0xffffffff, 0x00030002, + 0xf019, 0xffffffff, 0x00040007, + 0xf01a, 0xffffffff, 0x00060005, + 0xf01b, 0xffffffff, 0x00090008, + 0xf01c, 0xffffffff, 0x00010000, + 0xf01d, 0xffffffff, 0x00030002, + 0xf01e, 0xffffffff, 0x00040007, + 0xf01f, 0xffffffff, 0x00060005, + 0xf020, 0xffffffff, 0x00090008, + 0xf021, 0xffffffff, 0x00010000, + 0xf022, 0xffffffff, 0x00030002, + 0xf023, 0xffffffff, 0x00040007, + 0xf024, 0xffffffff, 0x00060005, + 0xf025, 0xffffffff, 0x00090008, + 0xf026, 0xffffffff, 0x00010000, + 0xf027, 0xffffffff, 0x00030002, + 0xf028, 0xffffffff, 0x00040007, + 0xf029, 0xffffffff, 0x00060005, + 0xf02a, 0xffffffff, 0x00090008, + 0xf02b, 0xffffffff, 0x00010000, + 0xf02c, 0xffffffff, 0x00030002, + 0xf02d, 0xffffffff, 0x00040007, + 0xf02e, 0xffffffff, 0x00060005, + 0xf02f, 0xffffffff, 0x00090008, + 0xf030, 0xffffffff, 0x00010000, + 0xf031, 0xffffffff, 0x00030002, + 0xf032, 0xffffffff, 0x00040007, + 0xf033, 0xffffffff, 0x00060005, + 0xf034, 0xffffffff, 0x00090008, + 0xf035, 0xffffffff, 0x00010000, + 0xf036, 0xffffffff, 0x00030002, + 0xf037, 0xffffffff, 0x00040007, + 0xf038, 0xffffffff, 0x00060005, + 0xf039, 0xffffffff, 0x00090008, + 0xf03a, 0xffffffff, 0x00010000, + 0xf03b, 0xffffffff, 0x00030002, + 0xf03c, 0xffffffff, 0x00040007, + 0xf03d, 0xffffffff, 0x00060005, + 0xf03e, 0xffffffff, 0x00090008, + 0x30c6, 0xffffffff, 0x00020200, + 0xcd4, 0xffffffff, 0x00000200, + 0x570, 0xffffffff, 0x00000400, + 0x157a, 0xffffffff, 0x00000000, + 0xbd4, 0xffffffff, 0x00000902, + 0xf000, 0xffffffff, 0x96940200, + 0x21c2, 0xffffffff, 0x00900100, + 0x3109, 0xffffffff, 0x0020003f, + 0xe, 0xffffffff, 0x0140001c, + 0xf, 0x000f0000, 0x000f0000, + 0x88, 0xffffffff, 0xc060000c, + 0x89, 0xc0000fff, 0x00000100, + 0x3e4, 0xffffffff, 0x00000100, + 0x3e6, 0x00000101, 0x00000000, + 0x82a, 0xffffffff, 0x00000104, + 0x1579, 0xff000fff, 0x00000100, + 0xc33, 0xc0000fff, 0x00000104, + 0x3079, 0x00000001, 0x00000001, + 0x3403, 0xff000ff0, 0x00000100, + 0x3603, 0xff000ff0, 0x00000100 +}; + +static const u32 godavari_golden_registers[] = +{ + 0x1579, 0xff607fff, 0xfc000100, + 0x1bb6, 0x00010101, 0x00010000, + 0x260c, 0xffffffff, 0x00000000, + 0x260c0, 0xf00fffff, 0x00000400, + 0x184c, 0xffffffff, 0x00010000, + 0x16ec, 0x000000f0, 0x00000070, + 0x16f0, 0xf0311fff, 0x80300000, + 0x263e, 0x73773777, 0x12010001, + 0x263f, 0xffffffff, 0x00000010, + 0x200c, 0x00001f0f, 0x0000100a, + 0xbd2, 0x73773777, 0x12010001, + 0x902, 0x000fffff, 0x000c007f, + 0x2285, 0xf000003f, 0x00000007, + 0x22c9, 0xffffffff, 0x00ff0fff, + 0xc281, 0x0000ff0f, 0x00000000, + 0xa293, 0x07ffffff, 0x06000000, + 0x136, 0x00000fff, 0x00000100, + 0x3405, 0x00010000, 0x00810001, + 0x3605, 0x00010000, 0x00810001, + 0xf9e, 0x00000001, 0x00000002, + 0x31da, 0x00000008, 0x00000008, + 0x31dc, 0x00000f00, 0x00000800, + 0x31dd, 0x00000f00, 0x00000800, + 0x31e6, 0x00ffffff, 0x00ff7fbf, + 0x31e7, 0x00ffffff, 0x00ff7faf, + 0x2300, 0x000000ff, 0x00000001, + 0x853e, 0x01ff01ff, 0x00000002, + 0x8526, 0x007ff800, 0x00200000, + 0x8057, 0xffffffff, 0x00000f40, + 0x2231, 0x001f3ae3, 0x00000082, + 0x2235, 0x0000001f, 0x00000010, + 0xc24d, 0xffffffff, 0x00000000 +}; + +static void cik_init_golden_registers(struct amdgpu_device *adev) +{ + /* Some of the registers might be dependent on GRBM_GFX_INDEX */ + mutex_lock(&adev->grbm_idx_mutex); + + switch (adev->asic_type) { + case CHIP_BONAIRE: + amdgpu_device_program_register_sequence(adev, + bonaire_mgcg_cgcg_init, + ARRAY_SIZE(bonaire_mgcg_cgcg_init)); + amdgpu_device_program_register_sequence(adev, + bonaire_golden_registers, + ARRAY_SIZE(bonaire_golden_registers)); + amdgpu_device_program_register_sequence(adev, + bonaire_golden_common_registers, + ARRAY_SIZE(bonaire_golden_common_registers)); + amdgpu_device_program_register_sequence(adev, + bonaire_golden_spm_registers, + ARRAY_SIZE(bonaire_golden_spm_registers)); + break; + case CHIP_KABINI: + amdgpu_device_program_register_sequence(adev, + kalindi_mgcg_cgcg_init, + ARRAY_SIZE(kalindi_mgcg_cgcg_init)); + amdgpu_device_program_register_sequence(adev, + kalindi_golden_registers, + ARRAY_SIZE(kalindi_golden_registers)); + amdgpu_device_program_register_sequence(adev, + kalindi_golden_common_registers, + ARRAY_SIZE(kalindi_golden_common_registers)); + amdgpu_device_program_register_sequence(adev, + kalindi_golden_spm_registers, + ARRAY_SIZE(kalindi_golden_spm_registers)); + break; + case CHIP_MULLINS: + amdgpu_device_program_register_sequence(adev, + kalindi_mgcg_cgcg_init, + ARRAY_SIZE(kalindi_mgcg_cgcg_init)); + amdgpu_device_program_register_sequence(adev, + godavari_golden_registers, + ARRAY_SIZE(godavari_golden_registers)); + amdgpu_device_program_register_sequence(adev, + kalindi_golden_common_registers, + ARRAY_SIZE(kalindi_golden_common_registers)); + amdgpu_device_program_register_sequence(adev, + kalindi_golden_spm_registers, + ARRAY_SIZE(kalindi_golden_spm_registers)); + break; + case CHIP_KAVERI: + amdgpu_device_program_register_sequence(adev, + spectre_mgcg_cgcg_init, + ARRAY_SIZE(spectre_mgcg_cgcg_init)); + amdgpu_device_program_register_sequence(adev, + spectre_golden_registers, + ARRAY_SIZE(spectre_golden_registers)); + amdgpu_device_program_register_sequence(adev, + spectre_golden_common_registers, + ARRAY_SIZE(spectre_golden_common_registers)); + amdgpu_device_program_register_sequence(adev, + spectre_golden_spm_registers, + ARRAY_SIZE(spectre_golden_spm_registers)); + break; + case CHIP_HAWAII: + amdgpu_device_program_register_sequence(adev, + hawaii_mgcg_cgcg_init, + ARRAY_SIZE(hawaii_mgcg_cgcg_init)); + amdgpu_device_program_register_sequence(adev, + hawaii_golden_registers, + ARRAY_SIZE(hawaii_golden_registers)); + amdgpu_device_program_register_sequence(adev, + hawaii_golden_common_registers, + ARRAY_SIZE(hawaii_golden_common_registers)); + amdgpu_device_program_register_sequence(adev, + hawaii_golden_spm_registers, + ARRAY_SIZE(hawaii_golden_spm_registers)); + break; + default: + break; + } + mutex_unlock(&adev->grbm_idx_mutex); +} + +/** + * cik_get_xclk - get the xclk + * + * @adev: amdgpu_device pointer + * + * Returns the reference clock used by the gfx engine + * (CIK). + */ +static u32 cik_get_xclk(struct amdgpu_device *adev) +{ + u32 reference_clock = adev->clock.spll.reference_freq; + + if (adev->flags & AMD_IS_APU) { + if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK) + return reference_clock / 2; + } else { + if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK) + return reference_clock / 4; + } + return reference_clock; +} + +/** + * cik_srbm_select - select specific register instances + * + * @adev: amdgpu_device pointer + * @me: selected ME (micro engine) + * @pipe: pipe + * @queue: queue + * @vmid: VMID + * + * Switches the currently active registers instances. Some + * registers are instanced per VMID, others are instanced per + * me/pipe/queue combination. + */ +void cik_srbm_select(struct amdgpu_device *adev, + u32 me, u32 pipe, u32 queue, u32 vmid) +{ + u32 srbm_gfx_cntl = + (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)| + ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)| + ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)| + ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK)); + WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl); +} + +static void cik_vga_set_state(struct amdgpu_device *adev, bool state) +{ + uint32_t tmp; + + tmp = RREG32(mmCONFIG_CNTL); + if (!state) + tmp |= CONFIG_CNTL__VGA_DIS_MASK; + else + tmp &= ~CONFIG_CNTL__VGA_DIS_MASK; + WREG32(mmCONFIG_CNTL, tmp); +} + +static bool cik_read_disabled_bios(struct amdgpu_device *adev) +{ + u32 bus_cntl; + u32 d1vga_control = 0; + u32 d2vga_control = 0; + u32 vga_render_control = 0; + u32 rom_cntl; + bool r; + + bus_cntl = RREG32(mmBUS_CNTL); + if (adev->mode_info.num_crtc) { + d1vga_control = RREG32(mmD1VGA_CONTROL); + d2vga_control = RREG32(mmD2VGA_CONTROL); + vga_render_control = RREG32(mmVGA_RENDER_CONTROL); + } + rom_cntl = RREG32_SMC(ixROM_CNTL); + + /* enable the rom */ + WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK)); + if (adev->mode_info.num_crtc) { + /* Disable VGA mode */ + WREG32(mmD1VGA_CONTROL, + (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK | + D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK))); + WREG32(mmD2VGA_CONTROL, + (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK | + D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK))); + WREG32(mmVGA_RENDER_CONTROL, + (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK)); + } + WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK); + + r = amdgpu_read_bios(adev); + + /* restore regs */ + WREG32(mmBUS_CNTL, bus_cntl); + if (adev->mode_info.num_crtc) { + WREG32(mmD1VGA_CONTROL, d1vga_control); + WREG32(mmD2VGA_CONTROL, d2vga_control); + WREG32(mmVGA_RENDER_CONTROL, vga_render_control); + } + WREG32_SMC(ixROM_CNTL, rom_cntl); + return r; +} + +static bool cik_read_bios_from_rom(struct amdgpu_device *adev, + u8 *bios, u32 length_bytes) +{ + u32 *dw_ptr; + unsigned long flags; + u32 i, length_dw; + + if (bios == NULL) + return false; + if (length_bytes == 0) + return false; + /* APU vbios image is part of sbios image */ + if (adev->flags & AMD_IS_APU) + return false; + + dw_ptr = (u32 *)bios; + length_dw = ALIGN(length_bytes, 4) / 4; + /* take the smc lock since we are using the smc index */ + spin_lock_irqsave(&adev->smc_idx_lock, flags); + /* set rom index to 0 */ + WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX); + WREG32(mmSMC_IND_DATA_0, 0); + /* set index to data for continous read */ + WREG32(mmSMC_IND_INDEX_0, ixROM_DATA); + for (i = 0; i < length_dw; i++) + dw_ptr[i] = RREG32(mmSMC_IND_DATA_0); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + + return true; +} + +static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = { + {mmGRBM_STATUS}, + {mmGB_ADDR_CONFIG}, + {mmMC_ARB_RAMCFG}, + {mmGB_TILE_MODE0}, + {mmGB_TILE_MODE1}, + {mmGB_TILE_MODE2}, + {mmGB_TILE_MODE3}, + {mmGB_TILE_MODE4}, + {mmGB_TILE_MODE5}, + {mmGB_TILE_MODE6}, + {mmGB_TILE_MODE7}, + {mmGB_TILE_MODE8}, + {mmGB_TILE_MODE9}, + {mmGB_TILE_MODE10}, + {mmGB_TILE_MODE11}, + {mmGB_TILE_MODE12}, + {mmGB_TILE_MODE13}, + {mmGB_TILE_MODE14}, + {mmGB_TILE_MODE15}, + {mmGB_TILE_MODE16}, + {mmGB_TILE_MODE17}, + {mmGB_TILE_MODE18}, + {mmGB_TILE_MODE19}, + {mmGB_TILE_MODE20}, + {mmGB_TILE_MODE21}, + {mmGB_TILE_MODE22}, + {mmGB_TILE_MODE23}, + {mmGB_TILE_MODE24}, + {mmGB_TILE_MODE25}, + {mmGB_TILE_MODE26}, + {mmGB_TILE_MODE27}, + {mmGB_TILE_MODE28}, + {mmGB_TILE_MODE29}, + {mmGB_TILE_MODE30}, + {mmGB_TILE_MODE31}, + {mmGB_MACROTILE_MODE0}, + {mmGB_MACROTILE_MODE1}, + {mmGB_MACROTILE_MODE2}, + {mmGB_MACROTILE_MODE3}, + {mmGB_MACROTILE_MODE4}, + {mmGB_MACROTILE_MODE5}, + {mmGB_MACROTILE_MODE6}, + {mmGB_MACROTILE_MODE7}, + {mmGB_MACROTILE_MODE8}, + {mmGB_MACROTILE_MODE9}, + {mmGB_MACROTILE_MODE10}, + {mmGB_MACROTILE_MODE11}, + {mmGB_MACROTILE_MODE12}, + {mmGB_MACROTILE_MODE13}, + {mmGB_MACROTILE_MODE14}, + {mmGB_MACROTILE_MODE15}, + {mmCC_RB_BACKEND_DISABLE, true}, + {mmGC_USER_RB_BACKEND_DISABLE, true}, + {mmGB_BACKEND_MAP, false}, + {mmPA_SC_RASTER_CONFIG, true}, + {mmPA_SC_RASTER_CONFIG_1, true}, +}; + + +static uint32_t cik_get_register_value(struct amdgpu_device *adev, + bool indexed, u32 se_num, + u32 sh_num, u32 reg_offset) +{ + if (indexed) { + uint32_t val; + unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num; + unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num; + + switch (reg_offset) { + case mmCC_RB_BACKEND_DISABLE: + return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable; + case mmGC_USER_RB_BACKEND_DISABLE: + return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable; + case mmPA_SC_RASTER_CONFIG: + return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config; + case mmPA_SC_RASTER_CONFIG_1: + return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1; + } + + mutex_lock(&adev->grbm_idx_mutex); + if (se_num != 0xffffffff || sh_num != 0xffffffff) + amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff); + + val = RREG32(reg_offset); + + if (se_num != 0xffffffff || sh_num != 0xffffffff) + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + mutex_unlock(&adev->grbm_idx_mutex); + return val; + } else { + unsigned idx; + + switch (reg_offset) { + case mmGB_ADDR_CONFIG: + return adev->gfx.config.gb_addr_config; + case mmMC_ARB_RAMCFG: + return adev->gfx.config.mc_arb_ramcfg; + case mmGB_TILE_MODE0: + case mmGB_TILE_MODE1: + case mmGB_TILE_MODE2: + case mmGB_TILE_MODE3: + case mmGB_TILE_MODE4: + case mmGB_TILE_MODE5: + case mmGB_TILE_MODE6: + case mmGB_TILE_MODE7: + case mmGB_TILE_MODE8: + case mmGB_TILE_MODE9: + case mmGB_TILE_MODE10: + case mmGB_TILE_MODE11: + case mmGB_TILE_MODE12: + case mmGB_TILE_MODE13: + case mmGB_TILE_MODE14: + case mmGB_TILE_MODE15: + case mmGB_TILE_MODE16: + case mmGB_TILE_MODE17: + case mmGB_TILE_MODE18: + case mmGB_TILE_MODE19: + case mmGB_TILE_MODE20: + case mmGB_TILE_MODE21: + case mmGB_TILE_MODE22: + case mmGB_TILE_MODE23: + case mmGB_TILE_MODE24: + case mmGB_TILE_MODE25: + case mmGB_TILE_MODE26: + case mmGB_TILE_MODE27: + case mmGB_TILE_MODE28: + case mmGB_TILE_MODE29: + case mmGB_TILE_MODE30: + case mmGB_TILE_MODE31: + idx = (reg_offset - mmGB_TILE_MODE0); + return adev->gfx.config.tile_mode_array[idx]; + case mmGB_MACROTILE_MODE0: + case mmGB_MACROTILE_MODE1: + case mmGB_MACROTILE_MODE2: + case mmGB_MACROTILE_MODE3: + case mmGB_MACROTILE_MODE4: + case mmGB_MACROTILE_MODE5: + case mmGB_MACROTILE_MODE6: + case mmGB_MACROTILE_MODE7: + case mmGB_MACROTILE_MODE8: + case mmGB_MACROTILE_MODE9: + case mmGB_MACROTILE_MODE10: + case mmGB_MACROTILE_MODE11: + case mmGB_MACROTILE_MODE12: + case mmGB_MACROTILE_MODE13: + case mmGB_MACROTILE_MODE14: + case mmGB_MACROTILE_MODE15: + idx = (reg_offset - mmGB_MACROTILE_MODE0); + return adev->gfx.config.macrotile_mode_array[idx]; + default: + return RREG32(reg_offset); + } + } +} + +static int cik_read_register(struct amdgpu_device *adev, u32 se_num, + u32 sh_num, u32 reg_offset, u32 *value) +{ + uint32_t i; + + *value = 0; + for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) { + bool indexed = cik_allowed_read_registers[i].grbm_indexed; + + if (reg_offset != cik_allowed_read_registers[i].reg_offset) + continue; + + *value = cik_get_register_value(adev, indexed, se_num, sh_num, + reg_offset); + return 0; + } + return -EINVAL; +} + +struct kv_reset_save_regs { + u32 gmcon_reng_execute; + u32 gmcon_misc; + u32 gmcon_misc3; +}; + +static void kv_save_regs_for_reset(struct amdgpu_device *adev, + struct kv_reset_save_regs *save) +{ + save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE); + save->gmcon_misc = RREG32(mmGMCON_MISC); + save->gmcon_misc3 = RREG32(mmGMCON_MISC3); + + WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute & + ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK); + WREG32(mmGMCON_MISC, save->gmcon_misc & + ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK | + GMCON_MISC__STCTRL_STUTTER_EN_MASK)); +} + +static void kv_restore_regs_for_reset(struct amdgpu_device *adev, + struct kv_reset_save_regs *save) +{ + int i; + + WREG32(mmGMCON_PGFSM_WRITE, 0); + WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0); + WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x210000); + WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x21003); + WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x2b00); + WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0); + WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x420000); + WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x120202); + WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36); + WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e); + WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff); + + for (i = 0; i < 5; i++) + WREG32(mmGMCON_PGFSM_WRITE, 0); + + WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332); + WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff); + + WREG32(mmGMCON_MISC3, save->gmcon_misc3); + WREG32(mmGMCON_MISC, save->gmcon_misc); + WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute); +} + +static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) +{ + struct kv_reset_save_regs kv_save = { 0 }; + u32 i; + int r = -EINVAL; + + dev_info(adev->dev, "GPU pci config reset\n"); + + if (adev->flags & AMD_IS_APU) + kv_save_regs_for_reset(adev, &kv_save); + + /* disable BM */ + pci_clear_master(adev->pdev); + /* reset */ + amdgpu_device_pci_config_reset(adev); + + udelay(100); + + /* wait for asic to come out of reset */ + for (i = 0; i < adev->usec_timeout; i++) { + if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) { + /* enable BM */ + pci_set_master(adev->pdev); + adev->has_hw_reset = true; + r = 0; + break; + } + udelay(1); + } + + /* does asic init need to be run first??? */ + if (adev->flags & AMD_IS_APU) + kv_restore_regs_for_reset(adev, &kv_save); + + return r; +} + +/** + * cik_asic_reset - soft reset GPU + * + * @adev: amdgpu_device pointer + * + * Look up which blocks are hung and attempt + * to reset them. + * Returns 0 for success. + */ +static int cik_asic_reset(struct amdgpu_device *adev) +{ + int r; + + amdgpu_atombios_scratch_regs_engine_hung(adev, true); + + r = cik_gpu_pci_config_reset(adev); + + amdgpu_atombios_scratch_regs_engine_hung(adev, false); + + return r; +} + +static u32 cik_get_config_memsize(struct amdgpu_device *adev) +{ + return RREG32(mmCONFIG_MEMSIZE); +} + +static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock, + u32 cntl_reg, u32 status_reg) +{ + int r, i; + struct atom_clock_dividers dividers; + uint32_t tmp; + + r = amdgpu_atombios_get_clock_dividers(adev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + clock, false, ÷rs); + if (r) + return r; + + tmp = RREG32_SMC(cntl_reg); + tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK | + CG_DCLK_CNTL__DCLK_DIVIDER_MASK); + tmp |= dividers.post_divider; + WREG32_SMC(cntl_reg, tmp); + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + return 0; +} + +static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk) +{ + int r = 0; + + r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS); + if (r) + return r; + + r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS); + return r; +} + +static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk) +{ + int r, i; + struct atom_clock_dividers dividers; + u32 tmp; + + r = amdgpu_atombios_get_clock_dividers(adev, + COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, + ecclk, false, ÷rs); + if (r) + return r; + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + tmp = RREG32_SMC(ixCG_ECLK_CNTL); + tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK | + CG_ECLK_CNTL__ECLK_DIVIDER_MASK); + tmp |= dividers.post_divider; + WREG32_SMC(ixCG_ECLK_CNTL, tmp); + + for (i = 0; i < 100; i++) { + if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK) + break; + mdelay(10); + } + if (i == 100) + return -ETIMEDOUT; + + return 0; +} + +static void cik_pcie_gen3_enable(struct amdgpu_device *adev) +{ + struct pci_dev *root = adev->pdev->bus->self; + int bridge_pos, gpu_pos; + u32 speed_cntl, current_data_rate; + int i; + u16 tmp16; + + if (pci_is_root_bus(adev->pdev->bus)) + return; + + if (amdgpu_pcie_gen2 == 0) + return; + + if (adev->flags & AMD_IS_APU) + return; + + if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3))) + return; + + speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL); + current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >> + PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT; + if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) { + if (current_data_rate == 2) { + DRM_INFO("PCIE gen 3 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n"); + } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) { + if (current_data_rate == 1) { + DRM_INFO("PCIE gen 2 link speeds already enabled\n"); + return; + } + DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n"); + } + + bridge_pos = pci_pcie_cap(root); + if (!bridge_pos) + return; + + gpu_pos = pci_pcie_cap(adev->pdev); + if (!gpu_pos) + return; + + if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) { + /* re-try equalization if gen3 is not already enabled */ + if (current_data_rate != 2) { + u16 bridge_cfg, gpu_cfg; + u16 bridge_cfg2, gpu_cfg2; + u32 max_lw, current_lw, tmp; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; + pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + tmp = RREG32_PCIE(ixPCIE_LC_STATUS1); + max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >> + PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT; + current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK) + >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT; + + if (current_lw < max_lw) { + tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL); + if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) { + tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK | + PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK); + tmp |= (max_lw << + PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT); + tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK | + PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK | + PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK; + WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp); + } + } + + for (i = 0; i < 10; i++) { + /* check status */ + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); + if (tmp16 & PCI_EXP_DEVSTA_TRPND) + break; + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); + + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); + + tmp = RREG32_PCIE(ixPCIE_LC_CNTL4); + tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK; + WREG32_PCIE(ixPCIE_LC_CNTL4, tmp); + + tmp = RREG32_PCIE(ixPCIE_LC_CNTL4); + tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK; + WREG32_PCIE(ixPCIE_LC_CNTL4, tmp); + + msleep(100); + + /* linkctl */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); + + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); + tmp16 &= ~PCI_EXP_LNKCTL_HAWD; + tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); + pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); + + /* linkctl2 */ + pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); + + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~((1 << 4) | (7 << 9)); + tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); + pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + tmp = RREG32_PCIE(ixPCIE_LC_CNTL4); + tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK; + WREG32_PCIE(ixPCIE_LC_CNTL4, tmp); + } + } + } + + /* set the link speed */ + speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK | + PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK; + speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK; + WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl); + + pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); + tmp16 &= ~0xf; + if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) + tmp16 |= 3; /* gen3 */ + else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) + tmp16 |= 2; /* gen2 */ + else + tmp16 |= 1; /* gen1 */ + pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); + + speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL); + speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK; + WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl); + + for (i = 0; i < adev->usec_timeout; i++) { + speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL); + if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0) + break; + udelay(1); + } +} + +static void cik_program_aspm(struct amdgpu_device *adev) +{ + u32 data, orig; + bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; + bool disable_clkreq = false; + + if (amdgpu_aspm == 0) + return; + + if (pci_is_root_bus(adev->pdev->bus)) + return; + + /* XXX double check APUs */ + if (adev->flags & AMD_IS_APU) + return; + + orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL); + data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK; + data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) | + PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data); + + orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3); + data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_LC_CNTL3, data); + + orig = data = RREG32_PCIE(ixPCIE_P_CNTL); + data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_P_CNTL, data); + + orig = data = RREG32_PCIE(ixPCIE_LC_CNTL); + data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK | + PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK); + data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK; + if (!disable_l0s) + data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT); + + if (!disable_l1) { + data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT); + data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_LC_CNTL, data); + + if (!disable_plloff_in_l1) { + bool clk_req_support; + + orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0); + data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | + PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | + (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT); + if (orig != data) + WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1); + data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | + PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | + (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT); + if (orig != data) + WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0); + data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | + PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK); + data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | + (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT); + if (orig != data) + WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data); + + orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1); + data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | + PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK); + data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | + (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT); + if (orig != data) + WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data); + + orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL); + data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK; + data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT); + if (orig != data) + WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data); + + if (!disable_clkreq) { + struct pci_dev *root = adev->pdev->bus->self; + u32 lnkcap; + + clk_req_support = false; + pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); + if (lnkcap & PCI_EXP_LNKCAP_CLKPM) + clk_req_support = true; + } else { + clk_req_support = false; + } + + if (clk_req_support) { + orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2); + data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | + PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_LC_CNTL2, data); + + orig = data = RREG32_SMC(ixTHM_CLK_CNTL); + data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK | + THM_CLK_CNTL__TMON_CLK_SEL_MASK); + data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) | + (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT); + if (orig != data) + WREG32_SMC(ixTHM_CLK_CNTL, data); + + orig = data = RREG32_SMC(ixMISC_CLK_CTRL); + data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK | + MISC_CLK_CTRL__ZCLK_SEL_MASK); + data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) | + (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT); + if (orig != data) + WREG32_SMC(ixMISC_CLK_CTRL, data); + + orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL); + data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK; + if (orig != data) + WREG32_SMC(ixCG_CLKPIN_CNTL, data); + + orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2); + data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK; + if (orig != data) + WREG32_SMC(ixCG_CLKPIN_CNTL_2, data); + + orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL); + data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK; + data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT); + if (orig != data) + WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data); + } + } + } else { + if (orig != data) + WREG32_PCIE(ixPCIE_LC_CNTL, data); + } + + orig = data = RREG32_PCIE(ixPCIE_CNTL2); + data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK | + PCIE_CNTL2__MST_MEM_LS_EN_MASK | + PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_CNTL2, data); + + if (!disable_l0s) { + data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL); + if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) == + PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) { + data = RREG32_PCIE(ixPCIE_LC_STATUS1); + if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) && + (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) { + orig = data = RREG32_PCIE(ixPCIE_LC_CNTL); + data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK; + if (orig != data) + WREG32_PCIE(ixPCIE_LC_CNTL, data); + } + } + } +} + +static uint32_t cik_get_rev_id(struct amdgpu_device *adev) +{ + return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK) + >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT; +} + +static void cik_detect_hw_virtualization(struct amdgpu_device *adev) +{ + if (is_virtual_machine()) /* passthrough mode */ + adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE; +} + +static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) { + WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1); + RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL); + } else { + amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1); + } +} + +static void cik_invalidate_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) { + WREG32(mmHDP_DEBUG0, 1); + RREG32(mmHDP_DEBUG0); + } else { + amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1); + } +} + +static bool cik_need_full_reset(struct amdgpu_device *adev) +{ + /* change this when we support soft reset */ + return true; +} + +static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, + uint64_t *count1) +{ + uint32_t perfctr = 0; + uint64_t cnt0_of, cnt1_of; + int tmp; + + /* This reports 0 on APUs, so return to avoid writing/reading registers + * that may or may not be different from their GPU counterparts + */ + if (adev->flags & AMD_IS_APU) + return; + + /* Set the 2 events that we wish to watch, defined above */ + /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */ + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); + perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); + + /* Write to enable desired perf counters */ + WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr); + /* Zero out and enable the perf counters + * Write 0x5: + * Bit 0 = Start all counters(1) + * Bit 2 = Global counter reset enable(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005); + + msleep(1000); + + /* Load the shadow and disable the perf counters + * Write 0x2: + * Bit 0 = Stop counters(0) + * Bit 1 = Load the shadow counters(1) + */ + WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002); + + /* Read register values to get any >32bit overflow */ + tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK); + cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); + cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); + + /* Get the values and add the overflow */ + *count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); + *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); +} + +static bool cik_need_reset_on_init(struct amdgpu_device *adev) +{ + u32 clock_cntl, pc; + + if (adev->flags & AMD_IS_APU) + return false; + + /* check if the SMC is already running */ + clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + pc = RREG32_SMC(ixSMC_PC_C); + if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) && + (0x20100 <= pc)) + return true; + + return false; +} + +static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev) +{ + uint64_t nak_r, nak_g; + + /* Get the number of NAKs received and generated */ + nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK); + nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED); + + /* Add the total number of NAKs, i.e the number of replays */ + return (nak_r + nak_g); +} + +static const struct amdgpu_asic_funcs cik_asic_funcs = +{ + .read_disabled_bios = &cik_read_disabled_bios, + .read_bios_from_rom = &cik_read_bios_from_rom, + .read_register = &cik_read_register, + .reset = &cik_asic_reset, + .set_vga_state = &cik_vga_set_state, + .get_xclk = &cik_get_xclk, + .set_uvd_clocks = &cik_set_uvd_clocks, + .set_vce_clocks = &cik_set_vce_clocks, + .get_config_memsize = &cik_get_config_memsize, + .flush_hdp = &cik_flush_hdp, + .invalidate_hdp = &cik_invalidate_hdp, + .need_full_reset = &cik_need_full_reset, + .init_doorbell_index = &legacy_doorbell_index_init, + .get_pcie_usage = &cik_get_pcie_usage, + .need_reset_on_init = &cik_need_reset_on_init, + .get_pcie_replay_count = &cik_get_pcie_replay_count, +}; + +static int cik_common_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + adev->smc_rreg = &cik_smc_rreg; + adev->smc_wreg = &cik_smc_wreg; + adev->pcie_rreg = &cik_pcie_rreg; + adev->pcie_wreg = &cik_pcie_wreg; + adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg; + adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg; + adev->didt_rreg = &cik_didt_rreg; + adev->didt_wreg = &cik_didt_wreg; + + adev->asic_funcs = &cik_asic_funcs; + + adev->rev_id = cik_get_rev_id(adev); + adev->external_rev_id = 0xFF; + switch (adev->asic_type) { + case CHIP_BONAIRE: + adev->cg_flags = + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_MGLS | + /*AMD_CG_SUPPORT_GFX_CGCG |*/ + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_CGTS | + AMD_CG_SUPPORT_GFX_CGTS_LS | + AMD_CG_SUPPORT_GFX_CP_LS | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_SDMA_MGCG | + AMD_CG_SUPPORT_SDMA_LS | + AMD_CG_SUPPORT_BIF_LS | + AMD_CG_SUPPORT_VCE_MGCG | + AMD_CG_SUPPORT_UVD_MGCG | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_MGCG; + adev->pg_flags = 0; + adev->external_rev_id = adev->rev_id + 0x14; + break; + case CHIP_HAWAII: + adev->cg_flags = + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_MGLS | + /*AMD_CG_SUPPORT_GFX_CGCG |*/ + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_CGTS | + AMD_CG_SUPPORT_GFX_CP_LS | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_SDMA_MGCG | + AMD_CG_SUPPORT_SDMA_LS | + AMD_CG_SUPPORT_BIF_LS | + AMD_CG_SUPPORT_VCE_MGCG | + AMD_CG_SUPPORT_UVD_MGCG | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_MGCG; + adev->pg_flags = 0; + adev->external_rev_id = 0x28; + break; + case CHIP_KAVERI: + adev->cg_flags = + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_MGLS | + /*AMD_CG_SUPPORT_GFX_CGCG |*/ + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_CGTS | + AMD_CG_SUPPORT_GFX_CGTS_LS | + AMD_CG_SUPPORT_GFX_CP_LS | + AMD_CG_SUPPORT_SDMA_MGCG | + AMD_CG_SUPPORT_SDMA_LS | + AMD_CG_SUPPORT_BIF_LS | + AMD_CG_SUPPORT_VCE_MGCG | + AMD_CG_SUPPORT_UVD_MGCG | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_MGCG; + adev->pg_flags = + /*AMD_PG_SUPPORT_GFX_PG | + AMD_PG_SUPPORT_GFX_SMG | + AMD_PG_SUPPORT_GFX_DMG |*/ + AMD_PG_SUPPORT_UVD | + AMD_PG_SUPPORT_VCE | + /* AMD_PG_SUPPORT_CP | + AMD_PG_SUPPORT_GDS | + AMD_PG_SUPPORT_RLC_SMU_HS | + AMD_PG_SUPPORT_ACP | + AMD_PG_SUPPORT_SAMU |*/ + 0; + if (adev->pdev->device == 0x1312 || + adev->pdev->device == 0x1316 || + adev->pdev->device == 0x1317) + adev->external_rev_id = 0x41; + else + adev->external_rev_id = 0x1; + break; + case CHIP_KABINI: + case CHIP_MULLINS: + adev->cg_flags = + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_MGLS | + /*AMD_CG_SUPPORT_GFX_CGCG |*/ + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_CGTS | + AMD_CG_SUPPORT_GFX_CGTS_LS | + AMD_CG_SUPPORT_GFX_CP_LS | + AMD_CG_SUPPORT_SDMA_MGCG | + AMD_CG_SUPPORT_SDMA_LS | + AMD_CG_SUPPORT_BIF_LS | + AMD_CG_SUPPORT_VCE_MGCG | + AMD_CG_SUPPORT_UVD_MGCG | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_MGCG; + adev->pg_flags = + /*AMD_PG_SUPPORT_GFX_PG | + AMD_PG_SUPPORT_GFX_SMG | */ + AMD_PG_SUPPORT_UVD | + /*AMD_PG_SUPPORT_VCE | + AMD_PG_SUPPORT_CP | + AMD_PG_SUPPORT_GDS | + AMD_PG_SUPPORT_RLC_SMU_HS | + AMD_PG_SUPPORT_SAMU |*/ + 0; + if (adev->asic_type == CHIP_KABINI) { + if (adev->rev_id == 0) + adev->external_rev_id = 0x81; + else if (adev->rev_id == 1) + adev->external_rev_id = 0x82; + else if (adev->rev_id == 2) + adev->external_rev_id = 0x85; + } else + adev->external_rev_id = adev->rev_id + 0xa1; + break; + default: + /* FIXME: not supported yet */ + return -EINVAL; + } + + return 0; +} + +static int cik_common_sw_init(void *handle) +{ + return 0; +} + +static int cik_common_sw_fini(void *handle) +{ + return 0; +} + +static int cik_common_hw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + /* move the golden regs per IP block */ + cik_init_golden_registers(adev); + /* enable pcie gen2/3 link */ + cik_pcie_gen3_enable(adev); + /* enable aspm */ + cik_program_aspm(adev); + + return 0; +} + +static int cik_common_hw_fini(void *handle) +{ + return 0; +} + +static int cik_common_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return cik_common_hw_fini(adev); +} + +static int cik_common_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return cik_common_hw_init(adev); +} + +static bool cik_common_is_idle(void *handle) +{ + return true; +} + +static int cik_common_wait_for_idle(void *handle) +{ + return 0; +} + +static int cik_common_soft_reset(void *handle) +{ + /* XXX hard reset?? */ + return 0; +} + +static int cik_common_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int cik_common_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +static const struct amd_ip_funcs cik_common_ip_funcs = { + .name = "cik_common", + .early_init = cik_common_early_init, + .late_init = NULL, + .sw_init = cik_common_sw_init, + .sw_fini = cik_common_sw_fini, + .hw_init = cik_common_hw_init, + .hw_fini = cik_common_hw_fini, + .suspend = cik_common_suspend, + .resume = cik_common_resume, + .is_idle = cik_common_is_idle, + .wait_for_idle = cik_common_wait_for_idle, + .soft_reset = cik_common_soft_reset, + .set_clockgating_state = cik_common_set_clockgating_state, + .set_powergating_state = cik_common_set_powergating_state, +}; + +static const struct amdgpu_ip_block_version cik_common_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_COMMON, + .major = 1, + .minor = 0, + .rev = 0, + .funcs = &cik_common_ip_funcs, +}; + +int cik_set_ip_blocks(struct amdgpu_device *adev) +{ + cik_detect_hw_virtualization(adev); + + switch (adev->asic_type) { + case CHIP_BONAIRE: + amdgpu_device_ip_block_add(adev, &cik_common_ip_block); + amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block); + amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); + amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block); + amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); + amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); + if (adev->enable_virtual_display) + amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif + else + amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block); + amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block); + amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block); + break; + case CHIP_HAWAII: + amdgpu_device_ip_block_add(adev, &cik_common_ip_block); + amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block); + amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); + amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block); + amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); + amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); + if (adev->enable_virtual_display) + amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif + else + amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block); + amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block); + amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block); + break; + case CHIP_KAVERI: + amdgpu_device_ip_block_add(adev, &cik_common_ip_block); + amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block); + amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); + amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block); + amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); + amdgpu_device_ip_block_add(adev, &kv_smu_ip_block); + if (adev->enable_virtual_display) + amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif + else + amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block); + + amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block); + amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block); + break; + case CHIP_KABINI: + case CHIP_MULLINS: + amdgpu_device_ip_block_add(adev, &cik_common_ip_block); + amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block); + amdgpu_device_ip_block_add(adev, &cik_ih_ip_block); + amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block); + amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block); + amdgpu_device_ip_block_add(adev, &kv_smu_ip_block); + if (adev->enable_virtual_display) + amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); +#if defined(CONFIG_DRM_AMD_DC) + else if (amdgpu_device_has_dc_support(adev)) + amdgpu_device_ip_block_add(adev, &dm_ip_block); +#endif + else + amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block); + amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block); + amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block); + break; + default: + /* FIXME: not supported yet */ + return -EINVAL; + } + return 0; +}
diff --git a/amdgpu/cik.h b/amdgpu/cik.h new file mode 100644 index 0000000..54c625a --- /dev/null +++ b/amdgpu/cik.h
@@ -0,0 +1,34 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __CIK_H__ +#define __CIK_H__ + +#define CIK_FLUSH_GPU_TLB_NUM_WREG 3 + +void cik_srbm_select(struct amdgpu_device *adev, + u32 me, u32 pipe, u32 queue, u32 vmid); +int cik_set_ip_blocks(struct amdgpu_device *adev); + +void legacy_doorbell_index_init(struct amdgpu_device *adev); +#endif
diff --git a/amdgpu/cik_dpm.h b/amdgpu/cik_dpm.h new file mode 100644 index 0000000..2fcc4b6 --- /dev/null +++ b/amdgpu/cik_dpm.h
@@ -0,0 +1,29 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __CIK_DPM_H__ +#define __CIK_DPM_H__ + +extern const struct amdgpu_ip_block_version kv_smu_ip_block; + +#endif
diff --git a/amdgpu/cik_ih.c b/amdgpu/cik_ih.c new file mode 100644 index 0000000..401c99f --- /dev/null +++ b/amdgpu/cik_ih.c
@@ -0,0 +1,456 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/pci.h> + +#include "amdgpu.h" +#include "amdgpu_ih.h" +#include "cikd.h" + +#include "bif/bif_4_1_d.h" +#include "bif/bif_4_1_sh_mask.h" + +#include "oss/oss_2_0_d.h" +#include "oss/oss_2_0_sh_mask.h" + +/* + * Interrupts + * Starting with r6xx, interrupts are handled via a ring buffer. + * Ring buffers are areas of GPU accessible memory that the GPU + * writes interrupt vectors into and the host reads vectors out of. + * There is a rptr (read pointer) that determines where the + * host is currently reading, and a wptr (write pointer) + * which determines where the GPU has written. When the + * pointers are equal, the ring is idle. When the GPU + * writes vectors to the ring buffer, it increments the + * wptr. When there is an interrupt, the host then starts + * fetching commands and processing them until the pointers are + * equal again at which point it updates the rptr. + */ + +static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev); + +/** + * cik_ih_enable_interrupts - Enable the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * + * Enable the interrupt ring buffer (CIK). + */ +static void cik_ih_enable_interrupts(struct amdgpu_device *adev) +{ + u32 ih_cntl = RREG32(mmIH_CNTL); + u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); + + ih_cntl |= IH_CNTL__ENABLE_INTR_MASK; + ih_rb_cntl |= IH_RB_CNTL__RB_ENABLE_MASK; + WREG32(mmIH_CNTL, ih_cntl); + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + adev->irq.ih.enabled = true; +} + +/** + * cik_ih_disable_interrupts - Disable the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * + * Disable the interrupt ring buffer (CIK). + */ +static void cik_ih_disable_interrupts(struct amdgpu_device *adev) +{ + u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); + u32 ih_cntl = RREG32(mmIH_CNTL); + + ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK; + ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK; + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + WREG32(mmIH_CNTL, ih_cntl); + /* set rptr, wptr to 0 */ + WREG32(mmIH_RB_RPTR, 0); + WREG32(mmIH_RB_WPTR, 0); + adev->irq.ih.enabled = false; + adev->irq.ih.rptr = 0; +} + +/** + * cik_ih_irq_init - init and enable the interrupt ring + * + * @adev: amdgpu_device pointer + * + * Allocate a ring buffer for the interrupt controller, + * enable the RLC, disable interrupts, enable the IH + * ring buffer and enable it (CIK). + * Called at device load and reume. + * Returns 0 for success, errors for failure. + */ +static int cik_ih_irq_init(struct amdgpu_device *adev) +{ + struct amdgpu_ih_ring *ih = &adev->irq.ih; + int rb_bufsz; + u32 interrupt_cntl, ih_cntl, ih_rb_cntl; + + /* disable irqs */ + cik_ih_disable_interrupts(adev); + + /* setup interrupt control */ + WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8); + interrupt_cntl = RREG32(mmINTERRUPT_CNTL); + /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi + * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN + */ + interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK; + /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ + interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK; + WREG32(mmINTERRUPT_CNTL, interrupt_cntl); + + WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); + rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); + + ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK | + IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK | + (rb_bufsz << 1)); + + ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK; + + /* set the writeback address whether it's enabled or not */ + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); + + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + + /* set rptr, wptr to 0 */ + WREG32(mmIH_RB_RPTR, 0); + WREG32(mmIH_RB_WPTR, 0); + + /* Default settings for IH_CNTL (disabled at first) */ + ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) | + (0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) | + (0 << IH_CNTL__MC_VMID__SHIFT); + /* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */ + if (adev->irq.msi_enabled) + ih_cntl |= IH_CNTL__RPTR_REARM_MASK; + WREG32(mmIH_CNTL, ih_cntl); + + pci_set_master(adev->pdev); + + /* enable irqs */ + cik_ih_enable_interrupts(adev); + + return 0; +} + +/** + * cik_ih_irq_disable - disable interrupts + * + * @adev: amdgpu_device pointer + * + * Disable interrupts on the hw (CIK). + */ +static void cik_ih_irq_disable(struct amdgpu_device *adev) +{ + cik_ih_disable_interrupts(adev); + /* Wait and acknowledge irq */ + mdelay(1); +} + +/** + * cik_ih_get_wptr - get the IH ring buffer wptr + * + * @adev: amdgpu_device pointer + * + * Get the IH ring buffer wptr from either the register + * or the writeback memory buffer (CIK). Also check for + * ring buffer overflow and deal with it. + * Used by cik_irq_process(). + * Returns the value of the wptr. + */ +static u32 cik_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + u32 wptr, tmp; + + wptr = le32_to_cpu(*ih->wptr_cpu); + + if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) { + wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK; + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK; + WREG32(mmIH_RB_CNTL, tmp); + } + return (wptr & ih->ptr_mask); +} + +/* CIK IV Ring + * Each IV ring entry is 128 bits: + * [7:0] - interrupt source id + * [31:8] - reserved + * [59:32] - interrupt source data + * [63:60] - reserved + * [71:64] - RINGID + * CP: + * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] + * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher + * - for gfx, hw shader state (0=PS...5=LS, 6=CS) + * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes + * PIPE_ID - ME0 0=3D + * - ME1&2 compute dispatcher (4 pipes each) + * SDMA: + * INSTANCE_ID [1:0], QUEUE_ID[1:0] + * INSTANCE_ID - 0 = sdma0, 1 = sdma1 + * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 + * [79:72] - VMID + * [95:80] - PASID + * [127:96] - reserved + */ + + /** + * cik_ih_decode_iv - decode an interrupt vector + * + * @adev: amdgpu_device pointer + * + * Decodes the interrupt vector at the current rptr + * position and also advance the position. + */ +static void cik_ih_decode_iv(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) +{ + /* wptr/rptr are in bytes! */ + u32 ring_index = ih->rptr >> 2; + uint32_t dw[4]; + + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); + + entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; + entry->src_id = dw[0] & 0xff; + entry->src_data[0] = dw[1] & 0xfffffff; + entry->ring_id = dw[2] & 0xff; + entry->vmid = (dw[2] >> 8) & 0xff; + entry->pasid = (dw[2] >> 16) & 0xffff; + + /* wptr/rptr are in bytes! */ + ih->rptr += 16; +} + +/** + * cik_ih_set_rptr - set the IH ring buffer rptr + * + * @adev: amdgpu_device pointer + * + * Set the IH ring buffer rptr. + */ +static void cik_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + WREG32(mmIH_RB_RPTR, ih->rptr); +} + +static int cik_ih_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int ret; + + ret = amdgpu_irq_add_domain(adev); + if (ret) + return ret; + + cik_ih_set_interrupt_funcs(adev); + + return 0; +} + +static int cik_ih_sw_init(void *handle) +{ + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false); + if (r) + return r; + + r = amdgpu_irq_init(adev); + + return r; +} + +static int cik_ih_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih); + amdgpu_irq_remove_domain(adev); + + return 0; +} + +static int cik_ih_hw_init(void *handle) +{ + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = cik_ih_irq_init(adev); + if (r) + return r; + + return 0; +} + +static int cik_ih_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + cik_ih_irq_disable(adev); + + return 0; +} + +static int cik_ih_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return cik_ih_hw_fini(adev); +} + +static int cik_ih_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return cik_ih_hw_init(adev); +} + +static bool cik_ih_is_idle(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 tmp = RREG32(mmSRBM_STATUS); + + if (tmp & SRBM_STATUS__IH_BUSY_MASK) + return false; + + return true; +} + +static int cik_ih_wait_for_idle(void *handle) +{ + unsigned i; + u32 tmp; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + for (i = 0; i < adev->usec_timeout; i++) { + /* read MC_STATUS */ + tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK; + if (!tmp) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +static int cik_ih_soft_reset(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + u32 srbm_soft_reset = 0; + u32 tmp = RREG32(mmSRBM_STATUS); + + if (tmp & SRBM_STATUS__IH_BUSY_MASK) + srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK; + + if (srbm_soft_reset) { + tmp = RREG32(mmSRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(mmSRBM_SOFT_RESET, tmp); + tmp = RREG32(mmSRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(mmSRBM_SOFT_RESET, tmp); + tmp = RREG32(mmSRBM_SOFT_RESET); + + /* Wait a little for things to settle down */ + udelay(50); + } + + return 0; +} + +static int cik_ih_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int cik_ih_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +static const struct amd_ip_funcs cik_ih_ip_funcs = { + .name = "cik_ih", + .early_init = cik_ih_early_init, + .late_init = NULL, + .sw_init = cik_ih_sw_init, + .sw_fini = cik_ih_sw_fini, + .hw_init = cik_ih_hw_init, + .hw_fini = cik_ih_hw_fini, + .suspend = cik_ih_suspend, + .resume = cik_ih_resume, + .is_idle = cik_ih_is_idle, + .wait_for_idle = cik_ih_wait_for_idle, + .soft_reset = cik_ih_soft_reset, + .set_clockgating_state = cik_ih_set_clockgating_state, + .set_powergating_state = cik_ih_set_powergating_state, +}; + +static const struct amdgpu_ih_funcs cik_ih_funcs = { + .get_wptr = cik_ih_get_wptr, + .decode_iv = cik_ih_decode_iv, + .set_rptr = cik_ih_set_rptr +}; + +static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev) +{ + adev->irq.ih_funcs = &cik_ih_funcs; +} + +const struct amdgpu_ip_block_version cik_ih_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_IH, + .major = 2, + .minor = 0, + .rev = 0, + .funcs = &cik_ih_ip_funcs, +};
diff --git a/amdgpu/cik_ih.h b/amdgpu/cik_ih.h new file mode 100644 index 0000000..1d9ddee --- /dev/null +++ b/amdgpu/cik_ih.h
@@ -0,0 +1,29 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __CIK_IH_H__ +#define __CIK_IH_H__ + +extern const struct amdgpu_ip_block_version cik_ih_ip_block; + +#endif
diff --git a/amdgpu/cik_sdma.c b/amdgpu/cik_sdma.c new file mode 100644 index 0000000..c45304f --- /dev/null +++ b/amdgpu/cik_sdma.c
@@ -0,0 +1,1394 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ + +#include <linux/firmware.h> +#include <linux/module.h> + +#include "amdgpu.h" +#include "amdgpu_ucode.h" +#include "amdgpu_trace.h" +#include "cikd.h" +#include "cik.h" + +#include "bif/bif_4_1_d.h" +#include "bif/bif_4_1_sh_mask.h" + +#include "gca/gfx_7_2_d.h" +#include "gca/gfx_7_2_enum.h" +#include "gca/gfx_7_2_sh_mask.h" + +#include "gmc/gmc_7_1_d.h" +#include "gmc/gmc_7_1_sh_mask.h" + +#include "oss/oss_2_0_d.h" +#include "oss/oss_2_0_sh_mask.h" + +static const u32 sdma_offsets[SDMA_MAX_INSTANCE] = +{ + SDMA0_REGISTER_OFFSET, + SDMA1_REGISTER_OFFSET +}; + +static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev); +static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev); +static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev); +static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev); +static int cik_sdma_soft_reset(void *handle); + +MODULE_FIRMWARE("amdgpu/bonaire_sdma.bin"); +MODULE_FIRMWARE("amdgpu/bonaire_sdma1.bin"); +MODULE_FIRMWARE("amdgpu/hawaii_sdma.bin"); +MODULE_FIRMWARE("amdgpu/hawaii_sdma1.bin"); +MODULE_FIRMWARE("amdgpu/kaveri_sdma.bin"); +MODULE_FIRMWARE("amdgpu/kaveri_sdma1.bin"); +MODULE_FIRMWARE("amdgpu/kabini_sdma.bin"); +MODULE_FIRMWARE("amdgpu/kabini_sdma1.bin"); +MODULE_FIRMWARE("amdgpu/mullins_sdma.bin"); +MODULE_FIRMWARE("amdgpu/mullins_sdma1.bin"); + +u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev); + + +static void cik_sdma_free_microcode(struct amdgpu_device *adev) +{ + int i; + for (i = 0; i < adev->sdma.num_instances; i++) { + release_firmware(adev->sdma.instance[i].fw); + adev->sdma.instance[i].fw = NULL; + } +} + +/* + * sDMA - System DMA + * Starting with CIK, the GPU has new asynchronous + * DMA engines. These engines are used for compute + * and gfx. There are two DMA engines (SDMA0, SDMA1) + * and each one supports 1 ring buffer used for gfx + * and 2 queues used for compute. + * + * The programming model is very similar to the CP + * (ring buffer, IBs, etc.), but sDMA has it's own + * packet format that is different from the PM4 format + * used by the CP. sDMA supports copying data, writing + * embedded data, solid fills, and a number of other + * things. It also has support for tiling/detiling of + * buffers. + */ + +/** + * cik_sdma_init_microcode - load ucode images from disk + * + * @adev: amdgpu_device pointer + * + * Use the firmware interface to load the ucode images into + * the driver (not loaded into hw). + * Returns 0 on success, error on failure. + */ +static int cik_sdma_init_microcode(struct amdgpu_device *adev) +{ + const char *chip_name; + char fw_name[30]; + int err = 0, i; + + DRM_DEBUG("\n"); + + switch (adev->asic_type) { + case CHIP_BONAIRE: + chip_name = "bonaire"; + break; + case CHIP_HAWAII: + chip_name = "hawaii"; + break; + case CHIP_KAVERI: + chip_name = "kaveri"; + break; + case CHIP_KABINI: + chip_name = "kabini"; + break; + case CHIP_MULLINS: + chip_name = "mullins"; + break; + default: BUG(); + } + + for (i = 0; i < adev->sdma.num_instances; i++) { + if (i == 0) + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); + else + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name); + err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); + if (err) + goto out; + err = amdgpu_ucode_validate(adev->sdma.instance[i].fw); + } +out: + if (err) { + pr_err("cik_sdma: Failed to load firmware \"%s\"\n", fw_name); + for (i = 0; i < adev->sdma.num_instances; i++) { + release_firmware(adev->sdma.instance[i].fw); + adev->sdma.instance[i].fw = NULL; + } + } + return err; +} + +/** + * cik_sdma_ring_get_rptr - get the current read pointer + * + * @ring: amdgpu ring pointer + * + * Get the current rptr from the hardware (CIK+). + */ +static uint64_t cik_sdma_ring_get_rptr(struct amdgpu_ring *ring) +{ + u32 rptr; + + rptr = ring->adev->wb.wb[ring->rptr_offs]; + + return (rptr & 0x3fffc) >> 2; +} + +/** + * cik_sdma_ring_get_wptr - get the current write pointer + * + * @ring: amdgpu ring pointer + * + * Get the current wptr from the hardware (CIK+). + */ +static uint64_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + + return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) & 0x3fffc) >> 2; +} + +/** + * cik_sdma_ring_set_wptr - commit the write pointer + * + * @ring: amdgpu ring pointer + * + * Write the wptr back to the hardware (CIK+). + */ +static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + + WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me], + (lower_32_bits(ring->wptr) << 2) & 0x3fffc); +} + +static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); + int i; + + for (i = 0; i < count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + amdgpu_ring_write(ring, ring->funcs->nop | + SDMA_NOP_COUNT(count - 1)); + else + amdgpu_ring_write(ring, ring->funcs->nop); +} + +/** + * cik_sdma_ring_emit_ib - Schedule an IB on the DMA engine + * + * @ring: amdgpu ring pointer + * @ib: IB object to schedule + * + * Schedule an IB in the DMA ring (CIK). + */ +static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring, + struct amdgpu_job *job, + struct amdgpu_ib *ib, + uint32_t flags) +{ + unsigned vmid = AMDGPU_JOB_GET_VMID(job); + u32 extra_bits = vmid & 0xf; + + /* IB packet must end on a 8 DW boundary */ + cik_sdma_ring_insert_nop(ring, (12 - (lower_32_bits(ring->wptr) & 7)) % 8); + + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits)); + amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */ + amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff); + amdgpu_ring_write(ring, ib->length_dw); + +} + +/** + * cik_sdma_ring_emit_hdp_flush - emit an hdp flush on the DMA ring + * + * @ring: amdgpu ring pointer + * + * Emit an hdp flush packet on the requested DMA ring. + */ +static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring) +{ + u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */ + u32 ref_and_mask; + + if (ring->me == 0) + ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK; + else + ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK; + + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); + amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2); + amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2); + amdgpu_ring_write(ring, ref_and_mask); /* reference */ + amdgpu_ring_write(ring, ref_and_mask); /* mask */ + amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ +} + +/** + * cik_sdma_ring_emit_fence - emit a fence on the DMA ring + * + * @ring: amdgpu ring pointer + * @fence: amdgpu fence object + * + * Add a DMA fence packet to the ring to write + * the fence seq number and DMA trap packet to generate + * an interrupt if needed (CIK). + */ +static void cik_sdma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, + unsigned flags) +{ + bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; + /* write the fence */ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0)); + amdgpu_ring_write(ring, lower_32_bits(addr)); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, lower_32_bits(seq)); + + /* optionally write high bits as well */ + if (write64bit) { + addr += 4; + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0)); + amdgpu_ring_write(ring, lower_32_bits(addr)); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, upper_32_bits(seq)); + } + + /* generate an interrupt */ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0)); +} + +/** + * cik_sdma_gfx_stop - stop the gfx async dma engines + * + * @adev: amdgpu_device pointer + * + * Stop the gfx async dma ring buffers (CIK). + */ +static void cik_sdma_gfx_stop(struct amdgpu_device *adev) +{ + struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; + struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; + u32 rb_cntl; + int i; + + if ((adev->mman.buffer_funcs_ring == sdma0) || + (adev->mman.buffer_funcs_ring == sdma1)) + amdgpu_ttm_set_buffer_funcs_status(adev, false); + + for (i = 0; i < adev->sdma.num_instances; i++) { + rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); + rb_cntl &= ~SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK; + WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); + WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], 0); + } + sdma0->sched.ready = false; + sdma1->sched.ready = false; +} + +/** + * cik_sdma_rlc_stop - stop the compute async dma engines + * + * @adev: amdgpu_device pointer + * + * Stop the compute async dma queues (CIK). + */ +static void cik_sdma_rlc_stop(struct amdgpu_device *adev) +{ + /* XXX todo */ +} + +/** + * cik_ctx_switch_enable - stop the async dma engines context switch + * + * @adev: amdgpu_device pointer + * @enable: enable/disable the DMA MEs context switch. + * + * Halt or unhalt the async dma engines context switch (VI). + */ +static void cik_ctx_switch_enable(struct amdgpu_device *adev, bool enable) +{ + u32 f32_cntl, phase_quantum = 0; + int i; + + if (amdgpu_sdma_phase_quantum) { + unsigned value = amdgpu_sdma_phase_quantum; + unsigned unit = 0; + + while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> + SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) { + value = (value + 1) >> 1; + unit++; + } + if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> + SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) { + value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> + SDMA0_PHASE0_QUANTUM__VALUE__SHIFT); + unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> + SDMA0_PHASE0_QUANTUM__UNIT__SHIFT); + WARN_ONCE(1, + "clamping sdma_phase_quantum to %uK clock cycles\n", + value << unit); + } + phase_quantum = + value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT | + unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT; + } + + for (i = 0; i < adev->sdma.num_instances; i++) { + f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]); + if (enable) { + f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, + AUTO_CTXSW_ENABLE, 1); + if (amdgpu_sdma_phase_quantum) { + WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i], + phase_quantum); + WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i], + phase_quantum); + } + } else { + f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, + AUTO_CTXSW_ENABLE, 0); + } + + WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl); + } +} + +/** + * cik_sdma_enable - stop the async dma engines + * + * @adev: amdgpu_device pointer + * @enable: enable/disable the DMA MEs. + * + * Halt or unhalt the async dma engines (CIK). + */ +static void cik_sdma_enable(struct amdgpu_device *adev, bool enable) +{ + u32 me_cntl; + int i; + + if (!enable) { + cik_sdma_gfx_stop(adev); + cik_sdma_rlc_stop(adev); + } + + for (i = 0; i < adev->sdma.num_instances; i++) { + me_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]); + if (enable) + me_cntl &= ~SDMA0_F32_CNTL__HALT_MASK; + else + me_cntl |= SDMA0_F32_CNTL__HALT_MASK; + WREG32(mmSDMA0_F32_CNTL + sdma_offsets[i], me_cntl); + } +} + +/** + * cik_sdma_gfx_resume - setup and start the async dma engines + * + * @adev: amdgpu_device pointer + * + * Set up the gfx DMA ring buffers and enable them (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_gfx_resume(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring; + u32 rb_cntl, ib_cntl; + u32 rb_bufsz; + u32 wb_offset; + int i, j, r; + + for (i = 0; i < adev->sdma.num_instances; i++) { + ring = &adev->sdma.instance[i].ring; + wb_offset = (ring->rptr_offs * 4); + + mutex_lock(&adev->srbm_mutex); + for (j = 0; j < 16; j++) { + cik_srbm_select(adev, 0, 0, 0, j); + /* SDMA GFX */ + WREG32(mmSDMA0_GFX_VIRTUAL_ADDR + sdma_offsets[i], 0); + WREG32(mmSDMA0_GFX_APE1_CNTL + sdma_offsets[i], 0); + /* XXX SDMA RLC - todo */ + } + cik_srbm_select(adev, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + + WREG32(mmSDMA0_TILING_CONFIG + sdma_offsets[i], + adev->gfx.config.gb_addr_config & 0x70); + + WREG32(mmSDMA0_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0); + WREG32(mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0); + + /* Set ring buffer size in dwords */ + rb_bufsz = order_base_2(ring->ring_size / 4); + rb_cntl = rb_bufsz << 1; +#ifdef __BIG_ENDIAN + rb_cntl |= SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK | + SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK; +#endif + WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(mmSDMA0_GFX_RB_RPTR + sdma_offsets[i], 0); + WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], 0); + WREG32(mmSDMA0_GFX_IB_RPTR + sdma_offsets[i], 0); + WREG32(mmSDMA0_GFX_IB_OFFSET + sdma_offsets[i], 0); + + /* set the wb address whether it's enabled or not */ + WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i], + upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF); + WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i], + ((adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)); + + rb_cntl |= SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK; + + WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8); + WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40); + + ring->wptr = 0; + WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], lower_32_bits(ring->wptr) << 2); + + /* enable DMA RB */ + WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], + rb_cntl | SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK); + + ib_cntl = SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK; +#ifdef __BIG_ENDIAN + ib_cntl |= SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK; +#endif + /* enable DMA IBs */ + WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl); + + ring->sched.ready = true; + } + + cik_sdma_enable(adev, true); + + for (i = 0; i < adev->sdma.num_instances; i++) { + ring = &adev->sdma.instance[i].ring; + r = amdgpu_ring_test_helper(ring); + if (r) + return r; + + if (adev->mman.buffer_funcs_ring == ring) + amdgpu_ttm_set_buffer_funcs_status(adev, true); + } + + return 0; +} + +/** + * cik_sdma_rlc_resume - setup and start the async dma engines + * + * @adev: amdgpu_device pointer + * + * Set up the compute DMA queues and enable them (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_rlc_resume(struct amdgpu_device *adev) +{ + /* XXX todo */ + return 0; +} + +/** + * cik_sdma_load_microcode - load the sDMA ME ucode + * + * @adev: amdgpu_device pointer + * + * Loads the sDMA0/1 ucode. + * Returns 0 for success, -EINVAL if the ucode is not available. + */ +static int cik_sdma_load_microcode(struct amdgpu_device *adev) +{ + const struct sdma_firmware_header_v1_0 *hdr; + const __le32 *fw_data; + u32 fw_size; + int i, j; + + /* halt the MEs */ + cik_sdma_enable(adev, false); + + for (i = 0; i < adev->sdma.num_instances; i++) { + if (!adev->sdma.instance[i].fw) + return -EINVAL; + hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; + amdgpu_ucode_print_sdma_hdr(&hdr->header); + fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; + adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version); + adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); + if (adev->sdma.instance[i].feature_version >= 20) + adev->sdma.instance[i].burst_nop = true; + fw_data = (const __le32 *) + (adev->sdma.instance[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0); + for (j = 0; j < fw_size; j++) + WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++)); + WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version); + } + + return 0; +} + +/** + * cik_sdma_start - setup and start the async dma engines + * + * @adev: amdgpu_device pointer + * + * Set up the DMA engines and enable them (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_start(struct amdgpu_device *adev) +{ + int r; + + r = cik_sdma_load_microcode(adev); + if (r) + return r; + + /* halt the engine before programing */ + cik_sdma_enable(adev, false); + /* enable sdma ring preemption */ + cik_ctx_switch_enable(adev, true); + + /* start the gfx rings and rlc compute queues */ + r = cik_sdma_gfx_resume(adev); + if (r) + return r; + r = cik_sdma_rlc_resume(adev); + if (r) + return r; + + return 0; +} + +/** + * cik_sdma_ring_test_ring - simple async dma engine test + * + * @ring: amdgpu_ring structure holding ring information + * + * Test the DMA engine by writing using it to write an + * value to memory. (CIK). + * Returns 0 for success, error for failure. + */ +static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + unsigned i; + unsigned index; + int r; + u32 tmp; + u64 gpu_addr; + + r = amdgpu_device_wb_get(adev, &index); + if (r) + return r; + + gpu_addr = adev->wb.gpu_addr + (index * 4); + tmp = 0xCAFEDEAD; + adev->wb.wb[index] = cpu_to_le32(tmp); + + r = amdgpu_ring_alloc(ring, 5); + if (r) + goto error_free_wb; + + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); + amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); + amdgpu_ring_write(ring, upper_32_bits(gpu_addr)); + amdgpu_ring_write(ring, 1); /* number of DWs to follow */ + amdgpu_ring_write(ring, 0xDEADBEEF); + amdgpu_ring_commit(ring); + + for (i = 0; i < adev->usec_timeout; i++) { + tmp = le32_to_cpu(adev->wb.wb[index]); + if (tmp == 0xDEADBEEF) + break; + udelay(1); + } + + if (i >= adev->usec_timeout) + r = -ETIMEDOUT; + +error_free_wb: + amdgpu_device_wb_free(adev, index); + return r; +} + +/** + * cik_sdma_ring_test_ib - test an IB on the DMA engine + * + * @ring: amdgpu_ring structure holding ring information + * + * Test a simple IB in the DMA ring (CIK). + * Returns 0 on success, error on failure. + */ +static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_ib ib; + struct dma_fence *f = NULL; + unsigned index; + u32 tmp = 0; + u64 gpu_addr; + long r; + + r = amdgpu_device_wb_get(adev, &index); + if (r) + return r; + + gpu_addr = adev->wb.gpu_addr + (index * 4); + tmp = 0xCAFEDEAD; + adev->wb.wb[index] = cpu_to_le32(tmp); + memset(&ib, 0, sizeof(ib)); + r = amdgpu_ib_get(adev, NULL, 256, &ib); + if (r) + goto err0; + + ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib.ptr[1] = lower_32_bits(gpu_addr); + ib.ptr[2] = upper_32_bits(gpu_addr); + ib.ptr[3] = 1; + ib.ptr[4] = 0xDEADBEEF; + ib.length_dw = 5; + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); + if (r) + goto err1; + + r = dma_fence_wait_timeout(f, false, timeout); + if (r == 0) { + r = -ETIMEDOUT; + goto err1; + } else if (r < 0) { + goto err1; + } + tmp = le32_to_cpu(adev->wb.wb[index]); + if (tmp == 0xDEADBEEF) + r = 0; + else + r = -EINVAL; + +err1: + amdgpu_ib_free(adev, &ib, NULL); + dma_fence_put(f); +err0: + amdgpu_device_wb_free(adev, index); + return r; +} + +/** + * cik_sdma_vm_copy_pages - update PTEs by copying them from the GART + * + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @src: src addr to copy from + * @count: number of page entries to update + * + * Update PTEs by copying them from the GART using sDMA (CIK). + */ +static void cik_sdma_vm_copy_pte(struct amdgpu_ib *ib, + uint64_t pe, uint64_t src, + unsigned count) +{ + unsigned bytes = count * 8; + + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = bytes; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src); + ib->ptr[ib->length_dw++] = upper_32_bits(src); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); + ib->ptr[ib->length_dw++] = upper_32_bits(pe); +} + +/** + * cik_sdma_vm_write_pages - update PTEs by writing them manually + * + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @value: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * + * Update PTEs by writing them manually using sDMA (CIK). + */ +static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, + uint64_t value, unsigned count, + uint32_t incr) +{ + unsigned ndw = count * 2; + + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); + ib->ptr[ib->length_dw++] = upper_32_bits(pe); + ib->ptr[ib->length_dw++] = ndw; + for (; ndw > 0; ndw -= 2) { + ib->ptr[ib->length_dw++] = lower_32_bits(value); + ib->ptr[ib->length_dw++] = upper_32_bits(value); + value += incr; + } +} + +/** + * cik_sdma_vm_set_pages - update the page tables using sDMA + * + * @ib: indirect buffer to fill with commands + * @pe: addr of the page entry + * @addr: dst addr to write into pe + * @count: number of page entries to update + * @incr: increase next addr by incr bytes + * @flags: access flags + * + * Update the page tables using sDMA (CIK). + */ +static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe, + uint64_t addr, unsigned count, + uint32_t incr, uint64_t flags) +{ + /* for physically contiguous pages (vram) */ + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0); + ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */ + ib->ptr[ib->length_dw++] = upper_32_bits(pe); + ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ + ib->ptr[ib->length_dw++] = upper_32_bits(flags); + ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */ + ib->ptr[ib->length_dw++] = upper_32_bits(addr); + ib->ptr[ib->length_dw++] = incr; /* increment size */ + ib->ptr[ib->length_dw++] = 0; + ib->ptr[ib->length_dw++] = count; /* number of entries */ +} + +/** + * cik_sdma_vm_pad_ib - pad the IB to the required number of dw + * + * @ib: indirect buffer to fill with padding + * + */ +static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) +{ + struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); + u32 pad_count; + int i; + + pad_count = (8 - (ib->length_dw & 0x7)) % 8; + for (i = 0; i < pad_count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + ib->ptr[ib->length_dw++] = + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0) | + SDMA_NOP_COUNT(pad_count - 1); + else + ib->ptr[ib->length_dw++] = + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0); +} + +/** + * cik_sdma_ring_emit_pipeline_sync - sync the pipeline + * + * @ring: amdgpu_ring pointer + * + * Make sure all previous operations are completed (CIK). + */ +static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring) +{ + uint32_t seq = ring->fence_drv.sync_seq; + uint64_t addr = ring->fence_drv.gpu_addr; + + /* wait for idle */ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, + SDMA_POLL_REG_MEM_EXTRA_OP(0) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */ + SDMA_POLL_REG_MEM_EXTRA_M)); + amdgpu_ring_write(ring, addr & 0xfffffffc); + amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); + amdgpu_ring_write(ring, seq); /* reference */ + amdgpu_ring_write(ring, 0xffffffff); /* mask */ + amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */ +} + +/** + * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA + * + * @ring: amdgpu_ring pointer + * @vm: amdgpu_vm pointer + * + * Update the page table base and flush the VM TLB + * using sDMA (CIK). + */ +static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vmid, uint64_t pd_addr) +{ + u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ + + amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); + + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits)); + amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, 0); /* reference */ + amdgpu_ring_write(ring, 0); /* mask */ + amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ +} + +static void cik_sdma_ring_emit_wreg(struct amdgpu_ring *ring, + uint32_t reg, uint32_t val) +{ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + amdgpu_ring_write(ring, reg); + amdgpu_ring_write(ring, val); +} + +static void cik_enable_sdma_mgcg(struct amdgpu_device *adev, + bool enable) +{ + u32 orig, data; + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { + WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100); + WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100); + } else { + orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET); + data |= 0xff000000; + if (data != orig) + WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET); + data |= 0xff000000; + if (data != orig) + WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data); + } +} + +static void cik_enable_sdma_mgls(struct amdgpu_device *adev, + bool enable) +{ + u32 orig, data; + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) { + orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); + data |= 0x100; + if (orig != data) + WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); + data |= 0x100; + if (orig != data) + WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); + } else { + orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET); + data &= ~0x100; + if (orig != data) + WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data); + + orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET); + data &= ~0x100; + if (orig != data) + WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data); + } +} + +static int cik_sdma_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + adev->sdma.num_instances = SDMA_MAX_INSTANCE; + + cik_sdma_set_ring_funcs(adev); + cik_sdma_set_irq_funcs(adev); + cik_sdma_set_buffer_funcs(adev); + cik_sdma_set_vm_pte_funcs(adev); + + return 0; +} + +static int cik_sdma_sw_init(void *handle) +{ + struct amdgpu_ring *ring; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r, i; + + r = cik_sdma_init_microcode(adev); + if (r) { + DRM_ERROR("Failed to load sdma firmware!\n"); + return r; + } + + /* SDMA trap event */ + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224, + &adev->sdma.trap_irq); + if (r) + return r; + + /* SDMA Privileged inst */ + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 241, + &adev->sdma.illegal_inst_irq); + if (r) + return r; + + /* SDMA Privileged inst */ + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 247, + &adev->sdma.illegal_inst_irq); + if (r) + return r; + + for (i = 0; i < adev->sdma.num_instances; i++) { + ring = &adev->sdma.instance[i].ring; + ring->ring_obj = NULL; + sprintf(ring->name, "sdma%d", i); + r = amdgpu_ring_init(adev, ring, 1024, + &adev->sdma.trap_irq, + (i == 0) ? + AMDGPU_SDMA_IRQ_INSTANCE0 : + AMDGPU_SDMA_IRQ_INSTANCE1); + if (r) + return r; + } + + return r; +} + +static int cik_sdma_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) + amdgpu_ring_fini(&adev->sdma.instance[i].ring); + + cik_sdma_free_microcode(adev); + return 0; +} + +static int cik_sdma_hw_init(void *handle) +{ + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = cik_sdma_start(adev); + if (r) + return r; + + return r; +} + +static int cik_sdma_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + cik_ctx_switch_enable(adev, false); + cik_sdma_enable(adev, false); + + return 0; +} + +static int cik_sdma_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return cik_sdma_hw_fini(adev); +} + +static int cik_sdma_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + cik_sdma_soft_reset(handle); + + return cik_sdma_hw_init(adev); +} + +static bool cik_sdma_is_idle(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 tmp = RREG32(mmSRBM_STATUS2); + + if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK | + SRBM_STATUS2__SDMA1_BUSY_MASK)) + return false; + + return true; +} + +static int cik_sdma_wait_for_idle(void *handle) +{ + unsigned i; + u32 tmp; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + for (i = 0; i < adev->usec_timeout; i++) { + tmp = RREG32(mmSRBM_STATUS2) & (SRBM_STATUS2__SDMA_BUSY_MASK | + SRBM_STATUS2__SDMA1_BUSY_MASK); + + if (!tmp) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +static int cik_sdma_soft_reset(void *handle) +{ + u32 srbm_soft_reset = 0; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 tmp = RREG32(mmSRBM_STATUS2); + + if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK) { + /* sdma0 */ + tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET); + tmp |= SDMA0_F32_CNTL__HALT_MASK; + WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp); + srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK; + } + if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK) { + /* sdma1 */ + tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET); + tmp |= SDMA0_F32_CNTL__HALT_MASK; + WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp); + srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK; + } + + if (srbm_soft_reset) { + tmp = RREG32(mmSRBM_SOFT_RESET); + tmp |= srbm_soft_reset; + dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); + WREG32(mmSRBM_SOFT_RESET, tmp); + tmp = RREG32(mmSRBM_SOFT_RESET); + + udelay(50); + + tmp &= ~srbm_soft_reset; + WREG32(mmSRBM_SOFT_RESET, tmp); + tmp = RREG32(mmSRBM_SOFT_RESET); + + /* Wait a little for things to settle down */ + udelay(50); + } + + return 0; +} + +static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned type, + enum amdgpu_interrupt_state state) +{ + u32 sdma_cntl; + + switch (type) { + case AMDGPU_SDMA_IRQ_INSTANCE0: + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET); + sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK; + WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl); + break; + case AMDGPU_IRQ_STATE_ENABLE: + sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET); + sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK; + WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl); + break; + default: + break; + } + break; + case AMDGPU_SDMA_IRQ_INSTANCE1: + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET); + sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK; + WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl); + break; + case AMDGPU_IRQ_STATE_ENABLE: + sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET); + sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK; + WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl); + break; + default: + break; + } + break; + default: + break; + } + return 0; +} + +static int cik_sdma_process_trap_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + u8 instance_id, queue_id; + + instance_id = (entry->ring_id & 0x3) >> 0; + queue_id = (entry->ring_id & 0xc) >> 2; + DRM_DEBUG("IH: SDMA trap\n"); + switch (instance_id) { + case 0: + switch (queue_id) { + case 0: + amdgpu_fence_process(&adev->sdma.instance[0].ring); + break; + case 1: + /* XXX compute */ + break; + case 2: + /* XXX compute */ + break; + } + break; + case 1: + switch (queue_id) { + case 0: + amdgpu_fence_process(&adev->sdma.instance[1].ring); + break; + case 1: + /* XXX compute */ + break; + case 2: + /* XXX compute */ + break; + } + break; + } + + return 0; +} + +static int cik_sdma_process_illegal_inst_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + u8 instance_id; + + DRM_ERROR("Illegal instruction in SDMA command stream\n"); + instance_id = (entry->ring_id & 0x3) >> 0; + drm_sched_fault(&adev->sdma.instance[instance_id].ring.sched); + return 0; +} + +static int cik_sdma_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + bool gate = false; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + if (state == AMD_CG_STATE_GATE) + gate = true; + + cik_enable_sdma_mgcg(adev, gate); + cik_enable_sdma_mgls(adev, gate); + + return 0; +} + +static int cik_sdma_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +static const struct amd_ip_funcs cik_sdma_ip_funcs = { + .name = "cik_sdma", + .early_init = cik_sdma_early_init, + .late_init = NULL, + .sw_init = cik_sdma_sw_init, + .sw_fini = cik_sdma_sw_fini, + .hw_init = cik_sdma_hw_init, + .hw_fini = cik_sdma_hw_fini, + .suspend = cik_sdma_suspend, + .resume = cik_sdma_resume, + .is_idle = cik_sdma_is_idle, + .wait_for_idle = cik_sdma_wait_for_idle, + .soft_reset = cik_sdma_soft_reset, + .set_clockgating_state = cik_sdma_set_clockgating_state, + .set_powergating_state = cik_sdma_set_powergating_state, +}; + +static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = { + .type = AMDGPU_RING_TYPE_SDMA, + .align_mask = 0xf, + .nop = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0), + .support_64bit_ptrs = false, + .get_rptr = cik_sdma_ring_get_rptr, + .get_wptr = cik_sdma_ring_get_wptr, + .set_wptr = cik_sdma_ring_set_wptr, + .emit_frame_size = + 6 + /* cik_sdma_ring_emit_hdp_flush */ + 3 + /* hdp invalidate */ + 6 + /* cik_sdma_ring_emit_pipeline_sync */ + CIK_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* cik_sdma_ring_emit_vm_flush */ + 9 + 9 + 9, /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */ + .emit_ib_size = 7 + 4, /* cik_sdma_ring_emit_ib */ + .emit_ib = cik_sdma_ring_emit_ib, + .emit_fence = cik_sdma_ring_emit_fence, + .emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync, + .emit_vm_flush = cik_sdma_ring_emit_vm_flush, + .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush, + .test_ring = cik_sdma_ring_test_ring, + .test_ib = cik_sdma_ring_test_ib, + .insert_nop = cik_sdma_ring_insert_nop, + .pad_ib = cik_sdma_ring_pad_ib, + .emit_wreg = cik_sdma_ring_emit_wreg, +}; + +static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev) +{ + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) { + adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs; + adev->sdma.instance[i].ring.me = i; + } +} + +static const struct amdgpu_irq_src_funcs cik_sdma_trap_irq_funcs = { + .set = cik_sdma_set_trap_irq_state, + .process = cik_sdma_process_trap_irq, +}; + +static const struct amdgpu_irq_src_funcs cik_sdma_illegal_inst_irq_funcs = { + .process = cik_sdma_process_illegal_inst_irq, +}; + +static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev) +{ + adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST; + adev->sdma.trap_irq.funcs = &cik_sdma_trap_irq_funcs; + adev->sdma.illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs; +} + +/** + * cik_sdma_emit_copy_buffer - copy buffer using the sDMA engine + * + * @ring: amdgpu_ring structure holding ring information + * @src_offset: src GPU address + * @dst_offset: dst GPU address + * @byte_count: number of bytes to xfer + * + * Copy GPU buffers using the DMA engine (CIK). + * Used by the amdgpu ttm implementation to move pages if + * registered as the asic copy callback. + */ +static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib, + uint64_t src_offset, + uint64_t dst_offset, + uint32_t byte_count) +{ + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); +} + +/** + * cik_sdma_emit_fill_buffer - fill buffer using the sDMA engine + * + * @ring: amdgpu_ring structure holding ring information + * @src_data: value to write to buffer + * @dst_offset: dst GPU address + * @byte_count: number of bytes to xfer + * + * Fill GPU buffers using the DMA engine (CIK). + */ +static void cik_sdma_emit_fill_buffer(struct amdgpu_ib *ib, + uint32_t src_data, + uint64_t dst_offset, + uint32_t byte_count) +{ + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = src_data; + ib->ptr[ib->length_dw++] = byte_count; +} + +static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = { + .copy_max_bytes = 0x1fffff, + .copy_num_dw = 7, + .emit_copy_buffer = cik_sdma_emit_copy_buffer, + + .fill_max_bytes = 0x1fffff, + .fill_num_dw = 5, + .emit_fill_buffer = cik_sdma_emit_fill_buffer, +}; + +static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev) +{ + adev->mman.buffer_funcs = &cik_sdma_buffer_funcs; + adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; +} + +static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = { + .copy_pte_num_dw = 7, + .copy_pte = cik_sdma_vm_copy_pte, + + .write_pte = cik_sdma_vm_write_pte, + .set_pte_pde = cik_sdma_vm_set_pte_pde, +}; + +static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev) +{ + struct drm_gpu_scheduler *sched; + unsigned i; + + adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs; + for (i = 0; i < adev->sdma.num_instances; i++) { + sched = &adev->sdma.instance[i].ring.sched; + adev->vm_manager.vm_pte_rqs[i] = + &sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL]; + } + adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances; +} + +const struct amdgpu_ip_block_version cik_sdma_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_SDMA, + .major = 2, + .minor = 0, + .rev = 0, + .funcs = &cik_sdma_ip_funcs, +};
diff --git a/amdgpu/cik_sdma.h b/amdgpu/cik_sdma.h new file mode 100644 index 0000000..a4a8fe0 --- /dev/null +++ b/amdgpu/cik_sdma.h
@@ -0,0 +1,29 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __CIK_SDMA_H__ +#define __CIK_SDMA_H__ + +extern const struct amdgpu_ip_block_version cik_sdma_ip_block; + +#endif
diff --git a/amdgpu/cikd.h b/amdgpu/cikd.h new file mode 100644 index 0000000..cee6e8a --- /dev/null +++ b/amdgpu/cikd.h
@@ -0,0 +1,609 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Alex Deucher + */ +#ifndef CIK_H +#define CIK_H + +#define MC_SEQ_MISC0__MT__MASK 0xf0000000 +#define MC_SEQ_MISC0__MT__GDDR1 0x10000000 +#define MC_SEQ_MISC0__MT__DDR2 0x20000000 +#define MC_SEQ_MISC0__MT__GDDR3 0x30000000 +#define MC_SEQ_MISC0__MT__GDDR4 0x40000000 +#define MC_SEQ_MISC0__MT__GDDR5 0x50000000 +#define MC_SEQ_MISC0__MT__HBM 0x60000000 +#define MC_SEQ_MISC0__MT__DDR3 0xB0000000 + +#define CP_ME_TABLE_SIZE 96 + +/* display controller offsets used for crtc/cur/lut/grph/viewport/etc. */ +#define CRTC0_REGISTER_OFFSET (0x1b7c - 0x1b7c) +#define CRTC1_REGISTER_OFFSET (0x1e7c - 0x1b7c) +#define CRTC2_REGISTER_OFFSET (0x417c - 0x1b7c) +#define CRTC3_REGISTER_OFFSET (0x447c - 0x1b7c) +#define CRTC4_REGISTER_OFFSET (0x477c - 0x1b7c) +#define CRTC5_REGISTER_OFFSET (0x4a7c - 0x1b7c) + +/* hpd instance offsets */ +#define HPD0_REGISTER_OFFSET (0x1807 - 0x1807) +#define HPD1_REGISTER_OFFSET (0x180a - 0x1807) +#define HPD2_REGISTER_OFFSET (0x180d - 0x1807) +#define HPD3_REGISTER_OFFSET (0x1810 - 0x1807) +#define HPD4_REGISTER_OFFSET (0x1813 - 0x1807) +#define HPD5_REGISTER_OFFSET (0x1816 - 0x1807) + +#define BONAIRE_GB_ADDR_CONFIG_GOLDEN 0x12010001 +#define HAWAII_GB_ADDR_CONFIG_GOLDEN 0x12011003 + +#define AMDGPU_NUM_OF_VMIDS 8 + +#define PIPEID(x) ((x) << 0) +#define MEID(x) ((x) << 2) +#define VMID(x) ((x) << 4) +#define QUEUEID(x) ((x) << 8) + +#define mmCC_DRM_ID_STRAPS 0x1559 +#define CC_DRM_ID_STRAPS__ATI_REV_ID_MASK 0xf0000000 + +#define mmCHUB_CONTROL 0x619 +#define BYPASS_VM (1 << 0) + +#define SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU (0 << 5) + +#define mmGRPH_LUT_10BIT_BYPASS_CONTROL 0x1a02 +#define LUT_10BIT_BYPASS_EN (1 << 8) + +# define CURSOR_MONO 0 +# define CURSOR_24_1 1 +# define CURSOR_24_8_PRE_MULT 2 +# define CURSOR_24_8_UNPRE_MULT 3 +# define CURSOR_URGENT_ALWAYS 0 +# define CURSOR_URGENT_1_8 1 +# define CURSOR_URGENT_1_4 2 +# define CURSOR_URGENT_3_8 3 +# define CURSOR_URGENT_1_2 4 + +# define GRPH_DEPTH_8BPP 0 +# define GRPH_DEPTH_16BPP 1 +# define GRPH_DEPTH_32BPP 2 +/* 8 BPP */ +# define GRPH_FORMAT_INDEXED 0 +/* 16 BPP */ +# define GRPH_FORMAT_ARGB1555 0 +# define GRPH_FORMAT_ARGB565 1 +# define GRPH_FORMAT_ARGB4444 2 +# define GRPH_FORMAT_AI88 3 +# define GRPH_FORMAT_MONO16 4 +# define GRPH_FORMAT_BGRA5551 5 +/* 32 BPP */ +# define GRPH_FORMAT_ARGB8888 0 +# define GRPH_FORMAT_ARGB2101010 1 +# define GRPH_FORMAT_32BPP_DIG 2 +# define GRPH_FORMAT_8B_ARGB2101010 3 +# define GRPH_FORMAT_BGRA1010102 4 +# define GRPH_FORMAT_8B_BGRA1010102 5 +# define GRPH_FORMAT_RGB111110 6 +# define GRPH_FORMAT_BGR101111 7 +# define ADDR_SURF_MACRO_TILE_ASPECT_1 0 +# define ADDR_SURF_MACRO_TILE_ASPECT_2 1 +# define ADDR_SURF_MACRO_TILE_ASPECT_4 2 +# define ADDR_SURF_MACRO_TILE_ASPECT_8 3 +# define GRPH_ARRAY_LINEAR_GENERAL 0 +# define GRPH_ARRAY_LINEAR_ALIGNED 1 +# define GRPH_ARRAY_1D_TILED_THIN1 2 +# define GRPH_ARRAY_2D_TILED_THIN1 4 +# define DISPLAY_MICRO_TILING 0 +# define THIN_MICRO_TILING 1 +# define DEPTH_MICRO_TILING 2 +# define ROTATED_MICRO_TILING 4 +# define GRPH_ENDIAN_NONE 0 +# define GRPH_ENDIAN_8IN16 1 +# define GRPH_ENDIAN_8IN32 2 +# define GRPH_ENDIAN_8IN64 3 +# define GRPH_RED_SEL_R 0 +# define GRPH_RED_SEL_G 1 +# define GRPH_RED_SEL_B 2 +# define GRPH_RED_SEL_A 3 +# define GRPH_GREEN_SEL_G 0 +# define GRPH_GREEN_SEL_B 1 +# define GRPH_GREEN_SEL_A 2 +# define GRPH_GREEN_SEL_R 3 +# define GRPH_BLUE_SEL_B 0 +# define GRPH_BLUE_SEL_A 1 +# define GRPH_BLUE_SEL_R 2 +# define GRPH_BLUE_SEL_G 3 +# define GRPH_ALPHA_SEL_A 0 +# define GRPH_ALPHA_SEL_R 1 +# define GRPH_ALPHA_SEL_G 2 +# define GRPH_ALPHA_SEL_B 3 +# define INPUT_GAMMA_USE_LUT 0 +# define INPUT_GAMMA_BYPASS 1 +# define INPUT_GAMMA_SRGB_24 2 +# define INPUT_GAMMA_XVYCC_222 3 + +# define INPUT_CSC_BYPASS 0 +# define INPUT_CSC_PROG_COEFF 1 +# define INPUT_CSC_PROG_SHARED_MATRIXA 2 + +# define OUTPUT_CSC_BYPASS 0 +# define OUTPUT_CSC_TV_RGB 1 +# define OUTPUT_CSC_YCBCR_601 2 +# define OUTPUT_CSC_YCBCR_709 3 +# define OUTPUT_CSC_PROG_COEFF 4 +# define OUTPUT_CSC_PROG_SHARED_MATRIXB 5 + +# define DEGAMMA_BYPASS 0 +# define DEGAMMA_SRGB_24 1 +# define DEGAMMA_XVYCC_222 2 +# define GAMUT_REMAP_BYPASS 0 +# define GAMUT_REMAP_PROG_COEFF 1 +# define GAMUT_REMAP_PROG_SHARED_MATRIXA 2 +# define GAMUT_REMAP_PROG_SHARED_MATRIXB 3 + +# define REGAMMA_BYPASS 0 +# define REGAMMA_SRGB_24 1 +# define REGAMMA_XVYCC_222 2 +# define REGAMMA_PROG_A 3 +# define REGAMMA_PROG_B 4 + +# define FMT_CLAMP_6BPC 0 +# define FMT_CLAMP_8BPC 1 +# define FMT_CLAMP_10BPC 2 + +# define HDMI_24BIT_DEEP_COLOR 0 +# define HDMI_30BIT_DEEP_COLOR 1 +# define HDMI_36BIT_DEEP_COLOR 2 +# define HDMI_ACR_HW 0 +# define HDMI_ACR_32 1 +# define HDMI_ACR_44 2 +# define HDMI_ACR_48 3 +# define HDMI_ACR_X1 1 +# define HDMI_ACR_X2 2 +# define HDMI_ACR_X4 4 +# define AFMT_AVI_INFO_Y_RGB 0 +# define AFMT_AVI_INFO_Y_YCBCR422 1 +# define AFMT_AVI_INFO_Y_YCBCR444 2 + +#define NO_AUTO 0 +#define ES_AUTO 1 +#define GS_AUTO 2 +#define ES_AND_GS_AUTO 3 + +# define ARRAY_MODE(x) ((x) << 2) +# define PIPE_CONFIG(x) ((x) << 6) +# define TILE_SPLIT(x) ((x) << 11) +# define MICRO_TILE_MODE_NEW(x) ((x) << 22) +# define SAMPLE_SPLIT(x) ((x) << 25) +# define BANK_WIDTH(x) ((x) << 0) +# define BANK_HEIGHT(x) ((x) << 2) +# define MACRO_TILE_ASPECT(x) ((x) << 4) +# define NUM_BANKS(x) ((x) << 6) + +#define MSG_ENTER_RLC_SAFE_MODE 1 +#define MSG_EXIT_RLC_SAFE_MODE 0 + +/* + * PM4 + */ +#define PACKET_TYPE0 0 +#define PACKET_TYPE1 1 +#define PACKET_TYPE2 2 +#define PACKET_TYPE3 3 + +#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3) +#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF) +#define CP_PACKET0_GET_REG(h) ((h) & 0xFFFF) +#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF) +#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \ + ((reg) & 0xFFFF) | \ + ((n) & 0x3FFF) << 16) +#define CP_PACKET2 0x80000000 +#define PACKET2_PAD_SHIFT 0 +#define PACKET2_PAD_MASK (0x3fffffff << 0) + +#define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v))) + +#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \ + (((op) & 0xFF) << 8) | \ + ((n) & 0x3FFF) << 16) + +#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1) + +/* Packet 3 types */ +#define PACKET3_NOP 0x10 +#define PACKET3_SET_BASE 0x11 +#define PACKET3_BASE_INDEX(x) ((x) << 0) +#define CE_PARTITION_BASE 3 +#define PACKET3_CLEAR_STATE 0x12 +#define PACKET3_INDEX_BUFFER_SIZE 0x13 +#define PACKET3_DISPATCH_DIRECT 0x15 +#define PACKET3_DISPATCH_INDIRECT 0x16 +#define PACKET3_ATOMIC_GDS 0x1D +#define PACKET3_ATOMIC_MEM 0x1E +#define PACKET3_OCCLUSION_QUERY 0x1F +#define PACKET3_SET_PREDICATION 0x20 +#define PACKET3_REG_RMW 0x21 +#define PACKET3_COND_EXEC 0x22 +#define PACKET3_PRED_EXEC 0x23 +#define PACKET3_DRAW_INDIRECT 0x24 +#define PACKET3_DRAW_INDEX_INDIRECT 0x25 +#define PACKET3_INDEX_BASE 0x26 +#define PACKET3_DRAW_INDEX_2 0x27 +#define PACKET3_CONTEXT_CONTROL 0x28 +#define PACKET3_INDEX_TYPE 0x2A +#define PACKET3_DRAW_INDIRECT_MULTI 0x2C +#define PACKET3_DRAW_INDEX_AUTO 0x2D +#define PACKET3_NUM_INSTANCES 0x2F +#define PACKET3_DRAW_INDEX_MULTI_AUTO 0x30 +#define PACKET3_INDIRECT_BUFFER_CONST 0x33 +#define PACKET3_STRMOUT_BUFFER_UPDATE 0x34 +#define PACKET3_DRAW_INDEX_OFFSET_2 0x35 +#define PACKET3_DRAW_PREAMBLE 0x36 +#define PACKET3_WRITE_DATA 0x37 +#define WRITE_DATA_DST_SEL(x) ((x) << 8) + /* 0 - register + * 1 - memory (sync - via GRBM) + * 2 - gl2 + * 3 - gds + * 4 - reserved + * 5 - memory (async - direct) + */ +#define WR_ONE_ADDR (1 << 16) +#define WR_CONFIRM (1 << 20) +#define WRITE_DATA_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + */ +#define WRITE_DATA_ENGINE_SEL(x) ((x) << 30) + /* 0 - me + * 1 - pfp + * 2 - ce + */ +#define PACKET3_DRAW_INDEX_INDIRECT_MULTI 0x38 +#define PACKET3_MEM_SEMAPHORE 0x39 +# define PACKET3_SEM_USE_MAILBOX (0x1 << 16) +# define PACKET3_SEM_SEL_SIGNAL_TYPE (0x1 << 20) /* 0 = increment, 1 = write 1 */ +# define PACKET3_SEM_CLIENT_CODE ((x) << 24) /* 0 = CP, 1 = CB, 2 = DB */ +# define PACKET3_SEM_SEL_SIGNAL (0x6 << 29) +# define PACKET3_SEM_SEL_WAIT (0x7 << 29) +#define PACKET3_COPY_DW 0x3B +#define PACKET3_WAIT_REG_MEM 0x3C +#define WAIT_REG_MEM_FUNCTION(x) ((x) << 0) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +#define WAIT_REG_MEM_MEM_SPACE(x) ((x) << 4) + /* 0 - reg + * 1 - mem + */ +#define WAIT_REG_MEM_OPERATION(x) ((x) << 6) + /* 0 - wait_reg_mem + * 1 - wr_wait_wr_reg + */ +#define WAIT_REG_MEM_ENGINE(x) ((x) << 8) + /* 0 - me + * 1 - pfp + */ +#define PACKET3_INDIRECT_BUFFER 0x3F +#define INDIRECT_BUFFER_TCL2_VOLATILE (1 << 22) +#define INDIRECT_BUFFER_VALID (1 << 23) +#define INDIRECT_BUFFER_CACHE_POLICY(x) ((x) << 28) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +#define PACKET3_COPY_DATA 0x40 +#define PACKET3_PFP_SYNC_ME 0x42 +#define PACKET3_SURFACE_SYNC 0x43 +# define PACKET3_DEST_BASE_0_ENA (1 << 0) +# define PACKET3_DEST_BASE_1_ENA (1 << 1) +# define PACKET3_CB0_DEST_BASE_ENA (1 << 6) +# define PACKET3_CB1_DEST_BASE_ENA (1 << 7) +# define PACKET3_CB2_DEST_BASE_ENA (1 << 8) +# define PACKET3_CB3_DEST_BASE_ENA (1 << 9) +# define PACKET3_CB4_DEST_BASE_ENA (1 << 10) +# define PACKET3_CB5_DEST_BASE_ENA (1 << 11) +# define PACKET3_CB6_DEST_BASE_ENA (1 << 12) +# define PACKET3_CB7_DEST_BASE_ENA (1 << 13) +# define PACKET3_DB_DEST_BASE_ENA (1 << 14) +# define PACKET3_TCL1_VOL_ACTION_ENA (1 << 15) +# define PACKET3_TC_VOL_ACTION_ENA (1 << 16) /* L2 */ +# define PACKET3_TC_WB_ACTION_ENA (1 << 18) /* L2 */ +# define PACKET3_DEST_BASE_2_ENA (1 << 19) +# define PACKET3_DEST_BASE_3_ENA (1 << 21) +# define PACKET3_TCL1_ACTION_ENA (1 << 22) +# define PACKET3_TC_ACTION_ENA (1 << 23) /* L2 */ +# define PACKET3_CB_ACTION_ENA (1 << 25) +# define PACKET3_DB_ACTION_ENA (1 << 26) +# define PACKET3_SH_KCACHE_ACTION_ENA (1 << 27) +# define PACKET3_SH_KCACHE_VOL_ACTION_ENA (1 << 28) +# define PACKET3_SH_ICACHE_ACTION_ENA (1 << 29) +#define PACKET3_COND_WRITE 0x45 +#define PACKET3_EVENT_WRITE 0x46 +#define EVENT_TYPE(x) ((x) << 0) +#define EVENT_INDEX(x) ((x) << 8) + /* 0 - any non-TS event + * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_* + * 2 - SAMPLE_PIPELINESTAT + * 3 - SAMPLE_STREAMOUTSTAT* + * 4 - *S_PARTIAL_FLUSH + * 5 - EOP events + * 6 - EOS events + */ +#define PACKET3_EVENT_WRITE_EOP 0x47 +#define EOP_TCL1_VOL_ACTION_EN (1 << 12) +#define EOP_TC_VOL_ACTION_EN (1 << 13) /* L2 */ +#define EOP_TC_WB_ACTION_EN (1 << 15) /* L2 */ +#define EOP_TCL1_ACTION_EN (1 << 16) +#define EOP_TC_ACTION_EN (1 << 17) /* L2 */ +#define EOP_TCL2_VOLATILE (1 << 24) +#define EOP_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +#define DATA_SEL(x) ((x) << 29) + /* 0 - discard + * 1 - send low 32bit data + * 2 - send 64bit data + * 3 - send 64bit GPU counter value + * 4 - send 64bit sys counter value + */ +#define INT_SEL(x) ((x) << 24) + /* 0 - none + * 1 - interrupt only (DATA_SEL = 0) + * 2 - interrupt when data write is confirmed + */ +#define DST_SEL(x) ((x) << 16) + /* 0 - MC + * 1 - TC/L2 + */ +#define PACKET3_EVENT_WRITE_EOS 0x48 +#define PACKET3_RELEASE_MEM 0x49 +#define PACKET3_PREAMBLE_CNTL 0x4A +# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28) +# define PACKET3_PREAMBLE_END_CLEAR_STATE (3 << 28) +#define PACKET3_DMA_DATA 0x50 +/* 1. header + * 2. CONTROL + * 3. SRC_ADDR_LO or DATA [31:0] + * 4. SRC_ADDR_HI [31:0] + * 5. DST_ADDR_LO [31:0] + * 6. DST_ADDR_HI [7:0] + * 7. COMMAND [30:21] | BYTE_COUNT [20:0] + */ +/* CONTROL */ +# define PACKET3_DMA_DATA_ENGINE(x) ((x) << 0) + /* 0 - ME + * 1 - PFP + */ +# define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +# define PACKET3_DMA_DATA_SRC_VOLATILE (1 << 15) +# define PACKET3_DMA_DATA_DST_SEL(x) ((x) << 20) + /* 0 - DST_ADDR using DAS + * 1 - GDS + * 3 - DST_ADDR using L2 + */ +# define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25) + /* 0 - LRU + * 1 - Stream + * 2 - Bypass + */ +# define PACKET3_DMA_DATA_DST_VOLATILE (1 << 27) +# define PACKET3_DMA_DATA_SRC_SEL(x) ((x) << 29) + /* 0 - SRC_ADDR using SAS + * 1 - GDS + * 2 - DATA + * 3 - SRC_ADDR using L2 + */ +# define PACKET3_DMA_DATA_CP_SYNC (1 << 31) +/* COMMAND */ +# define PACKET3_DMA_DATA_DIS_WC (1 << 21) +# define PACKET3_DMA_DATA_CMD_SRC_SWAP(x) ((x) << 22) + /* 0 - none + * 1 - 8 in 16 + * 2 - 8 in 32 + * 3 - 8 in 64 + */ +# define PACKET3_DMA_DATA_CMD_DST_SWAP(x) ((x) << 24) + /* 0 - none + * 1 - 8 in 16 + * 2 - 8 in 32 + * 3 - 8 in 64 + */ +# define PACKET3_DMA_DATA_CMD_SAS (1 << 26) + /* 0 - memory + * 1 - register + */ +# define PACKET3_DMA_DATA_CMD_DAS (1 << 27) + /* 0 - memory + * 1 - register + */ +# define PACKET3_DMA_DATA_CMD_SAIC (1 << 28) +# define PACKET3_DMA_DATA_CMD_DAIC (1 << 29) +# define PACKET3_DMA_DATA_CMD_RAW_WAIT (1 << 30) +#define PACKET3_AQUIRE_MEM 0x58 +#define PACKET3_REWIND 0x59 +#define PACKET3_LOAD_UCONFIG_REG 0x5E +#define PACKET3_LOAD_SH_REG 0x5F +#define PACKET3_LOAD_CONFIG_REG 0x60 +#define PACKET3_LOAD_CONTEXT_REG 0x61 +#define PACKET3_SET_CONFIG_REG 0x68 +#define PACKET3_SET_CONFIG_REG_START 0x00002000 +#define PACKET3_SET_CONFIG_REG_END 0x00002c00 +#define PACKET3_SET_CONTEXT_REG 0x69 +#define PACKET3_SET_CONTEXT_REG_START 0x0000a000 +#define PACKET3_SET_CONTEXT_REG_END 0x0000a400 +#define PACKET3_SET_CONTEXT_REG_INDIRECT 0x73 +#define PACKET3_SET_SH_REG 0x76 +#define PACKET3_SET_SH_REG_START 0x00002c00 +#define PACKET3_SET_SH_REG_END 0x00003000 +#define PACKET3_SET_SH_REG_OFFSET 0x77 +#define PACKET3_SET_QUEUE_REG 0x78 +#define PACKET3_SET_UCONFIG_REG 0x79 +#define PACKET3_SET_UCONFIG_REG_START 0x0000c000 +#define PACKET3_SET_UCONFIG_REG_END 0x0000c400 +#define PACKET3_SCRATCH_RAM_WRITE 0x7D +#define PACKET3_SCRATCH_RAM_READ 0x7E +#define PACKET3_LOAD_CONST_RAM 0x80 +#define PACKET3_WRITE_CONST_RAM 0x81 +#define PACKET3_DUMP_CONST_RAM 0x83 +#define PACKET3_INCREMENT_CE_COUNTER 0x84 +#define PACKET3_INCREMENT_DE_COUNTER 0x85 +#define PACKET3_WAIT_ON_CE_COUNTER 0x86 +#define PACKET3_WAIT_ON_DE_COUNTER_DIFF 0x88 +#define PACKET3_SWITCH_BUFFER 0x8B + +/* SDMA - first instance at 0xd000, second at 0xd800 */ +#define SDMA0_REGISTER_OFFSET 0x0 /* not a register */ +#define SDMA1_REGISTER_OFFSET 0x200 /* not a register */ +#define SDMA_MAX_INSTANCE 2 + +#define SDMA_PACKET(op, sub_op, e) ((((e) & 0xFFFF) << 16) | \ + (((sub_op) & 0xFF) << 8) | \ + (((op) & 0xFF) << 0)) +/* sDMA opcodes */ +#define SDMA_OPCODE_NOP 0 +# define SDMA_NOP_COUNT(x) (((x) & 0x3FFF) << 16) +#define SDMA_OPCODE_COPY 1 +# define SDMA_COPY_SUB_OPCODE_LINEAR 0 +# define SDMA_COPY_SUB_OPCODE_TILED 1 +# define SDMA_COPY_SUB_OPCODE_SOA 3 +# define SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW 4 +# define SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW 5 +# define SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW 6 +#define SDMA_OPCODE_WRITE 2 +# define SDMA_WRITE_SUB_OPCODE_LINEAR 0 +# define SDMA_WRITE_SUB_OPCODE_TILED 1 +#define SDMA_OPCODE_INDIRECT_BUFFER 4 +#define SDMA_OPCODE_FENCE 5 +#define SDMA_OPCODE_TRAP 6 +#define SDMA_OPCODE_SEMAPHORE 7 +# define SDMA_SEMAPHORE_EXTRA_O (1 << 13) + /* 0 - increment + * 1 - write 1 + */ +# define SDMA_SEMAPHORE_EXTRA_S (1 << 14) + /* 0 - wait + * 1 - signal + */ +# define SDMA_SEMAPHORE_EXTRA_M (1 << 15) + /* mailbox */ +#define SDMA_OPCODE_POLL_REG_MEM 8 +# define SDMA_POLL_REG_MEM_EXTRA_OP(x) ((x) << 10) + /* 0 - wait_reg_mem + * 1 - wr_wait_wr_reg + */ +# define SDMA_POLL_REG_MEM_EXTRA_FUNC(x) ((x) << 12) + /* 0 - always + * 1 - < + * 2 - <= + * 3 - == + * 4 - != + * 5 - >= + * 6 - > + */ +# define SDMA_POLL_REG_MEM_EXTRA_M (1 << 15) + /* 0 = register + * 1 = memory + */ +#define SDMA_OPCODE_COND_EXEC 9 +#define SDMA_OPCODE_CONSTANT_FILL 11 +# define SDMA_CONSTANT_FILL_EXTRA_SIZE(x) ((x) << 14) + /* 0 = byte fill + * 2 = DW fill + */ +#define SDMA_OPCODE_GENERATE_PTE_PDE 12 +#define SDMA_OPCODE_TIMESTAMP 13 +# define SDMA_TIMESTAMP_SUB_OPCODE_SET_LOCAL 0 +# define SDMA_TIMESTAMP_SUB_OPCODE_GET_LOCAL 1 +# define SDMA_TIMESTAMP_SUB_OPCODE_GET_GLOBAL 2 +#define SDMA_OPCODE_SRBM_WRITE 14 +# define SDMA_SRBM_WRITE_EXTRA_BYTE_ENABLE(x) ((x) << 12) + /* byte mask */ + +#define VCE_CMD_NO_OP 0x00000000 +#define VCE_CMD_END 0x00000001 +#define VCE_CMD_IB 0x00000002 +#define VCE_CMD_FENCE 0x00000003 +#define VCE_CMD_TRAP 0x00000004 +#define VCE_CMD_IB_AUTO 0x00000005 +#define VCE_CMD_SEMAPHORE 0x00000006 + +/* if PTR32, these are the bases for scratch and lds */ +#define PRIVATE_BASE(x) ((x) << 0) /* scratch */ +#define SHARED_BASE(x) ((x) << 16) /* LDS */ + +#define KFD_CIK_SDMA_QUEUE_OFFSET (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL) + +/* valid for both DEFAULT_MTYPE and APE1_MTYPE */ +enum { + MTYPE_CACHED = 0, + MTYPE_NONCACHED = 3 +}; + +/* mmPA_SC_RASTER_CONFIG mask */ +#define RB_MAP_PKR0(x) ((x) << 0) +#define RB_MAP_PKR0_MASK (0x3 << 0) +#define RB_MAP_PKR1(x) ((x) << 2) +#define RB_MAP_PKR1_MASK (0x3 << 2) +#define RB_XSEL2(x) ((x) << 4) +#define RB_XSEL2_MASK (0x3 << 4) +#define RB_XSEL (1 << 6) +#define RB_YSEL (1 << 7) +#define PKR_MAP(x) ((x) << 8) +#define PKR_MAP_MASK (0x3 << 8) +#define PKR_XSEL(x) ((x) << 10) +#define PKR_XSEL_MASK (0x3 << 10) +#define PKR_YSEL(x) ((x) << 12) +#define PKR_YSEL_MASK (0x3 << 12) +#define SC_MAP(x) ((x) << 16) +#define SC_MAP_MASK (0x3 << 16) +#define SC_XSEL(x) ((x) << 18) +#define SC_XSEL_MASK (0x3 << 18) +#define SC_YSEL(x) ((x) << 20) +#define SC_YSEL_MASK (0x3 << 20) +#define SE_MAP(x) ((x) << 24) +#define SE_MAP_MASK (0x3 << 24) +#define SE_XSEL(x) ((x) << 26) +#define SE_XSEL_MASK (0x3 << 26) +#define SE_YSEL(x) ((x) << 28) +#define SE_YSEL_MASK (0x3 << 28) + +/* mmPA_SC_RASTER_CONFIG_1 mask */ +#define SE_PAIR_MAP(x) ((x) << 0) +#define SE_PAIR_MAP_MASK (0x3 << 0) +#define SE_PAIR_XSEL(x) ((x) << 2) +#define SE_PAIR_XSEL_MASK (0x3 << 2) +#define SE_PAIR_YSEL(x) ((x) << 4) +#define SE_PAIR_YSEL_MASK (0x3 << 4) + +#endif
diff --git a/amdgpu/clearstate_ci.h b/amdgpu/clearstate_ci.h new file mode 100644 index 0000000..c3982f9 --- /dev/null +++ b/amdgpu/clearstate_ci.h
@@ -0,0 +1,944 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const unsigned int ci_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0x00000000, // TA_BC_BASE_ADDR_HI + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_HI_0 + 0x00000000, // COHER_DEST_BASE_HI_1 + 0x00000000, // COHER_DEST_BASE_HI_2 + 0x00000000, // COHER_DEST_BASE_HI_3 + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const unsigned int ci_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL + 0, // HOLE + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const unsigned int ci_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const unsigned int ci_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0x00000000, // VGT_GS_ONCHIP_CNTL + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const unsigned int ci_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // WD_ENHANCE + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const unsigned int ci_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const unsigned int ci_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0, // HOLE + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 +}; +static const struct cs_extent_def ci_SECT_CONTEXT_defs[] = +{ + {ci_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {ci_SECT_CONTEXT_def_2, 0x0000a0d6, 274 }, + {ci_SECT_CONTEXT_def_3, 0x0000a1f5, 6 }, + {ci_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {ci_SECT_CONTEXT_def_5, 0x0000a2a0, 2 }, + {ci_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {ci_SECT_CONTEXT_def_7, 0x0000a2a5, 233 }, + { 0, 0, 0 } +}; +static const struct cs_section_def ci_cs_data[] = { + { ci_SECT_CONTEXT_defs, SECT_CONTEXT }, + { 0, SECT_NONE } +};
diff --git a/amdgpu/clearstate_defs.h b/amdgpu/clearstate_defs.h new file mode 100644 index 0000000..3eda707 --- /dev/null +++ b/amdgpu/clearstate_defs.h
@@ -0,0 +1,44 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef CLEARSTATE_DEFS_H +#define CLEARSTATE_DEFS_H + +enum section_id { + SECT_NONE, + SECT_CONTEXT, + SECT_CLEAR, + SECT_CTRLCONST +}; + +struct cs_extent_def { + const unsigned int *extent; + const unsigned int reg_index; + const unsigned int reg_count; +}; + +struct cs_section_def { + const struct cs_extent_def *section; + const enum section_id id; +}; + +#endif
diff --git a/amdgpu/clearstate_gfx10.h b/amdgpu/clearstate_gfx10.h new file mode 100644 index 0000000..27a2ff0 --- /dev/null +++ b/amdgpu/clearstate_gfx10.h
@@ -0,0 +1,975 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const unsigned int gfx10_SECT_CONTEXT_def_1[] = { + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0x00000000, // HOLE + 0x00000000, // DB_DEPTH_SIZE_XY + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0x00000000, // DB_DFSM_CONTROL + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0x00000000, // DB_Z_INFO2 + 0x00000000, // DB_STENCIL_INFO2 + 0x00000000, // DB_Z_READ_BASE_HI + 0x00000000, // DB_STENCIL_READ_BASE_HI + 0x00000000, // DB_Z_WRITE_BASE_HI + 0x00000000, // DB_STENCIL_WRITE_BASE_HI + 0x00000000, // DB_HTILE_DATA_BASE_HI + 0x00150055, // DB_RMI_L2_CACHE_CONTROL + 0x00000000, // TA_BC_BASE_ADDR + 0x00000000, // TA_BC_BASE_ADDR_HI + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_HI_0 + 0x00000000, // COHER_DEST_BASE_HI_1 + 0x00000000, // COHER_DEST_BASE_HI_2 + 0x00000000, // COHER_DEST_BASE_HI_3 + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 + 0x00000000, // PA_SC_RASTER_CONFIG + 0x00000000, // PA_SC_RASTER_CONFIG_1 + 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL +}; +static const unsigned int gfx10_SECT_CONTEXT_def_2[] = { + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_RIGHT_VERT_GRID + 0x00000000, // PA_SC_LEFT_VERT_GRID + 0x00000000, // PA_SC_HORIZ_GRID + 0x00000000, // HOLE + 0x00000000, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0x00550055, // CB_RMI_GL2_CACHE_CONTROL + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0x00000000, // CB_DCC_CONTROL + 0x00000000, // CB_COVERAGE_OUT_CONTROL + 0x00000000, // DB_STENCIL_CONTROL + 0x01000000, // DB_STENCILREFMASK + 0x01000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0x00000000, // PA_CL_PROG_NEAR_CLIP_Z + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_SHADER_IDX_FORMAT + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SX_PS_DOWNCONVERT + 0x00000000, // SX_BLEND_OPT_EPSILON + 0x00000000, // SX_BLEND_OPT_CONTROL + 0x00000000, // SX_MRT0_BLEND_OPT + 0x00000000, // SX_MRT1_BLEND_OPT + 0x00000000, // SX_MRT2_BLEND_OPT + 0x00000000, // SX_MRT3_BLEND_OPT + 0x00000000, // SX_MRT4_BLEND_OPT + 0x00000000, // SX_MRT5_BLEND_OPT + 0x00000000, // SX_MRT6_BLEND_OPT + 0x00000000, // SX_MRT7_BLEND_OPT + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const unsigned int gfx10_SECT_CONTEXT_def_3[] = { + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD +}; +static const unsigned int gfx10_SECT_CONTEXT_def_4[] = { + 0x00000000, // VGT_GS_MAX_PRIMS_PER_SUBGROUP + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0x00000000, // PA_SU_SMALL_PRIM_FILTER_CNTL + 0x00000000, // PA_CL_OBJPRIM_ID_CNTL + 0x00000000, // PA_CL_NGG_CNTL + 0x00000000, // PA_SU_OVER_RASTERIZATION_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0x00000000, // VGT_GS_ONCHIP_CNTL + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const unsigned int gfx10_SECT_CONTEXT_def_5[] = { + 0x00000000, // WD_ENHANCE + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const unsigned int gfx10_SECT_CONTEXT_def_6[] = { + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const unsigned int gfx10_SECT_CONTEXT_def_7[] = { + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0x00000000, // VGT_DRAW_PAYLOAD_CNTL + 0x00000000, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_TESS_DISTRIBUTION + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0x00000000, // VGT_DISPATCH_DRAW_INDEX + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG +}; +static const unsigned int gfx10_SECT_CONTEXT_def_8[] = { + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0x00000000, // PA_SC_SHADER_CONTROL + 0x00000003, // PA_SC_BINNER_CNTL_0 + 0x00000000, // PA_SC_BINNER_CNTL_1 + 0x00100000, // PA_SC_CONSERVATIVE_RASTERIZATION_CNTL + 0x00000000, // PA_SC_NGG_MODE_CNTL + 0, // HOLE + 0x0000001e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000020, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0x00000000, // CB_COLOR0_DCC_CONTROL + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0x00000000, // CB_COLOR0_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0x00000000, // CB_COLOR1_DCC_CONTROL + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0x00000000, // CB_COLOR1_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0x00000000, // CB_COLOR2_DCC_CONTROL + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0x00000000, // CB_COLOR2_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0x00000000, // CB_COLOR3_DCC_CONTROL + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0x00000000, // CB_COLOR3_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0x00000000, // CB_COLOR4_DCC_CONTROL + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0x00000000, // CB_COLOR4_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0x00000000, // CB_COLOR5_DCC_CONTROL + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0x00000000, // CB_COLOR5_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0x00000000, // CB_COLOR6_DCC_CONTROL + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0x00000000, // CB_COLOR6_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0x00000000, // CB_COLOR7_DCC_CONTROL + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 + 0x00000000, // CB_COLOR7_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR0_BASE_EXT + 0x00000000, // CB_COLOR1_BASE_EXT + 0x00000000, // CB_COLOR2_BASE_EXT + 0x00000000, // CB_COLOR3_BASE_EXT + 0x00000000, // CB_COLOR4_BASE_EXT + 0x00000000, // CB_COLOR5_BASE_EXT + 0x00000000, // CB_COLOR6_BASE_EXT + 0x00000000, // CB_COLOR7_BASE_EXT + 0x00000000, // CB_COLOR0_CMASK_BASE_EXT + 0x00000000, // CB_COLOR1_CMASK_BASE_EXT + 0x00000000, // CB_COLOR2_CMASK_BASE_EXT + 0x00000000, // CB_COLOR3_CMASK_BASE_EXT + 0x00000000, // CB_COLOR4_CMASK_BASE_EXT + 0x00000000, // CB_COLOR5_CMASK_BASE_EXT + 0x00000000, // CB_COLOR6_CMASK_BASE_EXT + 0x00000000, // CB_COLOR7_CMASK_BASE_EXT + 0x00000000, // CB_COLOR0_FMASK_BASE_EXT + 0x00000000, // CB_COLOR1_FMASK_BASE_EXT + 0x00000000, // CB_COLOR2_FMASK_BASE_EXT + 0x00000000, // CB_COLOR3_FMASK_BASE_EXT + 0x00000000, // CB_COLOR4_FMASK_BASE_EXT + 0x00000000, // CB_COLOR5_FMASK_BASE_EXT + 0x00000000, // CB_COLOR6_FMASK_BASE_EXT + 0x00000000, // CB_COLOR7_FMASK_BASE_EXT + 0x00000000, // CB_COLOR0_DCC_BASE_EXT + 0x00000000, // CB_COLOR1_DCC_BASE_EXT + 0x00000000, // CB_COLOR2_DCC_BASE_EXT + 0x00000000, // CB_COLOR3_DCC_BASE_EXT + 0x00000000, // CB_COLOR4_DCC_BASE_EXT + 0x00000000, // CB_COLOR5_DCC_BASE_EXT + 0x00000000, // CB_COLOR6_DCC_BASE_EXT + 0x00000000, // CB_COLOR7_DCC_BASE_EXT + 0x00000000, // CB_COLOR0_ATTRIB2 + 0x00000000, // CB_COLOR1_ATTRIB2 + 0x00000000, // CB_COLOR2_ATTRIB2 + 0x00000000, // CB_COLOR3_ATTRIB2 + 0x00000000, // CB_COLOR4_ATTRIB2 + 0x00000000, // CB_COLOR5_ATTRIB2 + 0x00000000, // CB_COLOR6_ATTRIB2 + 0x00000000, // CB_COLOR7_ATTRIB2 + 0x00000000, // CB_COLOR0_ATTRIB3 + 0x00000000, // CB_COLOR1_ATTRIB3 + 0x00000000, // CB_COLOR2_ATTRIB3 + 0x00000000, // CB_COLOR3_ATTRIB3 + 0x00000000, // CB_COLOR4_ATTRIB3 + 0x00000000, // CB_COLOR5_ATTRIB3 + 0x00000000, // CB_COLOR6_ATTRIB3 + 0x00000000, // CB_COLOR7_ATTRIB3 +}; +static const struct cs_extent_def gfx10_SECT_CONTEXT_defs[] = { + {gfx10_SECT_CONTEXT_def_1, 0x0000a000, 215 }, + {gfx10_SECT_CONTEXT_def_2, 0x0000a0d8, 272 }, + {gfx10_SECT_CONTEXT_def_3, 0x0000a1f5, 4 }, + {gfx10_SECT_CONTEXT_def_4, 0x0000a1ff, 158 }, + {gfx10_SECT_CONTEXT_def_5, 0x0000a2a0, 2 }, + {gfx10_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {gfx10_SECT_CONTEXT_def_7, 0x0000a2a5, 66 }, + {gfx10_SECT_CONTEXT_def_8, 0x0000a2f5, 203 }, + { 0, 0, 0 } +}; +static const struct cs_section_def gfx10_cs_data[] = { + { gfx10_SECT_CONTEXT_defs, SECT_CONTEXT }, + { 0, SECT_NONE } +};
diff --git a/amdgpu/clearstate_gfx9.h b/amdgpu/clearstate_gfx9.h new file mode 100644 index 0000000..567a904 --- /dev/null +++ b/amdgpu/clearstate_gfx9.h
@@ -0,0 +1,942 @@ +/* + * Copyright 2017 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const unsigned int gfx9_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0x00000000, // DB_HTILE_DATA_BASE_HI + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_Z_READ_BASE_HI + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE_HI + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_Z_WRITE_BASE_HI + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE_HI + 0x00000000, // DB_DFSM_CONTROL + 0, // HOLE + 0x00000000, // DB_Z_INFO2 + 0x00000000, // DB_STENCIL_INFO2 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0x00000000, // TA_BC_BASE_ADDR_HI + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_HI_0 + 0x00000000, // COHER_DEST_BASE_HI_1 + 0x00000000, // COHER_DEST_BASE_HI_2 + 0x00000000, // COHER_DEST_BASE_HI_3 + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const unsigned int gfx9_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL + 0x00000000, // PA_SC_TILE_STEERING_OVERRIDE + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_RIGHT_VERT_GRID + 0x00000000, // PA_SC_LEFT_VERT_GRID + 0x00000000, // PA_SC_HORIZ_GRID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0x00000000, // CB_DCC_CONTROL + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x01000000, // DB_STENCILREFMASK + 0x01000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SX_PS_DOWNCONVERT + 0x00000000, // SX_BLEND_OPT_EPSILON + 0x00000000, // SX_BLEND_OPT_CONTROL + 0x00000000, // SX_MRT0_BLEND_OPT + 0x00000000, // SX_MRT1_BLEND_OPT + 0x00000000, // SX_MRT2_BLEND_OPT + 0x00000000, // SX_MRT3_BLEND_OPT + 0x00000000, // SX_MRT4_BLEND_OPT + 0x00000000, // SX_MRT5_BLEND_OPT + 0x00000000, // SX_MRT6_BLEND_OPT + 0x00000000, // SX_MRT7_BLEND_OPT + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL + 0x00000000, // CB_MRT0_EPITCH + 0x00000000, // CB_MRT1_EPITCH + 0x00000000, // CB_MRT2_EPITCH + 0x00000000, // CB_MRT3_EPITCH + 0x00000000, // CB_MRT4_EPITCH + 0x00000000, // CB_MRT5_EPITCH + 0x00000000, // CB_MRT6_EPITCH + 0x00000000, // CB_MRT7_EPITCH +}; +static const unsigned int gfx9_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD +}; +static const unsigned int gfx9_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0x00000000, // PA_SU_SMALL_PRIM_FILTER_CNTL + 0x00000000, // PA_CL_OBJPRIM_ID_CNTL + 0x00000000, // PA_CL_NGG_CNTL + 0x00000000, // PA_SU_OVER_RASTERIZATION_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0x00000000, // VGT_GS_ONCHIP_CNTL + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const unsigned int gfx9_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // WD_ENHANCE + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const unsigned int gfx9_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const unsigned int gfx9_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_GS_MAX_PRIMS_PER_SUBGROUP + 0x00000000, // VGT_DRAW_PAYLOAD_CNTL + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0, // HOLE + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_TESS_DISTRIBUTION + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0x00000000, // VGT_DISPATCH_DRAW_INDEX + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG +}; +static const unsigned int gfx9_SECT_CONTEXT_def_8[] = +{ + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0x00000000, // PA_SC_SHADER_CONTROL + 0x00000003, // PA_SC_BINNER_CNTL_0 + 0x00000000, // PA_SC_BINNER_CNTL_1 + 0x00000000, // PA_SC_CONSERVATIVE_RASTERIZATION_CNTL + 0x00000000, // PA_SC_NGG_MODE_CNTL + 0, // HOLE + 0x0000001e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000020, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_BASE_EXT + 0x00000000, // CB_COLOR0_ATTRIB2 + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0x00000000, // CB_COLOR0_DCC_CONTROL + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_BASE_EXT + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_BASE_EXT + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0x00000000, // CB_COLOR0_DCC_BASE + 0x00000000, // CB_COLOR0_DCC_BASE_EXT + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_BASE_EXT + 0x00000000, // CB_COLOR1_ATTRIB2 + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0x00000000, // CB_COLOR1_DCC_CONTROL + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_BASE_EXT + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_BASE_EXT + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0x00000000, // CB_COLOR1_DCC_BASE + 0x00000000, // CB_COLOR1_DCC_BASE_EXT + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_BASE_EXT + 0x00000000, // CB_COLOR2_ATTRIB2 + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0x00000000, // CB_COLOR2_DCC_CONTROL + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_BASE_EXT + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_BASE_EXT + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0x00000000, // CB_COLOR2_DCC_BASE + 0x00000000, // CB_COLOR2_DCC_BASE_EXT + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_BASE_EXT + 0x00000000, // CB_COLOR3_ATTRIB2 + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0x00000000, // CB_COLOR3_DCC_CONTROL + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_BASE_EXT + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_BASE_EXT + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0x00000000, // CB_COLOR3_DCC_BASE + 0x00000000, // CB_COLOR3_DCC_BASE_EXT + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_BASE_EXT + 0x00000000, // CB_COLOR4_ATTRIB2 + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0x00000000, // CB_COLOR4_DCC_CONTROL + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_BASE_EXT + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_BASE_EXT + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0x00000000, // CB_COLOR4_DCC_BASE + 0x00000000, // CB_COLOR4_DCC_BASE_EXT + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_BASE_EXT + 0x00000000, // CB_COLOR5_ATTRIB2 + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0x00000000, // CB_COLOR5_DCC_CONTROL + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_BASE_EXT + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_BASE_EXT + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0x00000000, // CB_COLOR5_DCC_BASE + 0x00000000, // CB_COLOR5_DCC_BASE_EXT + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_BASE_EXT + 0x00000000, // CB_COLOR6_ATTRIB2 + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0x00000000, // CB_COLOR6_DCC_CONTROL + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_BASE_EXT + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_BASE_EXT + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0x00000000, // CB_COLOR6_DCC_BASE + 0x00000000, // CB_COLOR6_DCC_BASE_EXT + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_BASE_EXT + 0x00000000, // CB_COLOR7_ATTRIB2 + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0x00000000, // CB_COLOR7_DCC_CONTROL + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_BASE_EXT + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_BASE_EXT + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 + 0x00000000, // CB_COLOR7_DCC_BASE + 0x00000000, // CB_COLOR7_DCC_BASE_EXT +}; +static const struct cs_extent_def gfx9_SECT_CONTEXT_defs[] = +{ + {gfx9_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {gfx9_SECT_CONTEXT_def_2, 0x0000a0d6, 282 }, + {gfx9_SECT_CONTEXT_def_3, 0x0000a1f5, 4 }, + {gfx9_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {gfx9_SECT_CONTEXT_def_5, 0x0000a2a0, 2 }, + {gfx9_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {gfx9_SECT_CONTEXT_def_7, 0x0000a2a5, 66 }, + {gfx9_SECT_CONTEXT_def_8, 0x0000a2f5, 155 }, + { 0, 0, 0 } +}; +static const struct cs_section_def gfx9_cs_data[] = { + { gfx9_SECT_CONTEXT_defs, SECT_CONTEXT }, + { 0, SECT_NONE } +};
diff --git a/amdgpu/clearstate_si.h b/amdgpu/clearstate_si.h new file mode 100644 index 0000000..66e39cd --- /dev/null +++ b/amdgpu/clearstate_si.h
@@ -0,0 +1,941 @@ +/* + * Copyright 2013 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const u32 si_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const u32 si_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_WAVE_MGMT_1 + 0x00000000, // SPI_WAVE_MGMT_2 + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const u32 si_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const u32 si_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0, // HOLE + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const u32 si_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const u32 si_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const u32 si_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0, // HOLE + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000010, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0, // HOLE + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 +}; +static const struct cs_extent_def si_SECT_CONTEXT_defs[] = +{ + {si_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {si_SECT_CONTEXT_def_2, 0x0000a0d8, 272 }, + {si_SECT_CONTEXT_def_3, 0x0000a1f5, 6 }, + {si_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {si_SECT_CONTEXT_def_5, 0x0000a2a1, 1 }, + {si_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {si_SECT_CONTEXT_def_7, 0x0000a2a5, 233 }, + { NULL, 0, 0 } +}; +static const struct cs_section_def si_cs_data[] = { + { si_SECT_CONTEXT_defs, SECT_CONTEXT }, + { NULL, SECT_NONE } +};
diff --git a/amdgpu/clearstate_vi.h b/amdgpu/clearstate_vi.h new file mode 100644 index 0000000..1aab9be --- /dev/null +++ b/amdgpu/clearstate_vi.h
@@ -0,0 +1,944 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +static const unsigned int vi_SECT_CONTEXT_def_1[] = +{ + 0x00000000, // DB_RENDER_CONTROL + 0x00000000, // DB_COUNT_CONTROL + 0x00000000, // DB_DEPTH_VIEW + 0x00000000, // DB_RENDER_OVERRIDE + 0x00000000, // DB_RENDER_OVERRIDE2 + 0x00000000, // DB_HTILE_DATA_BASE + 0, // HOLE + 0, // HOLE + 0x00000000, // DB_DEPTH_BOUNDS_MIN + 0x00000000, // DB_DEPTH_BOUNDS_MAX + 0x00000000, // DB_STENCIL_CLEAR + 0x00000000, // DB_DEPTH_CLEAR + 0x00000000, // PA_SC_SCREEN_SCISSOR_TL + 0x40004000, // PA_SC_SCREEN_SCISSOR_BR + 0, // HOLE + 0x00000000, // DB_DEPTH_INFO + 0x00000000, // DB_Z_INFO + 0x00000000, // DB_STENCIL_INFO + 0x00000000, // DB_Z_READ_BASE + 0x00000000, // DB_STENCIL_READ_BASE + 0x00000000, // DB_Z_WRITE_BASE + 0x00000000, // DB_STENCIL_WRITE_BASE + 0x00000000, // DB_DEPTH_SIZE + 0x00000000, // DB_DEPTH_SLICE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // TA_BC_BASE_ADDR + 0x00000000, // TA_BC_BASE_ADDR_HI + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // COHER_DEST_BASE_HI_0 + 0x00000000, // COHER_DEST_BASE_HI_1 + 0x00000000, // COHER_DEST_BASE_HI_2 + 0x00000000, // COHER_DEST_BASE_HI_3 + 0x00000000, // COHER_DEST_BASE_2 + 0x00000000, // COHER_DEST_BASE_3 + 0x00000000, // PA_SC_WINDOW_OFFSET + 0x80000000, // PA_SC_WINDOW_SCISSOR_TL + 0x40004000, // PA_SC_WINDOW_SCISSOR_BR + 0x0000ffff, // PA_SC_CLIPRECT_RULE + 0x00000000, // PA_SC_CLIPRECT_0_TL + 0x40004000, // PA_SC_CLIPRECT_0_BR + 0x00000000, // PA_SC_CLIPRECT_1_TL + 0x40004000, // PA_SC_CLIPRECT_1_BR + 0x00000000, // PA_SC_CLIPRECT_2_TL + 0x40004000, // PA_SC_CLIPRECT_2_BR + 0x00000000, // PA_SC_CLIPRECT_3_TL + 0x40004000, // PA_SC_CLIPRECT_3_BR + 0xaa99aaaa, // PA_SC_EDGERULE + 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET + 0xffffffff, // CB_TARGET_MASK + 0xffffffff, // CB_SHADER_MASK + 0x80000000, // PA_SC_GENERIC_SCISSOR_TL + 0x40004000, // PA_SC_GENERIC_SCISSOR_BR + 0x00000000, // COHER_DEST_BASE_0 + 0x00000000, // COHER_DEST_BASE_1 + 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR + 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL + 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR + 0x00000000, // PA_SC_VPORT_ZMIN_0 + 0x3f800000, // PA_SC_VPORT_ZMAX_0 + 0x00000000, // PA_SC_VPORT_ZMIN_1 + 0x3f800000, // PA_SC_VPORT_ZMAX_1 + 0x00000000, // PA_SC_VPORT_ZMIN_2 + 0x3f800000, // PA_SC_VPORT_ZMAX_2 + 0x00000000, // PA_SC_VPORT_ZMIN_3 + 0x3f800000, // PA_SC_VPORT_ZMAX_3 + 0x00000000, // PA_SC_VPORT_ZMIN_4 + 0x3f800000, // PA_SC_VPORT_ZMAX_4 + 0x00000000, // PA_SC_VPORT_ZMIN_5 + 0x3f800000, // PA_SC_VPORT_ZMAX_5 + 0x00000000, // PA_SC_VPORT_ZMIN_6 + 0x3f800000, // PA_SC_VPORT_ZMAX_6 + 0x00000000, // PA_SC_VPORT_ZMIN_7 + 0x3f800000, // PA_SC_VPORT_ZMAX_7 + 0x00000000, // PA_SC_VPORT_ZMIN_8 + 0x3f800000, // PA_SC_VPORT_ZMAX_8 + 0x00000000, // PA_SC_VPORT_ZMIN_9 + 0x3f800000, // PA_SC_VPORT_ZMAX_9 + 0x00000000, // PA_SC_VPORT_ZMIN_10 + 0x3f800000, // PA_SC_VPORT_ZMAX_10 + 0x00000000, // PA_SC_VPORT_ZMIN_11 + 0x3f800000, // PA_SC_VPORT_ZMAX_11 + 0x00000000, // PA_SC_VPORT_ZMIN_12 + 0x3f800000, // PA_SC_VPORT_ZMAX_12 + 0x00000000, // PA_SC_VPORT_ZMIN_13 + 0x3f800000, // PA_SC_VPORT_ZMAX_13 + 0x00000000, // PA_SC_VPORT_ZMIN_14 + 0x3f800000, // PA_SC_VPORT_ZMAX_14 + 0x00000000, // PA_SC_VPORT_ZMIN_15 + 0x3f800000, // PA_SC_VPORT_ZMAX_15 +}; +static const unsigned int vi_SECT_CONTEXT_def_2[] = +{ + 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL + 0, // HOLE + 0x00000000, // CP_PERFMON_CNTX_CNTL + 0x00000000, // CP_RINGID + 0x00000000, // CP_VMID + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0xffffffff, // VGT_MAX_VTX_INDX + 0x00000000, // VGT_MIN_VTX_INDX + 0x00000000, // VGT_INDX_OFFSET + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX + 0, // HOLE + 0x00000000, // CB_BLEND_RED + 0x00000000, // CB_BLEND_GREEN + 0x00000000, // CB_BLEND_BLUE + 0x00000000, // CB_BLEND_ALPHA + 0x00000000, // CB_DCC_CONTROL + 0, // HOLE + 0x00000000, // DB_STENCIL_CONTROL + 0x00000000, // DB_STENCILREFMASK + 0x00000000, // DB_STENCILREFMASK_BF + 0, // HOLE + 0x00000000, // PA_CL_VPORT_XSCALE + 0x00000000, // PA_CL_VPORT_XOFFSET + 0x00000000, // PA_CL_VPORT_YSCALE + 0x00000000, // PA_CL_VPORT_YOFFSET + 0x00000000, // PA_CL_VPORT_ZSCALE + 0x00000000, // PA_CL_VPORT_ZOFFSET + 0x00000000, // PA_CL_VPORT_XSCALE_1 + 0x00000000, // PA_CL_VPORT_XOFFSET_1 + 0x00000000, // PA_CL_VPORT_YSCALE_1 + 0x00000000, // PA_CL_VPORT_YOFFSET_1 + 0x00000000, // PA_CL_VPORT_ZSCALE_1 + 0x00000000, // PA_CL_VPORT_ZOFFSET_1 + 0x00000000, // PA_CL_VPORT_XSCALE_2 + 0x00000000, // PA_CL_VPORT_XOFFSET_2 + 0x00000000, // PA_CL_VPORT_YSCALE_2 + 0x00000000, // PA_CL_VPORT_YOFFSET_2 + 0x00000000, // PA_CL_VPORT_ZSCALE_2 + 0x00000000, // PA_CL_VPORT_ZOFFSET_2 + 0x00000000, // PA_CL_VPORT_XSCALE_3 + 0x00000000, // PA_CL_VPORT_XOFFSET_3 + 0x00000000, // PA_CL_VPORT_YSCALE_3 + 0x00000000, // PA_CL_VPORT_YOFFSET_3 + 0x00000000, // PA_CL_VPORT_ZSCALE_3 + 0x00000000, // PA_CL_VPORT_ZOFFSET_3 + 0x00000000, // PA_CL_VPORT_XSCALE_4 + 0x00000000, // PA_CL_VPORT_XOFFSET_4 + 0x00000000, // PA_CL_VPORT_YSCALE_4 + 0x00000000, // PA_CL_VPORT_YOFFSET_4 + 0x00000000, // PA_CL_VPORT_ZSCALE_4 + 0x00000000, // PA_CL_VPORT_ZOFFSET_4 + 0x00000000, // PA_CL_VPORT_XSCALE_5 + 0x00000000, // PA_CL_VPORT_XOFFSET_5 + 0x00000000, // PA_CL_VPORT_YSCALE_5 + 0x00000000, // PA_CL_VPORT_YOFFSET_5 + 0x00000000, // PA_CL_VPORT_ZSCALE_5 + 0x00000000, // PA_CL_VPORT_ZOFFSET_5 + 0x00000000, // PA_CL_VPORT_XSCALE_6 + 0x00000000, // PA_CL_VPORT_XOFFSET_6 + 0x00000000, // PA_CL_VPORT_YSCALE_6 + 0x00000000, // PA_CL_VPORT_YOFFSET_6 + 0x00000000, // PA_CL_VPORT_ZSCALE_6 + 0x00000000, // PA_CL_VPORT_ZOFFSET_6 + 0x00000000, // PA_CL_VPORT_XSCALE_7 + 0x00000000, // PA_CL_VPORT_XOFFSET_7 + 0x00000000, // PA_CL_VPORT_YSCALE_7 + 0x00000000, // PA_CL_VPORT_YOFFSET_7 + 0x00000000, // PA_CL_VPORT_ZSCALE_7 + 0x00000000, // PA_CL_VPORT_ZOFFSET_7 + 0x00000000, // PA_CL_VPORT_XSCALE_8 + 0x00000000, // PA_CL_VPORT_XOFFSET_8 + 0x00000000, // PA_CL_VPORT_YSCALE_8 + 0x00000000, // PA_CL_VPORT_YOFFSET_8 + 0x00000000, // PA_CL_VPORT_ZSCALE_8 + 0x00000000, // PA_CL_VPORT_ZOFFSET_8 + 0x00000000, // PA_CL_VPORT_XSCALE_9 + 0x00000000, // PA_CL_VPORT_XOFFSET_9 + 0x00000000, // PA_CL_VPORT_YSCALE_9 + 0x00000000, // PA_CL_VPORT_YOFFSET_9 + 0x00000000, // PA_CL_VPORT_ZSCALE_9 + 0x00000000, // PA_CL_VPORT_ZOFFSET_9 + 0x00000000, // PA_CL_VPORT_XSCALE_10 + 0x00000000, // PA_CL_VPORT_XOFFSET_10 + 0x00000000, // PA_CL_VPORT_YSCALE_10 + 0x00000000, // PA_CL_VPORT_YOFFSET_10 + 0x00000000, // PA_CL_VPORT_ZSCALE_10 + 0x00000000, // PA_CL_VPORT_ZOFFSET_10 + 0x00000000, // PA_CL_VPORT_XSCALE_11 + 0x00000000, // PA_CL_VPORT_XOFFSET_11 + 0x00000000, // PA_CL_VPORT_YSCALE_11 + 0x00000000, // PA_CL_VPORT_YOFFSET_11 + 0x00000000, // PA_CL_VPORT_ZSCALE_11 + 0x00000000, // PA_CL_VPORT_ZOFFSET_11 + 0x00000000, // PA_CL_VPORT_XSCALE_12 + 0x00000000, // PA_CL_VPORT_XOFFSET_12 + 0x00000000, // PA_CL_VPORT_YSCALE_12 + 0x00000000, // PA_CL_VPORT_YOFFSET_12 + 0x00000000, // PA_CL_VPORT_ZSCALE_12 + 0x00000000, // PA_CL_VPORT_ZOFFSET_12 + 0x00000000, // PA_CL_VPORT_XSCALE_13 + 0x00000000, // PA_CL_VPORT_XOFFSET_13 + 0x00000000, // PA_CL_VPORT_YSCALE_13 + 0x00000000, // PA_CL_VPORT_YOFFSET_13 + 0x00000000, // PA_CL_VPORT_ZSCALE_13 + 0x00000000, // PA_CL_VPORT_ZOFFSET_13 + 0x00000000, // PA_CL_VPORT_XSCALE_14 + 0x00000000, // PA_CL_VPORT_XOFFSET_14 + 0x00000000, // PA_CL_VPORT_YSCALE_14 + 0x00000000, // PA_CL_VPORT_YOFFSET_14 + 0x00000000, // PA_CL_VPORT_ZSCALE_14 + 0x00000000, // PA_CL_VPORT_ZOFFSET_14 + 0x00000000, // PA_CL_VPORT_XSCALE_15 + 0x00000000, // PA_CL_VPORT_XOFFSET_15 + 0x00000000, // PA_CL_VPORT_YSCALE_15 + 0x00000000, // PA_CL_VPORT_YOFFSET_15 + 0x00000000, // PA_CL_VPORT_ZSCALE_15 + 0x00000000, // PA_CL_VPORT_ZOFFSET_15 + 0x00000000, // PA_CL_UCP_0_X + 0x00000000, // PA_CL_UCP_0_Y + 0x00000000, // PA_CL_UCP_0_Z + 0x00000000, // PA_CL_UCP_0_W + 0x00000000, // PA_CL_UCP_1_X + 0x00000000, // PA_CL_UCP_1_Y + 0x00000000, // PA_CL_UCP_1_Z + 0x00000000, // PA_CL_UCP_1_W + 0x00000000, // PA_CL_UCP_2_X + 0x00000000, // PA_CL_UCP_2_Y + 0x00000000, // PA_CL_UCP_2_Z + 0x00000000, // PA_CL_UCP_2_W + 0x00000000, // PA_CL_UCP_3_X + 0x00000000, // PA_CL_UCP_3_Y + 0x00000000, // PA_CL_UCP_3_Z + 0x00000000, // PA_CL_UCP_3_W + 0x00000000, // PA_CL_UCP_4_X + 0x00000000, // PA_CL_UCP_4_Y + 0x00000000, // PA_CL_UCP_4_Z + 0x00000000, // PA_CL_UCP_4_W + 0x00000000, // PA_CL_UCP_5_X + 0x00000000, // PA_CL_UCP_5_Y + 0x00000000, // PA_CL_UCP_5_Z + 0x00000000, // PA_CL_UCP_5_W + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_CNTL_0 + 0x00000000, // SPI_PS_INPUT_CNTL_1 + 0x00000000, // SPI_PS_INPUT_CNTL_2 + 0x00000000, // SPI_PS_INPUT_CNTL_3 + 0x00000000, // SPI_PS_INPUT_CNTL_4 + 0x00000000, // SPI_PS_INPUT_CNTL_5 + 0x00000000, // SPI_PS_INPUT_CNTL_6 + 0x00000000, // SPI_PS_INPUT_CNTL_7 + 0x00000000, // SPI_PS_INPUT_CNTL_8 + 0x00000000, // SPI_PS_INPUT_CNTL_9 + 0x00000000, // SPI_PS_INPUT_CNTL_10 + 0x00000000, // SPI_PS_INPUT_CNTL_11 + 0x00000000, // SPI_PS_INPUT_CNTL_12 + 0x00000000, // SPI_PS_INPUT_CNTL_13 + 0x00000000, // SPI_PS_INPUT_CNTL_14 + 0x00000000, // SPI_PS_INPUT_CNTL_15 + 0x00000000, // SPI_PS_INPUT_CNTL_16 + 0x00000000, // SPI_PS_INPUT_CNTL_17 + 0x00000000, // SPI_PS_INPUT_CNTL_18 + 0x00000000, // SPI_PS_INPUT_CNTL_19 + 0x00000000, // SPI_PS_INPUT_CNTL_20 + 0x00000000, // SPI_PS_INPUT_CNTL_21 + 0x00000000, // SPI_PS_INPUT_CNTL_22 + 0x00000000, // SPI_PS_INPUT_CNTL_23 + 0x00000000, // SPI_PS_INPUT_CNTL_24 + 0x00000000, // SPI_PS_INPUT_CNTL_25 + 0x00000000, // SPI_PS_INPUT_CNTL_26 + 0x00000000, // SPI_PS_INPUT_CNTL_27 + 0x00000000, // SPI_PS_INPUT_CNTL_28 + 0x00000000, // SPI_PS_INPUT_CNTL_29 + 0x00000000, // SPI_PS_INPUT_CNTL_30 + 0x00000000, // SPI_PS_INPUT_CNTL_31 + 0x00000000, // SPI_VS_OUT_CONFIG + 0, // HOLE + 0x00000000, // SPI_PS_INPUT_ENA + 0x00000000, // SPI_PS_INPUT_ADDR + 0x00000000, // SPI_INTERP_CONTROL_0 + 0x00000002, // SPI_PS_IN_CONTROL + 0, // HOLE + 0x00000000, // SPI_BARYC_CNTL + 0, // HOLE + 0x00000000, // SPI_TMPRING_SIZE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // SPI_SHADER_POS_FORMAT + 0x00000000, // SPI_SHADER_Z_FORMAT + 0x00000000, // SPI_SHADER_COL_FORMAT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // CB_BLEND0_CONTROL + 0x00000000, // CB_BLEND1_CONTROL + 0x00000000, // CB_BLEND2_CONTROL + 0x00000000, // CB_BLEND3_CONTROL + 0x00000000, // CB_BLEND4_CONTROL + 0x00000000, // CB_BLEND5_CONTROL + 0x00000000, // CB_BLEND6_CONTROL + 0x00000000, // CB_BLEND7_CONTROL +}; +static const unsigned int vi_SECT_CONTEXT_def_3[] = +{ + 0x00000000, // PA_CL_POINT_X_RAD + 0x00000000, // PA_CL_POINT_Y_RAD + 0x00000000, // PA_CL_POINT_SIZE + 0x00000000, // PA_CL_POINT_CULL_RAD + 0x00000000, // VGT_DMA_BASE_HI + 0x00000000, // VGT_DMA_BASE +}; +static const unsigned int vi_SECT_CONTEXT_def_4[] = +{ + 0x00000000, // DB_DEPTH_CONTROL + 0x00000000, // DB_EQAA + 0x00000000, // CB_COLOR_CONTROL + 0x00000000, // DB_SHADER_CONTROL + 0x00090000, // PA_CL_CLIP_CNTL + 0x00000004, // PA_SU_SC_MODE_CNTL + 0x00000000, // PA_CL_VTE_CNTL + 0x00000000, // PA_CL_VS_OUT_CNTL + 0x00000000, // PA_CL_NANINF_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_CNTL + 0x00000000, // PA_SU_LINE_STIPPLE_SCALE + 0x00000000, // PA_SU_PRIM_FILTER_CNTL + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SU_POINT_SIZE + 0x00000000, // PA_SU_POINT_MINMAX + 0x00000000, // PA_SU_LINE_CNTL + 0x00000000, // PA_SC_LINE_STIPPLE + 0x00000000, // VGT_OUTPUT_PATH_CNTL + 0x00000000, // VGT_HOS_CNTL + 0x00000000, // VGT_HOS_MAX_TESS_LEVEL + 0x00000000, // VGT_HOS_MIN_TESS_LEVEL + 0x00000000, // VGT_HOS_REUSE_DEPTH + 0x00000000, // VGT_GROUP_PRIM_TYPE + 0x00000000, // VGT_GROUP_FIRST_DECR + 0x00000000, // VGT_GROUP_DECR + 0x00000000, // VGT_GROUP_VECT_0_CNTL + 0x00000000, // VGT_GROUP_VECT_1_CNTL + 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL + 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL + 0x00000000, // VGT_GS_MODE + 0x00000000, // VGT_GS_ONCHIP_CNTL + 0x00000000, // PA_SC_MODE_CNTL_0 + 0x00000000, // PA_SC_MODE_CNTL_1 + 0x00000000, // VGT_ENHANCE + 0x00000100, // VGT_GS_PER_ES + 0x00000080, // VGT_ES_PER_GS + 0x00000002, // VGT_GS_PER_VS + 0x00000000, // VGT_GSVS_RING_OFFSET_1 + 0x00000000, // VGT_GSVS_RING_OFFSET_2 + 0x00000000, // VGT_GSVS_RING_OFFSET_3 + 0x00000000, // VGT_GS_OUT_PRIM_TYPE + 0x00000000, // IA_ENHANCE +}; +static const unsigned int vi_SECT_CONTEXT_def_5[] = +{ + 0x00000000, // WD_ENHANCE + 0x00000000, // VGT_PRIMITIVEID_EN +}; +static const unsigned int vi_SECT_CONTEXT_def_6[] = +{ + 0x00000000, // VGT_PRIMITIVEID_RESET +}; +static const unsigned int vi_SECT_CONTEXT_def_7[] = +{ + 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_INSTANCE_STEP_RATE_0 + 0x00000000, // VGT_INSTANCE_STEP_RATE_1 + 0x000000ff, // IA_MULTI_VGT_PARAM + 0x00000000, // VGT_ESGS_RING_ITEMSIZE + 0x00000000, // VGT_GSVS_RING_ITEMSIZE + 0x00000000, // VGT_REUSE_OFF + 0x00000000, // VGT_VTX_CNT_EN + 0x00000000, // DB_HTILE_SURFACE + 0x00000000, // DB_SRESULTS_COMPARE_STATE0 + 0x00000000, // DB_SRESULTS_COMPARE_STATE1 + 0x00000000, // DB_PRELOAD_CONTROL + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2 + 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3 + 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3 + 0, // HOLE + 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE + 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE + 0, // HOLE + 0x00000000, // VGT_GS_MAX_VERT_OUT + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // VGT_TESS_DISTRIBUTION + 0x00000000, // VGT_SHADER_STAGES_EN + 0x00000000, // VGT_LS_HS_CONFIG + 0x00000000, // VGT_GS_VERT_ITEMSIZE + 0x00000000, // VGT_GS_VERT_ITEMSIZE_1 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_2 + 0x00000000, // VGT_GS_VERT_ITEMSIZE_3 + 0x00000000, // VGT_TF_PARAM + 0x00000000, // DB_ALPHA_TO_MASK + 0x00000000, // VGT_DISPATCH_DRAW_INDEX + 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL + 0x00000000, // PA_SU_POLY_OFFSET_CLAMP + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET + 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE + 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET + 0x00000000, // VGT_GS_INSTANCE_CNT + 0x00000000, // VGT_STRMOUT_CONFIG + 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x00000000, // PA_SC_CENTROID_PRIORITY_0 + 0x00000000, // PA_SC_CENTROID_PRIORITY_1 + 0x00001000, // PA_SC_LINE_CNTL + 0x00000000, // PA_SC_AA_CONFIG + 0x00000005, // PA_SU_VTX_CNTL + 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ + 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ + 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ + 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 + 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 + 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0 + 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1 + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0, // HOLE + 0x0000001e, // VGT_VERTEX_REUSE_BLOCK_CNTL + 0x00000020, // VGT_OUT_DEALLOC_CNTL + 0x00000000, // CB_COLOR0_BASE + 0x00000000, // CB_COLOR0_PITCH + 0x00000000, // CB_COLOR0_SLICE + 0x00000000, // CB_COLOR0_VIEW + 0x00000000, // CB_COLOR0_INFO + 0x00000000, // CB_COLOR0_ATTRIB + 0x00000000, // CB_COLOR0_DCC_CONTROL + 0x00000000, // CB_COLOR0_CMASK + 0x00000000, // CB_COLOR0_CMASK_SLICE + 0x00000000, // CB_COLOR0_FMASK + 0x00000000, // CB_COLOR0_FMASK_SLICE + 0x00000000, // CB_COLOR0_CLEAR_WORD0 + 0x00000000, // CB_COLOR0_CLEAR_WORD1 + 0x00000000, // CB_COLOR0_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR1_BASE + 0x00000000, // CB_COLOR1_PITCH + 0x00000000, // CB_COLOR1_SLICE + 0x00000000, // CB_COLOR1_VIEW + 0x00000000, // CB_COLOR1_INFO + 0x00000000, // CB_COLOR1_ATTRIB + 0x00000000, // CB_COLOR1_DCC_CONTROL + 0x00000000, // CB_COLOR1_CMASK + 0x00000000, // CB_COLOR1_CMASK_SLICE + 0x00000000, // CB_COLOR1_FMASK + 0x00000000, // CB_COLOR1_FMASK_SLICE + 0x00000000, // CB_COLOR1_CLEAR_WORD0 + 0x00000000, // CB_COLOR1_CLEAR_WORD1 + 0x00000000, // CB_COLOR1_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR2_BASE + 0x00000000, // CB_COLOR2_PITCH + 0x00000000, // CB_COLOR2_SLICE + 0x00000000, // CB_COLOR2_VIEW + 0x00000000, // CB_COLOR2_INFO + 0x00000000, // CB_COLOR2_ATTRIB + 0x00000000, // CB_COLOR2_DCC_CONTROL + 0x00000000, // CB_COLOR2_CMASK + 0x00000000, // CB_COLOR2_CMASK_SLICE + 0x00000000, // CB_COLOR2_FMASK + 0x00000000, // CB_COLOR2_FMASK_SLICE + 0x00000000, // CB_COLOR2_CLEAR_WORD0 + 0x00000000, // CB_COLOR2_CLEAR_WORD1 + 0x00000000, // CB_COLOR2_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR3_BASE + 0x00000000, // CB_COLOR3_PITCH + 0x00000000, // CB_COLOR3_SLICE + 0x00000000, // CB_COLOR3_VIEW + 0x00000000, // CB_COLOR3_INFO + 0x00000000, // CB_COLOR3_ATTRIB + 0x00000000, // CB_COLOR3_DCC_CONTROL + 0x00000000, // CB_COLOR3_CMASK + 0x00000000, // CB_COLOR3_CMASK_SLICE + 0x00000000, // CB_COLOR3_FMASK + 0x00000000, // CB_COLOR3_FMASK_SLICE + 0x00000000, // CB_COLOR3_CLEAR_WORD0 + 0x00000000, // CB_COLOR3_CLEAR_WORD1 + 0x00000000, // CB_COLOR3_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR4_BASE + 0x00000000, // CB_COLOR4_PITCH + 0x00000000, // CB_COLOR4_SLICE + 0x00000000, // CB_COLOR4_VIEW + 0x00000000, // CB_COLOR4_INFO + 0x00000000, // CB_COLOR4_ATTRIB + 0x00000000, // CB_COLOR4_DCC_CONTROL + 0x00000000, // CB_COLOR4_CMASK + 0x00000000, // CB_COLOR4_CMASK_SLICE + 0x00000000, // CB_COLOR4_FMASK + 0x00000000, // CB_COLOR4_FMASK_SLICE + 0x00000000, // CB_COLOR4_CLEAR_WORD0 + 0x00000000, // CB_COLOR4_CLEAR_WORD1 + 0x00000000, // CB_COLOR4_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR5_BASE + 0x00000000, // CB_COLOR5_PITCH + 0x00000000, // CB_COLOR5_SLICE + 0x00000000, // CB_COLOR5_VIEW + 0x00000000, // CB_COLOR5_INFO + 0x00000000, // CB_COLOR5_ATTRIB + 0x00000000, // CB_COLOR5_DCC_CONTROL + 0x00000000, // CB_COLOR5_CMASK + 0x00000000, // CB_COLOR5_CMASK_SLICE + 0x00000000, // CB_COLOR5_FMASK + 0x00000000, // CB_COLOR5_FMASK_SLICE + 0x00000000, // CB_COLOR5_CLEAR_WORD0 + 0x00000000, // CB_COLOR5_CLEAR_WORD1 + 0x00000000, // CB_COLOR5_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR6_BASE + 0x00000000, // CB_COLOR6_PITCH + 0x00000000, // CB_COLOR6_SLICE + 0x00000000, // CB_COLOR6_VIEW + 0x00000000, // CB_COLOR6_INFO + 0x00000000, // CB_COLOR6_ATTRIB + 0x00000000, // CB_COLOR6_DCC_CONTROL + 0x00000000, // CB_COLOR6_CMASK + 0x00000000, // CB_COLOR6_CMASK_SLICE + 0x00000000, // CB_COLOR6_FMASK + 0x00000000, // CB_COLOR6_FMASK_SLICE + 0x00000000, // CB_COLOR6_CLEAR_WORD0 + 0x00000000, // CB_COLOR6_CLEAR_WORD1 + 0x00000000, // CB_COLOR6_DCC_BASE + 0, // HOLE + 0x00000000, // CB_COLOR7_BASE + 0x00000000, // CB_COLOR7_PITCH + 0x00000000, // CB_COLOR7_SLICE + 0x00000000, // CB_COLOR7_VIEW + 0x00000000, // CB_COLOR7_INFO + 0x00000000, // CB_COLOR7_ATTRIB + 0x00000000, // CB_COLOR7_DCC_CONTROL + 0x00000000, // CB_COLOR7_CMASK + 0x00000000, // CB_COLOR7_CMASK_SLICE + 0x00000000, // CB_COLOR7_FMASK + 0x00000000, // CB_COLOR7_FMASK_SLICE + 0x00000000, // CB_COLOR7_CLEAR_WORD0 + 0x00000000, // CB_COLOR7_CLEAR_WORD1 +}; +static const struct cs_extent_def vi_SECT_CONTEXT_defs[] = +{ + {vi_SECT_CONTEXT_def_1, 0x0000a000, 212 }, + {vi_SECT_CONTEXT_def_2, 0x0000a0d6, 274 }, + {vi_SECT_CONTEXT_def_3, 0x0000a1f5, 6 }, + {vi_SECT_CONTEXT_def_4, 0x0000a200, 157 }, + {vi_SECT_CONTEXT_def_5, 0x0000a2a0, 2 }, + {vi_SECT_CONTEXT_def_6, 0x0000a2a3, 1 }, + {vi_SECT_CONTEXT_def_7, 0x0000a2a5, 233 }, + { 0, 0, 0 } +}; +static const struct cs_section_def vi_cs_data[] = { + { vi_SECT_CONTEXT_defs, SECT_CONTEXT }, + { 0, SECT_NONE } +};
diff --git a/amdgpu/cz_ih.c b/amdgpu/cz_ih.c new file mode 100644 index 0000000..1dca0ca --- /dev/null +++ b/amdgpu/cz_ih.c
@@ -0,0 +1,437 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include <linux/pci.h> + +#include "amdgpu.h" +#include "amdgpu_ih.h" +#include "vid.h" + +#include "oss/oss_3_0_1_d.h" +#include "oss/oss_3_0_1_sh_mask.h" + +#include "bif/bif_5_1_d.h" +#include "bif/bif_5_1_sh_mask.h" + +/* + * Interrupts + * Starting with r6xx, interrupts are handled via a ring buffer. + * Ring buffers are areas of GPU accessible memory that the GPU + * writes interrupt vectors into and the host reads vectors out of. + * There is a rptr (read pointer) that determines where the + * host is currently reading, and a wptr (write pointer) + * which determines where the GPU has written. When the + * pointers are equal, the ring is idle. When the GPU + * writes vectors to the ring buffer, it increments the + * wptr. When there is an interrupt, the host then starts + * fetching commands and processing them until the pointers are + * equal again at which point it updates the rptr. + */ + +static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev); + +/** + * cz_ih_enable_interrupts - Enable the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * + * Enable the interrupt ring buffer (VI). + */ +static void cz_ih_enable_interrupts(struct amdgpu_device *adev) +{ + u32 ih_cntl = RREG32(mmIH_CNTL); + u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); + + ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); + WREG32(mmIH_CNTL, ih_cntl); + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + adev->irq.ih.enabled = true; +} + +/** + * cz_ih_disable_interrupts - Disable the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * + * Disable the interrupt ring buffer (VI). + */ +static void cz_ih_disable_interrupts(struct amdgpu_device *adev) +{ + u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); + u32 ih_cntl = RREG32(mmIH_CNTL); + + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); + ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0); + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + WREG32(mmIH_CNTL, ih_cntl); + /* set rptr, wptr to 0 */ + WREG32(mmIH_RB_RPTR, 0); + WREG32(mmIH_RB_WPTR, 0); + adev->irq.ih.enabled = false; + adev->irq.ih.rptr = 0; +} + +/** + * cz_ih_irq_init - init and enable the interrupt ring + * + * @adev: amdgpu_device pointer + * + * Allocate a ring buffer for the interrupt controller, + * enable the RLC, disable interrupts, enable the IH + * ring buffer and enable it (VI). + * Called at device load and reume. + * Returns 0 for success, errors for failure. + */ +static int cz_ih_irq_init(struct amdgpu_device *adev) +{ + struct amdgpu_ih_ring *ih = &adev->irq.ih; + u32 interrupt_cntl, ih_cntl, ih_rb_cntl; + int rb_bufsz; + + /* disable irqs */ + cz_ih_disable_interrupts(adev); + + /* setup interrupt control */ + WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8); + interrupt_cntl = RREG32(mmINTERRUPT_CNTL); + /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi + * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN + */ + interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0); + /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ + interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0); + WREG32(mmINTERRUPT_CNTL, interrupt_cntl); + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); + + rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); + ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); + + /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */ + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1); + + /* set the writeback address whether it's enabled or not */ + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); + + WREG32(mmIH_RB_CNTL, ih_rb_cntl); + + /* set rptr, wptr to 0 */ + WREG32(mmIH_RB_RPTR, 0); + WREG32(mmIH_RB_WPTR, 0); + + /* Default settings for IH_CNTL (disabled at first) */ + ih_cntl = RREG32(mmIH_CNTL); + ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0); + + if (adev->irq.msi_enabled) + ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1); + WREG32(mmIH_CNTL, ih_cntl); + + pci_set_master(adev->pdev); + + /* enable interrupts */ + cz_ih_enable_interrupts(adev); + + return 0; +} + +/** + * cz_ih_irq_disable - disable interrupts + * + * @adev: amdgpu_device pointer + * + * Disable interrupts on the hw (VI). + */ +static void cz_ih_irq_disable(struct amdgpu_device *adev) +{ + cz_ih_disable_interrupts(adev); + + /* Wait and