diff --git a/drivers/Kconfig b/drivers/Kconfig index 645652c7a728ccce886bcd6b8f9d5886fe3a12be..bb4a932aad1e6f621093386ae890138627376c68 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -223,4 +223,6 @@ source "drivers/esoc/Kconfig" source "drivers/sensors/Kconfig" +source "drivers/gpu/msm/Kconfig" + endmenu diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index e9ed439a5b65329c33f450a1c82e78290581783a..febd265db7c823f39fcb1f106265e9ed4adfdc4f 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -1,6 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 # drm/tegra depends on host1x, so if both drivers are built-in care must be # taken to initialize them in the correct order. Link order is the only way # to ensure this currently. obj-$(CONFIG_TEGRA_HOST1X) += host1x/ obj-y += drm/ vga/ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ +obj-$(CONFIG_QCOM_KGSL) += msm/ diff --git a/drivers/gpu/msm/Kconfig b/drivers/gpu/msm/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..9cabb1c1eb014077fd43dabaa5b7c9e241a65aef --- /dev/null +++ b/drivers/gpu/msm/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0 +config QCOM_KGSL + tristate "Qualcomm Technologies, Inc. 3D Graphics driver" + depends on ARCH_QCOM + select GENERIC_ALLOCATOR + select FW_LOADER + select PM_DEVFREQ + select DEVFREQ_GOV_SIMPLE_ONDEMAND + select DEVFREQ_GOV_PERFORMANCE + select DEVFREQ_GOV_QCOM_ADRENO_TZ + select DEVFREQ_GOV_QCOM_GPUBW_MON + help + 3D graphics driver for the Adreno family of GPUs from QTI. + Required to use hardware accelerated OpenGL, compute and Vulkan + on QTI targets. This includes power management, memory management, + and scheduling for the Adreno GPUs. + +config QCOM_ADRENO_DEFAULT_GOVERNOR + string "devfreq governor for the adreno core" + default "msm-adreno-tz" if DEVFREQ_GOV_QCOM_ADRENO_TZ + default "simple_ondemand" + depends on QCOM_KGSL + +config QCOM_KGSL_IOMMU + bool + default y if QCOM_KGSL && (MSM_IOMMU || ARM_SMMU) diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..36ea2c2d8aa7935cf71edb3974fe0bf9f5ad0911 --- /dev/null +++ b/drivers/gpu/msm/Makefile @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0 +ccflags-y += -I$(src) + +msm_kgsl_core-y = \ + kgsl.o \ + kgsl_trace.o \ + kgsl_drawobj.o \ + kgsl_ioctl.o \ + kgsl_sharedmem.o \ + kgsl_pwrctrl.o \ + kgsl_pwrscale.o \ + kgsl_mmu.o \ + kgsl_snapshot.o \ + kgsl_events.o \ + kgsl_pool.o \ + kgsl_gmu_core.o \ + kgsl_gmu.o \ + kgsl_rgmu.o \ + kgsl_hfi.o + +msm_kgsl_core-$(CONFIG_QCOM_KGSL_IOMMU) += kgsl_iommu.o +msm_kgsl_core-$(CONFIG_DEBUG_FS) += kgsl_debugfs.o +msm_kgsl_core-$(CONFIG_SYNC_FILE) += kgsl_sync.o +msm_kgsl_core-$(CONFIG_COMPAT) += kgsl_compat.o + +msm_adreno-y += \ + adreno_ioctl.o \ + adreno_ringbuffer.o \ + adreno_drawctxt.o \ + adreno_dispatch.o \ + adreno_snapshot.o \ + adreno_coresight.o \ + adreno_trace.o \ + adreno_a3xx.o \ + adreno_a4xx.o \ + adreno_a5xx.o \ + adreno_a6xx.o \ + adreno_a3xx_snapshot.o \ + adreno_a4xx_snapshot.o \ + adreno_a5xx_snapshot.o \ + adreno_a6xx_snapshot.o \ + adreno_a4xx_preempt.o \ + adreno_a5xx_preempt.o \ + adreno_a6xx_preempt.o \ + adreno_a6xx_gmu.o \ + adreno_a6xx_rgmu.o \ + adreno_sysfs.o \ + adreno.o \ + adreno_cp_parser.o \ + adreno_perfcounter.o + +msm_adreno-$(CONFIG_QCOM_KGSL_IOMMU) += adreno_iommu.o +msm_adreno-$(CONFIG_DEBUG_FS) += adreno_debugfs.o adreno_profile.o +msm_adreno-$(CONFIG_COMPAT) += adreno_compat.o + +msm_kgsl_core-objs = $(msm_kgsl_core-y) +msm_adreno-objs = $(msm_adreno-y) + +obj-$(CONFIG_QCOM_KGSL) += msm_kgsl_core.o +obj-$(CONFIG_QCOM_KGSL) += msm_adreno.o diff --git a/drivers/gpu/msm/a3xx_reg.h b/drivers/gpu/msm/a3xx_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..b1f8fb3a6ac9d381bfc5f57739c3e59398fae8b6 --- /dev/null +++ b/drivers/gpu/msm/a3xx_reg.h @@ -0,0 +1,567 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2012-2017, The Linux Foundation. All rights reserved. + */ + +#ifndef _A300_REG_H +#define _A300_REG_H + +/* Interrupt bit positions within RBBM_INT_0 */ + +#define A3XX_INT_RBBM_GPU_IDLE 0 +#define A3XX_INT_RBBM_AHB_ERROR 1 +#define A3XX_INT_RBBM_REG_TIMEOUT 2 +#define A3XX_INT_RBBM_ME_MS_TIMEOUT 3 +#define A3XX_INT_RBBM_PFP_MS_TIMEOUT 4 +#define A3XX_INT_RBBM_ATB_BUS_OVERFLOW 5 +#define A3XX_INT_VFD_ERROR 6 +#define A3XX_INT_CP_SW_INT 7 +#define A3XX_INT_CP_T0_PACKET_IN_IB 8 +#define A3XX_INT_CP_OPCODE_ERROR 9 +#define A3XX_INT_CP_RESERVED_BIT_ERROR 10 +#define A3XX_INT_CP_HW_FAULT 11 +#define A3XX_INT_CP_DMA 12 +#define A3XX_INT_CP_IB2_INT 13 +#define A3XX_INT_CP_IB1_INT 14 +#define A3XX_INT_CP_RB_INT 15 +#define A3XX_INT_CP_REG_PROTECT_FAULT 16 +#define A3XX_INT_CP_RB_DONE_TS 17 +#define A3XX_INT_CP_VS_DONE_TS 18 +#define A3XX_INT_CP_PS_DONE_TS 19 +#define A3XX_INT_CACHE_FLUSH_TS 20 +#define A3XX_INT_CP_AHB_ERROR_HALT 21 +#define A3XX_INT_MISC_HANG_DETECT 24 +#define A3XX_INT_UCHE_OOB_ACCESS 25 + +/* CP_EVENT_WRITE events */ +#define CACHE_FLUSH_TS 4 + +/* Register definitions */ + +#define A3XX_RBBM_CLOCK_CTL 0x010 +#define A3XX_RBBM_SP_HYST_CNT 0x012 +#define A3XX_RBBM_SW_RESET_CMD 0x018 +#define A3XX_RBBM_AHB_CTL0 0x020 +#define A3XX_RBBM_AHB_CTL1 0x021 +#define A3XX_RBBM_AHB_CMD 0x022 +#define A3XX_RBBM_AHB_ERROR_STATUS 0x027 +#define A3XX_RBBM_GPR0_CTL 0x02E +/* This the same register as on A2XX, just in a different place */ +#define A3XX_RBBM_STATUS 0x030 +#define A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL 0x33 +#define A3XX_RBBM_INTERFACE_HANG_INT_CTL 0x50 +#define A3XX_RBBM_INT_CLEAR_CMD 0x061 +#define A3XX_RBBM_INT_0_MASK 0x063 +#define A3XX_RBBM_INT_0_STATUS 0x064 +#define A3XX_RBBM_PERFCTR_CTL 0x80 +#define A3XX_RBBM_PERFCTR_LOAD_CMD0 0x81 +#define A3XX_RBBM_PERFCTR_LOAD_CMD1 0x82 +#define A3XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x84 +#define A3XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x85 +#define A3XX_RBBM_PERFCOUNTER0_SELECT 0x86 +#define A3XX_RBBM_PERFCOUNTER1_SELECT 0x87 +#define A3XX_RBBM_GPU_BUSY_MASKED 0x88 +#define A3XX_RBBM_PERFCTR_CP_0_LO 0x90 +#define A3XX_RBBM_PERFCTR_CP_0_HI 0x91 +#define A3XX_RBBM_PERFCTR_RBBM_0_LO 0x92 +#define A3XX_RBBM_PERFCTR_RBBM_0_HI 0x93 +#define A3XX_RBBM_PERFCTR_RBBM_1_LO 0x94 +#define A3XX_RBBM_PERFCTR_RBBM_1_HI 0x95 +#define A3XX_RBBM_PERFCTR_PC_0_LO 0x96 +#define A3XX_RBBM_PERFCTR_PC_0_HI 0x97 +#define A3XX_RBBM_PERFCTR_PC_1_LO 0x98 +#define A3XX_RBBM_PERFCTR_PC_1_HI 0x99 +#define A3XX_RBBM_PERFCTR_PC_2_LO 0x9A +#define A3XX_RBBM_PERFCTR_PC_2_HI 0x9B +#define A3XX_RBBM_PERFCTR_PC_3_LO 0x9C +#define A3XX_RBBM_PERFCTR_PC_3_HI 0x9D +#define A3XX_RBBM_PERFCTR_VFD_0_LO 0x9E +#define A3XX_RBBM_PERFCTR_VFD_0_HI 0x9F +#define A3XX_RBBM_PERFCTR_VFD_1_LO 0xA0 +#define A3XX_RBBM_PERFCTR_VFD_1_HI 0xA1 +#define A3XX_RBBM_PERFCTR_HLSQ_0_LO 0xA2 +#define A3XX_RBBM_PERFCTR_HLSQ_0_HI 0xA3 +#define A3XX_RBBM_PERFCTR_HLSQ_1_LO 0xA4 +#define A3XX_RBBM_PERFCTR_HLSQ_1_HI 0xA5 +#define A3XX_RBBM_PERFCTR_HLSQ_2_LO 0xA6 +#define A3XX_RBBM_PERFCTR_HLSQ_2_HI 0xA7 +#define A3XX_RBBM_PERFCTR_HLSQ_3_LO 0xA8 +#define A3XX_RBBM_PERFCTR_HLSQ_3_HI 0xA9 +#define A3XX_RBBM_PERFCTR_HLSQ_4_LO 0xAA +#define A3XX_RBBM_PERFCTR_HLSQ_4_HI 0xAB +#define A3XX_RBBM_PERFCTR_HLSQ_5_LO 0xAC +#define A3XX_RBBM_PERFCTR_HLSQ_5_HI 0xAD +#define A3XX_RBBM_PERFCTR_VPC_0_LO 0xAE +#define A3XX_RBBM_PERFCTR_VPC_0_HI 0xAF +#define A3XX_RBBM_PERFCTR_VPC_1_LO 0xB0 +#define A3XX_RBBM_PERFCTR_VPC_1_HI 0xB1 +#define A3XX_RBBM_PERFCTR_TSE_0_LO 0xB2 +#define A3XX_RBBM_PERFCTR_TSE_0_HI 0xB3 +#define A3XX_RBBM_PERFCTR_TSE_1_LO 0xB4 +#define A3XX_RBBM_PERFCTR_TSE_1_HI 0xB5 +#define A3XX_RBBM_PERFCTR_RAS_0_LO 0xB6 +#define A3XX_RBBM_PERFCTR_RAS_0_HI 0xB7 +#define A3XX_RBBM_PERFCTR_RAS_1_LO 0xB8 +#define A3XX_RBBM_PERFCTR_RAS_1_HI 0xB9 +#define A3XX_RBBM_PERFCTR_UCHE_0_LO 0xBA +#define A3XX_RBBM_PERFCTR_UCHE_0_HI 0xBB +#define A3XX_RBBM_PERFCTR_UCHE_1_LO 0xBC +#define A3XX_RBBM_PERFCTR_UCHE_1_HI 0xBD +#define A3XX_RBBM_PERFCTR_UCHE_2_LO 0xBE +#define A3XX_RBBM_PERFCTR_UCHE_2_HI 0xBF +#define A3XX_RBBM_PERFCTR_UCHE_3_LO 0xC0 +#define A3XX_RBBM_PERFCTR_UCHE_3_HI 0xC1 +#define A3XX_RBBM_PERFCTR_UCHE_4_LO 0xC2 +#define A3XX_RBBM_PERFCTR_UCHE_4_HI 0xC3 +#define A3XX_RBBM_PERFCTR_UCHE_5_LO 0xC4 +#define A3XX_RBBM_PERFCTR_UCHE_5_HI 0xC5 +#define A3XX_RBBM_PERFCTR_TP_0_LO 0xC6 +#define A3XX_RBBM_PERFCTR_TP_0_HI 0xC7 +#define A3XX_RBBM_PERFCTR_TP_1_LO 0xC8 +#define A3XX_RBBM_PERFCTR_TP_1_HI 0xC9 +#define A3XX_RBBM_PERFCTR_TP_2_LO 0xCA +#define A3XX_RBBM_PERFCTR_TP_2_HI 0xCB +#define A3XX_RBBM_PERFCTR_TP_3_LO 0xCC +#define A3XX_RBBM_PERFCTR_TP_3_HI 0xCD +#define A3XX_RBBM_PERFCTR_TP_4_LO 0xCE +#define A3XX_RBBM_PERFCTR_TP_4_HI 0xCF +#define A3XX_RBBM_PERFCTR_TP_5_LO 0xD0 +#define A3XX_RBBM_PERFCTR_TP_5_HI 0xD1 +#define A3XX_RBBM_PERFCTR_SP_0_LO 0xD2 +#define A3XX_RBBM_PERFCTR_SP_0_HI 0xD3 +#define A3XX_RBBM_PERFCTR_SP_1_LO 0xD4 +#define A3XX_RBBM_PERFCTR_SP_1_HI 0xD5 +#define A3XX_RBBM_PERFCTR_SP_2_LO 0xD6 +#define A3XX_RBBM_PERFCTR_SP_2_HI 0xD7 +#define A3XX_RBBM_PERFCTR_SP_3_LO 0xD8 +#define A3XX_RBBM_PERFCTR_SP_3_HI 0xD9 +#define A3XX_RBBM_PERFCTR_SP_4_LO 0xDA +#define A3XX_RBBM_PERFCTR_SP_4_HI 0xDB +#define A3XX_RBBM_PERFCTR_SP_5_LO 0xDC +#define A3XX_RBBM_PERFCTR_SP_5_HI 0xDD +#define A3XX_RBBM_PERFCTR_SP_6_LO 0xDE +#define A3XX_RBBM_PERFCTR_SP_6_HI 0xDF +#define A3XX_RBBM_PERFCTR_SP_7_LO 0xE0 +#define A3XX_RBBM_PERFCTR_SP_7_HI 0xE1 +#define A3XX_RBBM_PERFCTR_RB_0_LO 0xE2 +#define A3XX_RBBM_PERFCTR_RB_0_HI 0xE3 +#define A3XX_RBBM_PERFCTR_RB_1_LO 0xE4 +#define A3XX_RBBM_PERFCTR_RB_1_HI 0xE5 + +#define A3XX_RBBM_RBBM_CTL 0x100 +#define A3XX_RBBM_PERFCTR_PWR_0_LO 0x0EA +#define A3XX_RBBM_PERFCTR_PWR_0_HI 0x0EB +#define A3XX_RBBM_PERFCTR_PWR_1_LO 0x0EC +#define A3XX_RBBM_PERFCTR_PWR_1_HI 0x0ED +#define A3XX_RBBM_DEBUG_BUS_CTL 0x111 +#define A3XX_RBBM_DEBUG_BUS_DATA_STATUS 0x112 +#define A3XX_RBBM_DEBUG_BUS_STB_CTL0 0x11B +#define A3XX_RBBM_DEBUG_BUS_STB_CTL1 0x11C +#define A3XX_RBBM_INT_TRACE_BUS_CTL 0x11D +#define A3XX_RBBM_EXT_TRACE_BUS_CTL 0x11E +#define A3XX_RBBM_EXT_TRACE_STOP_CNT 0x11F +#define A3XX_RBBM_EXT_TRACE_START_CNT 0x120 +#define A3XX_RBBM_EXT_TRACE_PERIOD_CNT 0x121 +#define A3XX_RBBM_EXT_TRACE_CMD 0x122 +#define A3XX_CP_RB_BASE 0x01C0 +#define A3XX_CP_RB_CNTL 0x01C1 +#define A3XX_CP_RB_RPTR 0x01C4 +#define A3XX_CP_RB_WPTR 0x01C5 +/* Following two are same as on A2XX, just in a different place */ +#define A3XX_CP_PFP_UCODE_ADDR 0x1C9 +#define A3XX_CP_PFP_UCODE_DATA 0x1CA +#define A3XX_CP_ROQ_ADDR 0x1CC +#define A3XX_CP_ROQ_DATA 0x1CD +#define A3XX_CP_MERCIU_ADDR 0x1D1 +#define A3XX_CP_MERCIU_DATA 0x1D2 +#define A3XX_CP_MERCIU_DATA2 0x1D3 +#define A3XX_CP_QUEUE_THRESHOLDS 0x01D5 +#define A3XX_CP_MEQ_ADDR 0x1DA +#define A3XX_CP_MEQ_DATA 0x1DB +#define A3XX_CP_STATE_DEBUG_INDEX 0x01EC +#define A3XX_CP_STATE_DEBUG_DATA 0x01ED +#define A3XX_CP_CNTL 0x01F4 +#define A3XX_CP_WFI_PEND_CTR 0x01F5 +#define A3XX_CP_ME_CNTL 0x01F6 +#define A3XX_CP_ME_STATUS 0x01F7 +#define A3XX_CP_ME_RAM_WADDR 0x01F8 +#define A3XX_CP_ME_RAM_RADDR 0x01F9 +#define A3XX_CP_ME_RAM_DATA 0x01FA +#define A3XX_CP_DEBUG 0x01FC + +#define A3XX_RBBM_PM_OVERRIDE2 0x039D + +#define A3XX_CP_PERFCOUNTER_SELECT 0x445 +#define A3XX_CP_IB1_BASE 0x0458 +#define A3XX_CP_IB1_BUFSZ 0x0459 +#define A3XX_CP_IB2_BASE 0x045A +#define A3XX_CP_IB2_BUFSZ 0x045B + +#define A3XX_CP_HW_FAULT 0x45C +#define A3XX_CP_PROTECT_CTRL 0x45E +#define A3XX_CP_PROTECT_STATUS 0x45F +#define A3XX_CP_PROTECT_REG_0 0x460 +#define A3XX_CP_STAT 0x047F +#define A3XX_CP_SCRATCH_REG0 0x578 +#define A3XX_CP_SCRATCH_REG6 0x57E +#define A3XX_CP_SCRATCH_REG7 0x57F +#define A3XX_VSC_SIZE_ADDRESS 0xC02 +#define A3XX_VSC_PIPE_DATA_ADDRESS_0 0xC07 +#define A3XX_VSC_PIPE_DATA_LENGTH_0 0xC08 +#define A3XX_VSC_PIPE_DATA_ADDRESS_1 0xC0A +#define A3XX_VSC_PIPE_DATA_LENGTH_1 0xC0B +#define A3XX_VSC_PIPE_DATA_ADDRESS_2 0xC0D +#define A3XX_VSC_PIPE_DATA_LENGTH_2 0xC0E +#define A3XX_VSC_PIPE_DATA_ADDRESS_3 0xC10 +#define A3XX_VSC_PIPE_DATA_LENGTH_3 0xC11 +#define A3XX_VSC_PIPE_DATA_ADDRESS_4 0xC13 +#define A3XX_VSC_PIPE_DATA_LENGTH_4 0xC14 +#define A3XX_VSC_PIPE_DATA_ADDRESS_5 0xC16 +#define A3XX_VSC_PIPE_DATA_LENGTH_5 0xC17 +#define A3XX_VSC_PIPE_DATA_ADDRESS_6 0xC19 +#define A3XX_VSC_PIPE_DATA_LENGTH_6 0xC1A +#define A3XX_VSC_PIPE_DATA_ADDRESS_7 0xC1C +#define A3XX_VSC_PIPE_DATA_LENGTH_7 0xC1D +#define A3XX_PC_PERFCOUNTER0_SELECT 0xC48 +#define A3XX_PC_PERFCOUNTER1_SELECT 0xC49 +#define A3XX_PC_PERFCOUNTER2_SELECT 0xC4A +#define A3XX_PC_PERFCOUNTER3_SELECT 0xC4B +#define A3XX_GRAS_TSE_DEBUG_ECO 0xC81 +#define A3XX_GRAS_PERFCOUNTER0_SELECT 0xC88 +#define A3XX_GRAS_PERFCOUNTER1_SELECT 0xC89 +#define A3XX_GRAS_PERFCOUNTER2_SELECT 0xC8A +#define A3XX_GRAS_PERFCOUNTER3_SELECT 0xC8B +#define A3XX_GRAS_CL_USER_PLANE_X0 0xCA0 +#define A3XX_GRAS_CL_USER_PLANE_Y0 0xCA1 +#define A3XX_GRAS_CL_USER_PLANE_Z0 0xCA2 +#define A3XX_GRAS_CL_USER_PLANE_W0 0xCA3 +#define A3XX_GRAS_CL_USER_PLANE_X1 0xCA4 +#define A3XX_GRAS_CL_USER_PLANE_Y1 0xCA5 +#define A3XX_GRAS_CL_USER_PLANE_Z1 0xCA6 +#define A3XX_GRAS_CL_USER_PLANE_W1 0xCA7 +#define A3XX_GRAS_CL_USER_PLANE_X2 0xCA8 +#define A3XX_GRAS_CL_USER_PLANE_Y2 0xCA9 +#define A3XX_GRAS_CL_USER_PLANE_Z2 0xCAA +#define A3XX_GRAS_CL_USER_PLANE_W2 0xCAB +#define A3XX_GRAS_CL_USER_PLANE_X3 0xCAC +#define A3XX_GRAS_CL_USER_PLANE_Y3 0xCAD +#define A3XX_GRAS_CL_USER_PLANE_Z3 0xCAE +#define A3XX_GRAS_CL_USER_PLANE_W3 0xCAF +#define A3XX_GRAS_CL_USER_PLANE_X4 0xCB0 +#define A3XX_GRAS_CL_USER_PLANE_Y4 0xCB1 +#define A3XX_GRAS_CL_USER_PLANE_Z4 0xCB2 +#define A3XX_GRAS_CL_USER_PLANE_W4 0xCB3 +#define A3XX_GRAS_CL_USER_PLANE_X5 0xCB4 +#define A3XX_GRAS_CL_USER_PLANE_Y5 0xCB5 +#define A3XX_GRAS_CL_USER_PLANE_Z5 0xCB6 +#define A3XX_GRAS_CL_USER_PLANE_W5 0xCB7 +#define A3XX_RB_GMEM_BASE_ADDR 0xCC0 +#define A3XX_RB_DEBUG_ECO_CONTROLS_ADDR 0xCC1 +#define A3XX_RB_PERFCOUNTER0_SELECT 0xCC6 +#define A3XX_RB_PERFCOUNTER1_SELECT 0xCC7 +#define A3XX_RB_FRAME_BUFFER_DIMENSION 0xCE0 +#define A3XX_SQ_GPR_MANAGEMENT 0x0D00 +#define A3XX_SQ_INST_STORE_MANAGEMENT 0x0D02 +#define A3XX_HLSQ_PERFCOUNTER0_SELECT 0xE00 +#define A3XX_HLSQ_PERFCOUNTER1_SELECT 0xE01 +#define A3XX_HLSQ_PERFCOUNTER2_SELECT 0xE02 +#define A3XX_HLSQ_PERFCOUNTER3_SELECT 0xE03 +#define A3XX_HLSQ_PERFCOUNTER4_SELECT 0xE04 +#define A3XX_HLSQ_PERFCOUNTER5_SELECT 0xE05 +#define A3XX_TP0_CHICKEN 0x0E1E +#define A3XX_VFD_PERFCOUNTER0_SELECT 0xE44 +#define A3XX_VFD_PERFCOUNTER1_SELECT 0xE45 +#define A3XX_VPC_VPC_DEBUG_RAM_SEL 0xE61 +#define A3XX_VPC_VPC_DEBUG_RAM_READ 0xE62 +#define A3XX_VPC_PERFCOUNTER0_SELECT 0xE64 +#define A3XX_VPC_PERFCOUNTER1_SELECT 0xE65 +#define A3XX_UCHE_CACHE_MODE_CONTROL_REG 0xE82 +#define A3XX_UCHE_PERFCOUNTER0_SELECT 0xE84 +#define A3XX_UCHE_PERFCOUNTER1_SELECT 0xE85 +#define A3XX_UCHE_PERFCOUNTER2_SELECT 0xE86 +#define A3XX_UCHE_PERFCOUNTER3_SELECT 0xE87 +#define A3XX_UCHE_PERFCOUNTER4_SELECT 0xE88 +#define A3XX_UCHE_PERFCOUNTER5_SELECT 0xE89 +#define A3XX_UCHE_CACHE_INVALIDATE0_REG 0xEA0 +#define A3XX_UCHE_CACHE_INVALIDATE1_REG 0xEA1 +#define A3XX_UCHE_CACHE_WAYS_VFD 0xEA6 +#define A3XX_SP_PERFCOUNTER0_SELECT 0xEC4 +#define A3XX_SP_PERFCOUNTER1_SELECT 0xEC5 +#define A3XX_SP_PERFCOUNTER2_SELECT 0xEC6 +#define A3XX_SP_PERFCOUNTER3_SELECT 0xEC7 +#define A3XX_SP_PERFCOUNTER4_SELECT 0xEC8 +#define A3XX_SP_PERFCOUNTER5_SELECT 0xEC9 +#define A3XX_SP_PERFCOUNTER6_SELECT 0xECA +#define A3XX_SP_PERFCOUNTER7_SELECT 0xECB +#define A3XX_TP_PERFCOUNTER0_SELECT 0xF04 +#define A3XX_TP_PERFCOUNTER1_SELECT 0xF05 +#define A3XX_TP_PERFCOUNTER2_SELECT 0xF06 +#define A3XX_TP_PERFCOUNTER3_SELECT 0xF07 +#define A3XX_TP_PERFCOUNTER4_SELECT 0xF08 +#define A3XX_TP_PERFCOUNTER5_SELECT 0xF09 +#define A3XX_GRAS_CL_CLIP_CNTL 0x2040 +#define A3XX_GRAS_CL_GB_CLIP_ADJ 0x2044 +#define A3XX_GRAS_CL_VPORT_XOFFSET 0x2048 +#define A3XX_GRAS_CL_VPORT_XSCALE 0x2049 +#define A3XX_GRAS_CL_VPORT_YOFFSET 0x204A +#define A3XX_GRAS_CL_VPORT_YSCALE 0x204B +#define A3XX_GRAS_CL_VPORT_ZOFFSET 0x204C +#define A3XX_GRAS_CL_VPORT_ZSCALE 0x204D +#define A3XX_GRAS_SU_POINT_MINMAX 0x2068 +#define A3XX_GRAS_SU_POINT_SIZE 0x2069 +#define A3XX_GRAS_SU_POLY_OFFSET_SCALE 0x206C +#define A3XX_GRAS_SU_POLY_OFFSET_OFFSET 0x206D +#define A3XX_GRAS_SU_MODE_CONTROL 0x2070 +#define A3XX_GRAS_SC_CONTROL 0x2072 +#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL 0x2074 +#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR 0x2075 +#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL 0x2079 +#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR 0x207A +#define A3XX_RB_MODE_CONTROL 0x20C0 +#define A3XX_RB_RENDER_CONTROL 0x20C1 +#define A3XX_RB_MSAA_CONTROL 0x20C2 +#define A3XX_RB_ALPHA_REFERENCE 0x20C3 +#define A3XX_RB_MRT_CONTROL0 0x20C4 +#define A3XX_RB_MRT_BUF_INFO0 0x20C5 +#define A3XX_RB_MRT_BUF_BASE0 0x20C6 +#define A3XX_RB_MRT_BLEND_CONTROL0 0x20C7 +#define A3XX_RB_MRT_CONTROL1 0x20C8 +#define A3XX_RB_MRT_BUF_INFO1 0x20C9 +#define A3XX_RB_MRT_BUF_BASE1 0x20CA +#define A3XX_RB_MRT_BLEND_CONTROL1 0x20CB +#define A3XX_RB_MRT_CONTROL2 0x20CC +#define A3XX_RB_MRT_BUF_INFO2 0x20CD +#define A3XX_RB_MRT_BUF_BASE2 0x20CE +#define A3XX_RB_MRT_BLEND_CONTROL2 0x20CF +#define A3XX_RB_MRT_CONTROL3 0x20D0 +#define A3XX_RB_MRT_BUF_INFO3 0x20D1 +#define A3XX_RB_MRT_BUF_BASE3 0x20D2 +#define A3XX_RB_MRT_BLEND_CONTROL3 0x20D3 +#define A3XX_RB_BLEND_RED 0x20E4 +#define A3XX_RB_BLEND_GREEN 0x20E5 +#define A3XX_RB_BLEND_BLUE 0x20E6 +#define A3XX_RB_BLEND_ALPHA 0x20E7 +#define A3XX_RB_CLEAR_COLOR_DW0 0x20E8 +#define A3XX_RB_CLEAR_COLOR_DW1 0x20E9 +#define A3XX_RB_CLEAR_COLOR_DW2 0x20EA +#define A3XX_RB_CLEAR_COLOR_DW3 0x20EB +#define A3XX_RB_COPY_CONTROL 0x20EC +#define A3XX_RB_COPY_DEST_BASE 0x20ED +#define A3XX_RB_COPY_DEST_PITCH 0x20EE +#define A3XX_RB_COPY_DEST_INFO 0x20EF +#define A3XX_RB_DEPTH_CONTROL 0x2100 +#define A3XX_RB_DEPTH_CLEAR 0x2101 +#define A3XX_RB_DEPTH_BUF_INFO 0x2102 +#define A3XX_RB_DEPTH_BUF_PITCH 0x2103 +#define A3XX_RB_STENCIL_CONTROL 0x2104 +#define A3XX_RB_STENCIL_CLEAR 0x2105 +#define A3XX_RB_STENCIL_BUF_INFO 0x2106 +#define A3XX_RB_STENCIL_BUF_PITCH 0x2107 +#define A3XX_RB_STENCIL_REF_MASK 0x2108 +#define A3XX_RB_STENCIL_REF_MASK_BF 0x2109 +#define A3XX_RB_LRZ_VSC_CONTROL 0x210C +#define A3XX_RB_WINDOW_OFFSET 0x210E +#define A3XX_RB_SAMPLE_COUNT_CONTROL 0x2110 +#define A3XX_RB_SAMPLE_COUNT_ADDR 0x2111 +#define A3XX_RB_Z_CLAMP_MIN 0x2114 +#define A3XX_RB_Z_CLAMP_MAX 0x2115 +#define A3XX_HLSQ_CONTROL_0_REG 0x2200 +#define A3XX_HLSQ_CONTROL_1_REG 0x2201 +#define A3XX_HLSQ_CONTROL_2_REG 0x2202 +#define A3XX_HLSQ_CONTROL_3_REG 0x2203 +#define A3XX_HLSQ_VS_CONTROL_REG 0x2204 +#define A3XX_HLSQ_FS_CONTROL_REG 0x2205 +#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG 0x2206 +#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG 0x2207 +#define A3XX_HLSQ_CL_NDRANGE_0_REG 0x220A +#define A3XX_HLSQ_CL_NDRANGE_1_REG 0x220B +#define A3XX_HLSQ_CL_NDRANGE_2_REG 0x220C +#define A3XX_HLSQ_CL_NDRANGE_3_REG 0x220D +#define A3XX_HLSQ_CL_NDRANGE_4_REG 0x220E +#define A3XX_HLSQ_CL_NDRANGE_5_REG 0x220F +#define A3XX_HLSQ_CL_NDRANGE_6_REG 0x2210 +#define A3XX_HLSQ_CL_CONTROL_0_REG 0x2211 +#define A3XX_HLSQ_CL_CONTROL_1_REG 0x2212 +#define A3XX_HLSQ_CL_KERNEL_CONST_REG 0x2214 +#define A3XX_HLSQ_CL_KERNEL_GROUP_X_REG 0x2215 +#define A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG 0x2216 +#define A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG 0x2217 +#define A3XX_HLSQ_CL_WG_OFFSET_REG 0x221A +#define A3XX_VFD_FETCH_INSTR_1_0 0x2247 +#define A3XX_VFD_FETCH_INSTR_1_1 0x2249 +#define A3XX_VFD_FETCH_INSTR_1_2 0x224B +#define A3XX_VFD_FETCH_INSTR_1_3 0x224D +#define A3XX_VFD_FETCH_INSTR_1_4 0x224F +#define A3XX_VFD_FETCH_INSTR_1_5 0x2251 +#define A3XX_VFD_FETCH_INSTR_1_6 0x2253 +#define A3XX_VFD_FETCH_INSTR_1_7 0x2255 +#define A3XX_VFD_FETCH_INSTR_1_8 0x2257 +#define A3XX_VFD_FETCH_INSTR_1_9 0x2259 +#define A3XX_VFD_FETCH_INSTR_1_A 0x225B +#define A3XX_VFD_FETCH_INSTR_1_B 0x225D +#define A3XX_VFD_FETCH_INSTR_1_C 0x225F +#define A3XX_VFD_FETCH_INSTR_1_D 0x2261 +#define A3XX_VFD_FETCH_INSTR_1_E 0x2263 +#define A3XX_VFD_FETCH_INSTR_1_F 0x2265 +#define A3XX_SP_SP_CTRL_REG 0x22C0 +#define A3XX_SP_VS_CTRL_REG0 0x22C4 +#define A3XX_SP_VS_CTRL_REG1 0x22C5 +#define A3XX_SP_VS_PARAM_REG 0x22C6 +#define A3XX_SP_VS_OUT_REG_0 0x22C7 +#define A3XX_SP_VS_OUT_REG_1 0x22C8 +#define A3XX_SP_VS_OUT_REG_2 0x22C9 +#define A3XX_SP_VS_OUT_REG_3 0x22CA +#define A3XX_SP_VS_OUT_REG_4 0x22CB +#define A3XX_SP_VS_OUT_REG_5 0x22CC +#define A3XX_SP_VS_OUT_REG_6 0x22CD +#define A3XX_SP_VS_OUT_REG_7 0x22CE +#define A3XX_SP_VS_VPC_DST_REG_0 0x22D0 +#define A3XX_SP_VS_VPC_DST_REG_1 0x22D1 +#define A3XX_SP_VS_VPC_DST_REG_2 0x22D2 +#define A3XX_SP_VS_VPC_DST_REG_3 0x22D3 +#define A3XX_SP_VS_OBJ_OFFSET_REG 0x22D4 +#define A3XX_SP_VS_OBJ_START_REG 0x22D5 +#define A3XX_SP_VS_PVT_MEM_PARAM_REG 0x22D6 +#define A3XX_SP_VS_PVT_MEM_ADDR_REG 0x22D7 +#define A3XX_SP_VS_PVT_MEM_SIZE_REG 0x22D8 +#define A3XX_SP_VS_LENGTH_REG 0x22DF +#define A3XX_SP_FS_CTRL_REG0 0x22E0 +#define A3XX_SP_FS_CTRL_REG1 0x22E1 +#define A3XX_SP_FS_OBJ_OFFSET_REG 0x22E2 +#define A3XX_SP_FS_OBJ_START_REG 0x22E3 +#define A3XX_SP_FS_PVT_MEM_PARAM_REG 0x22E4 +#define A3XX_SP_FS_PVT_MEM_ADDR_REG 0x22E5 +#define A3XX_SP_FS_PVT_MEM_SIZE_REG 0x22E6 +#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_0 0x22E8 +#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_1 0x22E9 +#define A3XX_SP_FS_OUTPUT_REG 0x22EC +#define A3XX_SP_FS_MRT_REG_0 0x22F0 +#define A3XX_SP_FS_MRT_REG_1 0x22F1 +#define A3XX_SP_FS_MRT_REG_2 0x22F2 +#define A3XX_SP_FS_MRT_REG_3 0x22F3 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_0 0x22F4 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_1 0x22F5 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_2 0x22F6 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_3 0x22F7 +#define A3XX_SP_FS_LENGTH_REG 0x22FF +#define A3XX_PA_SC_AA_CONFIG 0x2301 +#define A3XX_VBIF_CLKON 0x3001 +#define A3XX_VBIF_ABIT_SORT 0x301C +#define A3XX_VBIF_ABIT_SORT_CONF 0x301D +#define A3XX_VBIF_GATE_OFF_WRREQ_EN 0x302A +#define A3XX_VBIF_IN_RD_LIM_CONF0 0x302C +#define A3XX_VBIF_IN_RD_LIM_CONF1 0x302D +#define A3XX_VBIF_IN_WR_LIM_CONF0 0x3030 +#define A3XX_VBIF_IN_WR_LIM_CONF1 0x3031 +#define A3XX_VBIF_OUT_RD_LIM_CONF0 0x3034 +#define A3XX_VBIF_OUT_WR_LIM_CONF0 0x3035 +#define A3XX_VBIF_DDR_OUT_MAX_BURST 0x3036 +#define A3XX_VBIF_ARB_CTL 0x303C +#define A3XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049 +#define A3XX_VBIF_OUT_AXI_AOOO_EN 0x305E +#define A3XX_VBIF_OUT_AXI_AOOO 0x305F +#define A3XX_VBIF_PERF_CNT0_LO 0x3073 +#define A3XX_VBIF_PERF_CNT0_HI 0x3074 +#define A3XX_VBIF_PERF_CNT1_LO 0x3075 +#define A3XX_VBIF_PERF_CNT1_HI 0x3076 +#define A3XX_VBIF_PERF_PWR_CNT0_LO 0x3077 +#define A3XX_VBIF_PERF_PWR_CNT0_HI 0x3078 +#define A3XX_VBIF_PERF_PWR_CNT1_LO 0x3079 +#define A3XX_VBIF_PERF_PWR_CNT1_HI 0x307a +#define A3XX_VBIF_PERF_PWR_CNT2_LO 0x307b +#define A3XX_VBIF_PERF_PWR_CNT2_HI 0x307c + +#define A3XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A3XX_VBIF_XIN_HALT_CTRL0_MASK 0x3F +#define A30X_VBIF_XIN_HALT_CTRL0_MASK 0x7 + +#define A3XX_VBIF_XIN_HALT_CTRL1 0x3081 + +/* VBIF register offsets for A306 */ +#define A3XX_VBIF2_PERF_CNT_SEL0 0x30d0 +#define A3XX_VBIF2_PERF_CNT_SEL1 0x30d1 +#define A3XX_VBIF2_PERF_CNT_SEL2 0x30d2 +#define A3XX_VBIF2_PERF_CNT_SEL3 0x30d3 +#define A3XX_VBIF2_PERF_CNT_LOW0 0x30d8 +#define A3XX_VBIF2_PERF_CNT_LOW1 0x30d9 +#define A3XX_VBIF2_PERF_CNT_LOW2 0x30da +#define A3XX_VBIF2_PERF_CNT_LOW3 0x30db +#define A3XX_VBIF2_PERF_CNT_HIGH0 0x30e0 +#define A3XX_VBIF2_PERF_CNT_HIGH1 0x30e1 +#define A3XX_VBIF2_PERF_CNT_HIGH2 0x30e2 +#define A3XX_VBIF2_PERF_CNT_HIGH3 0x30e3 + +#define A3XX_VBIF2_PERF_PWR_CNT_EN0 0x3100 +#define A3XX_VBIF2_PERF_PWR_CNT_EN1 0x3101 +#define A3XX_VBIF2_PERF_PWR_CNT_EN2 0x3102 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW0 0x3110 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW1 0x3111 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW2 0x3112 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH0 0x3118 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH1 0x3119 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH2 0x311a + +#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0 0x3800 +#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL1 0x3801 + +/* RBBM Debug bus block IDs */ +#define RBBM_BLOCK_ID_CP 0x1 +#define RBBM_BLOCK_ID_RBBM 0x2 +#define RBBM_BLOCK_ID_VBIF 0x3 +#define RBBM_BLOCK_ID_HLSQ 0x4 +#define RBBM_BLOCK_ID_UCHE 0x5 +#define RBBM_BLOCK_ID_PC 0x8 +#define RBBM_BLOCK_ID_VFD 0x9 +#define RBBM_BLOCK_ID_VPC 0xa +#define RBBM_BLOCK_ID_TSE 0xb +#define RBBM_BLOCK_ID_RAS 0xc +#define RBBM_BLOCK_ID_VSC 0xd +#define RBBM_BLOCK_ID_SP_0 0x10 +#define RBBM_BLOCK_ID_SP_1 0x11 +#define RBBM_BLOCK_ID_SP_2 0x12 +#define RBBM_BLOCK_ID_SP_3 0x13 +#define RBBM_BLOCK_ID_TPL1_0 0x18 +#define RBBM_BLOCK_ID_TPL1_1 0x19 +#define RBBM_BLOCK_ID_TPL1_2 0x1a +#define RBBM_BLOCK_ID_TPL1_3 0x1b +#define RBBM_BLOCK_ID_RB_0 0x20 +#define RBBM_BLOCK_ID_RB_1 0x21 +#define RBBM_BLOCK_ID_RB_2 0x22 +#define RBBM_BLOCK_ID_RB_3 0x23 +#define RBBM_BLOCK_ID_MARB_0 0x28 +#define RBBM_BLOCK_ID_MARB_1 0x29 +#define RBBM_BLOCK_ID_MARB_2 0x2a +#define RBBM_BLOCK_ID_MARB_3 0x2b + +/* RBBM_CLOCK_CTL default value */ +#define A3XX_RBBM_CLOCK_CTL_DEFAULT 0xAAAAAAAA +#define A320_RBBM_CLOCK_CTL_DEFAULT 0xBFFFFFFF +#define A330_RBBM_CLOCK_CTL_DEFAULT 0xBFFCFFFF + +#define A330_RBBM_GPR0_CTL_DEFAULT 0x00000000 +#define A330v2_RBBM_GPR0_CTL_DEFAULT 0x05515455 +#define A310_RBBM_GPR0_CTL_DEFAULT 0x000000AA + +/* COUNTABLE FOR SP PERFCOUNTER */ +#define SP_ALU_ACTIVE_CYCLES 0x1D +#define SP0_ICL1_MISSES 0x1A +#define SP_FS_CFLOW_INSTRUCTIONS 0x0C + +/* COUNTABLE FOR TSE PERFCOUNTER */ +#define TSE_INPUT_PRIM_NUM 0x0 + +/* VBIF countables */ +#define VBIF_AXI_TOTAL_BEATS 85 + +/* VBIF Recoverable HALT bit value */ +#define VBIF_RECOVERABLE_HALT_CTRL 0x1 + +/* + * CP DEBUG settings for A3XX core: + * DYNAMIC_CLK_DISABLE [27] - turn off the dynamic clock control + * MIU_128BIT_WRITE_ENABLE [25] - Allow 128 bit writes to the VBIF + */ +#define A3XX_CP_DEBUG_DEFAULT ((1 << 27) | (1 << 25)) + + +#endif diff --git a/drivers/gpu/msm/a4xx_reg.h b/drivers/gpu/msm/a4xx_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..8b53750e512a8f90be11d991c4f3058ad65781d1 --- /dev/null +++ b/drivers/gpu/msm/a4xx_reg.h @@ -0,0 +1,839 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. + */ + +#ifndef _A4XX_REG_H +#define _A4XX_REG_H + +/* A4XX interrupt bits */ +#define A4XX_INT_RBBM_GPU_IDLE 0 +#define A4XX_INT_RBBM_AHB_ERROR 1 +#define A4XX_INT_RBBM_REG_TIMEOUT 2 +#define A4XX_INT_RBBM_ME_MS_TIMEOUT 3 +#define A4XX_INT_RBBM_PFP_MS_TIMEOUT 4 +#define A4XX_INT_RBBM_ETS_MS_TIMEOUT 5 +#define A4XX_INT_RBBM_ASYNC_OVERFLOW 6 +#define A4XX_INT_RBBM_GPC_ERR 7 +#define A4XX_INT_CP_SW 8 +#define A4XX_INT_CP_OPCODE_ERROR 9 +#define A4XX_INT_CP_RESERVED_BIT_ERROR 10 +#define A4XX_INT_CP_HW_FAULT 11 +#define A4XX_INT_CP_DMA 12 +#define A4XX_INT_CP_IB2_INT 13 +#define A4XX_INT_CP_IB1_INT 14 +#define A4XX_INT_CP_RB_INT 15 +#define A4XX_INT_CP_REG_PROTECT_FAULT 16 +#define A4XX_INT_CP_RB_DONE_TS 17 +#define A4XX_INT_CP_VS_DONE_TS 18 +#define A4XX_INT_CP_PS_DONE_TS 19 +#define A4XX_INT_CACHE_FLUSH_TS 20 +#define A4XX_INT_CP_AHB_ERROR_HALT 21 +#define A4XX_INT_RBBM_ATB_BUS_OVERFLOW 22 +#define A4XX_INT_MISC_HANG_DETECT 24 +#define A4XX_INT_UCHE_OOB_ACCESS 25 +#define A4XX_INT_RBBM_DPM_CALC_ERR 28 +#define A4XX_INT_RBBM_DPM_EPOCH_ERR 29 +#define A4XX_INT_RBBM_DPM_THERMAL_YELLOW_ERR 30 +#define A4XX_INT_RBBM_DPM_THERMAL_RED_ERR 31 + +/* RB registers */ +#define A4XX_RB_GMEM_BASE_ADDR 0xcc0 + +#define A4XX_RB_PERFCTR_RB_SEL_0 0xcc7 +#define A4XX_RB_PERFCTR_RB_SEL_1 0xcc8 +#define A4XX_RB_PERFCTR_RB_SEL_2 0xcc9 +#define A4XX_RB_PERFCTR_RB_SEL_3 0xcca +#define A4XX_RB_PERFCTR_RB_SEL_4 0xccb +#define A4XX_RB_PERFCTR_RB_SEL_5 0xccc +#define A4XX_RB_PERFCTR_RB_SEL_6 0xccd +#define A4XX_RB_PERFCTR_RB_SEL_7 0xcce + +enum a4xx_rb_perfctr_rb_sel { + RB_VALID_SAMPLES = 0x25, + RB_Z_FAIL = 0x28, + RB_S_FAIL = 0x29, +}; + +/* RBBM registers */ +#define A4XX_RBBM_CLOCK_CTL_TP0 0x4 +#define A4XX_RBBM_CLOCK_CTL_TP1 0x5 +#define A4XX_RBBM_CLOCK_CTL_TP2 0x6 +#define A4XX_RBBM_CLOCK_CTL_TP3 0x7 +#define A4XX_RBBM_CLOCK_CTL2_TP0 0x8 +#define A4XX_RBBM_CLOCK_CTL2_TP1 0x9 +#define A4XX_RBBM_CLOCK_CTL2_TP2 0xA +#define A4XX_RBBM_CLOCK_CTL2_TP3 0xB +#define A4XX_RBBM_CLOCK_HYST_TP0 0xC +#define A4XX_RBBM_CLOCK_HYST_TP1 0xD +#define A4XX_RBBM_CLOCK_HYST_TP2 0xE +#define A4XX_RBBM_CLOCK_HYST_TP3 0xF +#define A4XX_RBBM_CLOCK_DELAY_TP0 0x10 +#define A4XX_RBBM_CLOCK_DELAY_TP1 0x11 +#define A4XX_RBBM_CLOCK_DELAY_TP2 0x12 +#define A4XX_RBBM_CLOCK_DELAY_TP3 0x13 +#define A4XX_RBBM_CLOCK_CTL_UCHE 0x14 +#define A4XX_RBBM_CLOCK_CTL2_UCHE 0x15 +#define A4XX_RBBM_CLOCK_CTL3_UCHE 0x16 +#define A4XX_RBBM_CLOCK_CTL4_UCHE 0x17 +#define A4XX_RBBM_CLOCK_HYST_UCHE 0x18 +#define A4XX_RBBM_CLOCK_DELAY_UCHE 0x19 +#define A4XX_RBBM_CLOCK_MODE_GPC 0x1a +#define A4XX_RBBM_CLOCK_DELAY_GPC 0x1b +#define A4XX_RBBM_CLOCK_HYST_GPC 0x1c +#define A4XX_RBBM_CLOCK_CTL_TSE_RAS_RBBM 0x1d +#define A4XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x1e +#define A4XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x1f +#define A4XX_RBBM_CLOCK_CTL 0x20 +#define A4XX_RBBM_SP_HYST_CNT 0x21 +#define A4XX_RBBM_SW_RESET_CMD 0x22 +#define A4XX_RBBM_AHB_CTL0 0x23 +#define A4XX_RBBM_AHB_CTL1 0x24 +#define A4XX_RBBM_AHB_CMD 0x25 +#define A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL 0x2b +#define A4XX_RBBM_INTERFACE_HANG_INT_CTL 0x2f +#define A4XX_RBBM_INT_CLEAR_CMD 0x36 +#define A4XX_RBBM_INT_0_MASK 0x37 +#define A4XX_RBBM_RBBM_CTL 0x3e +#define A4XX_RBBM_CLOCK_CTL2 0x42 +#define A4XX_RBBM_BLOCK_SW_RESET_CMD 0x45 +#define A4XX_RBBM_EXT_TRACE_BUS_CTL 0x49 +#define A4XX_RBBM_CFG_DEBBUS_SEL_A 0x4a +#define A4XX_RBBM_CFG_DEBBUS_SEL_B 0x4b +#define A4XX_RBBM_CFG_DEBBUS_SEL_C 0x4c +#define A4XX_RBBM_CFG_DEBBUS_SEL_D 0x4d +#define A4XX_RBBM_CFG_DEBBUS_SEL_PING_INDEX_SHIFT 0 +#define A4XX_RBBM_CFG_DEBBUS_SEL_PING_BLK_SEL_SHIFT 8 + +#define A4XX_RBBM_CFG_DEBBUS_CTLT 0x4e + +#define A4XX_RBBM_CFG_DEBBUS_CTLM 0x4f +#define A4XX_RBBM_CFG_DEBBUS_CTLT_ENABLE_SHIFT 24 + +#define A4XX_RBBM_CFG_DEBBUS_OPL 0x50 +#define A4XX_RBBM_CFG_DEBBUS_OPE 0x51 +#define A4XX_RBBM_CFG_DEBBUS_IVTL_0 0x52 +#define A4XX_RBBM_CFG_DEBBUS_IVTL_1 0x53 +#define A4XX_RBBM_CFG_DEBBUS_IVTL_2 0x54 +#define A4XX_RBBM_CFG_DEBBUS_IVTL_3 0x55 + +#define A4XX_RBBM_CFG_DEBBUS_MASKL_0 0x56 +#define A4XX_RBBM_CFG_DEBBUS_MASKL_1 0x57 +#define A4XX_RBBM_CFG_DEBBUS_MASKL_2 0x58 +#define A4XX_RBBM_CFG_DEBBUS_MASKL_3 0x59 + + +#define A4XX_RBBM_CFG_DEBBUS_BYTEL_0 0x5a +#define A4XX_RBBM_CFG_DEBBUS_BYTEL_1 0x5b + +#define A4XX_RBBM_CFG_DEBBUS_IVTE_0 0x5c +#define A4XX_RBBM_CFG_DEBBUS_IVTE_1 0x5d +#define A4XX_RBBM_CFG_DEBBUS_IVTE_2 0x5e +#define A4XX_RBBM_CFG_DEBBUS_IVTE_3 0x5f +#define A4XX_RBBM_CFG_DEBBUS_MASKE_0 0x60 +#define A4XX_RBBM_CFG_DEBBUS_MASKE_1 0x61 +#define A4XX_RBBM_CFG_DEBBUS_MASKE_2 0x62 +#define A4XX_RBBM_CFG_DEBBUS_MASKE_3 0x63 +#define A4XX_RBBM_CFG_DEBBUS_NIBBLEE 0x64 +#define A4XX_RBBM_CFG_DEBBUS_PTRC0 0x65 +#define A4XX_RBBM_CFG_DEBBUS_PTRC1 0x66 +#define A4XX_RBBM_CFG_DEBBUS_LOADREG 0x67 +#define A4XX_RBBM_CLOCK_CTL_SP0 0x68 +#define A4XX_RBBM_CLOCK_CTL_SP1 0x69 +#define A4XX_RBBM_CLOCK_CTL_SP2 0x6A +#define A4XX_RBBM_CLOCK_CTL_SP3 0x6B +#define A4XX_RBBM_CLOCK_CTL2_SP0 0x6C +#define A4XX_RBBM_CLOCK_CTL2_SP1 0x6D +#define A4XX_RBBM_CLOCK_CTL2_SP2 0x6E +#define A4XX_RBBM_CLOCK_CTL2_SP3 0x6F +#define A4XX_RBBM_CLOCK_HYST_SP0 0x70 +#define A4XX_RBBM_CLOCK_HYST_SP1 0x71 +#define A4XX_RBBM_CLOCK_HYST_SP2 0x72 +#define A4XX_RBBM_CLOCK_HYST_SP3 0x73 +#define A4XX_RBBM_CLOCK_DELAY_SP0 0x74 +#define A4XX_RBBM_CLOCK_DELAY_SP1 0x75 +#define A4XX_RBBM_CLOCK_DELAY_SP2 0x76 +#define A4XX_RBBM_CLOCK_DELAY_SP3 0x77 +#define A4XX_RBBM_CLOCK_CTL_RB0 0x78 +#define A4XX_RBBM_CLOCK_CTL_RB1 0x79 +#define A4XX_RBBM_CLOCK_CTL_RB2 0x7A +#define A4XX_RBBM_CLOCK_CTL_RB3 0x7B +#define A4XX_RBBM_CLOCK_CTL2_RB0 0x7C +#define A4XX_RBBM_CLOCK_CTL2_RB1 0x7D +#define A4XX_RBBM_CLOCK_CTL2_RB2 0x7E +#define A4XX_RBBM_CLOCK_CTL2_RB3 0x7F +#define A4XX_RBBM_CLOCK_HYST_COM_DCOM 0x80 +#define A4XX_RBBM_CLOCK_CTL_COM_DCOM 0x81 +#define A4XX_RBBM_CLOCK_CTL_MARB_CCU0 0x82 +#define A4XX_RBBM_CLOCK_CTL_MARB_CCU1 0x83 +#define A4XX_RBBM_CLOCK_CTL_MARB_CCU2 0x84 +#define A4XX_RBBM_CLOCK_CTL_MARB_CCU3 0x85 +#define A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU0 0x86 +#define A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU1 0x87 +#define A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU2 0x88 +#define A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU3 0x89 +#define A4XX_RBBM_CLOCK_CTL_HLSQ 0x8a +#define A4XX_RBBM_CLOCK_HYST_HLSQ 0x8b + +#define A4XX_RBBM_CLOCK_DELAY_HLSQ 0x8c +#define A4XX_CGC_HLSQ_TP_EARLY_CYC_MASK 0x00700000 +#define A4XX_CGC_HLSQ_TP_EARLY_CYC_SHIFT 20 + +#define A4XX_RBBM_CLOCK_DELAY_COM_DCOM 0x8d +#define A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_0 0x8e +#define A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_1 0x8f +#define A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_2 0x90 +#define A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_3 0x91 + +#define A4XX_RBBM_CFG_DEBBUS_IDX 0x93 +#define A4XX_RBBM_CFG_DEBBUS_CLRC 0x94 +#define A4XX_RBBM_CFG_DEBBUS_LOADIVT 0x95 + +#define A4XX_RBBM_CLOCK_CTL_IP 0x97 +#define A4XX_RBBM_POWER_CNTL_IP 0x98 +#define A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_0 0x99 +#define A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_1 0x9a +#define A4XX_RBBM_PERFCTR_CP_0_LO 0x9c +#define A4XX_RBBM_PERFCTR_CP_0_HI 0x9d +#define A4XX_RBBM_PERFCTR_CP_1_LO 0x9e +#define A4XX_RBBM_PERFCTR_CP_1_HI 0x9f +#define A4XX_RBBM_PERFCTR_CP_2_LO 0xa0 +#define A4XX_RBBM_PERFCTR_CP_2_HI 0xa1 +#define A4XX_RBBM_PERFCTR_CP_3_LO 0xa2 +#define A4XX_RBBM_PERFCTR_CP_3_HI 0xa3 +#define A4XX_RBBM_PERFCTR_CP_4_LO 0xa4 +#define A4XX_RBBM_PERFCTR_CP_4_HI 0xa5 +#define A4XX_RBBM_PERFCTR_CP_5_LO 0xa6 +#define A4XX_RBBM_PERFCTR_CP_5_HI 0xa7 +#define A4XX_RBBM_PERFCTR_CP_6_LO 0xa8 +#define A4XX_RBBM_PERFCTR_CP_6_HI 0xa9 +#define A4XX_RBBM_PERFCTR_CP_7_LO 0xaa +#define A4XX_RBBM_PERFCTR_CP_7_HI 0xab +#define A4XX_RBBM_PERFCTR_RBBM_0_LO 0xac +#define A4XX_RBBM_PERFCTR_RBBM_0_HI 0xad +#define A4XX_RBBM_PERFCTR_RBBM_1_LO 0xae +#define A4XX_RBBM_PERFCTR_RBBM_1_HI 0xaf +#define A4XX_RBBM_PERFCTR_RBBM_2_LO 0xb0 +#define A4XX_RBBM_PERFCTR_RBBM_2_HI 0xb1 +#define A4XX_RBBM_PERFCTR_RBBM_3_LO 0xb2 +#define A4XX_RBBM_PERFCTR_RBBM_3_HI 0xb3 +#define A4XX_RBBM_PERFCTR_PC_0_LO 0xb4 +#define A4XX_RBBM_PERFCTR_PC_0_HI 0xb5 +#define A4XX_RBBM_PERFCTR_PC_1_LO 0xb6 +#define A4XX_RBBM_PERFCTR_PC_1_HI 0xb7 +#define A4XX_RBBM_PERFCTR_PC_2_LO 0xb8 +#define A4XX_RBBM_PERFCTR_PC_2_HI 0xb9 +#define A4XX_RBBM_PERFCTR_PC_3_LO 0xba +#define A4XX_RBBM_PERFCTR_PC_3_HI 0xbb +#define A4XX_RBBM_PERFCTR_PC_4_LO 0xbc +#define A4XX_RBBM_PERFCTR_PC_4_HI 0xbd +#define A4XX_RBBM_PERFCTR_PC_5_LO 0xbe +#define A4XX_RBBM_PERFCTR_PC_5_HI 0xbf +#define A4XX_RBBM_PERFCTR_PC_6_LO 0xc0 +#define A4XX_RBBM_PERFCTR_PC_6_HI 0xc1 +#define A4XX_RBBM_PERFCTR_PC_7_LO 0xc2 +#define A4XX_RBBM_PERFCTR_PC_7_HI 0xc3 +#define A4XX_RBBM_PERFCTR_VFD_0_LO 0xc4 +#define A4XX_RBBM_PERFCTR_VFD_0_HI 0xc5 +#define A4XX_RBBM_PERFCTR_VFD_1_LO 0xc6 +#define A4XX_RBBM_PERFCTR_VFD_1_HI 0xc7 +#define A4XX_RBBM_PERFCTR_VFD_2_LO 0xc8 +#define A4XX_RBBM_PERFCTR_VFD_2_HI 0xc9 +#define A4XX_RBBM_PERFCTR_VFD_3_LO 0xca +#define A4XX_RBBM_PERFCTR_VFD_3_HI 0xcb +#define A4XX_RBBM_PERFCTR_VFD_4_LO 0xcc +#define A4XX_RBBM_PERFCTR_VFD_4_HI 0xcd +#define A4XX_RBBM_PERFCTR_VFD_5_LO 0xce +#define A4XX_RBBM_PERFCTR_VFD_5_HI 0xcf +#define A4XX_RBBM_PERFCTR_VFD_6_LO 0xd0 +#define A4XX_RBBM_PERFCTR_VFD_6_HI 0xd1 +#define A4XX_RBBM_PERFCTR_VFD_7_LO 0xd2 +#define A4XX_RBBM_PERFCTR_VFD_7_HI 0xd3 +#define A4XX_RBBM_PERFCTR_HLSQ_0_LO 0xd4 +#define A4XX_RBBM_PERFCTR_HLSQ_0_HI 0xd5 +#define A4XX_RBBM_PERFCTR_HLSQ_1_LO 0xd6 +#define A4XX_RBBM_PERFCTR_HLSQ_1_HI 0xd7 +#define A4XX_RBBM_PERFCTR_HLSQ_2_LO 0xd8 +#define A4XX_RBBM_PERFCTR_HLSQ_2_HI 0xd9 +#define A4XX_RBBM_PERFCTR_HLSQ_3_LO 0xda +#define A4XX_RBBM_PERFCTR_HLSQ_3_HI 0xdb +#define A4XX_RBBM_PERFCTR_HLSQ_4_LO 0xdc +#define A4XX_RBBM_PERFCTR_HLSQ_4_HI 0xdd +#define A4XX_RBBM_PERFCTR_HLSQ_5_LO 0xde +#define A4XX_RBBM_PERFCTR_HLSQ_5_HI 0xdf +#define A4XX_RBBM_PERFCTR_HLSQ_6_LO 0xe0 +#define A4XX_RBBM_PERFCTR_HLSQ_6_HI 0xe1 +#define A4XX_RBBM_PERFCTR_HLSQ_7_LO 0xe2 +#define A4XX_RBBM_PERFCTR_HLSQ_7_HI 0xe3 +#define A4XX_RBBM_PERFCTR_VPC_0_LO 0xe4 +#define A4XX_RBBM_PERFCTR_VPC_0_HI 0xe5 +#define A4XX_RBBM_PERFCTR_VPC_1_LO 0xe6 +#define A4XX_RBBM_PERFCTR_VPC_1_HI 0xe7 +#define A4XX_RBBM_PERFCTR_VPC_2_LO 0xe8 +#define A4XX_RBBM_PERFCTR_VPC_2_HI 0xe9 +#define A4XX_RBBM_PERFCTR_VPC_3_LO 0xea +#define A4XX_RBBM_PERFCTR_VPC_3_HI 0xeb +#define A4XX_RBBM_PERFCTR_CCU_0_LO 0xec +#define A4XX_RBBM_PERFCTR_CCU_0_HI 0xed +#define A4XX_RBBM_PERFCTR_CCU_1_LO 0xee +#define A4XX_RBBM_PERFCTR_CCU_1_HI 0xef +#define A4XX_RBBM_PERFCTR_CCU_2_LO 0xf0 +#define A4XX_RBBM_PERFCTR_CCU_2_HI 0xf1 +#define A4XX_RBBM_PERFCTR_CCU_3_LO 0xf2 +#define A4XX_RBBM_PERFCTR_CCU_3_HI 0xf3 +#define A4XX_RBBM_PERFCTR_TSE_0_LO 0xf4 +#define A4XX_RBBM_PERFCTR_TSE_0_HI 0xf5 +#define A4XX_RBBM_PERFCTR_TSE_1_LO 0xf6 +#define A4XX_RBBM_PERFCTR_TSE_1_HI 0xf7 +#define A4XX_RBBM_PERFCTR_TSE_2_LO 0xf8 +#define A4XX_RBBM_PERFCTR_TSE_2_HI 0xf9 +#define A4XX_RBBM_PERFCTR_TSE_3_LO 0xfa +#define A4XX_RBBM_PERFCTR_TSE_3_HI 0xfb +#define A4XX_RBBM_PERFCTR_RAS_0_LO 0xfc +#define A4XX_RBBM_PERFCTR_RAS_0_HI 0xfd +#define A4XX_RBBM_PERFCTR_RAS_1_LO 0xfe +#define A4XX_RBBM_PERFCTR_RAS_1_HI 0xff +#define A4XX_RBBM_PERFCTR_RAS_2_LO 0x100 +#define A4XX_RBBM_PERFCTR_RAS_2_HI 0x101 +#define A4XX_RBBM_PERFCTR_RAS_3_LO 0x102 +#define A4XX_RBBM_PERFCTR_RAS_3_HI 0x103 +#define A4XX_RBBM_PERFCTR_UCHE_0_LO 0x104 +#define A4XX_RBBM_PERFCTR_UCHE_0_HI 0x105 +#define A4XX_RBBM_PERFCTR_UCHE_1_LO 0x106 +#define A4XX_RBBM_PERFCTR_UCHE_1_HI 0x107 +#define A4XX_RBBM_PERFCTR_UCHE_2_LO 0x108 +#define A4XX_RBBM_PERFCTR_UCHE_2_HI 0x109 +#define A4XX_RBBM_PERFCTR_UCHE_3_LO 0x10a +#define A4XX_RBBM_PERFCTR_UCHE_3_HI 0x10b +#define A4XX_RBBM_PERFCTR_UCHE_4_LO 0x10c +#define A4XX_RBBM_PERFCTR_UCHE_4_HI 0x10d +#define A4XX_RBBM_PERFCTR_UCHE_5_LO 0x10e +#define A4XX_RBBM_PERFCTR_UCHE_5_HI 0x10f +#define A4XX_RBBM_PERFCTR_UCHE_6_LO 0x110 +#define A4XX_RBBM_PERFCTR_UCHE_6_HI 0x111 +#define A4XX_RBBM_PERFCTR_UCHE_7_LO 0x112 +#define A4XX_RBBM_PERFCTR_UCHE_7_HI 0x113 +#define A4XX_RBBM_PERFCTR_TP_0_LO 0x114 +#define A4XX_RBBM_PERFCTR_TP_0_HI 0x115 +#define A4XX_RBBM_PERFCTR_TP_1_LO 0x116 +#define A4XX_RBBM_PERFCTR_TP_1_HI 0x117 +#define A4XX_RBBM_PERFCTR_TP_2_LO 0x118 +#define A4XX_RBBM_PERFCTR_TP_2_HI 0x119 +#define A4XX_RBBM_PERFCTR_TP_3_LO 0x11a +#define A4XX_RBBM_PERFCTR_TP_3_HI 0x11b +#define A4XX_RBBM_PERFCTR_TP_4_LO 0x11c +#define A4XX_RBBM_PERFCTR_TP_4_HI 0x11d +#define A4XX_RBBM_PERFCTR_TP_5_LO 0x11e +#define A4XX_RBBM_PERFCTR_TP_5_HI 0x11f +#define A4XX_RBBM_PERFCTR_TP_6_LO 0x120 +#define A4XX_RBBM_PERFCTR_TP_6_HI 0x121 +#define A4XX_RBBM_PERFCTR_TP_7_LO 0x122 +#define A4XX_RBBM_PERFCTR_TP_7_HI 0x123 +#define A4XX_RBBM_PERFCTR_SP_0_LO 0x124 +#define A4XX_RBBM_PERFCTR_SP_0_HI 0x125 +#define A4XX_RBBM_PERFCTR_SP_1_LO 0x126 +#define A4XX_RBBM_PERFCTR_SP_1_HI 0x127 +#define A4XX_RBBM_PERFCTR_SP_2_LO 0x128 +#define A4XX_RBBM_PERFCTR_SP_2_HI 0x129 +#define A4XX_RBBM_PERFCTR_SP_3_LO 0x12a +#define A4XX_RBBM_PERFCTR_SP_3_HI 0x12b +#define A4XX_RBBM_PERFCTR_SP_4_LO 0x12c +#define A4XX_RBBM_PERFCTR_SP_4_HI 0x12d +#define A4XX_RBBM_PERFCTR_SP_5_LO 0x12e +#define A4XX_RBBM_PERFCTR_SP_5_HI 0x12f +#define A4XX_RBBM_PERFCTR_SP_6_LO 0x130 +#define A4XX_RBBM_PERFCTR_SP_6_HI 0x131 +#define A4XX_RBBM_PERFCTR_SP_7_LO 0x132 +#define A4XX_RBBM_PERFCTR_SP_7_HI 0x133 +#define A4XX_RBBM_PERFCTR_SP_8_LO 0x134 +#define A4XX_RBBM_PERFCTR_SP_8_HI 0x135 +#define A4XX_RBBM_PERFCTR_SP_9_LO 0x136 +#define A4XX_RBBM_PERFCTR_SP_9_HI 0x137 +#define A4XX_RBBM_PERFCTR_SP_10_LO 0x138 +#define A4XX_RBBM_PERFCTR_SP_10_HI 0x139 +#define A4XX_RBBM_PERFCTR_SP_11_LO 0x13a +#define A4XX_RBBM_PERFCTR_SP_11_HI 0x13b +#define A4XX_RBBM_PERFCTR_RB_0_LO 0x13c +#define A4XX_RBBM_PERFCTR_RB_0_HI 0x13d +#define A4XX_RBBM_PERFCTR_RB_1_LO 0x13e +#define A4XX_RBBM_PERFCTR_RB_1_HI 0x13f +#define A4XX_RBBM_PERFCTR_RB_2_LO 0x140 +#define A4XX_RBBM_PERFCTR_RB_2_HI 0x141 +#define A4XX_RBBM_PERFCTR_RB_3_LO 0x142 +#define A4XX_RBBM_PERFCTR_RB_3_HI 0x143 +#define A4XX_RBBM_PERFCTR_RB_4_LO 0x144 +#define A4XX_RBBM_PERFCTR_RB_4_HI 0x145 +#define A4XX_RBBM_PERFCTR_RB_5_LO 0x146 +#define A4XX_RBBM_PERFCTR_RB_5_HI 0x147 +#define A4XX_RBBM_PERFCTR_RB_6_LO 0x148 +#define A4XX_RBBM_PERFCTR_RB_6_HI 0x149 +#define A4XX_RBBM_PERFCTR_RB_7_LO 0x14a +#define A4XX_RBBM_PERFCTR_RB_7_HI 0x14b +#define A4XX_RBBM_PERFCTR_VSC_0_LO 0x14c +#define A4XX_RBBM_PERFCTR_VSC_0_HI 0x14d +#define A4XX_RBBM_PERFCTR_VSC_1_LO 0x14e +#define A4XX_RBBM_PERFCTR_VSC_1_HI 0x14f +#define A4XX_RBBM_PERFCTR_PWR_0_LO 0x166 +#define A4XX_RBBM_PERFCTR_PWR_0_HI 0x167 +#define A4XX_RBBM_PERFCTR_PWR_1_LO 0x168 +#define A4XX_RBBM_PERFCTR_PWR_1_HI 0x169 +#define A4XX_RBBM_ALWAYSON_COUNTER_LO 0x16e +#define A4XX_RBBM_ALWAYSON_COUNTER_HI 0x16f +#define A4XX_RBBM_PERFCTR_CTL 0x170 +#define A4XX_RBBM_PERFCTR_LOAD_CMD0 0x171 +#define A4XX_RBBM_PERFCTR_LOAD_CMD1 0x172 +#define A4XX_RBBM_PERFCTR_LOAD_CMD2 0x173 +#define A4XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x174 +#define A4XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x175 +#define A4XX_RBBM_PERFCTR_RBBM_SEL_0 0x176 +#define A4XX_RBBM_PERFCTR_RBBM_SEL_1 0x177 +#define A4XX_RBBM_PERFCTR_RBBM_SEL_2 0x178 +#define A4XX_RBBM_PERFCTR_RBBM_SEL_3 0x179 +#define A4XX_RBBM_GPU_BUSY_MASKED 0x17a +#define A4XX_RBBM_INT_0_STATUS 0x17d +#define A4XX_RBBM_AHB_ME_SPLIT_STATUS 0x18c +#define A4XX_RBBM_AHB_PFP_SPLIT_STATUS 0x18d +#define A4XX_RBBM_AHB_ERROR_STATUS 0x18f +#define A4XX_RBBM_STATUS 0x191 +#define A4XX_RBBM_CFG_DEBBUS_TRACE_BUF4 0x1ad +#define A4XX_RBBM_POWER_STATUS 0x1b0 +#define A4XX_RBBM_PPD_V2_SP_PWR_WEIGHTS 0x1b2 +#define A4XX_RBBM_PPD_V2_SP_RB_EPOCH_TH 0x1b3 +#define A4XX_RBBM_PPD_V2_TP_CONFIG 0x1b4 +#define A4XX_RBBM_PPD_RAMP_V2_CONTROL 0x1b5 +#define A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL2 0x1b8 +#define A4XX_RBBM_PPD_CTRL 0x1b9 +#define A4XX_RBBM_PPD_EPOCH_INTER_TH_HIGH_CLEAR_THR 0x1bc +#define A4XX_RBBM_PPD_EPOCH_INTER_TH_LOW 0x1bd +/* SECVID registers */ +#define A4XX_RBBM_SECVID_TRUST_CONFIG 0xf000 +#define A4XX_RBBM_SECVID_TRUST_CONTROL 0xf400 +#define A4XX_RBBM_SECVID_TSB_TRUSTED_BASE 0xf800 +#define A4XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xf801 +#define A4XX_RBBM_SECVID_TSB_CONTROL 0xf802 + +/* CP registers */ +#define A4XX_CP_RB_BASE 0x200 +#define A4XX_CP_RB_CNTL 0x201 +#define A4XX_CP_RB_RPTR_ADDR 0x203 +#define A4XX_CP_RB_RPTR 0x204 +#define A4XX_CP_RB_WPTR 0x205 +#define A4XX_CP_IB1_BASE 0x206 +#define A4XX_CP_IB1_BUFSZ 0x207 +#define A4XX_CP_IB2_BASE 0x208 +#define A4XX_CP_IB2_BUFSZ 0x209 +#define A4XX_CP_ROQ_ADDR 0x21C +#define A4XX_CP_ROQ_DATA 0x21D +#define A4XX_CP_MEQ_ADDR 0x21E +#define A4XX_CP_MEQ_DATA 0x21F +#define A4XX_CP_MERCIU_ADDR 0x220 +#define A4XX_CP_MERCIU_DATA 0x221 +#define A4XX_CP_MERCIU_DATA2 0x222 +#define A4XX_CP_PFP_UCODE_ADDR 0x223 +#define A4XX_CP_PFP_UCODE_DATA 0x224 +#define A4XX_CP_ME_RAM_WADDR 0x225 +#define A4XX_CP_ME_RAM_RADDR 0x226 +#define A4XX_CP_ME_RAM_DATA 0x227 + +#define A4XX_CP_PREEMPT 0x22a + +#define A4XX_CP_PREEMPT_DISABLE 0x22b +#define A4XX_CP_CNTL 0x22c +#define A4XX_CP_ME_CNTL 0x22d +#define A4XX_CP_DEBUG 0x22e +#define A4XX_CP_STATE_DEBUG_INDEX 0x22f +#define A4XX_CP_STATE_DEBUG_DATA 0x230 +#define A4XX_CP_POWER_COLLAPSE_CNTL 0x234 +/* + * CP debug settings for A4xx cores + * MIU_128BIT_WRITE_ENABLE [25] - Allow 128 bit writes to the VBIF + */ +#define A4XX_CP_DEBUG_DEFAULT (1 << 25) + +#define A4XX_CP_PROTECT_REG_0 0x240 +#define A4XX_CP_PROTECT_CTRL 0x250 +#define A4XX_CP_PROTECT_REG_10 0x251 + +#define A4XX_CP_ME_STATUS 0x4d1 +#define A4XX_CP_CNTL 0x22c +#define A4XX_CP_WFI_PEND_CTR 0x4d2 +#define A4XX_CP_PREEMPT_DEBUG 0x4d6 +#define A4XX_CP_HW_FAULT 0x4d8 +#define A4XX_CP_PROTECT_STATUS 0x4da +#define A4XX_CP_PERFCTR_CP_SEL_0 0x500 +#define A4XX_CP_PERFCTR_CP_SEL_1 0x501 +#define A4XX_CP_PERFCTR_CP_SEL_2 0x502 +#define A4XX_CP_PERFCTR_CP_SEL_3 0x503 +#define A4XX_CP_PERFCTR_CP_SEL_4 0x504 +#define A4XX_CP_PERFCTR_CP_SEL_5 0x505 +#define A4XX_CP_PERFCTR_CP_SEL_6 0x506 +#define A4XX_CP_PERFCTR_CP_SEL_7 0x507 + +#define A4XX_CP_SCRATCH_REG6 0x57e +#define A4XX_CP_SCRATCH_REG7 0x57f +#define A4XX_CP_SCRATCH_REG8 0x580 +#define A4XX_CP_SCRATCH_REG9 0x581 +#define A4XX_CP_SCRATCH_REG10 0x582 +#define A4XX_CP_SCRATCH_REG11 0x583 +#define A4XX_CP_SCRATCH_REG12 0x584 +#define A4XX_CP_SCRATCH_REG13 0x585 +#define A4XX_CP_SCRATCH_REG14 0x586 +#define A4XX_CP_SCRATCH_REG15 0x587 +#define A4XX_CP_SCRATCH_REG16 0x588 +#define A4XX_CP_SCRATCH_REG17 0x589 +#define A4XX_CP_SCRATCH_REG18 0x58a +#define A4XX_CP_SCRATCH_REG23 0x58f + +/* SP registers */ +#define A4XX_SP_SP_CTRL 0x22C0 +#define A4XX_SP_INSTR_CACHE_CTRL 0x22c1 +#define A4XX_SP_VS_OBJ_START 0x22e1 +#define A4XX_SP_VS_PVT_MEM_ADDR 0x22e3 +#define A4XX_SP_FS_CTRL_1 0x22e9 +#define A4XX_SP_FS_OBJ_START 0x22eb +#define A4XX_SP_FS_PVT_MEM_ADDR 0x22ed +#define A4XX_SP_CS_CTRL_0 0x2300 +#define A4XX_SP_CS_OBJ_OFFSET 0x2301 +#define A4XX_SP_CS_OBJ_START 0x2302 +#define A4XX_SP_CS_LENGTH 0x2306 +#define A4XX_SP_MODE_CONTROL 0xec3 +#define A4XX_SP_PERFCTR_SP_SEL_0 0xec4 +#define A4XX_SP_PERFCTR_SP_SEL_1 0xec5 +#define A4XX_SP_PERFCTR_SP_SEL_2 0xec6 +#define A4XX_SP_PERFCTR_SP_SEL_3 0xec7 +#define A4XX_SP_PERFCTR_SP_SEL_4 0xec8 +#define A4XX_SP_PERFCTR_SP_SEL_5 0xec9 +#define A4XX_SP_PERFCTR_SP_SEL_6 0xeca +#define A4XX_SP_PERFCTR_SP_SEL_7 0xecb +#define A4XX_SP_PERFCTR_SP_SEL_8 0xecc +#define A4XX_SP_PERFCTR_SP_SEL_9 0xecd +#define A4XX_SP_PERFCTR_SP_SEL_10 0xece +#define A4XX_SP_PERFCTR_SP_SEL_11 0xecf +#define A4XX_SP_VS_PVT_MEM_ADDR 0x22e3 +#define A4XX_SP_FS_PVT_MEM_ADDR 0x22ed +#define A4XX_SP_VS_OBJ_START 0x22e1 +#define A4XX_SP_FS_OBJ_START 0x22eb + +/* COUNTABLE FOR SP PERFCOUNTER */ +#define A4XX_SP_ALU_ACTIVE_CYCLES 0x1D +#define A4XX_SP0_ICL1_MISSES 0x1A +#define A4XX_SP_FS_CFLOW_INSTRUCTIONS 0x0C + +/* COUNTABLE FOR TSE PERFCOUNTER */ +#define A4XX_TSE_INPUT_PRIM_NUM 0x0 + +enum a4xx_sp_perfctr_sp_sel { + SP_FS_STAGE_BARY_INSTRUCTIONS = 0x10, +}; + +/* VPC registers */ +#define A4XX_VPC_DEBUG_RAM_SEL 0xe60 +#define A4XX_VPC_DEBUG_RAM_READ 0xe61 +#define A4XX_VPC_PERFCTR_VPC_SEL_0 0xe65 +#define A4XX_VPC_PERFCTR_VPC_SEL_1 0xe66 +#define A4XX_VPC_PERFCTR_VPC_SEL_2 0xe67 +#define A4XX_VPC_PERFCTR_VPC_SEL_3 0xe68 + +/* UCHE register */ +#define UCHE_TRAP_BASE_LO 0xe83 +#define UCHE_TRAP_BASE_HI 0xe84 +#define A4XX_UCHE_INVALIDATE0 0xe8a +#define A4XX_UCHE_INVALIDATE1 0xe8b +#define A4XX_UCHE_CACHE_WAYS_VFD 0xe8c + +/* VSC registers */ +#define A4XX_VSC_SIZE_ADDRESS 0xc01 +#define A4XX_VSC_PIPE_DATA_ADDRESS_0 0xc10 +#define A4XX_VSC_PIPE_DATA_ADDRESS_1 0xc11 +#define A4XX_VSC_PIPE_DATA_ADDRESS_2 0xc12 +#define A4XX_VSC_PIPE_DATA_ADDRESS_3 0xc13 +#define A4XX_VSC_PIPE_DATA_ADDRESS_4 0xc14 +#define A4XX_VSC_PIPE_DATA_ADDRESS_5 0xc15 +#define A4XX_VSC_PIPE_DATA_ADDRESS_6 0xc16 +#define A4XX_VSC_PIPE_DATA_ADDRESS_7 0xc17 +#define A4XX_VSC_PIPE_DATA_LENGTH_0 0xc18 +#define A4XX_VSC_PIPE_DATA_LENGTH_1 0xc19 +#define A4XX_VSC_PIPE_DATA_LENGTH_2 0xc1a +#define A4XX_VSC_PIPE_DATA_LENGTH_3 0xc1b +#define A4XX_VSC_PIPE_DATA_LENGTH_4 0xc1c +#define A4XX_VSC_PIPE_DATA_LENGTH_5 0xc1d +#define A4XX_VSC_PIPE_DATA_LENGTH_6 0xc1e +#define A4XX_VSC_PIPE_DATA_LENGTH_7 0xc1f +#define A4XX_VSC_PERFCTR_VSC_SEL_0 0xc50 +#define A4XX_VSC_PERFCTR_VSC_SEL_1 0xc51 + +/* VFD registers */ +#define A4XX_VFD_FETCH_INSTR_1_31 0x2287 +#define A4XX_VFD_PERFCTR_VFD_SEL_0 0xe43 +#define A4XX_VFD_PERFCTR_VFD_SEL_1 0xe44 +#define A4XX_VFD_PERFCTR_VFD_SEL_2 0xe45 +#define A4XX_VFD_PERFCTR_VFD_SEL_3 0xe46 +#define A4XX_VFD_PERFCTR_VFD_SEL_4 0xe47 +#define A4XX_VFD_PERFCTR_VFD_SEL_5 0xe48 +#define A4XX_VFD_PERFCTR_VFD_SEL_6 0xe49 +#define A4XX_VFD_PERFCTR_VFD_SEL_7 0xe4a +#define A4XX_VFD_FETCH_INSTR_1_0 0x220b +#define A4XX_VFD_FETCH_INSTR_1_1 0x220f +#define A4XX_VFD_FETCH_INSTR_1_2 0x2213 +#define A4XX_VFD_FETCH_INSTR_1_3 0x2217 +#define A4XX_VFD_FETCH_INSTR_1_4 0x221b +#define A4XX_VFD_FETCH_INSTR_1_5 0x221f +#define A4XX_VFD_FETCH_INSTR_1_6 0x2223 +#define A4XX_VFD_FETCH_INSTR_1_7 0x2227 +#define A4XX_VFD_FETCH_INSTR_1_8 0x222b +#define A4XX_VFD_FETCH_INSTR_1_9 0x222f +#define A4XX_VFD_FETCH_INSTR_1_10 0x2233 +#define A4XX_VFD_FETCH_INSTR_1_11 0x2237 +#define A4XX_VFD_FETCH_INSTR_1_12 0x223b +#define A4XX_VFD_FETCH_INSTR_1_13 0x223f +#define A4XX_VFD_FETCH_INSTR_1_14 0x2243 +#define A4XX_VFD_FETCH_INSTR_1_15 0x2247 +#define A4XX_VFD_FETCH_INSTR_1_16 0x224b +#define A4XX_VFD_FETCH_INSTR_1_17 0x224f +#define A4XX_VFD_FETCH_INSTR_1_18 0x2253 +#define A4XX_VFD_FETCH_INSTR_1_19 0x2257 +#define A4XX_VFD_FETCH_INSTR_1_20 0x225b +#define A4XX_VFD_FETCH_INSTR_1_21 0x225f +#define A4XX_VFD_FETCH_INSTR_1_22 0x2263 +#define A4XX_VFD_FETCH_INSTR_1_23 0x2267 +#define A4XX_VFD_FETCH_INSTR_1_24 0x226b +#define A4XX_VFD_FETCH_INSTR_1_25 0x226f +#define A4XX_VFD_FETCH_INSTR_1_26 0x2273 +#define A4XX_VFD_FETCH_INSTR_1_27 0x2277 +#define A4XX_VFD_FETCH_INSTR_1_28 0x227b +#define A4XX_VFD_FETCH_INSTR_1_29 0x227f +#define A4XX_VFD_FETCH_INSTR_1_30 0x2283 +#define A4XX_VFD_FETCH_INSTR_1_31 0x2287 + + +enum a4xx_vfd_perfctr_vfd_sel { + VFD_VPC_BYPASS_TRANS = 0x2, + VFD_UPPER_SHADER_FIBERS = 0xb, + VFD_LOWER_SHADER_FIBERS = 0xc, +}; + +/* VBIF registers */ +#define A4XX_VBIF_VERSION 0x3000 +#define A4XX_VBIF_CLKON 0x3001 +#define A4XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1 +#define A4XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1 + +#define A4XX_VBIF_ABIT_SORT 0x301c +#define A4XX_VBIF_ABIT_SORT_CONF 0x301d +#define A4XX_VBIF_GATE_OFF_WRREQ_EN 0x302a +#define A4XX_VBIF_IN_RD_LIM_CONF0 0x302c +#define A4XX_VBIF_IN_RD_LIM_CONF1 0x302d +#define A4XX_VBIF_IN_WR_LIM_CONF0 0x3030 +#define A4XX_VBIF_IN_WR_LIM_CONF1 0x3031 +#define A4XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049 + +#define A4XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A4XX_VBIF_XIN_HALT_CTRL0_MASK 0x1F +#define A405_VBIF_XIN_HALT_CTRL0_MASK 0x3 + +#define A4XX_VBIF_XIN_HALT_CTRL1 0x3081 + +#define A4XX_VBIF_TEST_BUS_OUT_CTRL 0x3084 +#define A4XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1 +#define A4XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0 + +#define A4XX_VBIF_TEST_BUS1_CTRL0 0x3085 +#define A4XX_VBIF_TEST_BUS1_CTRL1 0x3086 +#define A4XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF +#define A4XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0 + +#define A4XX_VBIF_TEST_BUS2_CTRL0 0x3087 +#define A4XX_VBIF_TEST_BUS2_CTRL1 0x3088 +#define A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0xF +#define A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0 + +#define A4XX_VBIF_TEST_BUS_OUT 0x308c + +#define A4XX_VBIF_PERF_CNT_SEL0 0x30d0 +#define A4XX_VBIF_PERF_CNT_SEL1 0x30d1 +#define A4XX_VBIF_PERF_CNT_SEL2 0x30d2 +#define A4XX_VBIF_PERF_CNT_SEL3 0x30d3 +#define A4XX_VBIF_PERF_CNT_LOW0 0x30d8 +#define A4XX_VBIF_PERF_CNT_LOW1 0x30d9 +#define A4XX_VBIF_PERF_CNT_LOW2 0x30da +#define A4XX_VBIF_PERF_CNT_LOW3 0x30db +#define A4XX_VBIF_PERF_CNT_HIGH0 0x30e0 +#define A4XX_VBIF_PERF_CNT_HIGH1 0x30e1 +#define A4XX_VBIF_PERF_CNT_HIGH2 0x30e2 +#define A4XX_VBIF_PERF_CNT_HIGH3 0x30e3 + +#define A4XX_VBIF_PERF_PWR_CNT_EN0 0x3100 +#define A4XX_VBIF_PERF_PWR_CNT_EN1 0x3101 +#define A4XX_VBIF_PERF_PWR_CNT_EN2 0x3102 +#define A4XX_VBIF_PERF_PWR_CNT_EN3 0x3103 +#define A4XX_VBIF_PERF_PWR_CNT_LOW0 0x3110 +#define A4XX_VBIF_PERF_PWR_CNT_LOW1 0x3111 +#define A4XX_VBIF_PERF_PWR_CNT_LOW2 0x3112 +#define A4XX_VBIF_PERF_PWR_CNT_LOW3 0x3113 +#define A4XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118 +#define A4XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119 +#define A4XX_VBIF_PERF_PWR_CNT_HIGH2 0x311a +#define A4XX_VBIF_PERF_PWR_CNT_HIGH3 0x311b + +/* GRAS registers */ +#define A4XX_GRAS_PERFCTR_TSE_SEL_0 0xc88 +#define A4XX_GRAS_PERFCTR_TSE_SEL_1 0xc89 +#define A4XX_GRAS_PERFCTR_TSE_SEL_2 0xc8a +#define A4XX_GRAS_PERFCTR_TSE_SEL_3 0xc8b +#define A4XX_GRAS_PERFCTR_RAS_SEL_0 0xc8c +#define A4XX_GRAS_PERFCTR_RAS_SEL_1 0xc8d +#define A4XX_GRAS_PERFCTR_RAS_SEL_2 0xc8e +#define A4XX_GRAS_PERFCTR_RAS_SEL_3 0xc8f + +/* PC registers */ +#define A4XX_PC_PERFCTR_PC_SEL_0 0xd10 +#define A4XX_PC_PERFCTR_PC_SEL_1 0xd11 +#define A4XX_PC_PERFCTR_PC_SEL_2 0xd12 +#define A4XX_PC_PERFCTR_PC_SEL_3 0xd13 +#define A4XX_PC_PERFCTR_PC_SEL_4 0xd14 +#define A4XX_PC_PERFCTR_PC_SEL_5 0xd15 +#define A4XX_PC_PERFCTR_PC_SEL_6 0xd16 +#define A4XX_PC_PERFCTR_PC_SEL_7 0xd17 + +enum a4xx_pc_perfctr_pc_sel { + PC_INSTANCES = 0x1, + PC_VERTEX_HITS = 0x8, + PC_GENERATED_FIBERS = 0x12, + PC_GENERATED_WAVES = 0x13, +}; + +/* HLSQ registers */ +#define A4XX_HLSQ_TIMEOUT_THRESHOLD 0xe00 +#define A4XX_HLSQ_MODE_CONTROL 0xe05 +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xe06 +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xe07 +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xe08 +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xe09 +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xe0a +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xe0b +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_6 0xe0c +#define A4XX_HLSQ_PERFCTR_HLSQ_SEL_7 0xe0d +#define A4XX_HLSQ_SPTP_RDSEL 0xe30 +#define A4xx_HLSQ_CONTROL_0 0x23c0 +#define A4XX_HLSQ_CS_CONTROL 0x23ca +#define A4XX_HLSQ_CL_NDRANGE_0 0x23cd +#define A4XX_HLSQ_CL_CONTROL_0 0x23d4 +#define A4XX_HLSQ_CL_KERNEL_CONST 0x23d6 +#define A4XX_HLSQ_CL_KERNEL_GROUP_X 0x23d7 +#define A4XX_HLSQ_CL_WG_OFFSET 0x23da +#define A4XX_HLSQ_UPDATE_CONTROL 0x23db + +enum a4xx_hlsq_perfctr_hlsq_sel { + HLSQ_SP_VS_STAGE_CONSTANT = 0x0, + HLSQ_SP_VS_STAGE_INSTRUCTIONS = 0x1, + HLSQ_SP_FS_STAGE_CONSTANT = 0x2, + HLSQ_SP_FS_STAGE_INSTRUCTIONS = 0x3, + HLSQ_FS_STAGE_16_WAVES = 0x8, + HLSQ_FS_STAGE_32_WAVES = 0x9, + HLSQ_FS_STAGE_64_WAVES = 0xa, + HLSQ_VS_STAGE_16_WAVES = 0xb, + HLSQ_VS_STAGE_32_WAVES = 0xc, +}; + +/* CCU registers */ +#define A4XX_RB_PERFCTR_CCU_SEL_0 0xccf +#define A4XX_RB_PERFCTR_CCU_SEL_1 0xcd0 +#define A4XX_RB_PERFCTR_CCU_SEL_2 0xcd1 +#define A4XX_RB_PERFCTR_CCU_SEL_3 0xcd2 + +enum a4xx_cu_perfctr_ccu_sel { + CCU_VBIF_STALL = 0x1, + CCU_VBIF_LATENCY_CYCLES = 0x4, + CCU_VBIF_LATENCY_SAMPLES = 0x5, + CCU_Z_READ = 0x13, + CCU_Z_WRITE = 0x14, + CCU_C_READ = 0x15, + CCU_C_WRITE = 0x16, +}; + +/* UCHE registers */ +#define A4XX_UCHE_PERFCTR_UCHE_SEL_0 0xe8e +#define A4XX_UCHE_PERFCTR_UCHE_SEL_1 0xe8f +#define A4XX_UCHE_PERFCTR_UCHE_SEL_2 0xe90 +#define A4XX_UCHE_PERFCTR_UCHE_SEL_3 0xe91 +#define A4XX_UCHE_PERFCTR_UCHE_SEL_4 0xe92 +#define A4XX_UCHE_PERFCTR_UCHE_SEL_5 0xe93 +#define A4XX_UCHE_PERFCTR_UCHE_SEL_6 0xe94 +#define A4XX_UCHE_PERFCTR_UCHE_SEL_7 0xe95 + +/* TPL1 registers */ +enum a4xx_uche_perfctr_uche_sel { + UCHE_READ_REQUESTS_MARB = 0x8, + UCHE_READ_REQUESTS_SP = 0x9, + UCHE_WRITE_REQUESTS_MARB = 0xa, + UCHE_WRITE_REQUESTS_SP = 0xb, + UCHE_WRITE_REQUESTS_VPC = 0x14, +}; + +/* TPL1 registers */ +#define A4XX_TPL1_TP_MODE_CONTROL 0xf03 +#define A4XX_TPL1_PERFCTR_TP_SEL_0 0xf04 +#define A4XX_TPL1_PERFCTR_TP_SEL_1 0xf05 +#define A4XX_TPL1_PERFCTR_TP_SEL_2 0xf06 +#define A4XX_TPL1_PERFCTR_TP_SEL_3 0xf07 +#define A4XX_TPL1_PERFCTR_TP_SEL_4 0xf08 +#define A4XX_TPL1_PERFCTR_TP_SEL_5 0xf09 +#define A4XX_TPL1_PERFCTR_TP_SEL_6 0xf0a +#define A4XX_TPL1_PERFCTR_TP_SEL_7 0xf0b +#define A4XX_TPL1_TP_TEX_TSIZE_1 0x23a0 + +enum a4xx_tpl1_perfctr_tp_sel { + TP_OUTPUT_TEXELS_POINT = 0x2, + TP_OUTPUT_TEXELS_BILINEAR = 0x3, + TP_OUTPUT_TEXELS_MIP = 0x4, + TP_OUTPUT_TEXELS_ANISO = 0x5, + TP_OUTPUT_TEXELS_OPS16 = 0x6, + TP_OUTPUT_TEXELS_OPS32 = 0x7, + TP_ZERO_LOD = 0xe, + TP_LATENCY = 0x12, + TP_LATENCY_TRANS = 0x13, +}; + +/* Enum for debug bus */ +enum a4xx_rbbm_debbus_id { + A4XX_RBBM_DEBBUS_CP_ID = 0x1, + A4XX_RBBM_DEBBUS_RBBM_ID = 0x2, + A4XX_RBBM_DEBBUS_VBIF_ID = 0x3, + A4XX_RBBM_DEBBUS_HLSQ_ID = 0x4, + A4XX_RBBM_DEBBUS_UCHE_ID = 0x5, + A4XX_RBBM_DEBBUS_DPM_ID = 0x6, + A4XX_RBBM_DEBBUS_TESS_ID = 0x7, + A4XX_RBBM_DEBBUS_PC_ID = 0x8, + A4XX_RBBM_DEBBUS_VFD_ID = 0x9, + A4XX_RBBM_DEBBUS_VPC_ID = 0xa, + A4XX_RBBM_DEBBUS_TSE_ID = 0xb, + A4XX_RBBM_DEBBUS_RAS_ID = 0xc, + A4XX_RBBM_DEBBUS_VSC_ID = 0xd, + A4XX_RBBM_DEBBUS_COM_ID = 0xe, + A4XX_RBBM_DEBBUS_DCOM_ID = 0xf, + A4XX_RBBM_DEBBUS_SP_0_ID = 0x10, + A4XX_RBBM_DEBBUS_SP_1_ID = 0x11, + A4XX_RBBM_DEBBUS_SP_2_ID = 0x12, + A4XX_RBBM_DEBBUS_SP_3_ID = 0x13, + A4XX_RBBM_DEBBUS_TPL1_0_ID = 0x18, + A4XX_RBBM_DEBBUS_TPL1_1_ID = 0x19, + A4XX_RBBM_DEBBUS_TPL1_2_ID = 0x1a, + A4XX_RBBM_DEBBUS_TPL1_3_ID = 0x1b, + A4XX_RBBM_DEBBUS_RB_0_ID = 0x20, + A4XX_RBBM_DEBBUS_RB_1_ID = 0x21, + A4XX_RBBM_DEBBUS_RB_2_ID = 0x22, + A4XX_RBBM_DEBBUS_RB_3_ID = 0x23, + A4XX_RBBM_DEBBUS_MARB_0_ID = 0x28, + A4XX_RBBM_DEBBUS_MARB_1_ID = 0x29, + A4XX_RBBM_DEBBUS_MARB_2_ID = 0x2a, + A4XX_RBBM_DEBBUS_MARB_3_ID = 0x2b, + A4XX_RBBM_DEBBUS_CCU_0_ID = 0x30, + A4XX_RBBM_DEBBUS_CCU_1_ID = 0x31, + A4XX_RBBM_DEBBUS_CCU_2_ID = 0x32, + A4XX_RBBM_DEBBUS_CCU_3_ID = 0x33 +}; + +#define A4XX_NUM_AXI_ARB_BLOCKS 2 +#define A4XX_NUM_XIN_BLOCKS 5 + +#endif /* _A4XX_REG_H */ diff --git a/drivers/gpu/msm/a5xx_reg.h b/drivers/gpu/msm/a5xx_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..9bf1e46bede223cdc1aae1d2b81bc014495612b3 --- /dev/null +++ b/drivers/gpu/msm/a5xx_reg.h @@ -0,0 +1,902 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2014-2016, The Linux Foundation. All rights reserved. + */ + +#ifndef _A5XX_REG_H +#define _A5XX_REG_H + +/* A5XX interrupt bits */ +#define A5XX_INT_RBBM_GPU_IDLE 0 +#define A5XX_INT_RBBM_AHB_ERROR 1 +#define A5XX_INT_RBBM_TRANSFER_TIMEOUT 2 +#define A5XX_INT_RBBM_ME_MS_TIMEOUT 3 +#define A5XX_INT_RBBM_PFP_MS_TIMEOUT 4 +#define A5XX_INT_RBBM_ETS_MS_TIMEOUT 5 +#define A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW 6 +#define A5XX_INT_RBBM_GPC_ERROR 7 +#define A5XX_INT_CP_SW 8 +#define A5XX_INT_CP_HW_ERROR 9 +#define A5XX_INT_CP_CCU_FLUSH_DEPTH_TS 10 +#define A5XX_INT_CP_CCU_FLUSH_COLOR_TS 11 +#define A5XX_INT_CP_CCU_RESOLVE_TS 12 +#define A5XX_INT_CP_IB2 13 +#define A5XX_INT_CP_IB1 14 +#define A5XX_INT_CP_RB 15 +#define A5XX_INT_CP_UNUSED_1 16 +#define A5XX_INT_CP_RB_DONE_TS 17 +#define A5XX_INT_CP_WT_DONE_TS 18 +#define A5XX_INT_UNKNOWN_1 19 +#define A5XX_INT_CP_CACHE_FLUSH_TS 20 +#define A5XX_INT_UNUSED_2 21 +#define A5XX_INT_RBBM_ATB_BUS_OVERFLOW 22 +#define A5XX_INT_MISC_HANG_DETECT 23 +#define A5XX_INT_UCHE_OOB_ACCESS 24 +#define A5XX_INT_UCHE_TRAP_INTR 25 +#define A5XX_INT_DEBBUS_INTR_0 26 +#define A5XX_INT_DEBBUS_INTR_1 27 +#define A5XX_INT_GPMU_VOLTAGE_DROOP 28 +#define A5XX_INT_GPMU_FIRMWARE 29 +#define A5XX_INT_ISDB_CPU_IRQ 30 +#define A5XX_INT_ISDB_UNDER_DEBUG 31 + +/* CP Interrupt bits */ +#define A5XX_CP_OPCODE_ERROR 0 +#define A5XX_CP_RESERVED_BIT_ERROR 1 +#define A5XX_CP_HW_FAULT_ERROR 2 +#define A5XX_CP_DMA_ERROR 3 +#define A5XX_CP_REGISTER_PROTECTION_ERROR 4 +#define A5XX_CP_AHB_ERROR 5 + +/* CP registers */ +#define A5XX_CP_RB_BASE 0x800 +#define A5XX_CP_RB_BASE_HI 0x801 +#define A5XX_CP_RB_CNTL 0x802 +#define A5XX_CP_RB_RPTR_ADDR_LO 0x804 +#define A5XX_CP_RB_RPTR_ADDR_HI 0x805 +#define A5XX_CP_RB_RPTR 0x806 +#define A5XX_CP_RB_WPTR 0x807 +#define A5XX_CP_PFP_STAT_ADDR 0x808 +#define A5XX_CP_PFP_STAT_DATA 0x809 +#define A5XX_CP_DRAW_STATE_ADDR 0x80B +#define A5XX_CP_DRAW_STATE_DATA 0x80C +#define A5XX_CP_CRASH_SCRIPT_BASE_LO 0x817 +#define A5XX_CP_CRASH_SCRIPT_BASE_HI 0x818 +#define A5XX_CP_CRASH_DUMP_CNTL 0x819 +#define A5XX_CP_ME_STAT_ADDR 0x81A +#define A5XX_CP_ROQ_THRESHOLDS_1 0x81F +#define A5XX_CP_ROQ_THRESHOLDS_2 0x820 +#define A5XX_CP_ROQ_DBG_ADDR 0x821 +#define A5XX_CP_ROQ_DBG_DATA 0x822 +#define A5XX_CP_MEQ_DBG_ADDR 0x823 +#define A5XX_CP_MEQ_DBG_DATA 0x824 +#define A5XX_CP_MEQ_THRESHOLDS 0x825 +#define A5XX_CP_MERCIU_SIZE 0x826 +#define A5XX_CP_MERCIU_DBG_ADDR 0x827 +#define A5XX_CP_MERCIU_DBG_DATA_1 0x828 +#define A5XX_CP_MERCIU_DBG_DATA_2 0x829 +#define A5XX_CP_PFP_UCODE_DBG_ADDR 0x82A +#define A5XX_CP_PFP_UCODE_DBG_DATA 0x82B +#define A5XX_CP_ME_UCODE_DBG_ADDR 0x82F +#define A5XX_CP_ME_UCODE_DBG_DATA 0x830 +#define A5XX_CP_CNTL 0x831 +#define A5XX_CP_ME_CNTL 0x832 +#define A5XX_CP_CHICKEN_DBG 0x833 +#define A5XX_CP_PFP_INSTR_BASE_LO 0x835 +#define A5XX_CP_PFP_INSTR_BASE_HI 0x836 +#define A5XX_CP_PM4_INSTR_BASE_LO 0x838 +#define A5XX_CP_PM4_INSTR_BASE_HI 0x839 +#define A5XX_CP_CONTEXT_SWITCH_CNTL 0x83B +#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO 0x83C +#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI 0x83D +#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO 0x83E +#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI 0x83F +#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x840 +#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x841 +#define A5XX_CP_ADDR_MODE_CNTL 0x860 +#define A5XX_CP_ME_STAT_DATA 0xB14 +#define A5XX_CP_WFI_PEND_CTR 0xB15 +#define A5XX_CP_INTERRUPT_STATUS 0xB18 +#define A5XX_CP_HW_FAULT 0xB1A +#define A5XX_CP_PROTECT_STATUS 0xB1C +#define A5XX_CP_IB1_BASE 0xB1F +#define A5XX_CP_IB1_BASE_HI 0xB20 +#define A5XX_CP_IB1_BUFSZ 0xB21 +#define A5XX_CP_IB2_BASE 0xB22 +#define A5XX_CP_IB2_BASE_HI 0xB23 +#define A5XX_CP_IB2_BUFSZ 0xB24 +#define A5XX_CP_PROTECT_REG_0 0x880 +#define A5XX_CP_PROTECT_CNTL 0x8A0 +#define A5XX_CP_AHB_FAULT 0xB1B +#define A5XX_CP_PERFCTR_CP_SEL_0 0xBB0 +#define A5XX_CP_PERFCTR_CP_SEL_1 0xBB1 +#define A5XX_CP_PERFCTR_CP_SEL_2 0xBB2 +#define A5XX_CP_PERFCTR_CP_SEL_3 0xBB3 +#define A5XX_CP_PERFCTR_CP_SEL_4 0xBB4 +#define A5XX_CP_PERFCTR_CP_SEL_5 0xBB5 +#define A5XX_CP_PERFCTR_CP_SEL_6 0xBB6 +#define A5XX_CP_PERFCTR_CP_SEL_7 0xBB7 + +#define A5XX_VSC_ADDR_MODE_CNTL 0xBC1 + +/* CP Power Counter Registers Select */ +#define A5XX_CP_POWERCTR_CP_SEL_0 0xBBA +#define A5XX_CP_POWERCTR_CP_SEL_1 0xBBB +#define A5XX_CP_POWERCTR_CP_SEL_2 0xBBC +#define A5XX_CP_POWERCTR_CP_SEL_3 0xBBD + +/* RBBM registers */ +#define A5XX_RBBM_CFG_DBGBUS_SEL_A 0x4 +#define A5XX_RBBM_CFG_DBGBUS_SEL_B 0x5 +#define A5XX_RBBM_CFG_DBGBUS_SEL_C 0x6 +#define A5XX_RBBM_CFG_DBGBUS_SEL_D 0x7 +#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 + +#define A5XX_RBBM_CFG_DBGBUS_CNTLT 0x8 +#define A5XX_RBBM_CFG_DBGBUS_CNTLM 0x9 +#define A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT 0x18 +#define A5XX_RBBM_CFG_DBGBUS_OPL 0xA +#define A5XX_RBBM_CFG_DBGBUS_OPE 0xB +#define A5XX_RBBM_CFG_DBGBUS_IVTL_0 0xC +#define A5XX_RBBM_CFG_DBGBUS_IVTL_1 0xD +#define A5XX_RBBM_CFG_DBGBUS_IVTL_2 0xE +#define A5XX_RBBM_CFG_DBGBUS_IVTL_3 0xF +#define A5XX_RBBM_CFG_DBGBUS_MASKL_0 0x10 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_1 0x11 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_2 0x12 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_3 0x13 +#define A5XX_RBBM_CFG_DBGBUS_BYTEL_0 0x14 +#define A5XX_RBBM_CFG_DBGBUS_BYTEL_1 0x15 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_0 0x16 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_1 0x17 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_2 0x18 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_3 0x19 +#define A5XX_RBBM_CFG_DBGBUS_MASKE_0 0x1A +#define A5XX_RBBM_CFG_DBGBUS_MASKE_1 0x1B +#define A5XX_RBBM_CFG_DBGBUS_MASKE_2 0x1C +#define A5XX_RBBM_CFG_DBGBUS_MASKE_3 0x1D +#define A5XX_RBBM_CFG_DBGBUS_NIBBLEE 0x1E +#define A5XX_RBBM_CFG_DBGBUS_PTRC0 0x1F +#define A5XX_RBBM_CFG_DBGBUS_PTRC1 0x20 +#define A5XX_RBBM_CFG_DBGBUS_LOADREG 0x21 +#define A5XX_RBBM_CFG_DBGBUS_IDX 0x22 +#define A5XX_RBBM_CFG_DBGBUS_CLRC 0x23 +#define A5XX_RBBM_CFG_DBGBUS_LOADIVT 0x24 +#define A5XX_RBBM_INTERFACE_HANG_INT_CNTL 0x2F +#define A5XX_RBBM_INT_CLEAR_CMD 0x37 +#define A5XX_RBBM_INT_0_MASK 0x38 +#define A5XX_RBBM_AHB_DBG_CNTL 0x3F +#define A5XX_RBBM_EXT_VBIF_DBG_CNTL 0x41 +#define A5XX_RBBM_SW_RESET_CMD 0x43 +#define A5XX_RBBM_BLOCK_SW_RESET_CMD 0x45 +#define A5XX_RBBM_BLOCK_SW_RESET_CMD2 0x46 +#define A5XX_RBBM_DBG_LO_HI_GPIO 0x48 +#define A5XX_RBBM_EXT_TRACE_BUS_CNTL 0x49 +#define A5XX_RBBM_CLOCK_CNTL_TP0 0x4A +#define A5XX_RBBM_CLOCK_CNTL_TP1 0x4B +#define A5XX_RBBM_CLOCK_CNTL_TP2 0x4C +#define A5XX_RBBM_CLOCK_CNTL_TP3 0x4D +#define A5XX_RBBM_CLOCK_CNTL2_TP0 0x4E +#define A5XX_RBBM_CLOCK_CNTL2_TP1 0x4F +#define A5XX_RBBM_CLOCK_CNTL2_TP2 0x50 +#define A5XX_RBBM_CLOCK_CNTL2_TP3 0x51 +#define A5XX_RBBM_CLOCK_CNTL3_TP0 0x52 +#define A5XX_RBBM_CLOCK_CNTL3_TP1 0x53 +#define A5XX_RBBM_CLOCK_CNTL3_TP2 0x54 +#define A5XX_RBBM_CLOCK_CNTL3_TP3 0x55 +#define A5XX_RBBM_READ_AHB_THROUGH_DBG 0x59 +#define A5XX_RBBM_CLOCK_CNTL_UCHE 0x5A +#define A5XX_RBBM_CLOCK_CNTL2_UCHE 0x5B +#define A5XX_RBBM_CLOCK_CNTL3_UCHE 0x5C +#define A5XX_RBBM_CLOCK_CNTL4_UCHE 0x5D +#define A5XX_RBBM_CLOCK_HYST_UCHE 0x5E +#define A5XX_RBBM_CLOCK_DELAY_UCHE 0x5F +#define A5XX_RBBM_CLOCK_MODE_GPC 0x60 +#define A5XX_RBBM_CLOCK_DELAY_GPC 0x61 +#define A5XX_RBBM_CLOCK_HYST_GPC 0x62 +#define A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x63 +#define A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x64 +#define A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x65 +#define A5XX_RBBM_CLOCK_DELAY_HLSQ 0x66 +#define A5XX_RBBM_CLOCK_CNTL 0x67 +#define A5XX_RBBM_CLOCK_CNTL_SP0 0x68 +#define A5XX_RBBM_CLOCK_CNTL_SP1 0x69 +#define A5XX_RBBM_CLOCK_CNTL_SP2 0x6A +#define A5XX_RBBM_CLOCK_CNTL_SP3 0x6B +#define A5XX_RBBM_CLOCK_CNTL2_SP0 0x6C +#define A5XX_RBBM_CLOCK_CNTL2_SP1 0x6D +#define A5XX_RBBM_CLOCK_CNTL2_SP2 0x6E +#define A5XX_RBBM_CLOCK_CNTL2_SP3 0x6F +#define A5XX_RBBM_CLOCK_HYST_SP0 0x70 +#define A5XX_RBBM_CLOCK_HYST_SP1 0x71 +#define A5XX_RBBM_CLOCK_HYST_SP2 0x72 +#define A5XX_RBBM_CLOCK_HYST_SP3 0x73 +#define A5XX_RBBM_CLOCK_DELAY_SP0 0x74 +#define A5XX_RBBM_CLOCK_DELAY_SP1 0x75 +#define A5XX_RBBM_CLOCK_DELAY_SP2 0x76 +#define A5XX_RBBM_CLOCK_DELAY_SP3 0x77 +#define A5XX_RBBM_CLOCK_CNTL_RB0 0x78 +#define A5XX_RBBM_CLOCK_CNTL_RB1 0x79 +#define A5XX_RBBM_CLOCK_CNTL_RB2 0x7a +#define A5XX_RBBM_CLOCK_CNTL_RB3 0x7B +#define A5XX_RBBM_CLOCK_CNTL2_RB0 0x7C +#define A5XX_RBBM_CLOCK_CNTL2_RB1 0x7D +#define A5XX_RBBM_CLOCK_CNTL2_RB2 0x7E +#define A5XX_RBBM_CLOCK_CNTL2_RB3 0x7F +#define A5XX_RBBM_CLOCK_HYST_RAC 0x80 +#define A5XX_RBBM_CLOCK_DELAY_RAC 0x81 +#define A5XX_RBBM_CLOCK_CNTL_CCU0 0x82 +#define A5XX_RBBM_CLOCK_CNTL_CCU1 0x83 +#define A5XX_RBBM_CLOCK_CNTL_CCU2 0x84 +#define A5XX_RBBM_CLOCK_CNTL_CCU3 0x85 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU0 0x86 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU1 0x87 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU2 0x88 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU3 0x89 +#define A5XX_RBBM_CLOCK_CNTL_RAC 0x8A +#define A5XX_RBBM_CLOCK_CNTL2_RAC 0x8B +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0 0x8C +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1 0x8D +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2 0x8E +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3 0x8F +#define A5XX_RBBM_CLOCK_HYST_VFD 0x90 +#define A5XX_RBBM_CLOCK_MODE_VFD 0x91 +#define A5XX_RBBM_CLOCK_DELAY_VFD 0x92 +#define A5XX_RBBM_AHB_CNTL0 0x93 +#define A5XX_RBBM_AHB_CNTL1 0x94 +#define A5XX_RBBM_AHB_CNTL2 0x95 +#define A5XX_RBBM_AHB_CMD 0x96 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11 0x9C +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12 0x9D +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13 0x9E +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14 0x9F +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15 0xA0 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16 0xA1 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17 0xA2 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18 0xA3 +#define A5XX_RBBM_CLOCK_DELAY_TP0 0xA4 +#define A5XX_RBBM_CLOCK_DELAY_TP1 0xA5 +#define A5XX_RBBM_CLOCK_DELAY_TP2 0xA6 +#define A5XX_RBBM_CLOCK_DELAY_TP3 0xA7 +#define A5XX_RBBM_CLOCK_DELAY2_TP0 0xA8 +#define A5XX_RBBM_CLOCK_DELAY2_TP1 0xA9 +#define A5XX_RBBM_CLOCK_DELAY2_TP2 0xAA +#define A5XX_RBBM_CLOCK_DELAY2_TP3 0xAB +#define A5XX_RBBM_CLOCK_DELAY3_TP0 0xAC +#define A5XX_RBBM_CLOCK_DELAY3_TP1 0xAD +#define A5XX_RBBM_CLOCK_DELAY3_TP2 0xAE +#define A5XX_RBBM_CLOCK_DELAY3_TP3 0xAF +#define A5XX_RBBM_CLOCK_HYST_TP0 0xB0 +#define A5XX_RBBM_CLOCK_HYST_TP1 0xB1 +#define A5XX_RBBM_CLOCK_HYST_TP2 0xB2 +#define A5XX_RBBM_CLOCK_HYST_TP3 0xB3 +#define A5XX_RBBM_CLOCK_HYST2_TP0 0xB4 +#define A5XX_RBBM_CLOCK_HYST2_TP1 0xB5 +#define A5XX_RBBM_CLOCK_HYST2_TP2 0xB6 +#define A5XX_RBBM_CLOCK_HYST2_TP3 0xB7 +#define A5XX_RBBM_CLOCK_HYST3_TP0 0xB8 +#define A5XX_RBBM_CLOCK_HYST3_TP1 0xB9 +#define A5XX_RBBM_CLOCK_HYST3_TP2 0xBA +#define A5XX_RBBM_CLOCK_HYST3_TP3 0xBB +#define A5XX_RBBM_CLOCK_CNTL_GPMU 0xC8 +#define A5XX_RBBM_CLOCK_DELAY_GPMU 0xC9 +#define A5XX_RBBM_CLOCK_HYST_GPMU 0xCA +#define A5XX_RBBM_PERFCTR_CP_0_LO 0x3A0 +#define A5XX_RBBM_PERFCTR_CP_0_HI 0x3A1 +#define A5XX_RBBM_PERFCTR_CP_1_LO 0x3A2 +#define A5XX_RBBM_PERFCTR_CP_1_HI 0x3A3 +#define A5XX_RBBM_PERFCTR_CP_2_LO 0x3A4 +#define A5XX_RBBM_PERFCTR_CP_2_HI 0x3A5 +#define A5XX_RBBM_PERFCTR_CP_3_LO 0x3A6 +#define A5XX_RBBM_PERFCTR_CP_3_HI 0x3A7 +#define A5XX_RBBM_PERFCTR_CP_4_LO 0x3A8 +#define A5XX_RBBM_PERFCTR_CP_4_HI 0x3A9 +#define A5XX_RBBM_PERFCTR_CP_5_LO 0x3AA +#define A5XX_RBBM_PERFCTR_CP_5_HI 0x3AB +#define A5XX_RBBM_PERFCTR_CP_6_LO 0x3AC +#define A5XX_RBBM_PERFCTR_CP_6_HI 0x3AD +#define A5XX_RBBM_PERFCTR_CP_7_LO 0x3AE +#define A5XX_RBBM_PERFCTR_CP_7_HI 0x3AF +#define A5XX_RBBM_PERFCTR_RBBM_0_LO 0x3B0 +#define A5XX_RBBM_PERFCTR_RBBM_0_HI 0x3B1 +#define A5XX_RBBM_PERFCTR_RBBM_1_LO 0x3B2 +#define A5XX_RBBM_PERFCTR_RBBM_1_HI 0x3B3 +#define A5XX_RBBM_PERFCTR_RBBM_2_LO 0x3B4 +#define A5XX_RBBM_PERFCTR_RBBM_2_HI 0x3B5 +#define A5XX_RBBM_PERFCTR_RBBM_3_LO 0x3B6 +#define A5XX_RBBM_PERFCTR_RBBM_3_HI 0x3B7 +#define A5XX_RBBM_PERFCTR_PC_0_LO 0x3B8 +#define A5XX_RBBM_PERFCTR_PC_0_HI 0x3B9 +#define A5XX_RBBM_PERFCTR_PC_1_LO 0x3BA +#define A5XX_RBBM_PERFCTR_PC_1_HI 0x3BB +#define A5XX_RBBM_PERFCTR_PC_2_LO 0x3BC +#define A5XX_RBBM_PERFCTR_PC_2_HI 0x3BD +#define A5XX_RBBM_PERFCTR_PC_3_LO 0x3BE +#define A5XX_RBBM_PERFCTR_PC_3_HI 0x3BF +#define A5XX_RBBM_PERFCTR_PC_4_LO 0x3C0 +#define A5XX_RBBM_PERFCTR_PC_4_HI 0x3C1 +#define A5XX_RBBM_PERFCTR_PC_5_LO 0x3C2 +#define A5XX_RBBM_PERFCTR_PC_5_HI 0x3C3 +#define A5XX_RBBM_PERFCTR_PC_6_LO 0x3C4 +#define A5XX_RBBM_PERFCTR_PC_6_HI 0x3C5 +#define A5XX_RBBM_PERFCTR_PC_7_LO 0x3C6 +#define A5XX_RBBM_PERFCTR_PC_7_HI 0x3C7 +#define A5XX_RBBM_PERFCTR_VFD_0_LO 0x3C8 +#define A5XX_RBBM_PERFCTR_VFD_0_HI 0x3C9 +#define A5XX_RBBM_PERFCTR_VFD_1_LO 0x3CA +#define A5XX_RBBM_PERFCTR_VFD_1_HI 0x3CB +#define A5XX_RBBM_PERFCTR_VFD_2_LO 0x3CC +#define A5XX_RBBM_PERFCTR_VFD_2_HI 0x3CD +#define A5XX_RBBM_PERFCTR_VFD_3_LO 0x3CE +#define A5XX_RBBM_PERFCTR_VFD_3_HI 0x3CF +#define A5XX_RBBM_PERFCTR_VFD_4_LO 0x3D0 +#define A5XX_RBBM_PERFCTR_VFD_4_HI 0x3D1 +#define A5XX_RBBM_PERFCTR_VFD_5_LO 0x3D2 +#define A5XX_RBBM_PERFCTR_VFD_5_HI 0x3D3 +#define A5XX_RBBM_PERFCTR_VFD_6_LO 0x3D4 +#define A5XX_RBBM_PERFCTR_VFD_6_HI 0x3D5 +#define A5XX_RBBM_PERFCTR_VFD_7_LO 0x3D6 +#define A5XX_RBBM_PERFCTR_VFD_7_HI 0x3D7 +#define A5XX_RBBM_PERFCTR_HLSQ_0_LO 0x3D8 +#define A5XX_RBBM_PERFCTR_HLSQ_0_HI 0x3D9 +#define A5XX_RBBM_PERFCTR_HLSQ_1_LO 0x3DA +#define A5XX_RBBM_PERFCTR_HLSQ_1_HI 0x3DB +#define A5XX_RBBM_PERFCTR_HLSQ_2_LO 0x3DC +#define A5XX_RBBM_PERFCTR_HLSQ_2_HI 0x3DD +#define A5XX_RBBM_PERFCTR_HLSQ_3_LO 0x3DE +#define A5XX_RBBM_PERFCTR_HLSQ_3_HI 0x3DF +#define A5XX_RBBM_PERFCTR_HLSQ_4_LO 0x3E0 +#define A5XX_RBBM_PERFCTR_HLSQ_4_HI 0x3E1 +#define A5XX_RBBM_PERFCTR_HLSQ_5_LO 0x3E2 +#define A5XX_RBBM_PERFCTR_HLSQ_5_HI 0x3E3 +#define A5XX_RBBM_PERFCTR_HLSQ_6_LO 0x3E4 +#define A5XX_RBBM_PERFCTR_HLSQ_6_HI 0x3E5 +#define A5XX_RBBM_PERFCTR_HLSQ_7_LO 0x3E6 +#define A5XX_RBBM_PERFCTR_HLSQ_7_HI 0x3E7 +#define A5XX_RBBM_PERFCTR_VPC_0_LO 0x3E8 +#define A5XX_RBBM_PERFCTR_VPC_0_HI 0x3E9 +#define A5XX_RBBM_PERFCTR_VPC_1_LO 0x3EA +#define A5XX_RBBM_PERFCTR_VPC_1_HI 0x3EB +#define A5XX_RBBM_PERFCTR_VPC_2_LO 0x3EC +#define A5XX_RBBM_PERFCTR_VPC_2_HI 0x3ED +#define A5XX_RBBM_PERFCTR_VPC_3_LO 0x3EE +#define A5XX_RBBM_PERFCTR_VPC_3_HI 0x3EF +#define A5XX_RBBM_PERFCTR_CCU_0_LO 0x3F0 +#define A5XX_RBBM_PERFCTR_CCU_0_HI 0x3F1 +#define A5XX_RBBM_PERFCTR_CCU_1_LO 0x3F2 +#define A5XX_RBBM_PERFCTR_CCU_1_HI 0x3F3 +#define A5XX_RBBM_PERFCTR_CCU_2_LO 0x3F4 +#define A5XX_RBBM_PERFCTR_CCU_2_HI 0x3F5 +#define A5XX_RBBM_PERFCTR_CCU_3_LO 0x3F6 +#define A5XX_RBBM_PERFCTR_CCU_3_HI 0x3F7 +#define A5XX_RBBM_PERFCTR_TSE_0_LO 0x3F8 +#define A5XX_RBBM_PERFCTR_TSE_0_HI 0x3F9 +#define A5XX_RBBM_PERFCTR_TSE_1_LO 0x3FA +#define A5XX_RBBM_PERFCTR_TSE_1_HI 0x3FB +#define A5XX_RBBM_PERFCTR_TSE_2_LO 0x3FC +#define A5XX_RBBM_PERFCTR_TSE_2_HI 0x3FD +#define A5XX_RBBM_PERFCTR_TSE_3_LO 0x3FE +#define A5XX_RBBM_PERFCTR_TSE_3_HI 0x3FF +#define A5XX_RBBM_PERFCTR_RAS_0_LO 0x400 +#define A5XX_RBBM_PERFCTR_RAS_0_HI 0x401 +#define A5XX_RBBM_PERFCTR_RAS_1_LO 0x402 +#define A5XX_RBBM_PERFCTR_RAS_1_HI 0x403 +#define A5XX_RBBM_PERFCTR_RAS_2_LO 0x404 +#define A5XX_RBBM_PERFCTR_RAS_2_HI 0x405 +#define A5XX_RBBM_PERFCTR_RAS_3_LO 0x406 +#define A5XX_RBBM_PERFCTR_RAS_3_HI 0x407 +#define A5XX_RBBM_PERFCTR_UCHE_0_LO 0x408 +#define A5XX_RBBM_PERFCTR_UCHE_0_HI 0x409 +#define A5XX_RBBM_PERFCTR_UCHE_1_LO 0x40A +#define A5XX_RBBM_PERFCTR_UCHE_1_HI 0x40B +#define A5XX_RBBM_PERFCTR_UCHE_2_LO 0x40C +#define A5XX_RBBM_PERFCTR_UCHE_2_HI 0x40D +#define A5XX_RBBM_PERFCTR_UCHE_3_LO 0x40E +#define A5XX_RBBM_PERFCTR_UCHE_3_HI 0x40F +#define A5XX_RBBM_PERFCTR_UCHE_4_LO 0x410 +#define A5XX_RBBM_PERFCTR_UCHE_4_HI 0x411 +#define A5XX_RBBM_PERFCTR_UCHE_5_LO 0x412 +#define A5XX_RBBM_PERFCTR_UCHE_5_HI 0x413 +#define A5XX_RBBM_PERFCTR_UCHE_6_LO 0x414 +#define A5XX_RBBM_PERFCTR_UCHE_6_HI 0x415 +#define A5XX_RBBM_PERFCTR_UCHE_7_LO 0x416 +#define A5XX_RBBM_PERFCTR_UCHE_7_HI 0x417 +#define A5XX_RBBM_PERFCTR_TP_0_LO 0x418 +#define A5XX_RBBM_PERFCTR_TP_0_HI 0x419 +#define A5XX_RBBM_PERFCTR_TP_1_LO 0x41A +#define A5XX_RBBM_PERFCTR_TP_1_HI 0x41B +#define A5XX_RBBM_PERFCTR_TP_2_LO 0x41C +#define A5XX_RBBM_PERFCTR_TP_2_HI 0x41D +#define A5XX_RBBM_PERFCTR_TP_3_LO 0x41E +#define A5XX_RBBM_PERFCTR_TP_3_HI 0x41F +#define A5XX_RBBM_PERFCTR_TP_4_LO 0x420 +#define A5XX_RBBM_PERFCTR_TP_4_HI 0x421 +#define A5XX_RBBM_PERFCTR_TP_5_LO 0x422 +#define A5XX_RBBM_PERFCTR_TP_5_HI 0x423 +#define A5XX_RBBM_PERFCTR_TP_6_LO 0x424 +#define A5XX_RBBM_PERFCTR_TP_6_HI 0x425 +#define A5XX_RBBM_PERFCTR_TP_7_LO 0x426 +#define A5XX_RBBM_PERFCTR_TP_7_HI 0x427 +#define A5XX_RBBM_PERFCTR_SP_0_LO 0x428 +#define A5XX_RBBM_PERFCTR_SP_0_HI 0x429 +#define A5XX_RBBM_PERFCTR_SP_1_LO 0x42A +#define A5XX_RBBM_PERFCTR_SP_1_HI 0x42B +#define A5XX_RBBM_PERFCTR_SP_2_LO 0x42C +#define A5XX_RBBM_PERFCTR_SP_2_HI 0x42D +#define A5XX_RBBM_PERFCTR_SP_3_LO 0x42E +#define A5XX_RBBM_PERFCTR_SP_3_HI 0x42F +#define A5XX_RBBM_PERFCTR_SP_4_LO 0x430 +#define A5XX_RBBM_PERFCTR_SP_4_HI 0x431 +#define A5XX_RBBM_PERFCTR_SP_5_LO 0x432 +#define A5XX_RBBM_PERFCTR_SP_5_HI 0x433 +#define A5XX_RBBM_PERFCTR_SP_6_LO 0x434 +#define A5XX_RBBM_PERFCTR_SP_6_HI 0x435 +#define A5XX_RBBM_PERFCTR_SP_7_LO 0x436 +#define A5XX_RBBM_PERFCTR_SP_7_HI 0x437 +#define A5XX_RBBM_PERFCTR_SP_8_LO 0x438 +#define A5XX_RBBM_PERFCTR_SP_8_HI 0x439 +#define A5XX_RBBM_PERFCTR_SP_9_LO 0x43A +#define A5XX_RBBM_PERFCTR_SP_9_HI 0x43B +#define A5XX_RBBM_PERFCTR_SP_10_LO 0x43C +#define A5XX_RBBM_PERFCTR_SP_10_HI 0x43D +#define A5XX_RBBM_PERFCTR_SP_11_LO 0x43E +#define A5XX_RBBM_PERFCTR_SP_11_HI 0x43F +#define A5XX_RBBM_PERFCTR_RB_0_LO 0x440 +#define A5XX_RBBM_PERFCTR_RB_0_HI 0x441 +#define A5XX_RBBM_PERFCTR_RB_1_LO 0x442 +#define A5XX_RBBM_PERFCTR_RB_1_HI 0x443 +#define A5XX_RBBM_PERFCTR_RB_2_LO 0x444 +#define A5XX_RBBM_PERFCTR_RB_2_HI 0x445 +#define A5XX_RBBM_PERFCTR_RB_3_LO 0x446 +#define A5XX_RBBM_PERFCTR_RB_3_HI 0x447 +#define A5XX_RBBM_PERFCTR_RB_4_LO 0x448 +#define A5XX_RBBM_PERFCTR_RB_4_HI 0x449 +#define A5XX_RBBM_PERFCTR_RB_5_LO 0x44A +#define A5XX_RBBM_PERFCTR_RB_5_HI 0x44B +#define A5XX_RBBM_PERFCTR_RB_6_LO 0x44C +#define A5XX_RBBM_PERFCTR_RB_6_HI 0x44D +#define A5XX_RBBM_PERFCTR_RB_7_LO 0x44E +#define A5XX_RBBM_PERFCTR_RB_7_HI 0x44F +#define A5XX_RBBM_PERFCTR_VSC_0_LO 0x450 +#define A5XX_RBBM_PERFCTR_VSC_0_HI 0x451 +#define A5XX_RBBM_PERFCTR_VSC_1_LO 0x452 +#define A5XX_RBBM_PERFCTR_VSC_1_HI 0x453 +#define A5XX_RBBM_PERFCTR_LRZ_0_LO 0x454 +#define A5XX_RBBM_PERFCTR_LRZ_0_HI 0x455 +#define A5XX_RBBM_PERFCTR_LRZ_1_LO 0x456 +#define A5XX_RBBM_PERFCTR_LRZ_1_HI 0x457 +#define A5XX_RBBM_PERFCTR_LRZ_2_LO 0x458 +#define A5XX_RBBM_PERFCTR_LRZ_2_HI 0x459 +#define A5XX_RBBM_PERFCTR_LRZ_3_LO 0x45A +#define A5XX_RBBM_PERFCTR_LRZ_3_HI 0x45B +#define A5XX_RBBM_PERFCTR_CMP_0_LO 0x45C +#define A5XX_RBBM_PERFCTR_CMP_0_HI 0x45D +#define A5XX_RBBM_PERFCTR_CMP_1_LO 0x45E +#define A5XX_RBBM_PERFCTR_CMP_1_HI 0x45F +#define A5XX_RBBM_PERFCTR_CMP_2_LO 0x460 +#define A5XX_RBBM_PERFCTR_CMP_2_HI 0x461 +#define A5XX_RBBM_PERFCTR_CMP_3_LO 0x462 +#define A5XX_RBBM_PERFCTR_CMP_3_HI 0x463 +#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B +#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C +#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D +#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E +#define A5XX_RBBM_ALWAYSON_COUNTER_LO 0x4D2 +#define A5XX_RBBM_ALWAYSON_COUNTER_HI 0x4D3 +#define A5XX_RBBM_STATUS 0x4F5 +#define A5XX_RBBM_STATUS3 0x530 +#define A5XX_RBBM_INT_0_STATUS 0x4E1 +#define A5XX_RBBM_AHB_ME_SPLIT_STATUS 0x4F0 +#define A5XX_RBBM_AHB_PFP_SPLIT_STATUS 0x4F1 +#define A5XX_RBBM_AHB_ERROR_STATUS 0x4F4 +#define A5XX_RBBM_PERFCTR_CNTL 0x464 +#define A5XX_RBBM_PERFCTR_LOAD_CMD0 0x465 +#define A5XX_RBBM_PERFCTR_LOAD_CMD1 0x466 +#define A5XX_RBBM_PERFCTR_LOAD_CMD2 0x467 +#define A5XX_RBBM_PERFCTR_LOAD_CMD3 0x468 +#define A5XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x469 +#define A5XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x46A +#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B +#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C +#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D +#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E +#define A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x46F +#define A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC 0x504 +#define A5XX_RBBM_CFG_DBGBUS_OVER 0x505 +#define A5XX_RBBM_CFG_DBGBUS_COUNT0 0x506 +#define A5XX_RBBM_CFG_DBGBUS_COUNT1 0x507 +#define A5XX_RBBM_CFG_DBGBUS_COUNT2 0x508 +#define A5XX_RBBM_CFG_DBGBUS_COUNT3 0x509 +#define A5XX_RBBM_CFG_DBGBUS_COUNT4 0x50A +#define A5XX_RBBM_CFG_DBGBUS_COUNT5 0x50B +#define A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR 0x50C +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0 0x50D +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1 0x50E +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2 0x50F +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3 0x510 +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4 0x511 +#define A5XX_RBBM_CFG_DBGBUS_MISR0 0x512 +#define A5XX_RBBM_CFG_DBGBUS_MISR1 0x513 +#define A5XX_RBBM_ISDB_CNT 0x533 +#define A5XX_RBBM_SECVID_TRUST_CONFIG 0xF000 +#define A5XX_RBBM_SECVID_TRUST_CNTL 0xF400 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0xF800 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0xF801 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xF802 +#define A5XX_RBBM_SECVID_TSB_CNTL 0xF803 +#define A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0xF810 + +/* VSC registers */ +#define A5XX_VSC_PERFCTR_VSC_SEL_0 0xC60 +#define A5XX_VSC_PERFCTR_VSC_SEL_1 0xC61 + +#define A5XX_GRAS_ADDR_MODE_CNTL 0xC81 + +/* TSE registers */ +#define A5XX_GRAS_PERFCTR_TSE_SEL_0 0xC90 +#define A5XX_GRAS_PERFCTR_TSE_SEL_1 0xC91 +#define A5XX_GRAS_PERFCTR_TSE_SEL_2 0xC92 +#define A5XX_GRAS_PERFCTR_TSE_SEL_3 0xC93 + +/* RAS registers */ +#define A5XX_GRAS_PERFCTR_RAS_SEL_0 0xC94 +#define A5XX_GRAS_PERFCTR_RAS_SEL_1 0xC95 +#define A5XX_GRAS_PERFCTR_RAS_SEL_2 0xC96 +#define A5XX_GRAS_PERFCTR_RAS_SEL_3 0xC97 + +/* LRZ registers */ +#define A5XX_GRAS_PERFCTR_LRZ_SEL_0 0xC98 +#define A5XX_GRAS_PERFCTR_LRZ_SEL_1 0xC99 +#define A5XX_GRAS_PERFCTR_LRZ_SEL_2 0xC9A +#define A5XX_GRAS_PERFCTR_LRZ_SEL_3 0xC9B + + +/* RB registers */ +#define A5XX_RB_DBG_ECO_CNT 0xCC4 +#define A5XX_RB_ADDR_MODE_CNTL 0xCC5 +#define A5XX_RB_MODE_CNTL 0xCC6 +#define A5XX_RB_PERFCTR_RB_SEL_0 0xCD0 +#define A5XX_RB_PERFCTR_RB_SEL_1 0xCD1 +#define A5XX_RB_PERFCTR_RB_SEL_2 0xCD2 +#define A5XX_RB_PERFCTR_RB_SEL_3 0xCD3 +#define A5XX_RB_PERFCTR_RB_SEL_4 0xCD4 +#define A5XX_RB_PERFCTR_RB_SEL_5 0xCD5 +#define A5XX_RB_PERFCTR_RB_SEL_6 0xCD6 +#define A5XX_RB_PERFCTR_RB_SEL_7 0xCD7 + +/* CCU registers */ +#define A5XX_RB_PERFCTR_CCU_SEL_0 0xCD8 +#define A5XX_RB_PERFCTR_CCU_SEL_1 0xCD9 +#define A5XX_RB_PERFCTR_CCU_SEL_2 0xCDA +#define A5XX_RB_PERFCTR_CCU_SEL_3 0xCDB + +/* RB Power Counter RB Registers Select */ +#define A5XX_RB_POWERCTR_RB_SEL_0 0xCE0 +#define A5XX_RB_POWERCTR_RB_SEL_1 0xCE1 +#define A5XX_RB_POWERCTR_RB_SEL_2 0xCE2 +#define A5XX_RB_POWERCTR_RB_SEL_3 0xCE3 + +/* RB Power Counter CCU Registers Select */ +#define A5XX_RB_POWERCTR_CCU_SEL_0 0xCE4 +#define A5XX_RB_POWERCTR_CCU_SEL_1 0xCE5 + +/* CMP registers */ +#define A5XX_RB_PERFCTR_CMP_SEL_0 0xCEC +#define A5XX_RB_PERFCTR_CMP_SEL_1 0xCED +#define A5XX_RB_PERFCTR_CMP_SEL_2 0xCEE +#define A5XX_RB_PERFCTR_CMP_SEL_3 0xCEF + +/* PC registers */ +#define A5XX_PC_DBG_ECO_CNTL 0xD00 +#define A5XX_PC_ADDR_MODE_CNTL 0xD01 +#define A5XX_PC_PERFCTR_PC_SEL_0 0xD10 +#define A5XX_PC_PERFCTR_PC_SEL_1 0xD11 +#define A5XX_PC_PERFCTR_PC_SEL_2 0xD12 +#define A5XX_PC_PERFCTR_PC_SEL_3 0xD13 +#define A5XX_PC_PERFCTR_PC_SEL_4 0xD14 +#define A5XX_PC_PERFCTR_PC_SEL_5 0xD15 +#define A5XX_PC_PERFCTR_PC_SEL_6 0xD16 +#define A5XX_PC_PERFCTR_PC_SEL_7 0xD17 + +/* HLSQ registers */ +#define A5XX_HLSQ_DBG_ECO_CNTL 0xE04 +#define A5XX_HLSQ_ADDR_MODE_CNTL 0xE05 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xE10 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xE11 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xE12 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xE13 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xE14 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xE15 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 0xE16 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 0xE17 +#define A5XX_HLSQ_DBG_READ_SEL 0xBC00 +#define A5XX_HLSQ_DBG_AHB_READ_APERTURE 0xA000 + +/* VFD registers */ +#define A5XX_VFD_ADDR_MODE_CNTL 0xE41 +#define A5XX_VFD_PERFCTR_VFD_SEL_0 0xE50 +#define A5XX_VFD_PERFCTR_VFD_SEL_1 0xE51 +#define A5XX_VFD_PERFCTR_VFD_SEL_2 0xE52 +#define A5XX_VFD_PERFCTR_VFD_SEL_3 0xE53 +#define A5XX_VFD_PERFCTR_VFD_SEL_4 0xE54 +#define A5XX_VFD_PERFCTR_VFD_SEL_5 0xE55 +#define A5XX_VFD_PERFCTR_VFD_SEL_6 0xE56 +#define A5XX_VFD_PERFCTR_VFD_SEL_7 0xE57 + +/* VPC registers */ +#define A5XX_VPC_DBG_ECO_CNTL 0xE60 +#define A5XX_VPC_ADDR_MODE_CNTL 0xE61 +#define A5XX_VPC_PERFCTR_VPC_SEL_0 0xE64 +#define A5XX_VPC_PERFCTR_VPC_SEL_1 0xE65 +#define A5XX_VPC_PERFCTR_VPC_SEL_2 0xE66 +#define A5XX_VPC_PERFCTR_VPC_SEL_3 0xE67 + +/* UCHE registers */ +#define A5XX_UCHE_ADDR_MODE_CNTL 0xE80 +#define A5XX_UCHE_MODE_CNTL 0xE81 +#define A5XX_UCHE_WRITE_THRU_BASE_LO 0xE87 +#define A5XX_UCHE_WRITE_THRU_BASE_HI 0xE88 +#define A5XX_UCHE_TRAP_BASE_LO 0xE89 +#define A5XX_UCHE_TRAP_BASE_HI 0xE8A +#define A5XX_UCHE_GMEM_RANGE_MIN_LO 0xE8B +#define A5XX_UCHE_GMEM_RANGE_MIN_HI 0xE8C +#define A5XX_UCHE_GMEM_RANGE_MAX_LO 0xE8D +#define A5XX_UCHE_GMEM_RANGE_MAX_HI 0xE8E +#define A5XX_UCHE_DBG_ECO_CNTL_2 0xE8F +#define A5XX_UCHE_INVALIDATE0 0xE95 +#define A5XX_UCHE_CACHE_WAYS 0xE96 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_0 0xEA0 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_1 0xEA1 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_2 0xEA2 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_3 0xEA3 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_4 0xEA4 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_5 0xEA5 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_6 0xEA6 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_7 0xEA7 + +/* UCHE Power Counter UCHE Registers Select */ +#define A5XX_UCHE_POWERCTR_UCHE_SEL_0 0xEA8 +#define A5XX_UCHE_POWERCTR_UCHE_SEL_1 0xEA9 +#define A5XX_UCHE_POWERCTR_UCHE_SEL_2 0xEAA +#define A5XX_UCHE_POWERCTR_UCHE_SEL_3 0xEAB + +/* SP registers */ +#define A5XX_SP_DBG_ECO_CNTL 0xEC0 +#define A5XX_SP_ADDR_MODE_CNTL 0xEC1 +#define A5XX_SP_PERFCTR_SP_SEL_0 0xED0 +#define A5XX_SP_PERFCTR_SP_SEL_1 0xED1 +#define A5XX_SP_PERFCTR_SP_SEL_2 0xED2 +#define A5XX_SP_PERFCTR_SP_SEL_3 0xED3 +#define A5XX_SP_PERFCTR_SP_SEL_4 0xED4 +#define A5XX_SP_PERFCTR_SP_SEL_5 0xED5 +#define A5XX_SP_PERFCTR_SP_SEL_6 0xED6 +#define A5XX_SP_PERFCTR_SP_SEL_7 0xED7 +#define A5XX_SP_PERFCTR_SP_SEL_8 0xED8 +#define A5XX_SP_PERFCTR_SP_SEL_9 0xED9 +#define A5XX_SP_PERFCTR_SP_SEL_10 0xEDA +#define A5XX_SP_PERFCTR_SP_SEL_11 0xEDB + +/* SP Power Counter SP Registers Select */ +#define A5XX_SP_POWERCTR_SP_SEL_0 0xEDC +#define A5XX_SP_POWERCTR_SP_SEL_1 0xEDD +#define A5XX_SP_POWERCTR_SP_SEL_2 0xEDE +#define A5XX_SP_POWERCTR_SP_SEL_3 0xEDF + +/* TP registers */ +#define A5XX_TPL1_ADDR_MODE_CNTL 0xF01 +#define A5XX_TPL1_MODE_CNTL 0xF02 +#define A5XX_TPL1_PERFCTR_TP_SEL_0 0xF10 +#define A5XX_TPL1_PERFCTR_TP_SEL_1 0xF11 +#define A5XX_TPL1_PERFCTR_TP_SEL_2 0xF12 +#define A5XX_TPL1_PERFCTR_TP_SEL_3 0xF13 +#define A5XX_TPL1_PERFCTR_TP_SEL_4 0xF14 +#define A5XX_TPL1_PERFCTR_TP_SEL_5 0xF15 +#define A5XX_TPL1_PERFCTR_TP_SEL_6 0xF16 +#define A5XX_TPL1_PERFCTR_TP_SEL_7 0xF17 + +/* TP Power Counter TP Registers Select */ +#define A5XX_TPL1_POWERCTR_TP_SEL_0 0xF18 +#define A5XX_TPL1_POWERCTR_TP_SEL_1 0xF19 +#define A5XX_TPL1_POWERCTR_TP_SEL_2 0xF1A +#define A5XX_TPL1_POWERCTR_TP_SEL_3 0xF1B + +/* VBIF registers */ +#define A5XX_VBIF_VERSION 0x3000 +#define A5XX_VBIF_CLKON 0x3001 +#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1 +#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1 + +#define A5XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049 +#define A5XX_VBIF_GATE_OFF_WRREQ_EN 0x302A + +#define A5XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A5XX_VBIF_XIN_HALT_CTRL0_MASK 0xF +#define A510_VBIF_XIN_HALT_CTRL0_MASK 0x7 +#define A5XX_VBIF_XIN_HALT_CTRL1 0x3081 + +#define A5XX_VBIF_TEST_BUS_OUT_CTRL 0x3084 +#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1 +#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS1_CTRL0 0x3085 +#define A5XX_VBIF_TEST_BUS1_CTRL1 0x3086 +#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF +#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS2_CTRL0 0x3087 +#define A5XX_VBIF_TEST_BUS2_CTRL1 0x3088 +#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0x1FF +#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS_OUT 0x308c + +#define A5XX_VBIF_PERF_CNT_SEL0 0x30D0 +#define A5XX_VBIF_PERF_CNT_SEL1 0x30D1 +#define A5XX_VBIF_PERF_CNT_SEL2 0x30D2 +#define A5XX_VBIF_PERF_CNT_SEL3 0x30D3 +#define A5XX_VBIF_PERF_CNT_LOW0 0x30D8 +#define A5XX_VBIF_PERF_CNT_LOW1 0x30D9 +#define A5XX_VBIF_PERF_CNT_LOW2 0x30DA +#define A5XX_VBIF_PERF_CNT_LOW3 0x30DB +#define A5XX_VBIF_PERF_CNT_HIGH0 0x30E0 +#define A5XX_VBIF_PERF_CNT_HIGH1 0x30E1 +#define A5XX_VBIF_PERF_CNT_HIGH2 0x30E2 +#define A5XX_VBIF_PERF_CNT_HIGH3 0x30E3 + +#define A5XX_VBIF_PERF_PWR_CNT_EN0 0x3100 +#define A5XX_VBIF_PERF_PWR_CNT_EN1 0x3101 +#define A5XX_VBIF_PERF_PWR_CNT_EN2 0x3102 + +#define A5XX_VBIF_PERF_PWR_CNT_LOW0 0x3110 +#define A5XX_VBIF_PERF_PWR_CNT_LOW1 0x3111 +#define A5XX_VBIF_PERF_PWR_CNT_LOW2 0x3112 + +#define A5XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118 +#define A5XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119 +#define A5XX_VBIF_PERF_PWR_CNT_HIGH2 0x311A + +/* GPMU registers */ +#define A5XX_GPMU_INST_RAM_BASE 0x8800 +#define A5XX_GPMU_DATA_RAM_BASE 0x9800 +#define A5XX_GPMU_SP_POWER_CNTL 0xA881 +#define A5XX_GPMU_RBCCU_CLOCK_CNTL 0xA886 +#define A5XX_GPMU_RBCCU_POWER_CNTL 0xA887 +#define A5XX_GPMU_SP_PWR_CLK_STATUS 0xA88B +#define A5XX_GPMU_RBCCU_PWR_CLK_STATUS 0xA88D +#define A5XX_GPMU_PWR_COL_STAGGER_DELAY 0xA891 +#define A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL 0xA892 +#define A5XX_GPMU_PWR_COL_INTER_FRAME_HYST 0xA893 +#define A5XX_GPMU_PWR_COL_BINNING_CTRL 0xA894 +#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3 +#define A5XX_GPMU_WFI_CONFIG 0xA8C1 +#define A5XX_GPMU_RBBM_INTR_INFO 0xA8D6 +#define A5XX_GPMU_CM3_SYSRESET 0xA8D8 +#define A5XX_GPMU_GENERAL_0 0xA8E0 +#define A5XX_GPMU_GENERAL_1 0xA8E1 + +/* COUNTABLE FOR SP PERFCOUNTER */ +#define A5XX_SP_ALU_ACTIVE_CYCLES 0x1 +#define A5XX_SP0_ICL1_MISSES 0x35 +#define A5XX_SP_FS_CFLOW_INSTRUCTIONS 0x27 + +/* COUNTABLE FOR TSE PERFCOUNTER */ +#define A5XX_TSE_INPUT_PRIM_NUM 0x6 + +/* COUNTABLE FOR RBBM PERFCOUNTER */ +#define A5XX_RBBM_ALWAYS_COUNT 0x0 + +/* GPMU POWER COUNTERS */ +#define A5XX_SP_POWER_COUNTER_0_LO 0xA840 +#define A5XX_SP_POWER_COUNTER_0_HI 0xA841 +#define A5XX_SP_POWER_COUNTER_1_LO 0xA842 +#define A5XX_SP_POWER_COUNTER_1_HI 0xA843 +#define A5XX_SP_POWER_COUNTER_2_LO 0xA844 +#define A5XX_SP_POWER_COUNTER_2_HI 0xA845 +#define A5XX_SP_POWER_COUNTER_3_LO 0xA846 +#define A5XX_SP_POWER_COUNTER_3_HI 0xA847 + +#define A5XX_TP_POWER_COUNTER_0_LO 0xA848 +#define A5XX_TP_POWER_COUNTER_0_HI 0xA849 +#define A5XX_TP_POWER_COUNTER_1_LO 0xA84A +#define A5XX_TP_POWER_COUNTER_1_HI 0xA84B +#define A5XX_TP_POWER_COUNTER_2_LO 0xA84C +#define A5XX_TP_POWER_COUNTER_2_HI 0xA84D +#define A5XX_TP_POWER_COUNTER_3_LO 0xA84E +#define A5XX_TP_POWER_COUNTER_3_HI 0xA84F + +#define A5XX_RB_POWER_COUNTER_0_LO 0xA850 +#define A5XX_RB_POWER_COUNTER_0_HI 0xA851 +#define A5XX_RB_POWER_COUNTER_1_LO 0xA852 +#define A5XX_RB_POWER_COUNTER_1_HI 0xA853 +#define A5XX_RB_POWER_COUNTER_2_LO 0xA854 +#define A5XX_RB_POWER_COUNTER_2_HI 0xA855 +#define A5XX_RB_POWER_COUNTER_3_LO 0xA856 +#define A5XX_RB_POWER_COUNTER_3_HI 0xA857 + +#define A5XX_CCU_POWER_COUNTER_0_LO 0xA858 +#define A5XX_CCU_POWER_COUNTER_0_HI 0xA859 +#define A5XX_CCU_POWER_COUNTER_1_LO 0xA85A +#define A5XX_CCU_POWER_COUNTER_1_HI 0xA85B + +#define A5XX_UCHE_POWER_COUNTER_0_LO 0xA85C +#define A5XX_UCHE_POWER_COUNTER_0_HI 0xA85D +#define A5XX_UCHE_POWER_COUNTER_1_LO 0xA85E +#define A5XX_UCHE_POWER_COUNTER_1_HI 0xA85F +#define A5XX_UCHE_POWER_COUNTER_2_LO 0xA860 +#define A5XX_UCHE_POWER_COUNTER_2_HI 0xA861 +#define A5XX_UCHE_POWER_COUNTER_3_LO 0xA862 +#define A5XX_UCHE_POWER_COUNTER_3_HI 0xA863 + +#define A5XX_CP_POWER_COUNTER_0_LO 0xA864 +#define A5XX_CP_POWER_COUNTER_0_HI 0xA865 +#define A5XX_CP_POWER_COUNTER_1_LO 0xA866 +#define A5XX_CP_POWER_COUNTER_1_HI 0xA867 +#define A5XX_CP_POWER_COUNTER_2_LO 0xA868 +#define A5XX_CP_POWER_COUNTER_2_HI 0xA869 +#define A5XX_CP_POWER_COUNTER_3_LO 0xA86A +#define A5XX_CP_POWER_COUNTER_3_HI 0xA86B + +#define A5XX_GPMU_POWER_COUNTER_0_LO 0xA86C +#define A5XX_GPMU_POWER_COUNTER_0_HI 0xA86D +#define A5XX_GPMU_POWER_COUNTER_1_LO 0xA86E +#define A5XX_GPMU_POWER_COUNTER_1_HI 0xA86F +#define A5XX_GPMU_POWER_COUNTER_2_LO 0xA870 +#define A5XX_GPMU_POWER_COUNTER_2_HI 0xA871 +#define A5XX_GPMU_POWER_COUNTER_3_LO 0xA872 +#define A5XX_GPMU_POWER_COUNTER_3_HI 0xA873 +#define A5XX_GPMU_POWER_COUNTER_4_LO 0xA874 +#define A5XX_GPMU_POWER_COUNTER_4_HI 0xA875 +#define A5XX_GPMU_POWER_COUNTER_5_LO 0xA876 +#define A5XX_GPMU_POWER_COUNTER_5_HI 0xA877 + +#define A5XX_GPMU_POWER_COUNTER_ENABLE 0xA878 +#define A5XX_GPMU_ALWAYS_ON_COUNTER_LO 0xA879 +#define A5XX_GPMU_ALWAYS_ON_COUNTER_HI 0xA87A +#define A5XX_GPMU_ALWAYS_ON_COUNTER_RESET 0xA87B +#define A5XX_GPMU_POWER_COUNTER_SELECT_0 0xA87C +#define A5XX_GPMU_POWER_COUNTER_SELECT_1 0xA87D +#define A5XX_GPMU_GPMU_SP_CLOCK_CONTROL 0xA880 + +#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3 +#define A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL 0xA8A8 + +#define A5XX_GPMU_TEMP_SENSOR_ID 0xAC00 +#define A5XX_GPMU_TEMP_SENSOR_CONFIG 0xAC01 +#define A5XX_GPMU_DELTA_TEMP_THRESHOLD 0xAC03 +#define A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK 0xAC06 + +#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_0_1 0xAC40 +#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_2_3 0xAC41 +#define A5XX_GPMU_LEAKAGE_VTG_COEFF_0_1 0xAC42 +#define A5XX_GPMU_LEAKAGE_VTG_COEFF_2_3 0xAC43 +#define A5XX_GPMU_BASE_LEAKAGE 0xAC46 + +#define A5XX_GPMU_GPMU_VOLTAGE 0xAC60 +#define A5XX_GPMU_GPMU_VOLTAGE_INTR_STATUS 0xAC61 +#define A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK 0xAC62 +#define A5XX_GPMU_GPMU_PWR_THRESHOLD 0xAC80 +#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL 0xACC4 +#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS 0xACC5 +#define A5XX_GPMU_GPMU_ISENSE_CTRL 0xACD0 + +#define A5XX_GDPM_CONFIG1 0xB80C +#define A5XX_GDPM_INT_EN 0xB80F +#define A5XX_GDPM_INT_MASK 0xB811 +#define A5XX_GPMU_BEC_ENABLE 0xB9A0 + +/* ISENSE registers */ +#define A5XX_GPU_CS_DECIMAL_ALIGN 0xC16A +#define A5XX_GPU_CS_SENSOR_PARAM_CORE_1 0xC126 +#define A5XX_GPU_CS_SENSOR_PARAM_CORE_2 0xC127 +#define A5XX_GPU_CS_SW_OV_FUSE_EN 0xC168 +#define A5XX_GPU_CS_SENSOR_GENERAL_STATUS 0xC41A +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_0 0xC41D +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_2 0xC41F +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_4 0xC421 +#define A5XX_GPU_CS_ENABLE_REG 0xC520 +#define A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1 0xC557 +#define A5XX_GPU_CS_AMP_CALIBRATION_DONE 0xC565 +#define A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE 0xC556 +#endif /* _A5XX_REG_H */ + diff --git a/drivers/gpu/msm/a6xx_reg.h b/drivers/gpu/msm/a6xx_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..12fa49bf64190d19fcbbab44a50daa0fc14b864c --- /dev/null +++ b/drivers/gpu/msm/a6xx_reg.h @@ -0,0 +1,1107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + */ + +#ifndef _A6XX_REG_H +#define _A6XX_REG_H + +/* A6XX interrupt bits */ +#define A6XX_INT_RBBM_GPU_IDLE 0 +#define A6XX_INT_CP_AHB_ERROR 1 +#define A6XX_INT_ATB_ASYNCFIFO_OVERFLOW 6 +#define A6XX_INT_RBBM_GPC_ERROR 7 +#define A6XX_INT_CP_SW 8 +#define A6XX_INT_CP_HW_ERROR 9 +#define A6XX_INT_CP_CCU_FLUSH_DEPTH_TS 10 +#define A6XX_INT_CP_CCU_FLUSH_COLOR_TS 11 +#define A6XX_INT_CP_CCU_RESOLVE_TS 12 +#define A6XX_INT_CP_IB2 13 +#define A6XX_INT_CP_IB1 14 +#define A6XX_INT_CP_RB 15 +#define A6XX_INT_CP_RB_DONE_TS 17 +#define A6XX_INT_CP_WT_DONE_TS 18 +#define A6XX_INT_CP_CACHE_FLUSH_TS 20 +#define A6XX_INT_RBBM_ATB_BUS_OVERFLOW 22 +#define A6XX_INT_RBBM_HANG_DETECT 23 +#define A6XX_INT_UCHE_OOB_ACCESS 24 +#define A6XX_INT_UCHE_TRAP_INTR 25 +#define A6XX_INT_DEBBUS_INTR_0 26 +#define A6XX_INT_DEBBUS_INTR_1 27 +#define A6XX_INT_TSB_WRITE_ERROR 28 +#define A6XX_INT_ISDB_CPU_IRQ 30 +#define A6XX_INT_ISDB_UNDER_DEBUG 31 + +/* CP Interrupt bits */ +#define A6XX_CP_OPCODE_ERROR 0 +#define A6XX_CP_UCODE_ERROR 1 +#define A6XX_CP_HW_FAULT_ERROR 2 +#define A6XX_CP_REGISTER_PROTECTION_ERROR 4 +#define A6XX_CP_AHB_ERROR 5 +#define A6XX_CP_VSD_PARITY_ERROR 6 +#define A6XX_CP_ILLEGAL_INSTR_ERROR 7 + +/* CP registers */ +#define A6XX_CP_RB_BASE 0x800 +#define A6XX_CP_RB_BASE_HI 0x801 +#define A6XX_CP_RB_CNTL 0x802 +#define A6XX_CP_RB_RPTR_ADDR_LO 0x804 +#define A6XX_CP_RB_RPTR_ADDR_HI 0x805 +#define A6XX_CP_RB_RPTR 0x806 +#define A6XX_CP_RB_WPTR 0x807 +#define A6XX_CP_SQE_CNTL 0x808 +#define A6XX_CP_HW_FAULT 0x821 +#define A6XX_CP_INTERRUPT_STATUS 0x823 +#define A6XX_CP_PROTECT_STATUS 0X824 +#define A6XX_CP_SQE_INSTR_BASE_LO 0x830 +#define A6XX_CP_SQE_INSTR_BASE_HI 0x831 +#define A6XX_CP_MISC_CNTL 0x840 +#define A6XX_CP_ROQ_THRESHOLDS_1 0x8C1 +#define A6XX_CP_ROQ_THRESHOLDS_2 0x8C2 +#define A6XX_CP_MEM_POOL_SIZE 0x8C3 +#define A6XX_CP_CHICKEN_DBG 0x841 +#define A6XX_CP_ADDR_MODE_CNTL 0x842 +#define A6XX_CP_DBG_ECO_CNTL 0x843 +#define A6XX_CP_PROTECT_CNTL 0x84F +#define A6XX_CP_PROTECT_REG 0x850 +#define A6XX_CP_CONTEXT_SWITCH_CNTL 0x8A0 +#define A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x8A1 +#define A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x8A2 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO 0x8A3 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI 0x8A4 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO 0x8A5 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI 0x8A6 +#define A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO 0x8A7 +#define A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI 0x8A8 +#define A6XX_CP_CONTEXT_SWITCH_LEVEL_STATUS 0x8AB +#define A6XX_CP_PERFCTR_CP_SEL_0 0x8D0 +#define A6XX_CP_PERFCTR_CP_SEL_1 0x8D1 +#define A6XX_CP_PERFCTR_CP_SEL_2 0x8D2 +#define A6XX_CP_PERFCTR_CP_SEL_3 0x8D3 +#define A6XX_CP_PERFCTR_CP_SEL_4 0x8D4 +#define A6XX_CP_PERFCTR_CP_SEL_5 0x8D5 +#define A6XX_CP_PERFCTR_CP_SEL_6 0x8D6 +#define A6XX_CP_PERFCTR_CP_SEL_7 0x8D7 +#define A6XX_CP_PERFCTR_CP_SEL_8 0x8D8 +#define A6XX_CP_PERFCTR_CP_SEL_9 0x8D9 +#define A6XX_CP_PERFCTR_CP_SEL_10 0x8DA +#define A6XX_CP_PERFCTR_CP_SEL_11 0x8DB +#define A6XX_CP_PERFCTR_CP_SEL_12 0x8DC +#define A6XX_CP_PERFCTR_CP_SEL_13 0x8DD +#define A6XX_CP_CRASH_SCRIPT_BASE_LO 0x900 +#define A6XX_CP_CRASH_SCRIPT_BASE_HI 0x901 +#define A6XX_CP_CRASH_DUMP_CNTL 0x902 +#define A6XX_CP_CRASH_DUMP_STATUS 0x903 +#define A6XX_CP_SQE_STAT_ADDR 0x908 +#define A6XX_CP_SQE_STAT_DATA 0x909 +#define A6XX_CP_DRAW_STATE_ADDR 0x90A +#define A6XX_CP_DRAW_STATE_DATA 0x90B +#define A6XX_CP_ROQ_DBG_ADDR 0x90C +#define A6XX_CP_ROQ_DBG_DATA 0x90D +#define A6XX_CP_MEM_POOL_DBG_ADDR 0x90E +#define A6XX_CP_MEM_POOL_DBG_DATA 0x90F +#define A6XX_CP_SQE_UCODE_DBG_ADDR 0x910 +#define A6XX_CP_SQE_UCODE_DBG_DATA 0x911 +#define A6XX_CP_IB1_BASE 0x928 +#define A6XX_CP_IB1_BASE_HI 0x929 +#define A6XX_CP_IB1_REM_SIZE 0x92A +#define A6XX_CP_IB2_BASE 0x92B +#define A6XX_CP_IB2_BASE_HI 0x92C +#define A6XX_CP_IB2_REM_SIZE 0x92D +#define A6XX_CP_ALWAYS_ON_COUNTER_LO 0x980 +#define A6XX_CP_ALWAYS_ON_COUNTER_HI 0x981 +#define A6XX_CP_AHB_CNTL 0x98D +#define A6XX_CP_APERTURE_CNTL_HOST 0xA00 +#define A6XX_CP_APERTURE_CNTL_CD 0xA03 +#define A6XX_VSC_ADDR_MODE_CNTL 0xC01 + +/* RBBM registers */ +#define A6XX_RBBM_INT_0_STATUS 0x201 +#define A6XX_RBBM_STATUS 0x210 +#define A6XX_RBBM_STATUS3 0x213 +#define A6XX_RBBM_VBIF_GX_RESET_STATUS 0x215 +#define A6XX_RBBM_PERFCTR_CP_0_LO 0x400 +#define A6XX_RBBM_PERFCTR_CP_0_HI 0x401 +#define A6XX_RBBM_PERFCTR_CP_1_LO 0x402 +#define A6XX_RBBM_PERFCTR_CP_1_HI 0x403 +#define A6XX_RBBM_PERFCTR_CP_2_LO 0x404 +#define A6XX_RBBM_PERFCTR_CP_2_HI 0x405 +#define A6XX_RBBM_PERFCTR_CP_3_LO 0x406 +#define A6XX_RBBM_PERFCTR_CP_3_HI 0x407 +#define A6XX_RBBM_PERFCTR_CP_4_LO 0x408 +#define A6XX_RBBM_PERFCTR_CP_4_HI 0x409 +#define A6XX_RBBM_PERFCTR_CP_5_LO 0x40a +#define A6XX_RBBM_PERFCTR_CP_5_HI 0x40b +#define A6XX_RBBM_PERFCTR_CP_6_LO 0x40c +#define A6XX_RBBM_PERFCTR_CP_6_HI 0x40d +#define A6XX_RBBM_PERFCTR_CP_7_LO 0x40e +#define A6XX_RBBM_PERFCTR_CP_7_HI 0x40f +#define A6XX_RBBM_PERFCTR_CP_8_LO 0x410 +#define A6XX_RBBM_PERFCTR_CP_8_HI 0x411 +#define A6XX_RBBM_PERFCTR_CP_9_LO 0x412 +#define A6XX_RBBM_PERFCTR_CP_9_HI 0x413 +#define A6XX_RBBM_PERFCTR_CP_10_LO 0x414 +#define A6XX_RBBM_PERFCTR_CP_10_HI 0x415 +#define A6XX_RBBM_PERFCTR_CP_11_LO 0x416 +#define A6XX_RBBM_PERFCTR_CP_11_HI 0x417 +#define A6XX_RBBM_PERFCTR_CP_12_LO 0x418 +#define A6XX_RBBM_PERFCTR_CP_12_HI 0x419 +#define A6XX_RBBM_PERFCTR_CP_13_LO 0x41a +#define A6XX_RBBM_PERFCTR_CP_13_HI 0x41b +#define A6XX_RBBM_PERFCTR_RBBM_0_LO 0x41c +#define A6XX_RBBM_PERFCTR_RBBM_0_HI 0x41d +#define A6XX_RBBM_PERFCTR_RBBM_1_LO 0x41e +#define A6XX_RBBM_PERFCTR_RBBM_1_HI 0x41f +#define A6XX_RBBM_PERFCTR_RBBM_2_LO 0x420 +#define A6XX_RBBM_PERFCTR_RBBM_2_HI 0x421 +#define A6XX_RBBM_PERFCTR_RBBM_3_LO 0x422 +#define A6XX_RBBM_PERFCTR_RBBM_3_HI 0x423 +#define A6XX_RBBM_PERFCTR_PC_0_LO 0x424 +#define A6XX_RBBM_PERFCTR_PC_0_HI 0x425 +#define A6XX_RBBM_PERFCTR_PC_1_LO 0x426 +#define A6XX_RBBM_PERFCTR_PC_1_HI 0x427 +#define A6XX_RBBM_PERFCTR_PC_2_LO 0x428 +#define A6XX_RBBM_PERFCTR_PC_2_HI 0x429 +#define A6XX_RBBM_PERFCTR_PC_3_LO 0x42a +#define A6XX_RBBM_PERFCTR_PC_3_HI 0x42b +#define A6XX_RBBM_PERFCTR_PC_4_LO 0x42c +#define A6XX_RBBM_PERFCTR_PC_4_HI 0x42d +#define A6XX_RBBM_PERFCTR_PC_5_LO 0x42e +#define A6XX_RBBM_PERFCTR_PC_5_HI 0x42f +#define A6XX_RBBM_PERFCTR_PC_6_LO 0x430 +#define A6XX_RBBM_PERFCTR_PC_6_HI 0x431 +#define A6XX_RBBM_PERFCTR_PC_7_LO 0x432 +#define A6XX_RBBM_PERFCTR_PC_7_HI 0x433 +#define A6XX_RBBM_PERFCTR_VFD_0_LO 0x434 +#define A6XX_RBBM_PERFCTR_VFD_0_HI 0x435 +#define A6XX_RBBM_PERFCTR_VFD_1_LO 0x436 +#define A6XX_RBBM_PERFCTR_VFD_1_HI 0x437 +#define A6XX_RBBM_PERFCTR_VFD_2_LO 0x438 +#define A6XX_RBBM_PERFCTR_VFD_2_HI 0x439 +#define A6XX_RBBM_PERFCTR_VFD_3_LO 0x43a +#define A6XX_RBBM_PERFCTR_VFD_3_HI 0x43b +#define A6XX_RBBM_PERFCTR_VFD_4_LO 0x43c +#define A6XX_RBBM_PERFCTR_VFD_4_HI 0x43d +#define A6XX_RBBM_PERFCTR_VFD_5_LO 0x43e +#define A6XX_RBBM_PERFCTR_VFD_5_HI 0x43f +#define A6XX_RBBM_PERFCTR_VFD_6_LO 0x440 +#define A6XX_RBBM_PERFCTR_VFD_6_HI 0x441 +#define A6XX_RBBM_PERFCTR_VFD_7_LO 0x442 +#define A6XX_RBBM_PERFCTR_VFD_7_HI 0x443 +#define A6XX_RBBM_PERFCTR_HLSQ_0_LO 0x444 +#define A6XX_RBBM_PERFCTR_HLSQ_0_HI 0x445 +#define A6XX_RBBM_PERFCTR_HLSQ_1_LO 0x446 +#define A6XX_RBBM_PERFCTR_HLSQ_1_HI 0x447 +#define A6XX_RBBM_PERFCTR_HLSQ_2_LO 0x448 +#define A6XX_RBBM_PERFCTR_HLSQ_2_HI 0x449 +#define A6XX_RBBM_PERFCTR_HLSQ_3_LO 0x44a +#define A6XX_RBBM_PERFCTR_HLSQ_3_HI 0x44b +#define A6XX_RBBM_PERFCTR_HLSQ_4_LO 0x44c +#define A6XX_RBBM_PERFCTR_HLSQ_4_HI 0x44d +#define A6XX_RBBM_PERFCTR_HLSQ_5_LO 0x44e +#define A6XX_RBBM_PERFCTR_HLSQ_5_HI 0x44f +#define A6XX_RBBM_PERFCTR_VPC_0_LO 0x450 +#define A6XX_RBBM_PERFCTR_VPC_0_HI 0x451 +#define A6XX_RBBM_PERFCTR_VPC_1_LO 0x452 +#define A6XX_RBBM_PERFCTR_VPC_1_HI 0x453 +#define A6XX_RBBM_PERFCTR_VPC_2_LO 0x454 +#define A6XX_RBBM_PERFCTR_VPC_2_HI 0x455 +#define A6XX_RBBM_PERFCTR_VPC_3_LO 0x456 +#define A6XX_RBBM_PERFCTR_VPC_3_HI 0x457 +#define A6XX_RBBM_PERFCTR_VPC_4_LO 0x458 +#define A6XX_RBBM_PERFCTR_VPC_4_HI 0x459 +#define A6XX_RBBM_PERFCTR_VPC_5_LO 0x45a +#define A6XX_RBBM_PERFCTR_VPC_5_HI 0x45b +#define A6XX_RBBM_PERFCTR_CCU_0_LO 0x45c +#define A6XX_RBBM_PERFCTR_CCU_0_HI 0x45d +#define A6XX_RBBM_PERFCTR_CCU_1_LO 0x45e +#define A6XX_RBBM_PERFCTR_CCU_1_HI 0x45f +#define A6XX_RBBM_PERFCTR_CCU_2_LO 0x460 +#define A6XX_RBBM_PERFCTR_CCU_2_HI 0x461 +#define A6XX_RBBM_PERFCTR_CCU_3_LO 0x462 +#define A6XX_RBBM_PERFCTR_CCU_3_HI 0x463 +#define A6XX_RBBM_PERFCTR_CCU_4_LO 0x464 +#define A6XX_RBBM_PERFCTR_CCU_4_HI 0x465 +#define A6XX_RBBM_PERFCTR_TSE_0_LO 0x466 +#define A6XX_RBBM_PERFCTR_TSE_0_HI 0x467 +#define A6XX_RBBM_PERFCTR_TSE_1_LO 0x468 +#define A6XX_RBBM_PERFCTR_TSE_1_HI 0x469 +#define A6XX_RBBM_PERFCTR_TSE_2_LO 0x46a +#define A6XX_RBBM_PERFCTR_CCU_4_HI 0x465 +#define A6XX_RBBM_PERFCTR_TSE_0_LO 0x466 +#define A6XX_RBBM_PERFCTR_TSE_0_HI 0x467 +#define A6XX_RBBM_PERFCTR_TSE_1_LO 0x468 +#define A6XX_RBBM_PERFCTR_TSE_1_HI 0x469 +#define A6XX_RBBM_PERFCTR_TSE_2_LO 0x46a +#define A6XX_RBBM_PERFCTR_TSE_2_HI 0x46b +#define A6XX_RBBM_PERFCTR_TSE_3_LO 0x46c +#define A6XX_RBBM_PERFCTR_TSE_3_HI 0x46d +#define A6XX_RBBM_PERFCTR_RAS_0_LO 0x46e +#define A6XX_RBBM_PERFCTR_RAS_0_HI 0x46f +#define A6XX_RBBM_PERFCTR_RAS_1_LO 0x470 +#define A6XX_RBBM_PERFCTR_RAS_1_HI 0x471 +#define A6XX_RBBM_PERFCTR_RAS_2_LO 0x472 +#define A6XX_RBBM_PERFCTR_RAS_2_HI 0x473 +#define A6XX_RBBM_PERFCTR_RAS_3_LO 0x474 +#define A6XX_RBBM_PERFCTR_RAS_3_HI 0x475 +#define A6XX_RBBM_PERFCTR_UCHE_0_LO 0x476 +#define A6XX_RBBM_PERFCTR_UCHE_0_HI 0x477 +#define A6XX_RBBM_PERFCTR_UCHE_1_LO 0x478 +#define A6XX_RBBM_PERFCTR_UCHE_1_HI 0x479 +#define A6XX_RBBM_PERFCTR_UCHE_2_LO 0x47a +#define A6XX_RBBM_PERFCTR_UCHE_2_HI 0x47b +#define A6XX_RBBM_PERFCTR_UCHE_3_LO 0x47c +#define A6XX_RBBM_PERFCTR_UCHE_3_HI 0x47d +#define A6XX_RBBM_PERFCTR_UCHE_4_LO 0x47e +#define A6XX_RBBM_PERFCTR_UCHE_4_HI 0x47f +#define A6XX_RBBM_PERFCTR_UCHE_5_LO 0x480 +#define A6XX_RBBM_PERFCTR_UCHE_5_HI 0x481 +#define A6XX_RBBM_PERFCTR_UCHE_6_LO 0x482 +#define A6XX_RBBM_PERFCTR_UCHE_6_HI 0x483 +#define A6XX_RBBM_PERFCTR_UCHE_7_LO 0x484 +#define A6XX_RBBM_PERFCTR_UCHE_7_HI 0x485 +#define A6XX_RBBM_PERFCTR_UCHE_8_LO 0x486 +#define A6XX_RBBM_PERFCTR_UCHE_8_HI 0x487 +#define A6XX_RBBM_PERFCTR_UCHE_9_LO 0x488 +#define A6XX_RBBM_PERFCTR_UCHE_9_HI 0x489 +#define A6XX_RBBM_PERFCTR_UCHE_10_LO 0x48a +#define A6XX_RBBM_PERFCTR_UCHE_10_HI 0x48b +#define A6XX_RBBM_PERFCTR_UCHE_11_LO 0x48c +#define A6XX_RBBM_PERFCTR_UCHE_11_HI 0x48d +#define A6XX_RBBM_PERFCTR_TP_0_LO 0x48e +#define A6XX_RBBM_PERFCTR_TP_0_HI 0x48f +#define A6XX_RBBM_PERFCTR_TP_1_LO 0x490 +#define A6XX_RBBM_PERFCTR_TP_1_HI 0x491 +#define A6XX_RBBM_PERFCTR_TP_2_LO 0x492 +#define A6XX_RBBM_PERFCTR_TP_2_HI 0x493 +#define A6XX_RBBM_PERFCTR_TP_3_LO 0x494 +#define A6XX_RBBM_PERFCTR_TP_3_HI 0x495 +#define A6XX_RBBM_PERFCTR_TP_4_LO 0x496 +#define A6XX_RBBM_PERFCTR_TP_4_HI 0x497 +#define A6XX_RBBM_PERFCTR_TP_5_LO 0x498 +#define A6XX_RBBM_PERFCTR_TP_5_HI 0x499 +#define A6XX_RBBM_PERFCTR_TP_6_LO 0x49a +#define A6XX_RBBM_PERFCTR_TP_6_HI 0x49b +#define A6XX_RBBM_PERFCTR_TP_7_LO 0x49c +#define A6XX_RBBM_PERFCTR_TP_7_HI 0x49d +#define A6XX_RBBM_PERFCTR_TP_8_LO 0x49e +#define A6XX_RBBM_PERFCTR_TP_8_HI 0x49f +#define A6XX_RBBM_PERFCTR_TP_9_LO 0x4a0 +#define A6XX_RBBM_PERFCTR_TP_9_HI 0x4a1 +#define A6XX_RBBM_PERFCTR_TP_10_LO 0x4a2 +#define A6XX_RBBM_PERFCTR_TP_10_HI 0x4a3 +#define A6XX_RBBM_PERFCTR_TP_11_LO 0x4a4 +#define A6XX_RBBM_PERFCTR_TP_11_HI 0x4a5 +#define A6XX_RBBM_PERFCTR_SP_0_LO 0x4a6 +#define A6XX_RBBM_PERFCTR_SP_0_HI 0x4a7 +#define A6XX_RBBM_PERFCTR_SP_1_LO 0x4a8 +#define A6XX_RBBM_PERFCTR_SP_1_HI 0x4a9 +#define A6XX_RBBM_PERFCTR_SP_2_LO 0x4aa +#define A6XX_RBBM_PERFCTR_SP_2_HI 0x4ab +#define A6XX_RBBM_PERFCTR_SP_3_LO 0x4ac +#define A6XX_RBBM_PERFCTR_SP_3_HI 0x4ad +#define A6XX_RBBM_PERFCTR_SP_4_LO 0x4ae +#define A6XX_RBBM_PERFCTR_SP_4_HI 0x4af +#define A6XX_RBBM_PERFCTR_SP_5_LO 0x4b0 +#define A6XX_RBBM_PERFCTR_SP_5_HI 0x4b1 +#define A6XX_RBBM_PERFCTR_SP_6_LO 0x4b2 +#define A6XX_RBBM_PERFCTR_SP_6_HI 0x4b3 +#define A6XX_RBBM_PERFCTR_SP_7_LO 0x4b4 +#define A6XX_RBBM_PERFCTR_SP_7_HI 0x4b5 +#define A6XX_RBBM_PERFCTR_SP_8_LO 0x4b6 +#define A6XX_RBBM_PERFCTR_SP_8_HI 0x4b7 +#define A6XX_RBBM_PERFCTR_SP_9_LO 0x4b8 +#define A6XX_RBBM_PERFCTR_SP_9_HI 0x4b9 +#define A6XX_RBBM_PERFCTR_SP_10_LO 0x4ba +#define A6XX_RBBM_PERFCTR_SP_10_HI 0x4bb +#define A6XX_RBBM_PERFCTR_SP_11_LO 0x4bc +#define A6XX_RBBM_PERFCTR_SP_11_HI 0x4bd +#define A6XX_RBBM_PERFCTR_SP_12_LO 0x4be +#define A6XX_RBBM_PERFCTR_SP_12_HI 0x4bf +#define A6XX_RBBM_PERFCTR_SP_13_LO 0x4c0 +#define A6XX_RBBM_PERFCTR_SP_13_HI 0x4c1 +#define A6XX_RBBM_PERFCTR_SP_14_LO 0x4c2 +#define A6XX_RBBM_PERFCTR_SP_14_HI 0x4c3 +#define A6XX_RBBM_PERFCTR_SP_15_LO 0x4c4 +#define A6XX_RBBM_PERFCTR_SP_15_HI 0x4c5 +#define A6XX_RBBM_PERFCTR_SP_16_LO 0x4c6 +#define A6XX_RBBM_PERFCTR_SP_16_HI 0x4c7 +#define A6XX_RBBM_PERFCTR_SP_17_LO 0x4c8 +#define A6XX_RBBM_PERFCTR_SP_17_HI 0x4c9 +#define A6XX_RBBM_PERFCTR_SP_18_LO 0x4ca +#define A6XX_RBBM_PERFCTR_SP_18_HI 0x4cb +#define A6XX_RBBM_PERFCTR_SP_19_LO 0x4cc +#define A6XX_RBBM_PERFCTR_SP_19_HI 0x4cd +#define A6XX_RBBM_PERFCTR_SP_20_LO 0x4ce +#define A6XX_RBBM_PERFCTR_SP_20_HI 0x4cf +#define A6XX_RBBM_PERFCTR_SP_21_LO 0x4d0 +#define A6XX_RBBM_PERFCTR_SP_21_HI 0x4d1 +#define A6XX_RBBM_PERFCTR_SP_22_LO 0x4d2 +#define A6XX_RBBM_PERFCTR_SP_22_HI 0x4d3 +#define A6XX_RBBM_PERFCTR_SP_23_LO 0x4d4 +#define A6XX_RBBM_PERFCTR_SP_23_HI 0x4d5 +#define A6XX_RBBM_PERFCTR_RB_0_LO 0x4d6 +#define A6XX_RBBM_PERFCTR_RB_0_HI 0x4d7 +#define A6XX_RBBM_PERFCTR_RB_1_LO 0x4d8 +#define A6XX_RBBM_PERFCTR_RB_1_HI 0x4d9 +#define A6XX_RBBM_PERFCTR_RB_2_LO 0x4da +#define A6XX_RBBM_PERFCTR_RB_2_HI 0x4db +#define A6XX_RBBM_PERFCTR_RB_3_LO 0x4dc +#define A6XX_RBBM_PERFCTR_RB_3_HI 0x4dd +#define A6XX_RBBM_PERFCTR_RB_4_LO 0x4de +#define A6XX_RBBM_PERFCTR_RB_4_HI 0x4df +#define A6XX_RBBM_PERFCTR_RB_5_LO 0x4e0 +#define A6XX_RBBM_PERFCTR_RB_5_HI 0x4e1 +#define A6XX_RBBM_PERFCTR_RB_6_LO 0x4e2 +#define A6XX_RBBM_PERFCTR_RB_6_HI 0x4e3 +#define A6XX_RBBM_PERFCTR_RB_7_LO 0x4e4 +#define A6XX_RBBM_PERFCTR_RB_7_HI 0x4e5 +#define A6XX_RBBM_PERFCTR_VSC_0_LO 0x4e6 +#define A6XX_RBBM_PERFCTR_VSC_0_HI 0x4e7 +#define A6XX_RBBM_PERFCTR_VSC_1_LO 0x4e8 +#define A6XX_RBBM_PERFCTR_VSC_1_HI 0x4e9 +#define A6XX_RBBM_PERFCTR_LRZ_0_LO 0x4ea +#define A6XX_RBBM_PERFCTR_LRZ_0_HI 0x4eb +#define A6XX_RBBM_PERFCTR_LRZ_1_LO 0x4ec +#define A6XX_RBBM_PERFCTR_LRZ_1_HI 0x4ed +#define A6XX_RBBM_PERFCTR_LRZ_2_LO 0x4ee +#define A6XX_RBBM_PERFCTR_LRZ_2_HI 0x4ef +#define A6XX_RBBM_PERFCTR_LRZ_3_LO 0x4f0 +#define A6XX_RBBM_PERFCTR_LRZ_3_HI 0x4f1 +#define A6XX_RBBM_PERFCTR_CMP_0_LO 0x4f2 +#define A6XX_RBBM_PERFCTR_CMP_0_HI 0x4f3 +#define A6XX_RBBM_PERFCTR_CMP_1_LO 0x4f4 +#define A6XX_RBBM_PERFCTR_CMP_1_HI 0x4f5 +#define A6XX_RBBM_PERFCTR_CMP_2_LO 0x4f6 +#define A6XX_RBBM_PERFCTR_CMP_2_HI 0x4f7 +#define A6XX_RBBM_PERFCTR_CMP_3_LO 0x4f8 +#define A6XX_RBBM_PERFCTR_CMP_3_HI 0x4f9 +#define A6XX_RBBM_PERFCTR_CNTL 0x500 +#define A6XX_RBBM_PERFCTR_LOAD_CMD0 0x501 +#define A6XX_RBBM_PERFCTR_LOAD_CMD1 0x502 +#define A6XX_RBBM_PERFCTR_LOAD_CMD2 0x503 +#define A6XX_RBBM_PERFCTR_LOAD_CMD3 0x504 +#define A6XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x505 +#define A6XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x506 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_0 0x507 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_1 0x508 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_2 0x509 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_3 0x50A +#define A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x50B + +#define A6XX_RBBM_ISDB_CNT 0x533 + +#define A6XX_RBBM_SECVID_TRUST_CNTL 0xF400 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0xF800 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0xF801 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xF802 +#define A6XX_RBBM_SECVID_TSB_CNTL 0xF803 +#define A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0xF810 + +#define A6XX_RBBM_VBIF_CLIENT_QOS_CNTL 0x00010 +#define A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00011 +#define A6XX_RBBM_GPR0_CNTL 0x00018 +#define A6XX_RBBM_INTERFACE_HANG_INT_CNTL 0x0001f +#define A6XX_RBBM_INT_CLEAR_CMD 0x00037 +#define A6XX_RBBM_INT_0_MASK 0x00038 +#define A6XX_RBBM_INT_2_MASK 0x0003A +#define A6XX_RBBM_SP_HYST_CNT 0x00042 +#define A6XX_RBBM_SW_RESET_CMD 0x00043 +#define A6XX_RBBM_RAC_THRESHOLD_CNT 0x00044 +#define A6XX_RBBM_BLOCK_SW_RESET_CMD 0x00045 +#define A6XX_RBBM_BLOCK_SW_RESET_CMD2 0x00046 +#define A6XX_RBBM_BLOCK_GX_RETENTION_CNTL 0x00050 +#define A6XX_RBBM_CLOCK_CNTL 0x000ae +#define A6XX_RBBM_CLOCK_CNTL_SP0 0x000b0 +#define A6XX_RBBM_CLOCK_CNTL_SP1 0x000b1 +#define A6XX_RBBM_CLOCK_CNTL_SP2 0x000b2 +#define A6XX_RBBM_CLOCK_CNTL_SP3 0x000b3 +#define A6XX_RBBM_CLOCK_CNTL2_SP0 0x000b4 +#define A6XX_RBBM_CLOCK_CNTL2_SP1 0x000b5 +#define A6XX_RBBM_CLOCK_CNTL2_SP2 0x000b6 +#define A6XX_RBBM_CLOCK_CNTL2_SP3 0x000b7 +#define A6XX_RBBM_CLOCK_DELAY_SP0 0x000b8 +#define A6XX_RBBM_CLOCK_DELAY_SP1 0x000b9 +#define A6XX_RBBM_CLOCK_DELAY_SP2 0x000ba +#define A6XX_RBBM_CLOCK_DELAY_SP3 0x000bb +#define A6XX_RBBM_CLOCK_HYST_SP0 0x000bc +#define A6XX_RBBM_CLOCK_HYST_SP1 0x000bd +#define A6XX_RBBM_CLOCK_HYST_SP2 0x000be +#define A6XX_RBBM_CLOCK_HYST_SP3 0x000bf +#define A6XX_RBBM_CLOCK_CNTL_TP0 0x000c0 +#define A6XX_RBBM_CLOCK_CNTL_TP1 0x000c1 +#define A6XX_RBBM_CLOCK_CNTL_TP2 0x000c2 +#define A6XX_RBBM_CLOCK_CNTL_TP3 0x000c3 +#define A6XX_RBBM_CLOCK_CNTL2_TP0 0x000c4 +#define A6XX_RBBM_CLOCK_CNTL2_TP1 0x000c5 +#define A6XX_RBBM_CLOCK_CNTL2_TP2 0x000c6 +#define A6XX_RBBM_CLOCK_CNTL2_TP3 0x000c7 +#define A6XX_RBBM_CLOCK_CNTL3_TP0 0x000c8 +#define A6XX_RBBM_CLOCK_CNTL3_TP1 0x000c9 +#define A6XX_RBBM_CLOCK_CNTL3_TP2 0x000ca +#define A6XX_RBBM_CLOCK_CNTL3_TP3 0x000cb +#define A6XX_RBBM_CLOCK_CNTL4_TP0 0x000cc +#define A6XX_RBBM_CLOCK_CNTL4_TP1 0x000cd +#define A6XX_RBBM_CLOCK_CNTL4_TP2 0x000ce +#define A6XX_RBBM_CLOCK_CNTL4_TP3 0x000cf +#define A6XX_RBBM_CLOCK_DELAY_TP0 0x000d0 +#define A6XX_RBBM_CLOCK_DELAY_TP1 0x000d1 +#define A6XX_RBBM_CLOCK_DELAY_TP2 0x000d2 +#define A6XX_RBBM_CLOCK_DELAY_TP3 0x000d3 +#define A6XX_RBBM_CLOCK_DELAY2_TP0 0x000d4 +#define A6XX_RBBM_CLOCK_DELAY2_TP1 0x000d5 +#define A6XX_RBBM_CLOCK_DELAY2_TP2 0x000d6 +#define A6XX_RBBM_CLOCK_DELAY2_TP3 0x000d7 +#define A6XX_RBBM_CLOCK_DELAY3_TP0 0x000d8 +#define A6XX_RBBM_CLOCK_DELAY3_TP1 0x000d9 +#define A6XX_RBBM_CLOCK_DELAY3_TP2 0x000da +#define A6XX_RBBM_CLOCK_DELAY3_TP3 0x000db +#define A6XX_RBBM_CLOCK_DELAY4_TP0 0x000dc +#define A6XX_RBBM_CLOCK_DELAY4_TP1 0x000dd +#define A6XX_RBBM_CLOCK_DELAY4_TP2 0x000de +#define A6XX_RBBM_CLOCK_DELAY4_TP3 0x000df +#define A6XX_RBBM_CLOCK_HYST_TP0 0x000e0 +#define A6XX_RBBM_CLOCK_HYST_TP1 0x000e1 +#define A6XX_RBBM_CLOCK_HYST_TP2 0x000e2 +#define A6XX_RBBM_CLOCK_HYST_TP3 0x000e3 +#define A6XX_RBBM_CLOCK_HYST2_TP0 0x000e4 +#define A6XX_RBBM_CLOCK_HYST2_TP1 0x000e5 +#define A6XX_RBBM_CLOCK_HYST2_TP2 0x000e6 +#define A6XX_RBBM_CLOCK_HYST2_TP3 0x000e7 +#define A6XX_RBBM_CLOCK_HYST3_TP0 0x000e8 +#define A6XX_RBBM_CLOCK_HYST3_TP1 0x000e9 +#define A6XX_RBBM_CLOCK_HYST3_TP2 0x000ea +#define A6XX_RBBM_CLOCK_HYST3_TP3 0x000eb +#define A6XX_RBBM_CLOCK_HYST4_TP0 0x000ec +#define A6XX_RBBM_CLOCK_HYST4_TP1 0x000ed +#define A6XX_RBBM_CLOCK_HYST4_TP2 0x000ee +#define A6XX_RBBM_CLOCK_HYST4_TP3 0x000ef +#define A6XX_RBBM_CLOCK_CNTL_RB0 0x000f0 +#define A6XX_RBBM_CLOCK_CNTL_RB1 0x000f1 +#define A6XX_RBBM_CLOCK_CNTL_RB2 0x000f2 +#define A6XX_RBBM_CLOCK_CNTL_RB3 0x000f3 +#define A6XX_RBBM_CLOCK_CNTL2_RB0 0x000f4 +#define A6XX_RBBM_CLOCK_CNTL2_RB1 0x000f5 +#define A6XX_RBBM_CLOCK_CNTL2_RB2 0x000f6 +#define A6XX_RBBM_CLOCK_CNTL2_RB3 0x000f7 +#define A6XX_RBBM_CLOCK_CNTL_CCU0 0x000f8 +#define A6XX_RBBM_CLOCK_CNTL_CCU1 0x000f9 +#define A6XX_RBBM_CLOCK_CNTL_CCU2 0x000fa +#define A6XX_RBBM_CLOCK_CNTL_CCU3 0x000fb +#define A6XX_RBBM_CLOCK_HYST_RB_CCU0 0x00100 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU1 0x00101 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU2 0x00102 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU3 0x00103 +#define A6XX_RBBM_CLOCK_CNTL_RAC 0x00104 +#define A6XX_RBBM_CLOCK_CNTL2_RAC 0x00105 +#define A6XX_RBBM_CLOCK_DELAY_RAC 0x00106 +#define A6XX_RBBM_CLOCK_HYST_RAC 0x00107 +#define A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x00108 +#define A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x00109 +#define A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x0010a +#define A6XX_RBBM_CLOCK_CNTL_UCHE 0x0010b +#define A6XX_RBBM_CLOCK_CNTL2_UCHE 0x0010c +#define A6XX_RBBM_CLOCK_CNTL3_UCHE 0x0010d +#define A6XX_RBBM_CLOCK_CNTL4_UCHE 0x0010e +#define A6XX_RBBM_CLOCK_DELAY_UCHE 0x0010f +#define A6XX_RBBM_CLOCK_HYST_UCHE 0x00110 +#define A6XX_RBBM_CLOCK_MODE_VFD 0x00111 +#define A6XX_RBBM_CLOCK_DELAY_VFD 0x00112 +#define A6XX_RBBM_CLOCK_HYST_VFD 0x00113 +#define A6XX_RBBM_CLOCK_MODE_GPC 0x00114 +#define A6XX_RBBM_CLOCK_DELAY_GPC 0x00115 +#define A6XX_RBBM_CLOCK_HYST_GPC 0x00116 +#define A6XX_RBBM_CLOCK_DELAY_HLSQ_2 0x00117 +#define A6XX_RBBM_CLOCK_CNTL_GMU_GX 0x00118 +#define A6XX_RBBM_CLOCK_DELAY_GMU_GX 0x00119 +#define A6XX_RBBM_CLOCK_CNTL_TEX_FCHE 0x00120 +#define A6XX_RBBM_CLOCK_DELAY_TEX_FCHE 0x00121 +#define A6XX_RBBM_CLOCK_HYST_TEX_FCHE 0x00122 +#define A6XX_RBBM_CLOCK_HYST_GMU_GX 0x0011a +#define A6XX_RBBM_CLOCK_MODE_HLSQ 0x0011b +#define A6XX_RBBM_CLOCK_DELAY_HLSQ 0x0011c +#define A6XX_RBBM_CLOCK_HYST_HLSQ 0x0011d + +/* DBGC_CFG registers */ +#define A6XX_DBGC_CFG_DBGBUS_SEL_A 0x600 +#define A6XX_DBGC_CFG_DBGBUS_SEL_B 0x601 +#define A6XX_DBGC_CFG_DBGBUS_SEL_C 0x602 +#define A6XX_DBGC_CFG_DBGBUS_SEL_D 0x603 +#define A6XX_DBGC_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT 0x604 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_CNTLM 0x605 +#define A6XX_DBGC_CFG_DBGBUS_CTLTM_ENABLE_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_OPL 0x606 +#define A6XX_DBGC_CFG_DBGBUS_OPE 0x607 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_0 0x608 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_1 0x609 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_2 0x60a +#define A6XX_DBGC_CFG_DBGBUS_IVTL_3 0x60b +#define A6XX_DBGC_CFG_DBGBUS_MASKL_0 0x60c +#define A6XX_DBGC_CFG_DBGBUS_MASKL_1 0x60d +#define A6XX_DBGC_CFG_DBGBUS_MASKL_2 0x60e +#define A6XX_DBGC_CFG_DBGBUS_MASKL_3 0x60f +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0 0x610 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1 0x611 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL0_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL1_SHIFT 0x4 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL2_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL3_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_BYTEL4_SHIFT 0x10 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL5_SHIFT 0x14 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL6_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL7_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_BYTEL8_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL9_SHIFT 0x4 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL10_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL11_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_BYTEL12_SHIFT 0x10 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL13_SHIFT 0x14 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL14_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL15_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_IVTE_0 0x612 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_1 0x613 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_2 0x614 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_3 0x615 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_0 0x616 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_1 0x617 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_2 0x618 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_3 0x619 +#define A6XX_DBGC_CFG_DBGBUS_NIBBLEE 0x61a +#define A6XX_DBGC_CFG_DBGBUS_PTRC0 0x61b +#define A6XX_DBGC_CFG_DBGBUS_PTRC1 0x61c +#define A6XX_DBGC_CFG_DBGBUS_LOADREG 0x61d +#define A6XX_DBGC_CFG_DBGBUS_IDX 0x61e +#define A6XX_DBGC_CFG_DBGBUS_CLRC 0x61f +#define A6XX_DBGC_CFG_DBGBUS_LOADIVT 0x620 +#define A6XX_DBGC_VBIF_DBG_CNTL 0x621 +#define A6XX_DBGC_DBG_LO_HI_GPIO 0x622 +#define A6XX_DBGC_EXT_TRACE_BUS_CNTL 0x623 +#define A6XX_DBGC_READ_AHB_THROUGH_DBG 0x624 +#define A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x62f +#define A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x630 +#define A6XX_DBGC_EVT_CFG 0x640 +#define A6XX_DBGC_EVT_INTF_SEL_0 0x641 +#define A6XX_DBGC_EVT_INTF_SEL_1 0x642 +#define A6XX_DBGC_PERF_ATB_CFG 0x643 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_0 0x644 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_1 0x645 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_2 0x646 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_3 0x647 +#define A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 0x648 +#define A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 0x649 +#define A6XX_DBGC_PERF_ATB_DRAIN_CMD 0x64a +#define A6XX_DBGC_ECO_CNTL 0x650 +#define A6XX_DBGC_AHB_DBG_CNTL 0x651 + +/* VSC registers */ +#define A6XX_VSC_PERFCTR_VSC_SEL_0 0xCD8 +#define A6XX_VSC_PERFCTR_VSC_SEL_1 0xCD9 + +/* GRAS registers */ +#define A6XX_GRAS_ADDR_MODE_CNTL 0x8601 +#define A6XX_GRAS_PERFCTR_TSE_SEL_0 0x8610 +#define A6XX_GRAS_PERFCTR_TSE_SEL_1 0x8611 +#define A6XX_GRAS_PERFCTR_TSE_SEL_2 0x8612 +#define A6XX_GRAS_PERFCTR_TSE_SEL_3 0x8613 +#define A6XX_GRAS_PERFCTR_RAS_SEL_0 0x8614 +#define A6XX_GRAS_PERFCTR_RAS_SEL_1 0x8615 +#define A6XX_GRAS_PERFCTR_RAS_SEL_2 0x8616 +#define A6XX_GRAS_PERFCTR_RAS_SEL_3 0x8617 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_0 0x8618 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_1 0x8619 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_2 0x861A +#define A6XX_GRAS_PERFCTR_LRZ_SEL_3 0x861B + +/* RB registers */ +#define A6XX_RB_ADDR_MODE_CNTL 0x8E05 +#define A6XX_RB_NC_MODE_CNTL 0x8E08 +#define A6XX_RB_PERFCTR_RB_SEL_0 0x8E10 +#define A6XX_RB_PERFCTR_RB_SEL_1 0x8E11 +#define A6XX_RB_PERFCTR_RB_SEL_2 0x8E12 +#define A6XX_RB_PERFCTR_RB_SEL_3 0x8E13 +#define A6XX_RB_PERFCTR_RB_SEL_4 0x8E14 +#define A6XX_RB_PERFCTR_RB_SEL_5 0x8E15 +#define A6XX_RB_PERFCTR_RB_SEL_6 0x8E16 +#define A6XX_RB_PERFCTR_RB_SEL_7 0x8E17 +#define A6XX_RB_PERFCTR_CCU_SEL_0 0x8E18 +#define A6XX_RB_PERFCTR_CCU_SEL_1 0x8E19 +#define A6XX_RB_PERFCTR_CCU_SEL_2 0x8E1A +#define A6XX_RB_PERFCTR_CCU_SEL_3 0x8E1B +#define A6XX_RB_PERFCTR_CCU_SEL_4 0x8E1C +#define A6XX_RB_PERFCTR_CMP_SEL_0 0x8E2C +#define A6XX_RB_PERFCTR_CMP_SEL_1 0x8E2D +#define A6XX_RB_PERFCTR_CMP_SEL_2 0x8E2E +#define A6XX_RB_PERFCTR_CMP_SEL_3 0x8E2F +#define A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST 0x8E3B +#define A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD 0x8E3D +#define A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE 0x8E50 + +/* PC registers */ +#define A6XX_PC_DBG_ECO_CNTL 0x9E00 +#define A6XX_PC_ADDR_MODE_CNTL 0x9E01 +#define A6XX_PC_PERFCTR_PC_SEL_0 0x9E34 +#define A6XX_PC_PERFCTR_PC_SEL_1 0x9E35 +#define A6XX_PC_PERFCTR_PC_SEL_2 0x9E36 +#define A6XX_PC_PERFCTR_PC_SEL_3 0x9E37 +#define A6XX_PC_PERFCTR_PC_SEL_4 0x9E38 +#define A6XX_PC_PERFCTR_PC_SEL_5 0x9E39 +#define A6XX_PC_PERFCTR_PC_SEL_6 0x9E3A +#define A6XX_PC_PERFCTR_PC_SEL_7 0x9E3B + +/* HLSQ registers */ +#define A6XX_HLSQ_ADDR_MODE_CNTL 0xBE05 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xBE10 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xBE11 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xBE12 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xBE13 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xBE14 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xBE15 +#define A6XX_HLSQ_DBG_AHB_READ_APERTURE 0xC800 +#define A6XX_HLSQ_DBG_READ_SEL 0xD000 + +/* VFD registers */ +#define A6XX_VFD_ADDR_MODE_CNTL 0xA601 +#define A6XX_VFD_PERFCTR_VFD_SEL_0 0xA610 +#define A6XX_VFD_PERFCTR_VFD_SEL_1 0xA611 +#define A6XX_VFD_PERFCTR_VFD_SEL_2 0xA612 +#define A6XX_VFD_PERFCTR_VFD_SEL_3 0xA613 +#define A6XX_VFD_PERFCTR_VFD_SEL_4 0xA614 +#define A6XX_VFD_PERFCTR_VFD_SEL_5 0xA615 +#define A6XX_VFD_PERFCTR_VFD_SEL_6 0xA616 +#define A6XX_VFD_PERFCTR_VFD_SEL_7 0xA617 + +/* VPC registers */ +#define A6XX_VPC_ADDR_MODE_CNTL 0x9601 +#define A6XX_VPC_PERFCTR_VPC_SEL_0 0x9604 +#define A6XX_VPC_PERFCTR_VPC_SEL_1 0x9605 +#define A6XX_VPC_PERFCTR_VPC_SEL_2 0x9606 +#define A6XX_VPC_PERFCTR_VPC_SEL_3 0x9607 +#define A6XX_VPC_PERFCTR_VPC_SEL_4 0x9608 +#define A6XX_VPC_PERFCTR_VPC_SEL_5 0x9609 + +/* UCHE registers */ +#define A6XX_UCHE_ADDR_MODE_CNTL 0xE00 +#define A6XX_UCHE_MODE_CNTL 0xE01 +#define A6XX_UCHE_WRITE_RANGE_MAX_LO 0xE05 +#define A6XX_UCHE_WRITE_RANGE_MAX_HI 0xE06 +#define A6XX_UCHE_WRITE_THRU_BASE_LO 0xE07 +#define A6XX_UCHE_WRITE_THRU_BASE_HI 0xE08 +#define A6XX_UCHE_TRAP_BASE_LO 0xE09 +#define A6XX_UCHE_TRAP_BASE_HI 0xE0A +#define A6XX_UCHE_GMEM_RANGE_MIN_LO 0xE0B +#define A6XX_UCHE_GMEM_RANGE_MIN_HI 0xE0C +#define A6XX_UCHE_GMEM_RANGE_MAX_LO 0xE0D +#define A6XX_UCHE_GMEM_RANGE_MAX_HI 0xE0E +#define A6XX_UCHE_CACHE_WAYS 0xE17 +#define A6XX_UCHE_FILTER_CNTL 0xE18 +#define A6XX_UCHE_CLIENT_PF 0xE19 +#define A6XX_UCHE_CLIENT_PF_CLIENT_ID_MASK 0x7 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_0 0xE1C +#define A6XX_UCHE_PERFCTR_UCHE_SEL_1 0xE1D +#define A6XX_UCHE_PERFCTR_UCHE_SEL_2 0xE1E +#define A6XX_UCHE_PERFCTR_UCHE_SEL_3 0xE1F +#define A6XX_UCHE_PERFCTR_UCHE_SEL_4 0xE20 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_5 0xE21 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_6 0xE22 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_7 0xE23 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_8 0xE24 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_9 0xE25 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_10 0xE26 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_11 0xE27 +#define A6XX_UCHE_GBIF_GX_CONFIG 0xE3A + +/* SP registers */ +#define A6XX_SP_ADDR_MODE_CNTL 0xAE01 +#define A6XX_SP_NC_MODE_CNTL 0xAE02 +#define A6XX_SP_PERFCTR_SP_SEL_0 0xAE10 +#define A6XX_SP_PERFCTR_SP_SEL_1 0xAE11 +#define A6XX_SP_PERFCTR_SP_SEL_2 0xAE12 +#define A6XX_SP_PERFCTR_SP_SEL_3 0xAE13 +#define A6XX_SP_PERFCTR_SP_SEL_4 0xAE14 +#define A6XX_SP_PERFCTR_SP_SEL_5 0xAE15 +#define A6XX_SP_PERFCTR_SP_SEL_6 0xAE16 +#define A6XX_SP_PERFCTR_SP_SEL_7 0xAE17 +#define A6XX_SP_PERFCTR_SP_SEL_8 0xAE18 +#define A6XX_SP_PERFCTR_SP_SEL_9 0xAE19 +#define A6XX_SP_PERFCTR_SP_SEL_10 0xAE1A +#define A6XX_SP_PERFCTR_SP_SEL_11 0xAE1B +#define A6XX_SP_PERFCTR_SP_SEL_12 0xAE1C +#define A6XX_SP_PERFCTR_SP_SEL_13 0xAE1D +#define A6XX_SP_PERFCTR_SP_SEL_14 0xAE1E +#define A6XX_SP_PERFCTR_SP_SEL_15 0xAE1F +#define A6XX_SP_PERFCTR_SP_SEL_16 0xAE20 +#define A6XX_SP_PERFCTR_SP_SEL_17 0xAE21 +#define A6XX_SP_PERFCTR_SP_SEL_18 0xAE22 +#define A6XX_SP_PERFCTR_SP_SEL_19 0xAE23 +#define A6XX_SP_PERFCTR_SP_SEL_20 0xAE24 +#define A6XX_SP_PERFCTR_SP_SEL_21 0xAE25 +#define A6XX_SP_PERFCTR_SP_SEL_22 0xAE26 +#define A6XX_SP_PERFCTR_SP_SEL_23 0xAE27 + +/* TP registers */ +#define A6XX_TPL1_ADDR_MODE_CNTL 0xB601 +#define A6XX_TPL1_NC_MODE_CNTL 0xB604 +#define A6XX_TPL1_PERFCTR_TP_SEL_0 0xB610 +#define A6XX_TPL1_PERFCTR_TP_SEL_1 0xB611 +#define A6XX_TPL1_PERFCTR_TP_SEL_2 0xB612 +#define A6XX_TPL1_PERFCTR_TP_SEL_3 0xB613 +#define A6XX_TPL1_PERFCTR_TP_SEL_4 0xB614 +#define A6XX_TPL1_PERFCTR_TP_SEL_5 0xB615 +#define A6XX_TPL1_PERFCTR_TP_SEL_6 0xB616 +#define A6XX_TPL1_PERFCTR_TP_SEL_7 0xB617 +#define A6XX_TPL1_PERFCTR_TP_SEL_8 0xB618 +#define A6XX_TPL1_PERFCTR_TP_SEL_9 0xB619 +#define A6XX_TPL1_PERFCTR_TP_SEL_10 0xB61A +#define A6XX_TPL1_PERFCTR_TP_SEL_11 0xB61B + +/* VBIF registers */ +#define A6XX_VBIF_VERSION 0x3000 +#define A6XX_VBIF_CLKON 0x3001 +#define A6XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1 +#define A6XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1 +#define A6XX_VBIF_GATE_OFF_WRREQ_EN 0x302A +#define A6XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A6XX_VBIF_XIN_HALT_CTRL0_MASK 0xF +#define A6XX_VBIF_XIN_HALT_CTRL1 0x3081 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL 0x3084 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS1_CTRL0 0x3085 +#define A6XX_VBIF_TEST_BUS1_CTRL1 0x3086 +#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF +#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS2_CTRL0 0x3087 +#define A6XX_VBIF_TEST_BUS2_CTRL1 0x3088 +#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0x1FF +#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS_OUT 0x308C +#define A6XX_VBIF_PERF_CNT_SEL0 0x30d0 +#define A6XX_VBIF_PERF_CNT_SEL1 0x30d1 +#define A6XX_VBIF_PERF_CNT_SEL2 0x30d2 +#define A6XX_VBIF_PERF_CNT_SEL3 0x30d3 +#define A6XX_VBIF_PERF_CNT_LOW0 0x30d8 +#define A6XX_VBIF_PERF_CNT_LOW1 0x30d9 +#define A6XX_VBIF_PERF_CNT_LOW2 0x30da +#define A6XX_VBIF_PERF_CNT_LOW3 0x30db +#define A6XX_VBIF_PERF_CNT_HIGH0 0x30e0 +#define A6XX_VBIF_PERF_CNT_HIGH1 0x30e1 +#define A6XX_VBIF_PERF_CNT_HIGH2 0x30e2 +#define A6XX_VBIF_PERF_CNT_HIGH3 0x30e3 +#define A6XX_VBIF_PERF_PWR_CNT_EN0 0x3100 +#define A6XX_VBIF_PERF_PWR_CNT_EN1 0x3101 +#define A6XX_VBIF_PERF_PWR_CNT_EN2 0x3102 +#define A6XX_VBIF_PERF_PWR_CNT_LOW0 0x3110 +#define A6XX_VBIF_PERF_PWR_CNT_LOW1 0x3111 +#define A6XX_VBIF_PERF_PWR_CNT_LOW2 0x3112 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH2 0x311a + +/* GBIF countables */ +#define GBIF_AXI0_READ_DATA_TOTAL_BEATS 34 +#define GBIF_AXI1_READ_DATA_TOTAL_BEATS 35 +#define GBIF_AXI0_WRITE_DATA_TOTAL_BEATS 46 +#define GBIF_AXI1_WRITE_DATA_TOTAL_BEATS 47 + +/* GBIF registers */ +#define A6XX_GBIF_SCACHE_CNTL1 0x3c02 +#define A6XX_GBIF_QSB_SIDE0 0x3c03 +#define A6XX_GBIF_QSB_SIDE1 0x3c04 +#define A6XX_GBIF_QSB_SIDE2 0x3c05 +#define A6XX_GBIF_QSB_SIDE3 0x3c06 +#define A6XX_GBIF_HALT 0x3c45 +#define A6XX_GBIF_HALT_ACK 0x3c46 + +#define A6XX_GBIF_CLIENT_HALT_MASK BIT(0) +#define A6XX_GBIF_ARB_HALT_MASK BIT(1) + +#define A6XX_GBIF_PERF_PWR_CNT_EN 0x3cc0 +#define A6XX_GBIF_PERF_CNT_SEL 0x3cc2 +#define A6XX_GBIF_PERF_CNT_LOW0 0x3cc4 +#define A6XX_GBIF_PERF_CNT_LOW1 0x3cc5 +#define A6XX_GBIF_PERF_CNT_LOW2 0x3cc6 +#define A6XX_GBIF_PERF_CNT_LOW3 0x3cc7 +#define A6XX_GBIF_PERF_CNT_HIGH0 0x3cc8 +#define A6XX_GBIF_PERF_CNT_HIGH1 0x3cc9 +#define A6XX_GBIF_PERF_CNT_HIGH2 0x3cca +#define A6XX_GBIF_PERF_CNT_HIGH3 0x3ccb +#define A6XX_GBIF_PWR_CNT_LOW0 0x3ccc +#define A6XX_GBIF_PWR_CNT_LOW1 0x3ccd +#define A6XX_GBIF_PWR_CNT_LOW2 0x3cce +#define A6XX_GBIF_PWR_CNT_HIGH0 0x3ccf +#define A6XX_GBIF_PWR_CNT_HIGH1 0x3cd0 +#define A6XX_GBIF_PWR_CNT_HIGH2 0x3cd1 + + +/* CX_DBGC_CFG registers */ +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A 0x18400 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_B 0x18401 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_C 0x18402 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D 0x18403 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT 0x18404 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM 0x18405 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_OPL 0x18406 +#define A6XX_CX_DBGC_CFG_DBGBUS_OPE 0x18407 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0 0x18408 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1 0x18409 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2 0x1840A +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3 0x1840B +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0 0x1840C +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1 0x1840D +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2 0x1840E +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3 0x1840F +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0 0x18410 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1 0x18411 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL0_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL1_SHIFT 0x4 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL2_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL3_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL4_SHIFT 0x10 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL5_SHIFT 0x14 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL6_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL7_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL8_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL9_SHIFT 0x4 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL10_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL11_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL12_SHIFT 0x10 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL13_SHIFT 0x14 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL14_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL15_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_0 0x18412 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_1 0x18413 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_2 0x18414 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_3 0x18415 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_0 0x18416 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_1 0x18417 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_2 0x18418 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_3 0x18419 +#define A6XX_CX_DBGC_CFG_DBGBUS_NIBBLEE 0x1841A +#define A6XX_CX_DBGC_CFG_DBGBUS_PTRC0 0x1841B +#define A6XX_CX_DBGC_CFG_DBGBUS_PTRC1 0x1841C +#define A6XX_CX_DBGC_CFG_DBGBUS_LOADREG 0x1841D +#define A6XX_CX_DBGC_CFG_DBGBUS_IDX 0x1841E +#define A6XX_CX_DBGC_CFG_DBGBUS_CLRC 0x1841F +#define A6XX_CX_DBGC_CFG_DBGBUS_LOADIVT 0x18420 +#define A6XX_CX_DBGC_VBIF_DBG_CNTL 0x18421 +#define A6XX_CX_DBGC_DBG_LO_HI_GPIO 0x18422 +#define A6XX_CX_DBGC_EXT_TRACE_BUS_CNTL 0x18423 +#define A6XX_CX_DBGC_READ_AHB_THROUGH_DBG 0x18424 +#define A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x1842F +#define A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x18430 +#define A6XX_CX_DBGC_EVT_CFG 0x18440 +#define A6XX_CX_DBGC_EVT_INTF_SEL_0 0x18441 +#define A6XX_CX_DBGC_EVT_INTF_SEL_1 0x18442 +#define A6XX_CX_DBGC_PERF_ATB_CFG 0x18443 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_0 0x18444 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_1 0x18445 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_2 0x18446 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_3 0x18447 +#define A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 0x18448 +#define A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 0x18449 +#define A6XX_CX_DBGC_PERF_ATB_DRAIN_CMD 0x1844A +#define A6XX_CX_DBGC_ECO_CNTL 0x18450 +#define A6XX_CX_DBGC_AHB_DBG_CNTL 0x18451 + +/* GMU control registers */ +#define A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL 0x1A880 +#define A6XX_GMU_GX_SPTPRAC_POWER_CONTROL 0x1A881 +#define A6XX_GMU_CM3_ITCM_START 0x1B400 +#define A6XX_GMU_CM3_DTCM_START 0x1C400 +#define A6XX_GMU_NMI_CONTROL_STATUS 0x1CBF0 +#define A6XX_GMU_BOOT_SLUMBER_OPTION 0x1CBF8 +#define A6XX_GMU_GX_VOTE_IDX 0x1CBF9 +#define A6XX_GMU_MX_VOTE_IDX 0x1CBFA +#define A6XX_GMU_DCVS_ACK_OPTION 0x1CBFC +#define A6XX_GMU_DCVS_PERF_SETTING 0x1CBFD +#define A6XX_GMU_DCVS_BW_SETTING 0x1CBFE +#define A6XX_GMU_DCVS_RETURN 0x1CBFF +#define A6XX_GMU_ICACHE_CONFIG 0x1F400 +#define A6XX_GMU_DCACHE_CONFIG 0x1F401 +#define A6XX_GMU_SYS_BUS_CONFIG 0x1F40F +#define A6XX_GMU_CM3_SYSRESET 0x1F800 +#define A6XX_GMU_CM3_BOOT_CONFIG 0x1F801 +#define A6XX_GMU_CX_GMU_WFI_CONFIG 0x1F802 +#define A6XX_GMU_CM3_FW_BUSY 0x1F81A +#define A6XX_GMU_CM3_FW_INIT_RESULT 0x1F81C +#define A6XX_GMU_CM3_CFG 0x1F82D +#define A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE 0x1F840 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0 0x1F841 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1 0x1F842 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L 0x1F844 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H 0x1F845 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L 0x1F846 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H 0x1F847 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L 0x1F848 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H 0x1F849 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L 0x1F84A +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H 0x1F84B +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L 0x1F84C +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H 0x1F84D +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L 0x1F84E +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H 0x1F84F +#define A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_L 0x1F888 +#define A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_H 0x1F889 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL 0x1F8C0 +#define A6XX_GMU_PWR_COL_INTER_FRAME_HYST 0x1F8C1 +#define A6XX_GMU_PWR_COL_SPTPRAC_HYST 0x1F8C2 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS 0x1F8D0 +#define A6XX_GMU_GPU_NAP_CTRL 0x1F8E4 +#define A6XX_GMU_RPMH_CTRL 0x1F8E8 +#define A6XX_GMU_RPMH_HYST_CTRL 0x1F8E9 +#define A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE 0x1F8EC +#define A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG 0x1F900 +#define A6XX_GMU_BOOT_KMD_LM_HANDSHAKE 0x1F9F0 +#define A6XX_GMU_LLM_GLM_SLEEP_CTRL 0x1F957 +#define A6XX_GMU_LLM_GLM_SLEEP_STATUS 0x1F958 + +/* HFI registers*/ +#define A6XX_GMU_ALWAYS_ON_COUNTER_L 0x1F888 +#define A6XX_GMU_ALWAYS_ON_COUNTER_H 0x1F889 +#define A6XX_GMU_GMU_PWR_COL_KEEPALIVE 0x1F8C3 +#define A6XX_GMU_HFI_CTRL_STATUS 0x1F980 +#define A6XX_GMU_HFI_VERSION_INFO 0x1F981 +#define A6XX_GMU_HFI_SFR_ADDR 0x1F982 +#define A6XX_GMU_HFI_MMAP_ADDR 0x1F983 +#define A6XX_GMU_HFI_QTBL_INFO 0x1F984 +#define A6XX_GMU_HFI_QTBL_ADDR 0x1F985 +#define A6XX_GMU_HFI_CTRL_INIT 0x1F986 +#define A6XX_GMU_GMU2HOST_INTR_SET 0x1F990 +#define A6XX_GMU_GMU2HOST_INTR_CLR 0x1F991 +#define A6XX_GMU_GMU2HOST_INTR_INFO 0x1F992 +#define A6XX_GMU_GMU2HOST_INTR_MASK 0x1F993 +#define A6XX_GMU_HOST2GMU_INTR_SET 0x1F994 +#define A6XX_GMU_HOST2GMU_INTR_CLR 0x1F995 +#define A6XX_GMU_HOST2GMU_INTR_RAW_INFO 0x1F996 +#define A6XX_GMU_HOST2GMU_INTR_EN_0 0x1F997 +#define A6XX_GMU_HOST2GMU_INTR_EN_1 0x1F998 +#define A6XX_GMU_HOST2GMU_INTR_EN_2 0x1F999 +#define A6XX_GMU_HOST2GMU_INTR_EN_3 0x1F99A +#define A6XX_GMU_HOST2GMU_INTR_INFO_0 0x1F99B +#define A6XX_GMU_HOST2GMU_INTR_INFO_1 0x1F99C +#define A6XX_GMU_HOST2GMU_INTR_INFO_2 0x1F99D +#define A6XX_GMU_HOST2GMU_INTR_INFO_3 0x1F99E +#define A6XX_GMU_GENERAL_1 0x1F9C6 +#define A6XX_GMU_GENERAL_6 0x1F9CB +#define A6XX_GMU_GENERAL_7 0x1F9CC + +/* ISENSE registers */ +#define A6XX_GMU_ISENSE_CTRL 0x1F95D +#define A6XX_GPU_GMU_CX_GMU_ISENSE_CTRL 0x1f95d +#define A6XX_GPU_CS_ENABLE_REG 0x23120 + +/* LM registers */ +#define A6XX_GPU_GMU_CX_GMU_PWR_THRESHOLD 0x1F94D + + +#define A6XX_GMU_AO_INTERRUPT_EN 0x23B03 +#define A6XX_GMU_AO_HOST_INTERRUPT_CLR 0x23B04 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS 0x23B05 +#define A6XX_GMU_AO_HOST_INTERRUPT_MASK 0x23B06 +#define A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL 0x23B09 +#define A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL 0x23B0A +#define A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL 0x23B0B +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS 0x23B0C +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2 0x23B0D +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK 0x23B0E +#define A6XX_GMU_AO_AHB_FENCE_CTRL 0x23B10 +#define A6XX_GMU_AHB_FENCE_STATUS 0x23B13 +#define A6XX_GMU_RBBM_INT_UNMASKED_STATUS 0x23B15 +#define A6XX_GMU_AO_SPARE_CNTL 0x23B16 + +/* RGMU GLM registers */ +#define A6XX_GMU_AO_RGMU_GLM_SLEEP_CTRL 0x23B80 +#define A6XX_GMU_AO_RGMU_GLM_SLEEP_STATUS 0x23B81 +#define A6XX_GMU_AO_RGMU_GLM_HW_CRC_DISABLE 0x23B82 + +/* GMU RSC control registers */ +#define A6XX_GPU_RSCC_RSC_STATUS0_DRV0 0x23404 +#define A6XX_GMU_RSCC_CONTROL_REQ 0x23B07 +#define A6XX_GMU_RSCC_CONTROL_ACK 0x23B08 + +/* FENCE control registers */ +#define A6XX_GMU_AHB_FENCE_RANGE_0 0x23B11 +#define A6XX_GMU_AHB_FENCE_RANGE_1 0x23B12 + +/* GPUCC registers */ +#define A6XX_GPU_CC_GX_GDSCR 0x24403 +#define A6XX_GPU_CC_GX_DOMAIN_MISC 0x24542 + +/* GPU RSC sequencer registers */ +#define A6XX_RSCC_PDC_SEQ_START_ADDR 0x23408 +#define A6XX_RSCC_PDC_MATCH_VALUE_LO 0x23409 +#define A6XX_RSCC_PDC_MATCH_VALUE_HI 0x2340A +#define A6XX_RSCC_PDC_SLAVE_ID_DRV0 0x2340B +#define A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR 0x2340D +#define A6XX_RSCC_HIDDEN_TCS_CMD0_DATA 0x2340E +#define A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0 0x23482 +#define A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0 0x23483 +#define A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0 0x23489 +#define A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0 0x2348C +#define A6XX_RSCC_OVERRIDE_START_ADDR 0x23500 +#define A6XX_RSCC_SEQ_BUSY_DRV0 0x23501 +#define A6XX_RSCC_SEQ_MEM_0_DRV0 0x23580 +#define A6XX_RSCC_TCS0_DRV0_STATUS 0x23746 +#define A6XX_RSCC_TCS1_DRV0_STATUS 0x237EE +#define A6XX_RSCC_TCS2_DRV0_STATUS 0x23896 +#define A6XX_RSCC_TCS3_DRV0_STATUS 0x2393E + +/* GPU PDC sequencer registers in AOSS.RPMh domain */ +#define PDC_GPU_ENABLE_PDC 0x1140 +#define PDC_GPU_SEQ_START_ADDR 0x1148 +#define PDC_GPU_TCS0_CONTROL 0x1540 +#define PDC_GPU_TCS0_CMD_ENABLE_BANK 0x1541 +#define PDC_GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK 0x1542 +#define PDC_GPU_TCS0_CMD0_MSGID 0x1543 +#define PDC_GPU_TCS0_CMD0_ADDR 0x1544 +#define PDC_GPU_TCS0_CMD0_DATA 0x1545 +#define PDC_GPU_TCS1_CONTROL 0x1572 +#define PDC_GPU_TCS1_CMD_ENABLE_BANK 0x1573 +#define PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK 0x1574 +#define PDC_GPU_TCS1_CMD0_MSGID 0x1575 +#define PDC_GPU_TCS1_CMD0_ADDR 0x1576 +#define PDC_GPU_TCS1_CMD0_DATA 0x1577 +#define PDC_GPU_TCS2_CONTROL 0x15A4 +#define PDC_GPU_TCS2_CMD_ENABLE_BANK 0x15A5 +#define PDC_GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK 0x15A6 +#define PDC_GPU_TCS2_CMD0_MSGID 0x15A7 +#define PDC_GPU_TCS2_CMD0_ADDR 0x15A8 +#define PDC_GPU_TCS2_CMD0_DATA 0x15A9 +#define PDC_GPU_TCS3_CONTROL 0x15D6 +#define PDC_GPU_TCS3_CMD_ENABLE_BANK 0x15D7 +#define PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK 0x15D8 +#define PDC_GPU_TCS3_CMD0_MSGID 0x15D9 +#define PDC_GPU_TCS3_CMD0_ADDR 0x15DA +#define PDC_GPU_TCS3_CMD0_DATA 0x15DB + +/* + * Legacy DTSI used an offset from the start of the PDC resource + * for PDC SEQ programming. We are now using PDC subsections so + * start the PDC SEQ offset at zero. + */ +#define PDC_GPU_SEQ_MEM_0 0x0 + +/* RGMU(PCC) registers in A6X_GMU_CX_0_NON_CONTEXT_DEC domain */ +#define A6XX_RGMU_CX_INTR_GEN_EN 0x1F80F +#define A6XX_RGMU_CX_RGMU_TIMER0 0x1F834 +#define A6XX_RGMU_CX_RGMU_TIMER1 0x1F835 +#define A6XX_RGMU_CX_PCC_CTRL 0x1F838 +#define A6XX_RGMU_CX_PCC_INIT_RESULT 0x1F839 +#define A6XX_RGMU_CX_PCC_BKPT_CFG 0x1F83A +#define A6XX_RGMU_CX_PCC_BKPT_ADDR 0x1F83B +#define A6XX_RGMU_CX_PCC_STATUS 0x1F83C +#define A6XX_RGMU_CX_PCC_DEBUG 0x1F83D + +#endif /* _A6XX_REG_H */ + diff --git a/drivers/gpu/msm/adreno-gpulist.h b/drivers/gpu/msm/adreno-gpulist.h new file mode 100644 index 0000000000000000000000000000000000000000..f7216cf7880ca45c0a6af25ab2ccd649467d2282 --- /dev/null +++ b/drivers/gpu/msm/adreno-gpulist.h @@ -0,0 +1,472 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2002,2007-2018, The Linux Foundation. All rights reserved. + */ + +#define ANY_ID (~0) + +static const struct adreno_gpu_core adreno_gpulist[] = { + { + .gpurev = ADRENO_REV_A306, + .core = 3, + .major = 0, + .minor = 6, + .patchid = 0x00, + .features = ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .gpudev = &adreno_a3xx_gpudev, + .gmem_size = SZ_128K, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A306A, + .core = 3, + .major = 0, + .minor = 6, + .patchid = 0x20, + .features = ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .gpudev = &adreno_a3xx_gpudev, + .gmem_size = SZ_128K, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A304, + .core = 3, + .major = 0, + .minor = 4, + .patchid = 0x00, + .features = ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .gpudev = &adreno_a3xx_gpudev, + .gmem_size = (SZ_64K + SZ_32K), + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A405, + .core = 4, + .major = 0, + .minor = 5, + .patchid = ANY_ID, + .features = ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a420_pm4.fw", + .pfpfw_name = "a420_pfp.fw", + .gpudev = &adreno_a4xx_gpudev, + .gmem_size = SZ_256K, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A420, + .core = 4, + .major = 2, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_USES_OCMEM | ADRENO_WARM_START | + ADRENO_USE_BOOTSTRAP | ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a420_pm4.fw", + .pfpfw_name = "a420_pfp.fw", + .gpudev = &adreno_a4xx_gpudev, + .gmem_size = (SZ_1M + SZ_512K), + .pm4_jt_idx = 0x901, + .pm4_jt_addr = 0x300, + .pfp_jt_idx = 0x401, + .pfp_jt_addr = 0x400, + .pm4_bstrp_size = 0x06, + .pfp_bstrp_size = 0x28, + .pfp_bstrp_ver = 0x4ff083, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A430, + .core = 4, + .major = 3, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_USES_OCMEM | ADRENO_WARM_START | + ADRENO_USE_BOOTSTRAP | ADRENO_SPTP_PC | ADRENO_PPD | + ADRENO_CONTENT_PROTECTION | ADRENO_PREEMPTION | + ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a420_pm4.fw", + .pfpfw_name = "a420_pfp.fw", + .gpudev = &adreno_a4xx_gpudev, + .gmem_size = (SZ_1M + SZ_512K), + .pm4_jt_idx = 0x901, + .pm4_jt_addr = 0x300, + .pfp_jt_idx = 0x401, + .pfp_jt_addr = 0x400, + .pm4_bstrp_size = 0x06, + .pfp_bstrp_size = 0x28, + .pfp_bstrp_ver = 0x4ff083, + .shader_offset = 0x20000, + .shader_size = 0x10000, + .num_protected_regs = 0x18, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A418, + .core = 4, + .major = 1, + .minor = 8, + .patchid = ANY_ID, + .features = ADRENO_USES_OCMEM | ADRENO_WARM_START | + ADRENO_USE_BOOTSTRAP | ADRENO_SPTP_PC | + ADRENO_SOFT_FAULT_DETECT, + .pm4fw_name = "a420_pm4.fw", + .pfpfw_name = "a420_pfp.fw", + .gpudev = &adreno_a4xx_gpudev, + .gmem_size = (SZ_512K), + .pm4_jt_idx = 0x901, + .pm4_jt_addr = 0x300, + .pfp_jt_idx = 0x401, + .pfp_jt_addr = 0x400, + .pm4_bstrp_size = 0x06, + .pfp_bstrp_size = 0x28, + .pfp_bstrp_ver = 0x4ff083, + .shader_offset = 0x20000, /* SP and TP addresses */ + .shader_size = 0x10000, + .num_protected_regs = 0x18, + .busy_mask = 0x7FFFFFFE, + }, + { + .gpurev = ADRENO_REV_A530, + .core = 5, + .major = 3, + .minor = 0, + .patchid = 0, + .pm4fw_name = "a530v1_pm4.fw", + .pfpfw_name = "a530v1_pfp.fw", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A530, + .core = 5, + .major = 3, + .minor = 0, + .patchid = 1, + .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM | + ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a530_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .gpmufw_name = "a530_gpmu.fw2", + .gpmu_major = 1, + .gpmu_minor = 0, + .busy_mask = 0xFFFFFFFE, + .lm_major = 3, + .lm_minor = 0, + .gpmu_tsens = 0x00060007, + .max_power = 5448, + .regfw_name = "a530v2_seq.fw2", + }, + { + .gpurev = ADRENO_REV_A530, + .core = 5, + .major = 3, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_GPMU | ADRENO_SPTP_PC | ADRENO_LM | + ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a530_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .gpmufw_name = "a530v3_gpmu.fw2", + .gpmu_major = 1, + .gpmu_minor = 0, + .busy_mask = 0xFFFFFFFE, + .lm_major = 1, + .lm_minor = 0, + .gpmu_tsens = 0x00060007, + .max_power = 5448, + .regfw_name = "a530v3_seq.fw2", + }, + { + .gpurev = ADRENO_REV_A505, + .core = 5, + .major = 0, + .minor = 5, + .patchid = ANY_ID, + .features = ADRENO_PREEMPTION | ADRENO_64BIT, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = (SZ_128K + SZ_8K), + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A506, + .core = 5, + .major = 0, + .minor = 6, + .patchid = ANY_ID, + .features = ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a506_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = (SZ_128K + SZ_8K), + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A510, + .core = 5, + .major = 1, + .minor = 0, + .patchid = ANY_ID, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_256K, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A540, + .core = 5, + .major = 4, + .minor = 0, + .patchid = 0, + .features = ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION | + ADRENO_GPMU | ADRENO_SPTP_PC, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a540_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a540_gpmu.fw2", + .gpmu_major = 3, + .gpmu_minor = 0, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A540, + .core = 5, + .major = 4, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION | + ADRENO_GPMU | ADRENO_SPTP_PC, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a540_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a540_gpmu.fw2", + .gpmu_major = 3, + .gpmu_minor = 0, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A512, + .core = 5, + .major = 1, + .minor = 2, + .patchid = ANY_ID, + .features = ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a512_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = (SZ_256K + SZ_16K), + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A508, + .core = 5, + .major = 0, + .minor = 8, + .patchid = ANY_ID, + .features = ADRENO_PREEMPTION | ADRENO_64BIT | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a508_zap", + .gpudev = &adreno_a5xx_gpudev, + .gmem_size = (SZ_128K + SZ_8K), + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + }, + { + .gpurev = ADRENO_REV_A630, + .core = 6, + .major = 3, + .minor = 0, + .patchid = 0, + .features = ADRENO_64BIT | ADRENO_RPMH | + ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_LM, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a630_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a630_gmu.bin", + .gpmu_major = 0x1, + .gpmu_minor = 0x003, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A630, + .core = 6, + .major = 3, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_IFPC | + ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | + ADRENO_IOCOHERENT | ADRENO_PREEMPTION, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a630_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_1M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a630_gmu.bin", + .gpmu_major = 0x1, + .gpmu_minor = 0x003, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A615, + .core = 6, + .major = 1, + .minor = 5, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_PREEMPTION | + ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a615_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_512K, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a630_gmu.bin", + .gpmu_major = 0x1, + .gpmu_minor = 0x003, + }, + { + .gpurev = ADRENO_REV_A640, + .core = 6, + .major = 4, + .minor = 0, + .patchid = 0, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_GPMU | + ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | + ADRENO_IFPC, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a640_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_1M, //Verified 1MB + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a640_gmu.bin", + .gpmu_major = 0x2, + .gpmu_minor = 0x000, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + .va_padding = SZ_64K, + }, + { + .gpurev = ADRENO_REV_A640, + .core = 6, + .major = 4, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_GPMU | + ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | + ADRENO_IFPC | ADRENO_PREEMPTION, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a640_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_1M, //Verified 1MB + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a640_gmu.bin", + .gpmu_major = 0x2, + .gpmu_minor = 0x000, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A680, + .core = 6, + .major = 8, + .minor = 0, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_GPMU, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a640_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_2M, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a640_gmu.bin", + .gpmu_major = 0x2, + .gpmu_minor = 0x000, + .gpmu_tsens = 0x000C000D, + .max_power = 5448, + }, + { + .gpurev = ADRENO_REV_A608, + .core = 6, + .major = 0, + .minor = 8, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_CONTENT_PROTECTION | + ADRENO_IOCOHERENT | ADRENO_PREEMPTION | ADRENO_GPMU, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a608_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = (SZ_128K + SZ_4K), + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a608_rgmu.bin", + }, + { + .gpurev = ADRENO_REV_A616, + .core = 6, + .major = 1, + .minor = 6, + .patchid = ANY_ID, + .features = ADRENO_64BIT | ADRENO_RPMH | ADRENO_PREEMPTION | + ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a615_zap", + .gpudev = &adreno_a6xx_gpudev, + .gmem_size = SZ_512K, + .num_protected_regs = 0x20, + .busy_mask = 0xFFFFFFFE, + .gpmufw_name = "a630_gmu.bin", + .gpmu_major = 0x1, + .gpmu_minor = 0x003, + }, +}; diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c new file mode 100644 index 0000000000000000000000000000000000000000..e19b57d4dac607a1b24a288dbc6ca6eda0e49854 --- /dev/null +++ b/drivers/gpu/msm/adreno.c @@ -0,0 +1,3714 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2002,2007-2018, The Linux Foundation. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kgsl.h" +#include "kgsl_gmu_core.h" +#include "kgsl_pwrscale.h" +#include "kgsl_sharedmem.h" +#include "kgsl_iommu.h" +#include "kgsl_trace.h" +#include "adreno_llc.h" + +#include "adreno.h" +#include "adreno_iommu.h" +#include "adreno_compat.h" +#include "adreno_pm4types.h" +#include "adreno_trace.h" + +#include "a3xx_reg.h" +#include "a6xx_reg.h" +#include "adreno_snapshot.h" + +/* Include the master list of GPU cores that are supported */ +#include "adreno-gpulist.h" +#include "adreno_dispatch.h" + +#define DRIVER_VERSION_MAJOR 3 +#define DRIVER_VERSION_MINOR 1 + +static void adreno_input_work(struct work_struct *work); +static unsigned int counter_delta(struct kgsl_device *device, + unsigned int reg, unsigned int *counter); + +static struct devfreq_msm_adreno_tz_data adreno_tz_data = { + .bus = { + .max = 350, + }, + .device_id = KGSL_DEVICE_3D0, +}; + +static const struct kgsl_functable adreno_functable; + +static struct adreno_device device_3d0 = { + .dev = { + KGSL_DEVICE_COMMON_INIT(device_3d0.dev), + .pwrscale = KGSL_PWRSCALE_INIT(&adreno_tz_data), + .name = DEVICE_3D0_NAME, + .id = KGSL_DEVICE_3D0, + .pwrctrl = { + .irq_name = "kgsl_3d0_irq", + }, + .iomemname = "kgsl_3d0_reg_memory", + .shadermemname = "kgsl_3d0_shader_memory", + .ftbl = &adreno_functable, + }, + .gmem_size = SZ_256K, + .ft_policy = KGSL_FT_DEFAULT_POLICY, + .ft_pf_policy = KGSL_FT_PAGEFAULT_DEFAULT_POLICY, + .long_ib_detect = 1, + .input_work = __WORK_INITIALIZER(device_3d0.input_work, + adreno_input_work), + .pwrctrl_flag = BIT(ADRENO_SPTP_PC_CTRL) | BIT(ADRENO_PPD_CTRL) | + BIT(ADRENO_LM_CTRL) | BIT(ADRENO_HWCG_CTRL) | + BIT(ADRENO_THROTTLING_CTRL), + .profile.enabled = false, + .active_list = LIST_HEAD_INIT(device_3d0.active_list), + .active_list_lock = __SPIN_LOCK_UNLOCKED(device_3d0.active_list_lock), + .gpu_llc_slice_enable = true, + .gpuhtw_llc_slice_enable = true, + .preempt = { + .preempt_level = 1, + .skipsaverestore = 1, + .usesgmem = 1, + }, +}; + +/* Ptr to array for the current set of fault detect registers */ +unsigned int *adreno_ft_regs; +/* Total number of fault detect registers */ +unsigned int adreno_ft_regs_num; +/* Ptr to array for the current fault detect registers values */ +unsigned int *adreno_ft_regs_val; +/* Array of default fault detect registers */ +static unsigned int adreno_ft_regs_default[] = { + ADRENO_REG_RBBM_STATUS, + ADRENO_REG_CP_RB_RPTR, + ADRENO_REG_CP_IB1_BASE, + ADRENO_REG_CP_IB1_BUFSZ, + ADRENO_REG_CP_IB2_BASE, + ADRENO_REG_CP_IB2_BUFSZ, +}; + +/* Nice level for the higher priority GPU start thread */ +int adreno_wake_nice = -7; + +/* Number of milliseconds to stay active active after a wake on touch */ +unsigned int adreno_wake_timeout = 100; + +/** + * adreno_readreg64() - Read a 64bit register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @lo: lower 32bit register enum that is to be read + * @hi: higher 32bit register enum that is to be read + * @val: 64 bit Register value read is placed here + */ +void adreno_readreg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t *val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int val_lo = 0, val_hi = 0; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, lo)) + kgsl_regread(device, gpudev->reg_offsets->offsets[lo], &val_lo); + if (adreno_checkreg_off(adreno_dev, hi)) + kgsl_regread(device, gpudev->reg_offsets->offsets[hi], &val_hi); + + *val = (val_lo | ((uint64_t)val_hi << 32)); +} + +/** + * adreno_writereg64() - Write a 64bit register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @lo: lower 32bit register enum that is to be written + * @hi: higher 32bit register enum that is to be written + * @val: 64 bit value to write + */ +void adreno_writereg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, lo)) + kgsl_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[lo], lower_32_bits(val)); + if (adreno_checkreg_off(adreno_dev, hi)) + kgsl_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[hi], upper_32_bits(val)); +} + +/** + * adreno_get_rptr() - Get the current ringbuffer read pointer + * @rb: Pointer the ringbuffer to query + * + * Get the latest rptr + */ +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + unsigned int rptr = 0; + + if (adreno_is_a3xx(adreno_dev)) + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, + &rptr); + else { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_sharedmem_readl(&device->scratch, &rptr, + SCRATCH_RPTR_OFFSET(rb->id)); + } + + return rptr; +} + +/** + * adreno_of_read_property() - Adreno read property + * @node: Device node + * + * Read a u32 property. + */ +static inline int adreno_of_read_property(struct device *dev, + struct device_node *node, const char *prop, unsigned int *ptr) +{ + int ret = of_property_read_u32(node, prop, ptr); + + if (ret) + dev_err(dev, "Unable to read '%s'\n", prop); + return ret; +} + +static void __iomem *efuse_base; +static size_t efuse_len; + +int adreno_efuse_map(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct resource *res; + + if (efuse_base != NULL) + return 0; + + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "qfprom_memory"); + + if (res == NULL) + return -ENODEV; + + efuse_base = ioremap(res->start, resource_size(res)); + if (efuse_base == NULL) + return -ENODEV; + + efuse_len = resource_size(res); + return 0; +} + +void adreno_efuse_unmap(struct adreno_device *adreno_dev) +{ + if (efuse_base != NULL) { + iounmap(efuse_base); + efuse_base = NULL; + efuse_len = 0; + } +} + +int adreno_efuse_read_u32(struct adreno_device *adreno_dev, unsigned int offset, + unsigned int *val) +{ + if (efuse_base == NULL) + return -ENODEV; + + if (offset >= efuse_len) + return -ERANGE; + + if (val != NULL) { + *val = readl_relaxed(efuse_base + offset); + /* Make sure memory is updated before returning */ + rmb(); + } + + return 0; +} + +static int _get_counter(struct adreno_device *adreno_dev, + int group, int countable, unsigned int *lo, + unsigned int *hi) +{ + int ret = 0; + + if (*lo == 0) { + + ret = adreno_perfcounter_get(adreno_dev, group, countable, + lo, hi, PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to allocate fault detect performance counter %d/%d\n", + group, countable); + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPU fault detect will be less reliable\n"); + } + } + + return ret; +} + +static inline void _put_counter(struct adreno_device *adreno_dev, + int group, int countable, unsigned int *lo, + unsigned int *hi) +{ + if (*lo != 0) + adreno_perfcounter_put(adreno_dev, group, countable, + PERFCOUNTER_FLAG_KERNEL); + + *lo = 0; + *hi = 0; +} + +/** + * adreno_fault_detect_start() - Allocate performance counters + * used for fast fault detection + * @adreno_dev: Pointer to an adreno_device structure + * + * Allocate the series of performance counters that should be periodically + * checked to verify that the GPU is still moving + */ +void adreno_fault_detect_start(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int i, j = ARRAY_SIZE(adreno_ft_regs_default); + + if (!test_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv)) + return; + + if (adreno_dev->fast_hang_detect == 1) + return; + + for (i = 0; i < gpudev->ft_perf_counters_count; i++) { + _get_counter(adreno_dev, gpudev->ft_perf_counters[i].counter, + gpudev->ft_perf_counters[i].countable, + &adreno_ft_regs[j + (i * 2)], + &adreno_ft_regs[j + ((i * 2) + 1)]); + } + + adreno_dev->fast_hang_detect = 1; +} + +/** + * adreno_fault_detect_stop() - Release performance counters + * used for fast fault detection + * @adreno_dev: Pointer to an adreno_device structure + * + * Release the counters allocated in adreno_fault_detect_start + */ +void adreno_fault_detect_stop(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int i, j = ARRAY_SIZE(adreno_ft_regs_default); + + if (!test_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv)) + return; + + if (!adreno_dev->fast_hang_detect) + return; + + for (i = 0; i < gpudev->ft_perf_counters_count; i++) { + _put_counter(adreno_dev, gpudev->ft_perf_counters[i].counter, + gpudev->ft_perf_counters[i].countable, + &adreno_ft_regs[j + (i * 2)], + &adreno_ft_regs[j + ((i * 2) + 1)]); + + } + + adreno_dev->fast_hang_detect = 0; +} + +/* + * A workqueue callback responsible for actually turning on the GPU after a + * touch event. kgsl_pwrctrl_change_state(ACTIVE) is used without any + * active_count protection to avoid the need to maintain state. Either + * somebody will start using the GPU or the idle timer will fire and put the + * GPU back into slumber. + */ +static void adreno_input_work(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, input_work); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + mutex_lock(&device->mutex); + + device->flags |= KGSL_FLAG_WAKE_ON_TOUCH; + + /* + * Don't schedule adreno_start in a high priority workqueue, we are + * already in a workqueue which should be sufficient + */ + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + + /* + * When waking up from a touch event we want to stay active long enough + * for the user to send a draw command. The default idle timer timeout + * is shorter than we want so go ahead and push the idle timer out + * further for this special case + */ + mod_timer(&device->idle_timer, + jiffies + msecs_to_jiffies(adreno_wake_timeout)); + mutex_unlock(&device->mutex); +} + +/* + * Process input events and schedule work if needed. At this point we are only + * interested in groking EV_ABS touchscreen events + */ +static void adreno_input_event(struct input_handle *handle, unsigned int type, + unsigned int code, int value) +{ + struct kgsl_device *device = handle->handler->private; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* Only consider EV_ABS (touch) events */ + if (type != EV_ABS) + return; + + /* + * Don't do anything if anything hasn't been rendered since we've been + * here before + */ + + if (device->flags & KGSL_FLAG_WAKE_ON_TOUCH) + return; + + /* + * If the device is in nap, kick the idle timer to make sure that we + * don't go into slumber before the first render. If the device is + * already in slumber schedule the wake. + */ + + if (device->state == KGSL_STATE_NAP) { + /* + * Set the wake on touch bit to keep from coming back here and + * keeping the device in nap without rendering + */ + + device->flags |= KGSL_FLAG_WAKE_ON_TOUCH; + + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } else if (device->state == KGSL_STATE_SLUMBER) { + schedule_work(&adreno_dev->input_work); + } +} + +#ifdef CONFIG_INPUT +static int adreno_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + struct input_handle *handle; + int ret; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (handle == NULL) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = handler->name; + + ret = input_register_handle(handle); + if (ret) { + kfree(handle); + return ret; + } + + ret = input_open_device(handle); + if (ret) { + input_unregister_handle(handle); + kfree(handle); + } + + return ret; +} + +static void adreno_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} +#else +static int adreno_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + return 0; +} +static void adreno_input_disconnect(struct input_handle *handle) {} +#endif + +/* + * We are only interested in EV_ABS events so only register handlers for those + * input devices that have EV_ABS events + */ +static const struct input_device_id adreno_input_ids[] = { + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_ABS) }, + /* assumption: MT_.._X & MT_.._Y are in the same long */ + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) }, + }, + { }, +}; + +static struct input_handler adreno_input_handler = { + .event = adreno_input_event, + .connect = adreno_input_connect, + .disconnect = adreno_input_disconnect, + .name = "kgsl", + .id_table = adreno_input_ids, +}; + +/* + * _soft_reset() - Soft reset GPU + * @adreno_dev: Pointer to adreno device + * + * Soft reset the GPU by doing a AHB write of value 1 to RBBM_SW_RESET + * register. This is used when we want to reset the GPU without + * turning off GFX power rail. The reset when asserted resets + * all the HW logic, restores GPU registers to default state and + * flushes out pending VBIF transactions. + */ +static int _soft_reset(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int reg; + + /* + * On a530 v1 RBBM cannot be reset in soft reset. + * Reset all blocks except RBBM for a530v1. + */ + if (adreno_is_a530v1(adreno_dev)) { + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD, + 0xFFDFFC0); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD2, + 0x1FFFFFFF); + } else { + + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 1); + /* + * Do a dummy read to get a brief read cycle delay for the + * reset to take effect + */ + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, ®); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 0); + } + + /* The SP/TP regulator gets turned off after a soft reset */ + + if (gpudev->regulator_enable) + gpudev->regulator_enable(adreno_dev); + + return 0; +} + +/** + * adreno_irqctrl() - Enables/disables the RBBM interrupt mask + * @adreno_dev: Pointer to an adreno_device + * @state: 1 for masked or 0 for unmasked + * Power: The caller of this function must make sure to use OOBs + * so that we know that the GPU is powered on + */ +void adreno_irqctrl(struct adreno_device *adreno_dev, int state) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int mask = state ? gpudev->irq->mask : 0; + + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_0_MASK, mask); +} + +/* + * adreno_hang_int_callback() - Isr for fatal interrupts that hang GPU + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +void adreno_hang_int_callback(struct adreno_device *adreno_dev, int bit) +{ + dev_crit_ratelimited(KGSL_DEVICE(adreno_dev)->dev, + "MISC: GPU hang detected\n"); + adreno_irqctrl(adreno_dev, 0); + + /* Trigger a fault in the dispatcher - this will effect a restart */ + adreno_set_gpu_fault(adreno_dev, ADRENO_HARD_FAULT); + adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev)); +} + +/* + * adreno_cp_callback() - CP interrupt handler + * @adreno_dev: Adreno device pointer + * @irq: irq number + * + * Handle the cp interrupt generated by GPU. + */ +void adreno_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + adreno_dispatcher_schedule(device); +} + +static irqreturn_t adreno_irq_handler(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_irq *irq_params = gpudev->irq; + irqreturn_t ret = IRQ_NONE; + unsigned int status = 0, fence = 0, fence_retries = 0, tmp, int_bit; + unsigned int shadow_status = 0; + int i; + + atomic_inc(&adreno_dev->pending_irq_refcnt); + /* Ensure this increment is done before the IRQ status is updated */ + smp_mb__after_atomic(); + + /* + * On A6xx, the GPU can power down once the INT_0_STATUS is read + * below. But there still might be some register reads required + * so force the GMU/GPU into KEEPALIVE mode until done with the ISR. + */ + if (gpudev->gpu_keepalive) + gpudev->gpu_keepalive(adreno_dev, true); + + /* + * If the AHB fence is not in ALLOW mode when we receive an RBBM + * interrupt, something went wrong. This means that we cannot proceed + * since the IRQ status and clear registers are not accessible. + * This is usually harmless because the GMU will abort power collapse + * and change the fence back to ALLOW. Poll so that this can happen. + */ + if (gmu_core_isenabled(device)) { + adreno_readreg(adreno_dev, + ADRENO_REG_GMU_AO_AHB_FENCE_CTRL, + &fence); + + while (fence != 0) { + /* Wait for small time before trying again */ + udelay(1); + adreno_readreg(adreno_dev, + ADRENO_REG_GMU_AO_AHB_FENCE_CTRL, + &fence); + + if (fence_retries == FENCE_RETRY_MAX && fence != 0) { + adreno_readreg(adreno_dev, + ADRENO_REG_GMU_RBBM_INT_UNMASKED_STATUS, + &shadow_status); + + dev_crit_ratelimited(device->dev, + "AHB fence stuck in ISR: Shadow INT status=%8.8X\n", + shadow_status & irq_params->mask); + goto done; + } + fence_retries++; + } + } + + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_INT_0_STATUS, &status); + + /* + * Clear all the interrupt bits but ADRENO_INT_RBBM_AHB_ERROR. Because + * even if we clear it here, it will stay high until it is cleared + * in its respective handler. Otherwise, the interrupt handler will + * fire again. + */ + int_bit = ADRENO_INT_BIT(adreno_dev, ADRENO_INT_RBBM_AHB_ERROR); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_CLEAR_CMD, + status & ~int_bit); + + /* Loop through all set interrupts and call respective handlers */ + for (tmp = status; tmp != 0;) { + i = fls(tmp) - 1; + + if (irq_params->funcs[i].func != NULL) { + if (irq_params->mask & BIT(i)) + irq_params->funcs[i].func(adreno_dev, i); + } else + dev_crit_ratelimited(device->dev, + "Unhandled interrupt bit %x\n", + i); + + ret = IRQ_HANDLED; + + tmp &= ~BIT(i); + } + + gpudev->irq_trace(adreno_dev, status); + + /* + * Clear ADRENO_INT_RBBM_AHB_ERROR bit after this interrupt has been + * cleared in its respective handler + */ + if (status & int_bit) + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_CLEAR_CMD, + int_bit); + +done: + /* Turn off the KEEPALIVE vote from earlier unless hard fault set */ + if (gpudev->gpu_keepalive) { + /* If hard fault, then let snapshot turn off the keepalive */ + if (!(adreno_gpu_fault(adreno_dev) & ADRENO_HARD_FAULT)) + gpudev->gpu_keepalive(adreno_dev, false); + } + + /* Make sure the regwrites are done before the decrement */ + smp_mb__before_atomic(); + atomic_dec(&adreno_dev->pending_irq_refcnt); + /* Ensure other CPUs see the decrement */ + smp_mb__after_atomic(); + + return ret; + +} + +static inline bool _rev_match(unsigned int id, unsigned int entry) +{ + return (entry == ANY_ID || entry == id); +} + +static inline const struct adreno_gpu_core *_get_gpu_core(unsigned int chipid) +{ + unsigned int core = ADRENO_CHIPID_CORE(chipid); + unsigned int major = ADRENO_CHIPID_MAJOR(chipid); + unsigned int minor = ADRENO_CHIPID_MINOR(chipid); + unsigned int patchid = ADRENO_CHIPID_PATCH(chipid); + int i; + + for (i = 0; i < ARRAY_SIZE(adreno_gpulist); i++) { + if (core == adreno_gpulist[i].core && + _rev_match(major, adreno_gpulist[i].major) && + _rev_match(minor, adreno_gpulist[i].minor) && + _rev_match(patchid, adreno_gpulist[i].patchid)) + return &adreno_gpulist[i]; + } + + return NULL; +} + +static struct { + unsigned int quirk; + const char *prop; +} adreno_quirks[] = { + { ADRENO_QUIRK_TWO_PASS_USE_WFI, "qcom,gpu-quirk-two-pass-use-wfi" }, + { ADRENO_QUIRK_IOMMU_SYNC, "qcom,gpu-quirk-iommu-sync" }, + { ADRENO_QUIRK_CRITICAL_PACKETS, "qcom,gpu-quirk-critical-packets" }, + { ADRENO_QUIRK_FAULT_DETECT_MASK, "qcom,gpu-quirk-fault-detect-mask" }, + { ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING, + "qcom,gpu-quirk-dp2clockgating-disable" }, + { ADRENO_QUIRK_DISABLE_LMLOADKILL, + "qcom,gpu-quirk-lmloadkill-disable" }, + { ADRENO_QUIRK_HFI_USE_REG, "qcom,gpu-quirk-hfi-use-reg" }, + { ADRENO_QUIRK_SECVID_SET_ONCE, "qcom,gpu-quirk-secvid-set-once" }, + { ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW, + "qcom,gpu-quirk-limit-uche-gbif-rw" }, + { ADRENO_QUIRK_MMU_SECURE_CB_ALT, "qcom,gpu-quirk-mmu-secure-cb-alt" }, + { ADRENO_QUIRK_CX_GDSC, "qcom,gpu-quirk-cx-gdsc" }, +}; + +static struct device_node * +adreno_get_soc_hw_revision_node(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + struct device_node *node, *child; + unsigned int rev; + + node = of_find_node_by_name(pdev->dev.of_node, "qcom,soc-hw-revisions"); + if (node == NULL) + return NULL; + + for_each_child_of_node(node, child) { + if (of_property_read_u32(child, "qcom,soc-hw-revision", &rev)) + continue; + + if (rev == adreno_dev->soc_hw_rev) + return child; + } + + dev_warn(KGSL_DEVICE(adreno_dev)->dev, + "No matching SOC HW revision found for efused HW rev=%u\n", + adreno_dev->soc_hw_rev); + return NULL; +} + +static void adreno_update_soc_hw_revision_quirks( + struct adreno_device *adreno_dev, struct platform_device *pdev) +{ + struct device_node *node; + int i; + + node = adreno_get_soc_hw_revision_node(adreno_dev, pdev); + if (node == NULL) + node = pdev->dev.of_node; + + /* get chip id, fall back to parent if revision node does not have it */ + if (of_property_read_u32(node, "qcom,chipid", &adreno_dev->chipid)) { + if (of_property_read_u32(pdev->dev.of_node, + "qcom,chipid", &adreno_dev->chipid)) { + dev_crit(KGSL_DEVICE(adreno_dev)->dev, + "No GPU chip ID was specified\n"); + BUG(); + return; + } + } + + /* update quirk */ + for (i = 0; i < ARRAY_SIZE(adreno_quirks); i++) { + if (of_property_read_bool(node, adreno_quirks[i].prop)) + adreno_dev->quirks |= adreno_quirks[i].quirk; + } +} + +static void +adreno_identify_gpu(struct adreno_device *adreno_dev) +{ + const struct adreno_reg_offsets *reg_offsets; + struct adreno_gpudev *gpudev; + int i; + + adreno_dev->gpucore = _get_gpu_core(adreno_dev->chipid); + + if (adreno_dev->gpucore == NULL) { + dev_crit(KGSL_DEVICE(adreno_dev)->dev, + "Unknown GPU chip ID %8.8X\n", adreno_dev->chipid); + BUG(); + return; + } + + /* + * The gmem size might be dynamic when ocmem is involved so copy it out + * of the gpu device + */ + + adreno_dev->gmem_size = adreno_dev->gpucore->gmem_size; + + /* + * Initialize uninitialzed gpu registers, only needs to be done once + * Make all offsets that are not initialized to ADRENO_REG_UNUSED + */ + + gpudev = ADRENO_GPU_DEVICE(adreno_dev); + reg_offsets = gpudev->reg_offsets; + + for (i = 0; i < ADRENO_REG_REGISTER_MAX; i++) { + if (reg_offsets->offset_0 != i && !reg_offsets->offsets[i]) + reg_offsets->offsets[i] = ADRENO_REG_UNUSED; + } + + /* Do target specific identification */ + if (gpudev->platform_setup != NULL) + gpudev->platform_setup(adreno_dev); +} + +static const struct platform_device_id adreno_id_table[] = { + { DEVICE_3D0_NAME, (unsigned long) &device_3d0, }, + {}, +}; + +MODULE_DEVICE_TABLE(platform, adreno_id_table); + +static const struct of_device_id adreno_match_table[] = { + { .compatible = "qcom,kgsl-3d0", .data = &device_3d0 }, + {} +}; + +static void adreno_of_get_ca_target_pwrlevel(struct adreno_device *adreno_dev, + struct device_node *node) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int ca_target_pwrlevel = 1; + + of_property_read_u32(node, "qcom,ca-target-pwrlevel", + &ca_target_pwrlevel); + + if (ca_target_pwrlevel > device->pwrctrl.num_pwrlevels - 2) + ca_target_pwrlevel = 1; + + device->pwrscale.ctxt_aware_target_pwrlevel = ca_target_pwrlevel; +} + +static void adreno_of_get_ca_aware_properties(struct adreno_device *adreno_dev, + struct device_node *parent) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrscale *pwrscale = &device->pwrscale; + struct device_node *node, *child; + unsigned int bin = 0; + + pwrscale->ctxt_aware_enable = + of_property_read_bool(parent, "qcom,enable-ca-jump"); + + if (pwrscale->ctxt_aware_enable) { + if (of_property_read_u32(parent, "qcom,ca-busy-penalty", + &pwrscale->ctxt_aware_busy_penalty)) + pwrscale->ctxt_aware_busy_penalty = 12000; + + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevel-bins"); + if (node == NULL) { + adreno_of_get_ca_target_pwrlevel(adreno_dev, parent); + return; + } + + for_each_child_of_node(node, child) { + if (of_property_read_u32(child, "qcom,speed-bin", &bin)) + continue; + + if (bin == adreno_dev->speed_bin) { + adreno_of_get_ca_target_pwrlevel(adreno_dev, + child); + return; + } + } + + pwrscale->ctxt_aware_target_pwrlevel = 1; + } +} + +static int adreno_of_parse_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *node) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct device_node *child; + int ret; + + /* ADD the GPU OPP table if we define it */ + if (of_find_property(device->pdev->dev.of_node, + "operating-points-v2", NULL)) { + ret = dev_pm_opp_of_add_table(&device->pdev->dev); + if (ret) { + dev_err(device->dev, + "Unable to set the GPU OPP table: %d\n", ret); + return ret; + } + } + + pwr->num_pwrlevels = 0; + + for_each_child_of_node(node, child) { + unsigned int index; + struct kgsl_pwrlevel *level; + + if (adreno_of_read_property(device->dev, child, "reg", &index)) + return -EINVAL; + + if (index >= KGSL_MAX_PWRLEVELS) { + dev_err(device->dev, + "Pwrlevel index %d is out of range\n", index); + continue; + } + + if (index >= pwr->num_pwrlevels) + pwr->num_pwrlevels = index + 1; + + level = &pwr->pwrlevels[index]; + + if (adreno_of_read_property(device->dev, child, "qcom,gpu-freq", + &level->gpu_freq)) + return -EINVAL; + + if (adreno_of_read_property(device->dev, child, "qcom,bus-freq", + &level->bus_freq)) + return -EINVAL; + + if (of_property_read_u32(child, "qcom,bus-min", + &level->bus_min)) + level->bus_min = level->bus_freq; + + if (of_property_read_u32(child, "qcom,bus-max", + &level->bus_max)) + level->bus_max = level->bus_freq; + + if (of_property_read_u32(child, "qcom,dvm-val", + &level->acd_dvm_val)) + level->acd_dvm_val = 0xFFFFFFFF; + } + + return 0; +} + + +static void adreno_of_get_initial_pwrlevel(struct adreno_device *adreno_dev, + struct device_node *node) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int init_level = 1; + + of_property_read_u32(node, "qcom,initial-pwrlevel", &init_level); + + if (init_level < 0 || init_level > pwr->num_pwrlevels) + init_level = 1; + + pwr->active_pwrlevel = init_level; + pwr->default_pwrlevel = init_level; +} + +static int adreno_of_get_legacy_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *parent) +{ + struct device_node *node; + int ret; + + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevels"); + + if (node == NULL) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to find 'qcom,gpu-pwrlevels'\n"); + return -EINVAL; + } + + ret = adreno_of_parse_pwrlevels(adreno_dev, node); + if (ret == 0) + adreno_of_get_initial_pwrlevel(adreno_dev, parent); + return ret; +} + +static int adreno_of_get_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *parent) +{ + struct device_node *node, *child; + unsigned int bin = 0; + + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevel-bins"); + if (node == NULL) + return adreno_of_get_legacy_pwrlevels(adreno_dev, parent); + + for_each_child_of_node(node, child) { + + if (of_property_read_u32(child, "qcom,speed-bin", &bin)) + continue; + + if (bin == adreno_dev->speed_bin) { + int ret; + + ret = adreno_of_parse_pwrlevels(adreno_dev, child); + if (ret == 0) + adreno_of_get_initial_pwrlevel(adreno_dev, + child); + return ret; + } + } + + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPU speed_bin:%d mismatch for efused bin:%d\n", + adreno_dev->speed_bin, bin); + return -ENODEV; +} + +static void +l3_pwrlevel_probe(struct kgsl_device *device, struct device_node *node) +{ + struct device_node *pwrlevel_node, *child; + + pwrlevel_node = of_find_node_by_name(node, "qcom,l3-pwrlevels"); + + if (pwrlevel_node == NULL) + return; + + for_each_available_child_of_node(pwrlevel_node, child) { + unsigned int index; + + if (of_property_read_u32(child, "reg", &index)) + return; + if (index >= MAX_L3_LEVELS) + continue; + + if (index >= device->num_l3_pwrlevels) + device->num_l3_pwrlevels = index + 1; + + if (of_property_read_u32(child, "qcom,l3-freq", + &device->l3_freq[index])) + continue; + } + + device->l3_clk = devm_clk_get(&device->pdev->dev, "l3_vote"); + + if (IS_ERR_OR_NULL(device->l3_clk)) { + dev_err(&device->pdev->dev, + "Unable to get the l3_vote clock\n"); + device->l3_clk = NULL; + } +} + +static int adreno_of_get_power(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct device_node *node = pdev->dev.of_node; + struct resource *res; + unsigned int timeout; + + if (of_property_read_string(node, "label", &pdev->name)) { + dev_err(device->dev, "Unable to read 'label'\n"); + return -EINVAL; + } + + if (adreno_of_read_property(device->dev, node, "qcom,id", &pdev->id)) + return -EINVAL; + + /* Get starting physical address of device registers */ + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + device->iomemname); + if (res == NULL) { + dev_err(device->dev, + "platform_get_resource_byname failed\n"); + return -EINVAL; + } + if (res->start == 0 || resource_size(res) == 0) { + dev_err(device->dev, "dev %d invalid register region\n", + device->id); + return -EINVAL; + } + + device->reg_phys = res->start; + device->reg_len = resource_size(res); + + if (adreno_of_get_pwrlevels(adreno_dev, node)) + return -EINVAL; + + /* Get context aware DCVS properties */ + adreno_of_get_ca_aware_properties(adreno_dev, node); + + l3_pwrlevel_probe(device, node); + + /* get pm-qos-active-latency, set it to default if not found */ + if (of_property_read_u32(node, "qcom,pm-qos-active-latency", + &device->pwrctrl.pm_qos_active_latency)) + device->pwrctrl.pm_qos_active_latency = 501; + + /* get pm-qos-cpu-mask-latency, set it to default if not found */ + if (of_property_read_u32(node, "qcom,l2pc-cpu-mask-latency", + &device->pwrctrl.pm_qos_cpu_mask_latency)) + device->pwrctrl.pm_qos_cpu_mask_latency = 501; + + /* get pm-qos-wakeup-latency, set it to default if not found */ + if (of_property_read_u32(node, "qcom,pm-qos-wakeup-latency", + &device->pwrctrl.pm_qos_wakeup_latency)) + device->pwrctrl.pm_qos_wakeup_latency = 101; + + if (of_property_read_u32(node, "qcom,idle-timeout", &timeout)) + timeout = 80; + + device->pwrctrl.interval_timeout = msecs_to_jiffies(timeout); + + device->pwrctrl.bus_control = of_property_read_bool(node, + "qcom,bus-control"); + + device->pwrctrl.input_disable = of_property_read_bool(node, + "qcom,disable-wake-on-touch"); + + return 0; +} + +#ifdef CONFIG_QCOM_OCMEM +static int +adreno_ocmem_malloc(struct adreno_device *adreno_dev) +{ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_USES_OCMEM)) + return 0; + + if (adreno_dev->ocmem_hdl == NULL) { + adreno_dev->ocmem_hdl = + ocmem_allocate(OCMEM_GRAPHICS, adreno_dev->gmem_size); + if (IS_ERR_OR_NULL(adreno_dev->ocmem_hdl)) { + adreno_dev->ocmem_hdl = NULL; + return -ENOMEM; + } + + adreno_dev->gmem_size = adreno_dev->ocmem_hdl->len; + adreno_dev->gmem_base = adreno_dev->ocmem_hdl->addr; + } + + return 0; +} + +static void +adreno_ocmem_free(struct adreno_device *adreno_dev) +{ + if (adreno_dev->ocmem_hdl != NULL) { + ocmem_free(OCMEM_GRAPHICS, adreno_dev->ocmem_hdl); + adreno_dev->ocmem_hdl = NULL; + } +} +#else +static int +adreno_ocmem_malloc(struct adreno_device *adreno_dev) +{ + return 0; +} + +static void +adreno_ocmem_free(struct adreno_device *adreno_dev) +{ +} +#endif + +static void adreno_cx_dbgc_probe(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct resource *res; + + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "cx_dbgc"); + + if (res == NULL) + return; + + adreno_dev->cx_dbgc_base = res->start - device->reg_phys; + adreno_dev->cx_dbgc_len = resource_size(res); + adreno_dev->cx_dbgc_virt = devm_ioremap(device->dev, + device->reg_phys + + adreno_dev->cx_dbgc_base, + adreno_dev->cx_dbgc_len); + + if (adreno_dev->cx_dbgc_virt == NULL) + dev_warn(device->dev, "cx_dbgc ioremap failed\n"); +} + +static void adreno_efuse_read_soc_hw_rev(struct adreno_device *adreno_dev) +{ + unsigned int val; + unsigned int soc_hw_rev[3]; + int ret; + + if (of_property_read_u32_array( + KGSL_DEVICE(adreno_dev)->pdev->dev.of_node, + "qcom,soc-hw-rev-efuse", soc_hw_rev, 3)) + return; + + ret = adreno_efuse_map(adreno_dev); + if (ret) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to map hardware revision fuse: ret=%d\n", ret); + return; + } + + ret = adreno_efuse_read_u32(adreno_dev, soc_hw_rev[0], &val); + adreno_efuse_unmap(adreno_dev); + + if (ret) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to read hardware revision fuse: ret=%d\n", ret); + return; + } + + adreno_dev->soc_hw_rev = (val >> soc_hw_rev[1]) & soc_hw_rev[2]; +} + +static bool adreno_is_gpu_disabled(struct adreno_device *adreno_dev) +{ + unsigned int row0; + unsigned int pte_row0_msb[3]; + int ret; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (of_property_read_u32_array(device->pdev->dev.of_node, + "qcom,gpu-disable-fuse", pte_row0_msb, 3)) + return false; + /* + * Read the fuse value to disable GPU driver if fuse + * is blown. By default(fuse value is 0) GPU is enabled. + */ + if (adreno_efuse_map(adreno_dev)) + return false; + + ret = adreno_efuse_read_u32(adreno_dev, pte_row0_msb[0], &row0); + adreno_efuse_unmap(adreno_dev); + + if (ret) + return false; + + return (row0 >> pte_row0_msb[2]) & + pte_row0_msb[1] ? true : false; +} + +static int adreno_probe(struct platform_device *pdev) +{ + const struct of_device_id *of_id = + of_match_device(adreno_match_table, &pdev->dev); + struct adreno_device *adreno_dev = (struct adreno_device *) of_id->data; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int status; + + device->pdev = pdev; + + if (adreno_is_gpu_disabled(adreno_dev)) { + dev_err(&pdev->dev, "adreno: GPU is disabled on this device\n"); + return -ENODEV; + } + + /* Identify SOC hardware revision to be used */ + adreno_efuse_read_soc_hw_rev(adreno_dev); + + adreno_update_soc_hw_revision_quirks(adreno_dev, pdev); + + /* Get the chip ID from the DT and set up target specific parameters */ + adreno_identify_gpu(adreno_dev); + + status = adreno_of_get_power(adreno_dev, pdev); + if (status) { + device->pdev = NULL; + return status; + } + + /* + * Probe/init GMU after initial gpu power probe + * Another part of GPU power probe in platform_probe + * needs GMU initialized. + */ + status = gmu_core_probe(device); + if (status) { + device->pdev = NULL; + return status; + } + + /* + * The SMMU APIs use unsigned long for virtual addresses which means + * that we cannot use 64 bit virtual addresses on a 32 bit kernel even + * though the hardware and the rest of the KGSL driver supports it. + */ + if (adreno_support_64bit(adreno_dev)) + device->mmu.features |= KGSL_MMU_64BIT; + + /* Default to 4K alignment (in other words, no additional padding) */ + device->mmu.va_padding = PAGE_SIZE; + + if (adreno_dev->gpucore->va_padding) { + device->mmu.features |= KGSL_MMU_PAD_VA; + device->mmu.va_padding = adreno_dev->gpucore->va_padding; + } + + status = kgsl_device_platform_probe(device); + if (status) { + device->pdev = NULL; + return status; + } + + /* Probe for the optional CX_DBGC block */ + adreno_cx_dbgc_probe(device); + + /* + * qcom,iommu-secure-id is used to identify MMUs that can handle secure + * content but that is only part of the story - the GPU also has to be + * able to handle secure content. Unfortunately in a classic catch-22 + * we cannot identify the GPU until after the DT is parsed. tl;dr - + * check the GPU capabilities here and modify mmu->secured accordingly + */ + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_CONTENT_PROTECTION)) + device->mmu.secured = false; + + if (ADRENO_FEATURE(adreno_dev, ADRENO_IOCOHERENT)) + device->mmu.features |= KGSL_MMU_IO_COHERENT; + + status = adreno_ringbuffer_probe(adreno_dev); + if (status) + goto out; + + status = adreno_dispatcher_init(adreno_dev); + if (status) + goto out; + + adreno_debugfs_init(adreno_dev); + adreno_profile_init(adreno_dev); + + adreno_sysfs_init(adreno_dev); + + kgsl_pwrscale_init(&pdev->dev, CONFIG_QCOM_ADRENO_DEFAULT_GOVERNOR); + + /* Initialize coresight for the target */ + adreno_coresight_init(adreno_dev); + + /* Get the system cache slice descriptor for GPU */ + adreno_dev->gpu_llc_slice = adreno_llc_getd(LLCC_GPU); + if (IS_ERR(adreno_dev->gpu_llc_slice) && + PTR_ERR(adreno_dev->gpu_llc_slice) != -ENOENT) + dev_warn(device->dev, + "Failed to get GPU LLC slice descriptor %ld\n", + PTR_ERR(adreno_dev->gpu_llc_slice)); + + /* Get the system cache slice descriptor for GPU pagetables */ + adreno_dev->gpuhtw_llc_slice = adreno_llc_getd(LLCC_GPUHTW); + if (IS_ERR(adreno_dev->gpuhtw_llc_slice) && + PTR_ERR(adreno_dev->gpuhtw_llc_slice) != -ENOENT) + dev_warn(device->dev, + "Failed to get gpuhtw LLC slice descriptor %ld\n", + PTR_ERR(adreno_dev->gpuhtw_llc_slice)); + +#ifdef CONFIG_INPUT + if (!device->pwrctrl.input_disable) { + adreno_input_handler.private = device; + /* + * It isn't fatal if we cannot register the input handler. Sad, + * perhaps, but not fatal + */ + if (input_register_handler(&adreno_input_handler)) { + adreno_input_handler.private = NULL; + dev_err(device->dev, + "Unable to register the input handler\n"); + } + } +#endif +out: + if (status) { + adreno_ringbuffer_close(adreno_dev); + kgsl_device_platform_remove(device); + device->pdev = NULL; + } + + return status; +} + +static void _adreno_free_memories(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + + if (test_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, &adreno_dev->priv)) + kgsl_free_global(device, &adreno_dev->profile_buffer); + + /* Free local copies of firmware and other command streams */ + kfree(pfp_fw->fwvirt); + pfp_fw->fwvirt = NULL; + + kfree(pm4_fw->fwvirt); + pm4_fw->fwvirt = NULL; + + kfree(adreno_dev->gpmu_cmds); + adreno_dev->gpmu_cmds = NULL; + + kgsl_free_global(device, &pfp_fw->memdesc); + kgsl_free_global(device, &pm4_fw->memdesc); +} + +static int adreno_remove(struct platform_device *pdev) +{ + const struct of_device_id *of_id = + of_match_device(adreno_match_table, &pdev->dev); + struct adreno_device *adreno_dev = (struct adreno_device *) of_id->data; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->remove != NULL) + gpudev->remove(adreno_dev); + + /* The memory is fading */ + _adreno_free_memories(adreno_dev); + +#ifdef CONFIG_INPUT + if (adreno_input_handler.private) + input_unregister_handler(&adreno_input_handler); +#endif + adreno_sysfs_close(adreno_dev); + + adreno_coresight_remove(adreno_dev); + adreno_profile_close(adreno_dev); + + /* Release the system cache slice descriptor */ + adreno_llc_putd(adreno_dev->gpu_llc_slice); + adreno_llc_putd(adreno_dev->gpuhtw_llc_slice); + + kgsl_pwrscale_close(device); + + adreno_dispatcher_close(adreno_dev); + adreno_ringbuffer_close(adreno_dev); + + adreno_fault_detect_stop(adreno_dev); + + kfree(adreno_ft_regs); + adreno_ft_regs = NULL; + + kfree(adreno_ft_regs_val); + adreno_ft_regs_val = NULL; + + if (efuse_base != NULL) + iounmap(efuse_base); + + adreno_perfcounter_close(adreno_dev); + kgsl_device_platform_remove(device); + + gmu_core_remove(device); + + if (test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv)) { + kgsl_free_global(device, &adreno_dev->pwron_fixup); + clear_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv); + } + clear_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv); + + return 0; +} + +static void adreno_fault_detect_init(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int i; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SOFT_FAULT_DETECT)) + return; + + /* Disable the fast hang detect bit until we know its a go */ + adreno_dev->fast_hang_detect = 0; + + adreno_ft_regs_num = (ARRAY_SIZE(adreno_ft_regs_default) + + gpudev->ft_perf_counters_count*2); + + adreno_ft_regs = kcalloc(adreno_ft_regs_num, sizeof(unsigned int), + GFP_KERNEL); + adreno_ft_regs_val = kcalloc(adreno_ft_regs_num, sizeof(unsigned int), + GFP_KERNEL); + + if (adreno_ft_regs == NULL || adreno_ft_regs_val == NULL) { + kfree(adreno_ft_regs); + kfree(adreno_ft_regs_val); + + adreno_ft_regs = NULL; + adreno_ft_regs_val = NULL; + + return; + } + + for (i = 0; i < ARRAY_SIZE(adreno_ft_regs_default); i++) + adreno_ft_regs[i] = adreno_getreg(adreno_dev, + adreno_ft_regs_default[i]); + + set_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv); + + adreno_fault_detect_start(adreno_dev); +} + +/** + * adreno_clear_pending_transactions() - Clear transactions in GBIF/VBIF pipe + * @device: Pointer to the device whose GBIF/VBIF pipe is to be cleared + */ +int adreno_clear_pending_transactions(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret = 0; + + if (adreno_has_gbif(adreno_dev)) { + + /* Halt new client requests */ + adreno_writereg(adreno_dev, ADRENO_REG_GBIF_HALT, + gpudev->gbif_client_halt_mask); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_GBIF_HALT_ACK, + gpudev->gbif_client_halt_mask); + + /* Halt all AXI requests */ + adreno_writereg(adreno_dev, ADRENO_REG_GBIF_HALT, + gpudev->gbif_arb_halt_mask); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_GBIF_HALT_ACK, + gpudev->gbif_arb_halt_mask); + } else { + unsigned int mask = gpudev->vbif_xin_halt_ctrl0_mask; + + adreno_writereg(adreno_dev, ADRENO_REG_VBIF_XIN_HALT_CTRL0, + mask); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_VBIF_XIN_HALT_CTRL1, mask); + adreno_writereg(adreno_dev, ADRENO_REG_VBIF_XIN_HALT_CTRL0, 0); + } + return ret; +} + +static int adreno_init(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret; + + if (!adreno_is_a3xx(adreno_dev)) + kgsl_sharedmem_set(device, &device->scratch, 0, 0, + device->scratch.size); + + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + if (ret) + return ret; + + /* + * initialization only needs to be done once initially until + * device is shutdown + */ + if (test_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv)) + return 0; + + /* + * Either the microcode read failed because the usermodehelper isn't + * available or the microcode was corrupted. Fail the init and force + * the user to try the open() again + */ + + ret = gpudev->microcode_read(adreno_dev); + if (ret) + return ret; + + /* Put the GPU in a responsive state */ + if (ADRENO_GPUREV(adreno_dev) < 600) { + /* No need for newer generation architectures */ + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + if (ret) + return ret; + } + + ret = adreno_iommu_init(adreno_dev); + if (ret) + return ret; + + adreno_perfcounter_init(adreno_dev); + adreno_fault_detect_init(adreno_dev); + + /* Power down the device */ + if (ADRENO_GPUREV(adreno_dev) < 600) + kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); + + if (gpudev->init != NULL) + gpudev->init(adreno_dev); + + set_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv); + + /* Use shader offset and length defined in gpudev */ + if (adreno_dev->gpucore->shader_offset && + adreno_dev->gpucore->shader_size) { + + if (device->shader_mem_phys || device->shader_mem_virt) + dev_err(device->dev, + "Shader memory already specified in device tree\n"); + else { + device->shader_mem_phys = device->reg_phys + + adreno_dev->gpucore->shader_offset; + device->shader_mem_virt = device->reg_virt + + adreno_dev->gpucore->shader_offset; + device->shader_mem_len = + adreno_dev->gpucore->shader_size; + } + } + + /* + * Allocate a small chunk of memory for precise drawobj profiling for + * those targets that have the always on timer + */ + + if (!adreno_is_a3xx(adreno_dev)) { + int r = kgsl_allocate_global(device, + &adreno_dev->profile_buffer, PAGE_SIZE, + 0, 0, "alwayson"); + + adreno_dev->profile_index = 0; + + if (r == 0) { + set_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, + &adreno_dev->priv); + kgsl_sharedmem_set(device, + &adreno_dev->profile_buffer, 0, 0, + PAGE_SIZE); + } + + } + + if (ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION)) { + int r = 0; + + if (gpudev->preemption_init) + r = gpudev->preemption_init(adreno_dev); + + if (!WARN(r, "adreno: GPU preemption is disabled")) + set_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); + } + + return 0; +} + +static bool regulators_left_on(struct kgsl_device *device) +{ + int i; + + if (gmu_core_gpmu_isenabled(device)) + return false; + + for (i = 0; i < KGSL_MAX_REGULATORS; i++) { + struct kgsl_regulator *regulator = + &device->pwrctrl.regulators[i]; + + if (IS_ERR_OR_NULL(regulator->reg)) + break; + + if (regulator_is_enabled(regulator->reg)) + return true; + } + + return false; +} + +static void _set_secvid(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + static bool set; + + /* Program GPU contect protection init values */ + if (device->mmu.secured && !set) { + if (adreno_is_a4xx(adreno_dev)) + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_SECVID_TRUST_CONFIG, 0x2); + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_SECVID_TSB_CONTROL, 0x0); + + adreno_writereg64(adreno_dev, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + KGSL_IOMMU_SECURE_BASE(&device->mmu)); + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_SIZE, + KGSL_IOMMU_SECURE_SIZE); + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_SECVID_SET_ONCE)) + set = true; + } +} + +static int adreno_switch_to_unsecure_mode(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 2); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + cmds += cp_secure_mode(adreno_dev, cmds, 0); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "Switch to unsecure failed to idle\n"); + + return ret; +} + +int adreno_set_unsecured_mode(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + int ret = 0; + + if (!adreno_is_a5xx(adreno_dev) && !adreno_is_a6xx(adreno_dev)) + return -EINVAL; + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS) && + adreno_is_a5xx(adreno_dev)) { + ret = a5xx_critical_packet_submit(adreno_dev, rb); + if (ret) + return ret; + } + + /* GPU comes up in secured mode, make it unsecured by default */ + if (adreno_dev->zap_loaded) + ret = adreno_switch_to_unsecure_mode(adreno_dev, rb); + else + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_SECVID_TRUST_CONTROL, 0x0); + + return ret; +} + +static void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev) +{ + int i; + struct adreno_ringbuffer *rb; + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + if (rb->drawctxt_active) + kgsl_context_put(&(rb->drawctxt_active->base)); + rb->drawctxt_active = NULL; + + kgsl_sharedmem_writel(KGSL_DEVICE(adreno_dev), + &rb->pagetable_desc, PT_INFO_OFFSET(current_rb_ptname), + 0); + } +} + +/** + * _adreno_start - Power up the GPU and prepare to accept commands + * @adreno_dev: Pointer to an adreno_device structure + * + * The core function that powers up and initalizes the GPU. This function is + * called at init and after coming out of SLUMBER + */ +static int _adreno_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int status = -EINVAL, ret; + unsigned int state = device->state; + bool regulator_left_on; + unsigned int pmqos_wakeup_vote = device->pwrctrl.pm_qos_wakeup_latency; + unsigned int pmqos_active_vote = device->pwrctrl.pm_qos_active_latency; + + /* make sure ADRENO_DEVICE_STARTED is not set here */ + WARN_ON(test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)); + + /* disallow l2pc during wake up to improve GPU wake up time */ + kgsl_pwrctrl_update_l2pc(&adreno_dev->dev, + KGSL_L2PC_WAKEUP_TIMEOUT); + + pm_qos_update_request(&device->pwrctrl.pm_qos_req_dma, + pmqos_wakeup_vote); + + regulator_left_on = regulators_left_on(device); + + /* Clear any GPU faults that might have been left over */ + adreno_clear_gpu_fault(adreno_dev); + + /* Put the GPU in a responsive state */ + status = kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + if (status) + goto error_pwr_off; + + /* Set any stale active contexts to NULL */ + adreno_set_active_ctxs_null(adreno_dev); + + /* Set the bit to indicate that we've just powered on */ + set_bit(ADRENO_DEVICE_PWRON, &adreno_dev->priv); + + /* Soft reset the GPU if a regulator is stuck on*/ + if (regulator_left_on) + _soft_reset(adreno_dev); + + if ((adreno_is_a640v1(adreno_dev)) && + scm_is_call_available(SCM_SVC_MP, CP_SMMU_APERTURE_ID)) { + ret = kgsl_program_smmu_aperture(); + if (ret) { + dev_err(device->dev, + "SMMU aperture programming call failed with error %d\n", + ret); + goto error_pwr_off; + } + } + adreno_ringbuffer_set_global(adreno_dev, 0); + + status = kgsl_mmu_start(device); + if (status) + goto error_pwr_off; + + status = adreno_ocmem_malloc(adreno_dev); + if (status) { + dev_err(device->dev, "OCMEM malloc failed\n"); + goto error_mmu_off; + } + + /* Send OOB request to turn on the GX */ + status = gmu_core_dev_oob_set(device, oob_gpu); + if (status) + goto error_mmu_off; + + status = gmu_core_dev_hfi_start_msg(device); + if (status) + goto error_oob_clear; + + _set_secvid(device); + + /* Enable 64 bit gpu addr if feature is set */ + if (gpudev->enable_64bit && + adreno_support_64bit(adreno_dev)) + gpudev->enable_64bit(adreno_dev); + + if (adreno_dev->perfctr_pwr_lo == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_PWR, 1, + &adreno_dev->perfctr_pwr_lo, NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (WARN_ONCE(ret, "Unable to get perfcounters for DCVS\n")) + adreno_dev->perfctr_pwr_lo = 0; + } + + + if (device->pwrctrl.bus_control) { + /* VBIF waiting for RAM */ + if (adreno_dev->starved_ram_lo == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 0, + &adreno_dev->starved_ram_lo, NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->starved_ram_lo = 0; + } + } + + if (adreno_has_gbif(adreno_dev)) { + if (adreno_dev->starved_ram_lo_ch1 == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 1, + &adreno_dev->starved_ram_lo_ch1, NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->starved_ram_lo_ch1 = 0; + } + } + + if (adreno_dev->ram_cycles_lo == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->ram_cycles_lo = 0; + } + } + + if (adreno_dev->ram_cycles_lo_ch1_read == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_read, + NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->ram_cycles_lo_ch1_read = 0; + } + } + + if (adreno_dev->ram_cycles_lo_ch0_write == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch0_write, + NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->ram_cycles_lo_ch0_write = 0; + } + } + + if (adreno_dev->ram_cycles_lo_ch1_write == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_write, + NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->ram_cycles_lo_ch1_write = 0; + } + } + } else { + /* VBIF DDR cycles */ + if (adreno_dev->ram_cycles_lo == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + VBIF_AXI_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL, + PERFCOUNTER_FLAG_KERNEL); + + if (ret) { + dev_err(device->dev, + "Unable to get perf counters for bus DCVS\n"); + adreno_dev->ram_cycles_lo = 0; + } + } + } + } + + if (gmu_core_isenabled(device) && adreno_dev->perfctr_ifpc_lo == 0) { + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 4, + &adreno_dev->perfctr_ifpc_lo, NULL, + PERFCOUNTER_FLAG_KERNEL); + if (WARN_ONCE(ret, "Unable to get perf counter for IFPC\n")) + adreno_dev->perfctr_ifpc_lo = 0; + } + + /* Clear the busy_data stats - we're starting over from scratch */ + adreno_dev->busy_data.gpu_busy = 0; + adreno_dev->busy_data.bif_ram_cycles = 0; + adreno_dev->busy_data.bif_ram_cycles_read_ch1 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch0 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch1 = 0; + adreno_dev->busy_data.bif_starved_ram = 0; + adreno_dev->busy_data.bif_starved_ram_ch1 = 0; + adreno_dev->busy_data.num_ifpc = 0; + + /* Restore performance counter registers with saved values */ + adreno_perfcounter_restore(adreno_dev); + + /* Start the GPU */ + gpudev->start(adreno_dev); + + /* + * The system cache control registers + * live on the CX/GX rail. Hence need + * reprogramming everytime the GPU + * comes out of power collapse. + */ + adreno_llc_setup(device); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + adreno_irqctrl(adreno_dev, 1); + + adreno_perfcounter_start(adreno_dev); + + /* Clear FSR here in case it is set from a previous pagefault */ + kgsl_mmu_clear_fsr(&device->mmu); + + status = adreno_ringbuffer_start(adreno_dev, ADRENO_START_COLD); + if (status) + goto error_oob_clear; + + /* Start the dispatcher */ + adreno_dispatcher_start(device); + + device->reset_counter++; + + set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + if (pmqos_active_vote != pmqos_wakeup_vote) + pm_qos_update_request(&device->pwrctrl.pm_qos_req_dma, + pmqos_active_vote); + + /* Send OOB request to allow IFPC */ + gmu_core_dev_oob_clear(device, oob_gpu); + + /* If we made it this far, the BOOT OOB was sent to the GMU */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) + gmu_core_dev_oob_clear(device, oob_boot_slumber); + + return 0; + +error_oob_clear: + gmu_core_dev_oob_clear(device, oob_gpu); + +error_mmu_off: + kgsl_mmu_stop(&device->mmu); + +error_pwr_off: + /* set the state back to original state */ + kgsl_pwrctrl_change_state(device, state); + + if (pmqos_active_vote != pmqos_wakeup_vote) + pm_qos_update_request(&device->pwrctrl.pm_qos_req_dma, + pmqos_active_vote); + + return status; +} + +/** + * adreno_start() - Power up and initialize the GPU + * @device: Pointer to the KGSL device to power up + * @priority: Boolean flag to specify of the start should be scheduled in a low + * latency work queue + * + * Power up the GPU and initialize it. If priority is specified then elevate + * the thread priority for the duration of the start operation + */ +int adreno_start(struct kgsl_device *device, int priority) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int nice = task_nice(current); + int ret; + + if (priority && (adreno_wake_nice < nice)) + set_user_nice(current, adreno_wake_nice); + + ret = _adreno_start(adreno_dev); + + if (priority) + set_user_nice(current, nice); + + return ret; +} + +static int adreno_stop(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int error = 0; + + if (!test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)) + return 0; + + error = gmu_core_dev_oob_set(device, oob_gpu); + if (error) { + gmu_core_dev_oob_clear(device, oob_gpu); + + if (gmu_core_regulator_isenabled(device)) { + /* GPU is on. Try recovery */ + set_bit(GMU_FAULT, &device->gmu_core.flags); + gmu_core_snapshot(device); + error = -EINVAL; + } + } + + adreno_dispatcher_stop(adreno_dev); + + adreno_ringbuffer_stop(adreno_dev); + + kgsl_pwrscale_update_stats(device); + + adreno_irqctrl(adreno_dev, 0); + + adreno_ocmem_free(adreno_dev); + + adreno_llc_deactivate_slice(adreno_dev->gpu_llc_slice); + adreno_llc_deactivate_slice(adreno_dev->gpuhtw_llc_slice); + + /* Save active coresight registers if applicable */ + adreno_coresight_stop(adreno_dev); + + /* Save physical performance counter values before GPU power down*/ + adreno_perfcounter_save(adreno_dev); + + gmu_core_dev_prepare_stop(device); + gmu_core_dev_oob_clear(device, oob_gpu); + + /* + * Saving perfcounters will use an OOB to put the GMU into + * active state. Before continuing, we should wait for the + * GMU to return to the lowest idle level. This is + * because some idle level transitions require VBIF and MMU. + */ + + if (!error && gmu_core_dev_wait_for_lowest_idle(device)) { + set_bit(GMU_FAULT, &device->gmu_core.flags); + gmu_core_snapshot(device); + /* + * Assume GMU hang after 10ms without responding. + * It shall be relative safe to clear vbif and stop + * MMU later. Early return in adreno_stop function + * will result in kernel panic in adreno_start + */ + error = -EINVAL; + } + + adreno_clear_pending_transactions(device); + + /* The halt is not cleared in the above function if we have GBIF */ + adreno_deassert_gbif_halt(adreno_dev); + + kgsl_mmu_stop(&device->mmu); + + /* + * At this point, MMU is turned off so we can safely + * destroy any pending contexts and their pagetables + */ + adreno_set_active_ctxs_null(adreno_dev); + + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + return error; +} + +static inline bool adreno_try_soft_reset(struct kgsl_device *device, int fault) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* + * Do not do soft reset for a IOMMU fault (because the IOMMU hardware + * needs a reset too) or for the A304 because it can't do SMMU + * programming of any kind after a soft reset + */ + + if ((fault & ADRENO_IOMMU_PAGE_FAULT) || adreno_is_a304(adreno_dev)) + return false; + + return true; +} + +/** + * adreno_reset() - Helper function to reset the GPU + * @device: Pointer to the KGSL device structure for the GPU + * @fault: Type of fault. Needed to skip soft reset for MMU fault + * + * Try to reset the GPU to recover from a fault. First, try to do a low latency + * soft reset. If the soft reset fails for some reason, then bring out the big + * guns and toggle the footswitch. + */ +int adreno_reset(struct kgsl_device *device, int fault) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int ret = -EINVAL; + int i = 0; + + /* Try soft reset first */ + if (adreno_try_soft_reset(device, fault)) { + /* Make sure VBIF is cleared before resetting */ + ret = adreno_clear_pending_transactions(device); + + if (ret == 0) { + ret = adreno_soft_reset(device); + if (ret) + dev_err_once(device->dev, + "Device soft reset failed\n"); + } + } + if (ret) { + /* If soft reset failed/skipped, then pull the power */ + kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + /* since device is officially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* Try to reset the device */ + ret = adreno_start(device, 0); + + /* On some GPUS, keep trying until it works */ + if (ret && ADRENO_GPUREV(adreno_dev) < 600) { + for (i = 0; i < NUM_TIMES_RESET_RETRY; i++) { + msleep(20); + ret = adreno_start(device, 0); + if (!ret) + break; + } + } + } + if (ret) + return ret; + + if (i != 0) + dev_warn(device->dev, + "Device hard reset tried %d tries\n", i); + + /* + * If active_cnt is non-zero then the system was active before + * going into a reset - put it back in that state + */ + + if (atomic_read(&device->active_cnt)) + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + else + kgsl_pwrctrl_change_state(device, KGSL_STATE_NAP); + + return ret; +} + +static int copy_prop(void __user *dst, size_t count, void *src, size_t size) +{ + if (count != size) + return -EINVAL; + + if (copy_to_user(dst, src, count)) + return -EFAULT; + + return 0; +} + +static int adreno_prop_device_info(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_devinfo devinfo = { + .device_id = device->id + 1, + .chip_id = adreno_dev->chipid, + .mmu_enabled = MMU_FEATURE(&device->mmu, KGSL_MMU_PAGED), + .gmem_gpubaseaddr = adreno_dev->gmem_base, + .gmem_sizebytes = adreno_dev->gmem_size, + }; + + return copy_prop(value, count, &devinfo, sizeof(devinfo)); +} + +static int adreno_prop_device_shadow(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct kgsl_shadowprop shadowprop = { 0 }; + + if (device->memstore.hostptr) { + /* + * NOTE: with mmu enabled, gpuaddr doesn't mean + * anything to mmap(). + */ + + shadowprop.gpuaddr = (unsigned long)device->memstore.gpuaddr; + shadowprop.size = device->memstore.size; + + shadowprop.flags = KGSL_FLAGS_INITIALIZED | + KGSL_FLAGS_PER_CONTEXT_TIMESTAMPS; + } + + return copy_prop(value, count, &shadowprop, sizeof(shadowprop)); +} + +static int adreno_prop_device_qdss_stm(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct kgsl_qdss_stm_prop qdssprop = {0}; + struct kgsl_memdesc *qdss_desc = kgsl_mmu_get_qdss_global_entry(device); + + if (qdss_desc) { + qdssprop.gpuaddr = qdss_desc->gpuaddr; + qdssprop.size = qdss_desc->size; + } + + return copy_prop(value, count, &qdssprop, sizeof(qdssprop)); +} + +static int adreno_prop_device_qtimer(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct kgsl_qtimer_prop qtimerprop = {0}; + struct kgsl_memdesc *qtimer_desc = + kgsl_mmu_get_qtimer_global_entry(device); + + if (qtimer_desc) { + qtimerprop.gpuaddr = qtimer_desc->gpuaddr; + qtimerprop.size = qtimer_desc->size; + } + + return copy_prop(value, count, &qtimerprop, sizeof(qtimerprop)); +} + +static int adreno_prop_s32(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + int val = 0; + + if (type == KGSL_PROP_MMU_ENABLE) + val = MMU_FEATURE(&device->mmu, KGSL_MMU_PAGED); + else if (type == KGSL_PROP_INTERRUPT_WAITS) + val = 1; + + return copy_prop(value, count, &val, sizeof(val)); +} + +static int adreno_prop_uche_gmem_addr(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + u64 vaddr; + + vaddr = (ADRENO_GPUREV(adreno_dev) >= 500) ? ADRENO_UCHE_GMEM_BASE : 0; + + return copy_prop(value, count, &vaddr, sizeof(vaddr)); +} + +static int adreno_prop_sp_generic_mem(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_sp_generic_mem sp_mem = { + .local = adreno_dev->sp_local_gpuaddr, + .pvt = adreno_dev->sp_pvt_gpuaddr, + }; + + return copy_prop(value, count, &sp_mem, sizeof(sp_mem)); +} + +static int adreno_prop_ucode_version(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_ucode_version ucode = { + .pfp = adreno_dev->fw[ADRENO_FW_PFP].version, + .pm4 = adreno_dev->fw[ADRENO_FW_PM4].version, + }; + + return copy_prop(value, count, &ucode, sizeof(ucode)); +} + +static int adreno_prop_gpmu_version(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_gpmu_version gpmu = { 0 }; + + if (!adreno_dev->gpucore) + return -EINVAL; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + return -EOPNOTSUPP; + + gpmu.major = adreno_dev->gpucore->gpmu_major; + gpmu.minor = adreno_dev->gpucore->gpmu_minor; + gpmu.features = adreno_dev->gpucore->gpmu_features; + + return copy_prop(value, count, &gpmu, sizeof(gpmu)); +} + +static int adreno_prop_u32(struct kgsl_device *device, + u32 type, void __user *value, size_t count) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + u32 val = 0; + + if (type == KGSL_PROP_HIGHEST_BANK_BIT) { + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,highest-bank-bit", &val)) + return -EINVAL; + } else if (type == KGSL_PROP_MIN_ACCESS_LENGTH) + of_property_read_u32(device->pdev->dev.of_node, + "qcom,min-access-length", &val); + else if (type == KGSL_PROP_UBWC_MODE) + of_property_read_u32(device->pdev->dev.of_node, + "qcom,ubwc-mode", &val); + else if (type == KGSL_PROP_DEVICE_BITNESS) + val = adreno_support_64bit(adreno_dev) ? 48 : 32; + + return copy_prop(value, count, &val, sizeof(val)); +} + +static struct { + int (*func)(struct kgsl_device *device, u32 type, void __user *value, + size_t count); +} adreno_property_funcs[] = { + [KGSL_PROP_DEVICE_INFO] = { .func = adreno_prop_device_info }, + [KGSL_PROP_DEVICE_SHADOW] = { .func = adreno_prop_device_shadow }, + [KGSL_PROP_DEVICE_QDSS_STM] = { .func = adreno_prop_device_qdss_stm }, + [KGSL_PROP_DEVICE_QTIMER] = { .func = adreno_prop_device_qtimer }, + [KGSL_PROP_MMU_ENABLE] = { .func = adreno_prop_s32 }, + [KGSL_PROP_INTERRUPT_WAITS] = { .func = adreno_prop_s32 }, + [KGSL_PROP_UCHE_GMEM_VADDR] = { .func = adreno_prop_uche_gmem_addr }, + [KGSL_PROP_SP_GENERIC_MEM] = { .func = adreno_prop_sp_generic_mem }, + [KGSL_PROP_UCODE_VERSION] = { .func = adreno_prop_ucode_version }, + [KGSL_PROP_GPMU_VERSION] = { .func = adreno_prop_gpmu_version }, + [KGSL_PROP_HIGHEST_BANK_BIT] = { .func = adreno_prop_u32 }, + [KGSL_PROP_MIN_ACCESS_LENGTH] = { .func = adreno_prop_u32 }, + [KGSL_PROP_UBWC_MODE] = { .func = adreno_prop_u32 }, + [KGSL_PROP_DEVICE_BITNESS] = { .func = adreno_prop_u32 }, +}; + +static int adreno_getproperty(struct kgsl_device *device, + unsigned int type, + void __user *value, + size_t sizebytes) +{ + if (type >= ARRAY_SIZE(adreno_property_funcs) || + !adreno_property_funcs[type].func) + return -EINVAL; + + return adreno_property_funcs[type].func(device, type, value, sizebytes); +} + +int adreno_set_constraint(struct kgsl_device *device, + struct kgsl_context *context, + struct kgsl_device_constraint *constraint) +{ + int status = 0; + + switch (constraint->type) { + case KGSL_CONSTRAINT_PWRLEVEL: { + struct kgsl_device_constraint_pwrlevel pwr; + + if (constraint->size != sizeof(pwr)) { + status = -EINVAL; + break; + } + + if (copy_from_user(&pwr, + (void __user *)constraint->data, + sizeof(pwr))) { + status = -EFAULT; + break; + } + if (pwr.level >= KGSL_CONSTRAINT_PWR_MAXLEVELS) { + status = -EINVAL; + break; + } + + context->pwr_constraint.type = + KGSL_CONSTRAINT_PWRLEVEL; + context->pwr_constraint.sub_type = pwr.level; + trace_kgsl_user_pwrlevel_constraint(device, + context->id, + context->pwr_constraint.type, + context->pwr_constraint.sub_type); + } + break; + case KGSL_CONSTRAINT_NONE: + if (context->pwr_constraint.type == KGSL_CONSTRAINT_PWRLEVEL) + trace_kgsl_user_pwrlevel_constraint(device, + context->id, + KGSL_CONSTRAINT_NONE, + context->pwr_constraint.sub_type); + context->pwr_constraint.type = KGSL_CONSTRAINT_NONE; + break; + case KGSL_CONSTRAINT_L3_PWRLEVEL: { + struct kgsl_device_constraint_pwrlevel pwr; + + if (constraint->size != sizeof(pwr)) { + status = -EINVAL; + break; + } + + if (copy_from_user(&pwr, constraint->data, sizeof(pwr))) { + status = -EFAULT; + break; + } + if (pwr.level >= KGSL_CONSTRAINT_PWR_MAXLEVELS) + pwr.level = KGSL_CONSTRAINT_PWR_MAXLEVELS - 1; + + context->l3_pwr_constraint.type = KGSL_CONSTRAINT_L3_PWRLEVEL; + context->l3_pwr_constraint.sub_type = pwr.level; + trace_kgsl_user_pwrlevel_constraint(device, context->id, + context->l3_pwr_constraint.type, + context->l3_pwr_constraint.sub_type); + } + break; + case KGSL_CONSTRAINT_L3_NONE: { + unsigned int type = context->l3_pwr_constraint.type; + + if (type == KGSL_CONSTRAINT_L3_PWRLEVEL) + trace_kgsl_user_pwrlevel_constraint(device, context->id, + KGSL_CONSTRAINT_L3_NONE, + context->l3_pwr_constraint.sub_type); + context->l3_pwr_constraint.type = KGSL_CONSTRAINT_L3_NONE; + } + break; + default: + status = -EINVAL; + break; + } + + /* If a new constraint has been set for a context, cancel the old one */ + if ((status == 0) && + (context->id == device->pwrctrl.constraint.owner_id)) { + trace_kgsl_constraint(device, device->pwrctrl.constraint.type, + device->pwrctrl.active_pwrlevel, 0); + device->pwrctrl.constraint.type = KGSL_CONSTRAINT_NONE; + } + + return status; +} + +static int adreno_setproperty(struct kgsl_device_private *dev_priv, + unsigned int type, + void __user *value, + unsigned int sizebytes) +{ + int status = -EINVAL; + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + switch (type) { + case KGSL_PROP_PWRCTRL: { + unsigned int enable; + + if (sizebytes != sizeof(enable)) + break; + + if (copy_from_user(&enable, value, sizeof(enable))) { + status = -EFAULT; + break; + } + + mutex_lock(&device->mutex); + + if (enable) { + device->pwrctrl.ctrl_flags = 0; + + if (!kgsl_active_count_get(device)) { + adreno_fault_detect_start(adreno_dev); + kgsl_active_count_put(device); + } + + kgsl_pwrscale_enable(device); + } else { + kgsl_pwrctrl_change_state(device, + KGSL_STATE_ACTIVE); + device->pwrctrl.ctrl_flags = KGSL_PWR_ON; + adreno_fault_detect_stop(adreno_dev); + kgsl_pwrscale_disable(device, true); + } + + mutex_unlock(&device->mutex); + status = 0; + } + break; + case KGSL_PROP_PWR_CONSTRAINT: + case KGSL_PROP_L3_PWR_CONSTRAINT: { + struct kgsl_device_constraint constraint; + struct kgsl_context *context; + + if (sizebytes != sizeof(constraint)) + break; + + if (copy_from_user(&constraint, value, + sizeof(constraint))) { + status = -EFAULT; + break; + } + + context = kgsl_context_get_owner(dev_priv, + constraint.context_id); + + if (context == NULL) + break; + + status = adreno_set_constraint(device, context, + &constraint); + + kgsl_context_put(context); + } + break; + default: + break; + } + + return status; +} + +/* + * adreno_irq_pending() - Checks if interrupt is generated by h/w + * @adreno_dev: Pointer to device whose interrupts are checked + * + * Returns true if interrupts are pending from device else 0. + */ +inline unsigned int adreno_irq_pending(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int status; + + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_INT_0_STATUS, &status); + + /* + * IRQ handler clears the RBBM INT0 status register immediately + * entering the ISR before actually serving the interrupt because + * of this we can't rely only on RBBM INT0 status only. + * Use pending_irq_refcnt along with RBBM INT0 to correctly + * determine whether any IRQ is pending or not. + */ + if ((status & gpudev->irq->mask) || + atomic_read(&adreno_dev->pending_irq_refcnt)) + return 1; + else + return 0; +} + + +/** + * adreno_hw_isidle() - Check if the GPU core is idle + * @adreno_dev: Pointer to the Adreno device structure for the GPU + * + * Return true if the RBBM status register for the GPU type indicates that the + * hardware is idle + */ +bool adreno_hw_isidle(struct adreno_device *adreno_dev) +{ + const struct adreno_gpu_core *gpucore = adreno_dev->gpucore; + unsigned int reg_rbbm_status; + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + /* if hw driver implements idle check - use it */ + if (gpudev->hw_isidle) + return gpudev->hw_isidle(adreno_dev); + + if (adreno_is_a540(adreno_dev)) + /** + * Due to CRC idle throttling GPU + * idle hysteresys can take up to + * 3usec for expire - account for it + */ + udelay(5); + + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS, + ®_rbbm_status); + + if (reg_rbbm_status & gpucore->busy_mask) + return false; + + /* Don't consider ourselves idle if there is an IRQ pending */ + if (adreno_irq_pending(adreno_dev)) + return false; + + return true; +} + +/** + * adreno_soft_reset() - Do a soft reset of the GPU hardware + * @device: KGSL device to soft reset + * + * "soft reset" the GPU hardware - this is a fast path GPU reset + * The GPU hardware is reset but we never pull power so we can skip + * a lot of the standard adreno_stop/adreno_start sequence + */ +int adreno_soft_reset(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret; + + ret = gmu_core_dev_oob_set(device, oob_gpu); + if (ret) + return ret; + + kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + adreno_set_active_ctxs_null(adreno_dev); + + adreno_irqctrl(adreno_dev, 0); + + adreno_clear_gpu_fault(adreno_dev); + /* since device is oficially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* save physical performance counter values before GPU soft reset */ + adreno_perfcounter_save(adreno_dev); + + /* Reset the GPU */ + if (gpudev->soft_reset) + ret = gpudev->soft_reset(adreno_dev); + else + ret = _soft_reset(adreno_dev); + if (ret) { + gmu_core_dev_oob_clear(device, oob_gpu); + return ret; + } + + /* Clear the busy_data stats - we're starting over from scratch */ + adreno_dev->busy_data.gpu_busy = 0; + adreno_dev->busy_data.bif_ram_cycles = 0; + adreno_dev->busy_data.bif_ram_cycles_read_ch1 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch0 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch1 = 0; + adreno_dev->busy_data.bif_starved_ram = 0; + adreno_dev->busy_data.bif_starved_ram_ch1 = 0; + + /* Set the page table back to the default page table */ + adreno_ringbuffer_set_global(adreno_dev, 0); + kgsl_mmu_set_pt(&device->mmu, device->mmu.defaultpagetable); + + _set_secvid(device); + + /* Enable 64 bit gpu addr if feature is set */ + if (gpudev->enable_64bit && + adreno_support_64bit(adreno_dev)) + gpudev->enable_64bit(adreno_dev); + + + /* Reinitialize the GPU */ + gpudev->start(adreno_dev); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + /* Enable IRQ */ + adreno_irqctrl(adreno_dev, 1); + + /* stop all ringbuffers to cancel RB events */ + adreno_ringbuffer_stop(adreno_dev); + /* + * If we have offsets for the jump tables we can try to do a warm start, + * otherwise do a full ringbuffer restart + */ + + if (ADRENO_FEATURE(adreno_dev, ADRENO_WARM_START)) + ret = adreno_ringbuffer_start(adreno_dev, ADRENO_START_WARM); + else + ret = adreno_ringbuffer_start(adreno_dev, ADRENO_START_COLD); + if (ret == 0) { + device->reset_counter++; + set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + } + + /* Restore physical performance counter values after soft reset */ + adreno_perfcounter_restore(adreno_dev); + + gmu_core_dev_oob_clear(device, oob_gpu); + + return ret; +} + +/* + * adreno_isidle() - return true if the GPU hardware is idle + * @device: Pointer to the KGSL device structure for the GPU + * + * Return true if the GPU hardware is idle and there are no commands pending in + * the ringbuffer + */ +bool adreno_isidle(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_ringbuffer *rb; + int i; + + if (!kgsl_state_is_awake(device)) + return true; + + /* + * wptr is updated when we add commands to ringbuffer, add a barrier + * to make sure updated wptr is compared to rptr + */ + smp_mb(); + + /* + * ringbuffer is truly idle when all ringbuffers read and write + * pointers are equal + */ + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + if (!adreno_rb_empty(rb)) + return false; + } + + return adreno_hw_isidle(adreno_dev); +} + +/* Print some key registers if a spin-for-idle times out */ +void adreno_spin_idle_debug(struct adreno_device *adreno_dev, + const char *str) +{ + struct kgsl_device *device = &adreno_dev->dev; + unsigned int rptr, wptr; + unsigned int status, status3, intstatus; + unsigned int hwfault; + + dev_err(device->dev, str); + + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &rptr); + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); + + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS, &status); + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS3, &status3); + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_INT_0_STATUS, &intstatus); + adreno_readreg(adreno_dev, ADRENO_REG_CP_HW_FAULT, &hwfault); + + dev_err(device->dev, + "rb=%d pos=%X/%X rbbm_status=%8.8X/%8.8X int_0_status=%8.8X\n", + adreno_dev->cur_rb->id, rptr, wptr, status, status3, intstatus); + + dev_err(device->dev, " hwfault=%8.8X\n", hwfault); + + kgsl_device_snapshot(device, NULL, adreno_gmu_gpu_fault(adreno_dev)); +} + +/** + * adreno_spin_idle() - Spin wait for the GPU to idle + * @adreno_dev: Pointer to an adreno device + * @timeout: milliseconds to wait before returning error + * + * Spin the CPU waiting for the RBBM status to return idle + */ +int adreno_spin_idle(struct adreno_device *adreno_dev, unsigned int timeout) +{ + unsigned long wait = jiffies + msecs_to_jiffies(timeout); + + do { + /* + * If we fault, stop waiting and return an error. The dispatcher + * will clean up the fault from the work queue, but we need to + * make sure we don't block it by waiting for an idle that + * will never come. + */ + + if (adreno_gpu_fault(adreno_dev) != 0) + return -EDEADLK; + + if (adreno_isidle(KGSL_DEVICE(adreno_dev))) + return 0; + + } while (time_before(jiffies, wait)); + + /* + * Under rare conditions, preemption can cause the while loop to exit + * without checking if the gpu is idle. check one last time before we + * return failure. + */ + if (adreno_gpu_fault(adreno_dev) != 0) + return -EDEADLK; + + if (adreno_isidle(KGSL_DEVICE(adreno_dev))) + return 0; + + return -ETIMEDOUT; +} + +/** + * adreno_idle() - wait for the GPU hardware to go idle + * @device: Pointer to the KGSL device structure for the GPU + * + * Wait up to ADRENO_IDLE_TIMEOUT milliseconds for the GPU hardware to go quiet. + * Caller must hold the device mutex, and must not hold the dispatcher mutex. + */ + +int adreno_idle(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int ret; + + /* + * Make sure the device mutex is held so the dispatcher can't send any + * more commands to the hardware + */ + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EDEADLK; + + /* Check if we are already idle before idling dispatcher */ + if (adreno_isidle(device)) + return 0; + /* + * Wait for dispatcher to finish completing commands + * already submitted + */ + ret = adreno_dispatcher_idle(adreno_dev); + if (ret) + return ret; + + return adreno_spin_idle(adreno_dev, ADRENO_IDLE_TIMEOUT); +} + +/** + * adreno_drain() - Drain the dispatch queue + * @device: Pointer to the KGSL device structure for the GPU + * + * Drain the dispatcher of existing drawobjs. This halts + * additional commands from being issued until the gate is completed. + */ +static int adreno_drain(struct kgsl_device *device) +{ + reinit_completion(&device->halt_gate); + + return 0; +} + +/* Caller must hold the device mutex. */ +static int adreno_suspend_context(struct kgsl_device *device) +{ + /* process any profiling results that are available */ + adreno_profile_process_results(ADRENO_DEVICE(device)); + + /* Wait for the device to go idle */ + return adreno_idle(device); +} + +/** + * adreno_read - General read function to read adreno device memory + * @device - Pointer to the GPU device struct (for adreno device) + * @base - Base address (kernel virtual) where the device memory is mapped + * @offsetwords - Offset in words from the base address, of the memory that + * is to be read + * @value - Value read from the device memory + * @mem_len - Length of the device memory mapped to the kernel + */ +static void adreno_read(struct kgsl_device *device, void __iomem *base, + unsigned int offsetwords, unsigned int *value, + unsigned int mem_len) +{ + + void __iomem *reg; + + /* Make sure we're not reading from invalid memory */ + if (WARN(offsetwords * sizeof(uint32_t) >= mem_len, + "Out of bounds register read: 0x%x/0x%x\n", + offsetwords, mem_len >> 2)) + return; + + reg = (base + (offsetwords << 2)); + + if (!in_interrupt()) + kgsl_pre_hwaccess(device); + + *value = __raw_readl(reg); + /* + * ensure this read finishes before the next one. + * i.e. act like normal readl() + */ + rmb(); +} + +/** + * adreno_regread - Used to read adreno device registers + * @offsetwords - Word (4 Bytes) offset to the register to be read + * @value - Value read from device register + */ +static void adreno_regread(struct kgsl_device *device, unsigned int offsetwords, + unsigned int *value) +{ + adreno_read(device, device->reg_virt, offsetwords, value, + device->reg_len); +} + +/** + * adreno_shadermem_regread - Used to read GPU (adreno) shader memory + * @device - GPU device whose shader memory is to be read + * @offsetwords - Offset in words, of the shader memory address to be read + * @value - Pointer to where the read shader mem value is to be stored + */ +void adreno_shadermem_regread(struct kgsl_device *device, + unsigned int offsetwords, unsigned int *value) +{ + adreno_read(device, device->shader_mem_virt, offsetwords, value, + device->shader_mem_len); +} + +static void adreno_regwrite(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int value) +{ + void __iomem *reg; + + /* Make sure we're not writing to an invalid register */ + if (WARN(offsetwords * sizeof(uint32_t) >= device->reg_len, + "Out of bounds register write: 0x%x/0x%x\n", + offsetwords, device->reg_len >> 2)) + return; + + if (!in_interrupt()) + kgsl_pre_hwaccess(device); + + trace_kgsl_regwrite(device, offsetwords, value); + + reg = (device->reg_virt + (offsetwords << 2)); + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + __raw_writel(value, reg); +} + +/** + * adreno_gmu_clear_and_unmask_irqs() - Clear pending IRQs and Unmask IRQs + * @adreno_dev: Pointer to the Adreno device that owns the GMU + */ +void adreno_gmu_clear_and_unmask_irqs(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); + + /* Clear any pending IRQs before unmasking on GMU */ + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR, + 0xFFFFFFFF); + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, + 0xFFFFFFFF); + + /* Unmask needed IRQs on GMU */ + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + (unsigned int) ~(gmu_dev_ops->gmu2host_intr_mask)); + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + (unsigned int) ~(gmu_dev_ops->gmu_ao_intr_mask)); +} + +/** + * adreno_gmu_mask_and_clear_irqs() - Mask all IRQs and clear pending IRQs + * @adreno_dev: Pointer to the Adreno device that owns the GMU + */ +void adreno_gmu_mask_and_clear_irqs(struct adreno_device *adreno_dev) +{ + /* Mask all IRQs on GMU */ + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + 0xFFFFFFFF); + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + 0xFFFFFFFF); + + /* Clear any pending IRQs before disabling */ + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, + 0xFFFFFFFF); + adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR, + 0xFFFFFFFF); +} + +/* + * adreno_gmu_fenced_write() - Check if there is a GMU and it is enabled + * @adreno_dev: Pointer to the Adreno device that owns the GMU + * @offset: 32bit register enum that is to be written + * @val: The value to be written to the register + * @fence_mask: The value to poll the fence status register + * + * Check the WRITEDROPPED0/1 bit in the FENCE_STATUS register to check if + * the write to the fenced register went through. If it didn't then we retry + * the write until it goes through or we time out. + */ +int adreno_gmu_fenced_write(struct adreno_device *adreno_dev, + enum adreno_regs offset, unsigned int val, + unsigned int fence_mask) +{ + unsigned int status, i; + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int reg_offset = gpudev->reg_offsets->offsets[offset]; + + adreno_writereg(adreno_dev, offset, val); + + if (!gmu_core_isenabled(KGSL_DEVICE(adreno_dev))) + return 0; + + for (i = 0; i < GMU_CORE_LONG_WAKEUP_RETRY_LIMIT; i++) { + adreno_read_gmureg(adreno_dev, ADRENO_REG_GMU_AHB_FENCE_STATUS, + &status); + + /* + * If !writedropped0/1, then the write to fenced register + * was successful + */ + if (!(status & fence_mask)) + return 0; + /* Wait a small amount of time before trying again */ + udelay(GMU_CORE_WAKEUP_DELAY_US); + + /* Try to write the fenced register again */ + adreno_writereg(adreno_dev, offset, val); + + if (i == GMU_CORE_SHORT_WAKEUP_RETRY_LIMIT) + dev_err(adreno_dev->dev.dev, + "Waited %d usecs to write fenced register 0x%x. Continuing to wait...\n", + (GMU_CORE_SHORT_WAKEUP_RETRY_LIMIT * + GMU_CORE_WAKEUP_DELAY_US), + reg_offset); + } + + dev_err(adreno_dev->dev.dev, + "Timed out waiting %d usecs to write fenced register 0x%x\n", + GMU_CORE_LONG_WAKEUP_RETRY_LIMIT * GMU_CORE_WAKEUP_DELAY_US, + reg_offset); + return -ETIMEDOUT; +} + +bool adreno_is_cx_dbgc_register(struct kgsl_device *device, + unsigned int offsetwords) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + return adreno_dev->cx_dbgc_virt && + (offsetwords >= (adreno_dev->cx_dbgc_base >> 2)) && + (offsetwords < (adreno_dev->cx_dbgc_base + + adreno_dev->cx_dbgc_len) >> 2); +} + +void adreno_cx_dbgc_regread(struct kgsl_device *device, + unsigned int offsetwords, unsigned int *value) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int cx_dbgc_offset; + + if (!adreno_is_cx_dbgc_register(device, offsetwords)) + return; + + cx_dbgc_offset = (offsetwords << 2) - adreno_dev->cx_dbgc_base; + *value = __raw_readl(adreno_dev->cx_dbgc_virt + cx_dbgc_offset); + + /* + * ensure this read finishes before the next one. + * i.e. act like normal readl() + */ + rmb(); +} + +void adreno_cx_dbgc_regwrite(struct kgsl_device *device, + unsigned int offsetwords, unsigned int value) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int cx_dbgc_offset; + + if (!adreno_is_cx_dbgc_register(device, offsetwords)) + return; + + cx_dbgc_offset = (offsetwords << 2) - adreno_dev->cx_dbgc_base; + trace_kgsl_regwrite(device, offsetwords, value); + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + __raw_writel(value, adreno_dev->cx_dbgc_virt + cx_dbgc_offset); +} + +/** + * adreno_waittimestamp - sleep while waiting for the specified timestamp + * @device - pointer to a KGSL device structure + * @context - pointer to the active kgsl context + * @timestamp - GPU timestamp to wait for + * @msecs - amount of time to wait (in milliseconds) + * + * Wait up to 'msecs' milliseconds for the specified timestamp to expire. + */ +static int adreno_waittimestamp(struct kgsl_device *device, + struct kgsl_context *context, + unsigned int timestamp, + unsigned int msecs) +{ + int ret; + + if (context == NULL) { + /* If they are doing then complain once */ + dev_WARN_ONCE(device->dev, 1, + "IOCTL_KGSL_DEVICE_WAITTIMESTAMP is deprecated\n"); + return -ENOTTY; + } + + /* Return -ENOENT if the context has been detached */ + if (kgsl_context_detached(context)) + return -ENOENT; + + ret = adreno_drawctxt_wait(ADRENO_DEVICE(device), context, + timestamp, msecs); + + /* If the context got invalidated then return a specific error */ + if (kgsl_context_invalid(context)) + ret = -EDEADLK; + + /* + * Return -EPROTO if the device has faulted since the last time we + * checked. Userspace uses this as a marker for performing post + * fault activities + */ + + if (!ret && test_and_clear_bit(ADRENO_CONTEXT_FAULT, &context->priv)) + ret = -EPROTO; + + return ret; +} + +/** + * __adreno_readtimestamp() - Reads the timestamp from memstore memory + * @adreno_dev: Pointer to an adreno device + * @index: Index into the memstore memory + * @type: Type of timestamp to read + * @timestamp: The out parameter where the timestamp is read + */ +static int __adreno_readtimestamp(struct adreno_device *adreno_dev, int index, + int type, unsigned int *timestamp) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int status = 0; + + switch (type) { + case KGSL_TIMESTAMP_CONSUMED: + kgsl_sharedmem_readl(&device->memstore, timestamp, + KGSL_MEMSTORE_OFFSET(index, soptimestamp)); + break; + case KGSL_TIMESTAMP_RETIRED: + kgsl_sharedmem_readl(&device->memstore, timestamp, + KGSL_MEMSTORE_OFFSET(index, eoptimestamp)); + break; + default: + status = -EINVAL; + *timestamp = 0; + break; + } + return status; +} + +/** + * adreno_rb_readtimestamp(): Return the value of given type of timestamp + * for a RB + * @adreno_dev: adreno device whose timestamp values are being queried + * @priv: The object being queried for a timestamp (expected to be a rb pointer) + * @type: The type of timestamp (one of 3) to be read + * @timestamp: Pointer to where the read timestamp is to be written to + * + * CONSUMED and RETIRED type timestamps are sorted by id and are constantly + * updated by the GPU through shared memstore memory. QUEUED type timestamps + * are read directly from context struct. + + * The function returns 0 on success and timestamp value at the *timestamp + * address and returns -EINVAL on any read error/invalid type and timestamp = 0. + */ +int adreno_rb_readtimestamp(struct adreno_device *adreno_dev, + void *priv, enum kgsl_timestamp_type type, + unsigned int *timestamp) +{ + int status = 0; + struct adreno_ringbuffer *rb = priv; + + if (type == KGSL_TIMESTAMP_QUEUED) + *timestamp = rb->timestamp; + else + status = __adreno_readtimestamp(adreno_dev, + rb->id + KGSL_MEMSTORE_MAX, + type, timestamp); + + return status; +} + +/** + * adreno_readtimestamp(): Return the value of given type of timestamp + * @device: GPU device whose timestamp values are being queried + * @priv: The object being queried for a timestamp (expected to be a context) + * @type: The type of timestamp (one of 3) to be read + * @timestamp: Pointer to where the read timestamp is to be written to + * + * CONSUMED and RETIRED type timestamps are sorted by id and are constantly + * updated by the GPU through shared memstore memory. QUEUED type timestamps + * are read directly from context struct. + + * The function returns 0 on success and timestamp value at the *timestamp + * address and returns -EINVAL on any read error/invalid type and timestamp = 0. + */ +static int adreno_readtimestamp(struct kgsl_device *device, + void *priv, enum kgsl_timestamp_type type, + unsigned int *timestamp) +{ + int status = 0; + struct kgsl_context *context = priv; + + if (type == KGSL_TIMESTAMP_QUEUED) { + struct adreno_context *ctxt = ADRENO_CONTEXT(context); + + *timestamp = ctxt->timestamp; + } else + status = __adreno_readtimestamp(ADRENO_DEVICE(device), + context->id, type, timestamp); + + return status; +} + +/** + * adreno_device_private_create(): Allocate an adreno_device_private structure + */ +struct kgsl_device_private *adreno_device_private_create(void) +{ + struct adreno_device_private *adreno_priv = + kzalloc(sizeof(*adreno_priv), GFP_KERNEL); + + if (adreno_priv) { + INIT_LIST_HEAD(&adreno_priv->perfcounter_list); + return &adreno_priv->dev_priv; + } + return NULL; +} + +/** + * adreno_device_private_destroy(): Destroy an adreno_device_private structure + * and release the perfcounters held by the kgsl fd. + * @dev_priv: The kgsl device private structure + */ +void adreno_device_private_destroy(struct kgsl_device_private *dev_priv) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_device_private *adreno_priv = + container_of(dev_priv, struct adreno_device_private, + dev_priv); + struct adreno_perfcounter_list_node *p, *tmp; + + mutex_lock(&device->mutex); + list_for_each_entry_safe(p, tmp, &adreno_priv->perfcounter_list, node) { + adreno_perfcounter_put(adreno_dev, p->groupid, + p->countable, PERFCOUNTER_FLAG_NONE); + list_del(&p->node); + kfree(p); + } + mutex_unlock(&device->mutex); + + kfree(adreno_priv); +} + +static inline s64 adreno_ticks_to_us(u32 ticks, u32 freq) +{ + freq /= 1000000; + return ticks / freq; +} + +/** + * adreno_power_stats() - Reads the counters needed for freq decisions + * @device: Pointer to device whose counters are read + * @stats: Pointer to stats set that needs updating + * Power: The caller is expected to be in a clock enabled state as this + * function does reg reads + */ +static void adreno_power_stats(struct kgsl_device *device, + struct kgsl_power_stats *stats) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + int64_t adj = 0; + + memset(stats, 0, sizeof(*stats)); + + /* Get the busy cycles counted since the counter was last reset */ + if (adreno_dev->perfctr_pwr_lo != 0) { + uint64_t gpu_busy; + + gpu_busy = counter_delta(device, adreno_dev->perfctr_pwr_lo, + &busy->gpu_busy); + + if (gpudev->read_throttling_counters) { + adj = gpudev->read_throttling_counters(adreno_dev); + if (adj < 0 && -adj > gpu_busy) + adj = -gpu_busy; + + gpu_busy += adj; + } + + if (adreno_is_a6xx(adreno_dev)) { + /* clock sourced from XO */ + stats->busy_time = gpu_busy * 10; + do_div(stats->busy_time, 192); + } else { + /* clock sourced from GFX3D */ + stats->busy_time = adreno_ticks_to_us(gpu_busy, + kgsl_pwrctrl_active_freq(pwr)); + } + } + + if (device->pwrctrl.bus_control) { + uint64_t ram_cycles = 0, starved_ram = 0; + + if (adreno_dev->ram_cycles_lo != 0) + ram_cycles = counter_delta(device, + adreno_dev->ram_cycles_lo, + &busy->bif_ram_cycles); + + if (adreno_has_gbif(adreno_dev)) { + if (adreno_dev->ram_cycles_lo_ch1_read != 0) + ram_cycles += counter_delta(device, + adreno_dev->ram_cycles_lo_ch1_read, + &busy->bif_ram_cycles_read_ch1); + + if (adreno_dev->ram_cycles_lo_ch0_write != 0) + ram_cycles += counter_delta(device, + adreno_dev->ram_cycles_lo_ch0_write, + &busy->bif_ram_cycles_write_ch0); + + if (adreno_dev->ram_cycles_lo_ch1_write != 0) + ram_cycles += counter_delta(device, + adreno_dev->ram_cycles_lo_ch1_write, + &busy->bif_ram_cycles_write_ch1); + } + + if (adreno_dev->starved_ram_lo != 0) + starved_ram = counter_delta(device, + adreno_dev->starved_ram_lo, + &busy->bif_starved_ram); + + if (adreno_has_gbif(adreno_dev)) { + if (adreno_dev->starved_ram_lo_ch1 != 0) + starved_ram += counter_delta(device, + adreno_dev->starved_ram_lo_ch1, + &busy->bif_starved_ram_ch1); + } + + stats->ram_time = ram_cycles; + stats->ram_wait = starved_ram; + } + + if (adreno_dev->perfctr_ifpc_lo != 0) { + uint32_t num_ifpc; + + num_ifpc = counter_delta(device, adreno_dev->perfctr_ifpc_lo, + &busy->num_ifpc); + adreno_dev->ifpc_count += num_ifpc; + if (num_ifpc > 0) + trace_adreno_ifpc_count(adreno_dev->ifpc_count); + } + + if (adreno_dev->lm_threshold_count && + gpudev->count_throttles) + gpudev->count_throttles(adreno_dev, adj); +} + +static unsigned int adreno_gpuid(struct kgsl_device *device, + unsigned int *chipid) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* + * Some applications need to know the chip ID too, so pass + * that as a parameter + */ + + if (chipid != NULL) + *chipid = adreno_dev->chipid; + + /* + * Standard KGSL gpuid format: + * top word is 0x0002 for 2D or 0x0003 for 3D + * Bottom word is core specific identifer + */ + + return (0x0003 << 16) | ADRENO_GPUREV(adreno_dev); +} + +static int adreno_regulator_enable(struct kgsl_device *device) +{ + int ret = 0; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->regulator_enable && + !test_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv)) { + ret = gpudev->regulator_enable(adreno_dev); + if (!ret) + set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv); + } + return ret; +} + +static bool adreno_is_hw_collapsible(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + /* + * Skip power collapse for A304, if power ctrl flag is set to + * non zero. As A304 soft_reset will not work, power collapse + * needs to disable to avoid soft_reset. + */ + if (adreno_is_a304(adreno_dev) && + device->pwrctrl.ctrl_flags) + return false; + + return adreno_isidle(device) && (gpudev->is_sptp_idle ? + gpudev->is_sptp_idle(adreno_dev) : true); +} + +static void adreno_regulator_disable(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->regulator_disable && + test_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv)) { + gpudev->regulator_disable(adreno_dev); + clear_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv); + } +} + +static void adreno_pwrlevel_change_settings(struct kgsl_device *device, + unsigned int prelevel, unsigned int postlevel, bool post) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->pwrlevel_change_settings) + gpudev->pwrlevel_change_settings(adreno_dev, prelevel, + postlevel, post); +} + +static void adreno_clk_set_options(struct kgsl_device *device, const char *name, + struct clk *clk, bool on) +{ + if (ADRENO_GPU_DEVICE(ADRENO_DEVICE(device))->clk_set_options) + ADRENO_GPU_DEVICE(ADRENO_DEVICE(device))->clk_set_options( + ADRENO_DEVICE(device), name, clk, on); +} + +static void adreno_iommu_sync(struct kgsl_device *device, bool sync) +{ + struct scm_desc desc = {0}; + int ret; + + if (sync) { + mutex_lock(&kgsl_mmu_sync); + desc.args[0] = true; + desc.arginfo = SCM_ARGS(1); + ret = scm_call2_atomic(SCM_SIP_FNID(SCM_SVC_PWR, 0x8), &desc); + if (ret) + dev_err(device->dev, + "MMU sync with Hypervisor off %x\n", ret); + } else { + desc.args[0] = false; + desc.arginfo = SCM_ARGS(1); + scm_call2_atomic(SCM_SIP_FNID(SCM_SVC_PWR, 0x8), &desc); + mutex_unlock(&kgsl_mmu_sync); + } +} + +static void _regulator_disable(struct device *dev, + struct kgsl_regulator *regulator) +{ + unsigned long wait_time = jiffies + msecs_to_jiffies(200); + + if (IS_ERR_OR_NULL(regulator->reg)) + return; + + regulator_disable(regulator->reg); + + while (!time_after(jiffies, wait_time)) { + if (!regulator_is_enabled(regulator->reg)) + return; + cpu_relax(); + } + + dev_err(dev, "regulator '%s' still on after 200ms\n", regulator->name); +} + +static void adreno_regulator_disable_poll(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int i; + + /* Fast path - hopefully we don't need this quirk */ + if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_IOMMU_SYNC)) { + for (i = KGSL_MAX_REGULATORS - 1; i >= 0; i--) { + if (!IS_ERR_OR_NULL(pwr->regulators[i].reg)) + regulator_disable(pwr->regulators[i].reg); + } + return; + } + + adreno_iommu_sync(device, true); + + for (i = 0; i < KGSL_MAX_REGULATORS; i++) + _regulator_disable(device->dev, &pwr->regulators[i]); + + adreno_iommu_sync(device, false); +} + +static void adreno_gpu_model(struct kgsl_device *device, char *str, + size_t bufsz) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + snprintf(str, bufsz, "Adreno%d%d%dv%d", + ADRENO_CHIPID_CORE(adreno_dev->chipid), + ADRENO_CHIPID_MAJOR(adreno_dev->chipid), + ADRENO_CHIPID_MINOR(adreno_dev->chipid), + ADRENO_CHIPID_PATCH(adreno_dev->chipid) + 1); +} + +static const struct kgsl_functable adreno_functable = { + /* Mandatory functions */ + .regread = adreno_regread, + .regwrite = adreno_regwrite, + .idle = adreno_idle, + .isidle = adreno_isidle, + .suspend_context = adreno_suspend_context, + .init = adreno_init, + .start = adreno_start, + .stop = adreno_stop, + .getproperty = adreno_getproperty, + .getproperty_compat = adreno_getproperty_compat, + .waittimestamp = adreno_waittimestamp, + .readtimestamp = adreno_readtimestamp, + .queue_cmds = adreno_dispatcher_queue_cmds, + .ioctl = adreno_ioctl, + .compat_ioctl = adreno_compat_ioctl, + .power_stats = adreno_power_stats, + .gpuid = adreno_gpuid, + .snapshot = adreno_snapshot, + .irq_handler = adreno_irq_handler, + .drain = adreno_drain, + .device_private_create = adreno_device_private_create, + .device_private_destroy = adreno_device_private_destroy, + /* Optional functions */ + .drawctxt_create = adreno_drawctxt_create, + .drawctxt_detach = adreno_drawctxt_detach, + .drawctxt_destroy = adreno_drawctxt_destroy, + .drawctxt_dump = adreno_drawctxt_dump, + .setproperty = adreno_setproperty, + .setproperty_compat = adreno_setproperty_compat, + .drawctxt_sched = adreno_drawctxt_sched, + .resume = adreno_dispatcher_start, + .regulator_enable = adreno_regulator_enable, + .is_hw_collapsible = adreno_is_hw_collapsible, + .regulator_disable = adreno_regulator_disable, + .pwrlevel_change_settings = adreno_pwrlevel_change_settings, + .regulator_disable_poll = adreno_regulator_disable_poll, + .clk_set_options = adreno_clk_set_options, + .gpu_model = adreno_gpu_model, + .stop_fault_timer = adreno_dispatcher_stop_fault_timer, + .dispatcher_halt = adreno_dispatcher_halt, + .dispatcher_unhalt = adreno_dispatcher_unhalt, +}; + +static struct platform_driver adreno_platform_driver = { + .probe = adreno_probe, + .remove = adreno_remove, + .suspend = kgsl_suspend_driver, + .resume = kgsl_resume_driver, + .id_table = adreno_id_table, + .driver = { + .name = DEVICE_3D_NAME, + .pm = &kgsl_pm_ops, + .of_match_table = adreno_match_table, + } +}; + +static const struct of_device_id busmon_match_table[] = { + { .compatible = "qcom,kgsl-busmon", .data = &device_3d0 }, + {} +}; + +static int adreno_busmon_probe(struct platform_device *pdev) +{ + struct kgsl_device *device; + const struct of_device_id *pdid = + of_match_device(busmon_match_table, &pdev->dev); + + if (pdid == NULL) + return -ENXIO; + + device = (struct kgsl_device *)pdid->data; + device->busmondev = &pdev->dev; + dev_set_drvdata(device->busmondev, device); + + return 0; +} + +static struct platform_driver kgsl_bus_platform_driver = { + .probe = adreno_busmon_probe, + .driver = { + .name = "kgsl-busmon", + .of_match_table = busmon_match_table, + } +}; + +static int __init kgsl_3d_init(void) +{ + int ret; + + ret = platform_driver_register(&kgsl_bus_platform_driver); + if (ret) + return ret; + + ret = platform_driver_register(&adreno_platform_driver); + if (ret) + platform_driver_unregister(&kgsl_bus_platform_driver); + + return ret; +} + +static void __exit kgsl_3d_exit(void) +{ + platform_driver_unregister(&adreno_platform_driver); + platform_driver_unregister(&kgsl_bus_platform_driver); +} + +module_init(kgsl_3d_init); +module_exit(kgsl_3d_exit); + +MODULE_DESCRIPTION("3D Graphics driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:kgsl_3d"); diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h new file mode 100644 index 0000000000000000000000000000000000000000..f99ea8cbd7ba437018e7c4eabb778bfe841ae977 --- /dev/null +++ b/drivers/gpu/msm/adreno.h @@ -0,0 +1,1973 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2008-2018, The Linux Foundation. All rights reserved. + */ +#ifndef __ADRENO_H +#define __ADRENO_H + +#include "kgsl_device.h" +#include "kgsl_sharedmem.h" +#include "adreno_drawctxt.h" +#include "adreno_ringbuffer.h" +#include "adreno_profile.h" +#include "adreno_dispatch.h" +#include "kgsl_iommu.h" +#include "adreno_perfcounter.h" +#include +#include +#include "kgsl_gmu_core.h" + +#include "a4xx_reg.h" + +#ifdef CONFIG_QCOM_OCMEM +#include +#endif + +#define DEVICE_3D_NAME "kgsl-3d" +#define DEVICE_3D0_NAME "kgsl-3d0" + +/* ADRENO_DEVICE - Given a kgsl_device return the adreno device struct */ +#define ADRENO_DEVICE(device) \ + container_of(device, struct adreno_device, dev) + +/* KGSL_DEVICE - given an adreno_device, return the KGSL device struct */ +#define KGSL_DEVICE(_dev) (&((_dev)->dev)) + +/* ADRENO_CONTEXT - Given a context return the adreno context struct */ +#define ADRENO_CONTEXT(context) \ + container_of(context, struct adreno_context, base) + +/* ADRENO_GPU_DEVICE - Given an adreno device return the GPU specific struct */ +#define ADRENO_GPU_DEVICE(_a) ((_a)->gpucore->gpudev) + +#define ADRENO_CHIPID_CORE(_id) (((_id) >> 24) & 0xFF) +#define ADRENO_CHIPID_MAJOR(_id) (((_id) >> 16) & 0xFF) +#define ADRENO_CHIPID_MINOR(_id) (((_id) >> 8) & 0xFF) +#define ADRENO_CHIPID_PATCH(_id) ((_id) & 0xFF) + +/* ADRENO_GPUREV - Return the GPU ID for the given adreno_device */ +#define ADRENO_GPUREV(_a) ((_a)->gpucore->gpurev) + +/* + * ADRENO_FEATURE - return true if the specified feature is supported by the GPU + * core + */ +#define ADRENO_FEATURE(_dev, _bit) \ + ((_dev)->gpucore->features & (_bit)) + +/** + * ADRENO_QUIRK - return true if the specified quirk is required by the GPU + */ +#define ADRENO_QUIRK(_dev, _bit) \ + ((_dev)->quirks & (_bit)) + +/* + * ADRENO_PREEMPT_STYLE - return preemption style + */ +#define ADRENO_PREEMPT_STYLE(flags) \ + ((flags & KGSL_CONTEXT_PREEMPT_STYLE_MASK) >> \ + KGSL_CONTEXT_PREEMPT_STYLE_SHIFT) + +/* + * return the dispatcher drawqueue in which the given drawobj should + * be submitted + */ +#define ADRENO_DRAWOBJ_DISPATCH_DRAWQUEUE(c) \ + (&((ADRENO_CONTEXT(c->context))->rb->dispatch_q)) + +#define ADRENO_DRAWOBJ_RB(c) \ + ((ADRENO_CONTEXT(c->context))->rb) + +#define ADRENO_FW(a, f) (&(a->fw[f])) + +/* Adreno core features */ +/* The core uses OCMEM for GMEM/binning memory */ +#define ADRENO_USES_OCMEM BIT(0) +/* The core supports an accelerated warm start */ +#define ADRENO_WARM_START BIT(1) +/* The core supports the microcode bootstrap functionality */ +#define ADRENO_USE_BOOTSTRAP BIT(2) +/* The core supports SP/TP hw controlled power collapse */ +#define ADRENO_SPTP_PC BIT(3) +/* The core supports Peak Power Detection(PPD)*/ +#define ADRENO_PPD BIT(4) +/* The GPU supports content protection */ +#define ADRENO_CONTENT_PROTECTION BIT(5) +/* The GPU supports preemption */ +#define ADRENO_PREEMPTION BIT(6) +/* The core uses GPMU for power and limit management */ +#define ADRENO_GPMU BIT(7) +/* The GPMU supports Limits Management */ +#define ADRENO_LM BIT(8) +/* The core uses 64 bit GPU addresses */ +#define ADRENO_64BIT BIT(9) +/* The GPU supports retention for cpz registers */ +#define ADRENO_CPZ_RETENTION BIT(10) +/* The core has soft fault detection available */ +#define ADRENO_SOFT_FAULT_DETECT BIT(11) +/* The GMU supports RPMh for power management*/ +#define ADRENO_RPMH BIT(12) +/* The GMU supports IFPC power management*/ +#define ADRENO_IFPC BIT(13) +/* The GMU supports HW based NAP */ +#define ADRENO_HW_NAP BIT(14) +/* The GMU supports min voltage*/ +#define ADRENO_MIN_VOLT BIT(15) +/* The core supports IO-coherent memory */ +#define ADRENO_IOCOHERENT BIT(16) +/* + * The GMU supports Adaptive Clock Distribution (ACD) + * for droop mitigation + */ +#define ADRENO_ACD BIT(17) + +/* + * Adreno GPU quirks - control bits for various workarounds + */ + +/* Set TWOPASSUSEWFI in PC_DBG_ECO_CNTL (5XX/6XX) */ +#define ADRENO_QUIRK_TWO_PASS_USE_WFI BIT(0) +/* Lock/unlock mutex to sync with the IOMMU */ +#define ADRENO_QUIRK_IOMMU_SYNC BIT(1) +/* Submit critical packets at GPU wake up */ +#define ADRENO_QUIRK_CRITICAL_PACKETS BIT(2) +/* Mask out RB1-3 activity signals from HW hang detection logic */ +#define ADRENO_QUIRK_FAULT_DETECT_MASK BIT(3) +/* Disable RB sampler datapath clock gating optimization */ +#define ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING BIT(4) +/* Disable local memory(LM) feature to avoid corner case error */ +#define ADRENO_QUIRK_DISABLE_LMLOADKILL BIT(5) +/* Allow HFI to use registers to send message to GMU */ +#define ADRENO_QUIRK_HFI_USE_REG BIT(6) +/* Only set protected SECVID registers once */ +#define ADRENO_QUIRK_SECVID_SET_ONCE BIT(7) +/* + * Limit number of read and write transactions from + * UCHE block to GBIF to avoid possible deadlock + * between GBIF, SMMU and MEMNOC. + */ +#define ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW BIT(8) +/* Select alternate secure context bank for mmu */ +#define ADRENO_QUIRK_MMU_SECURE_CB_ALT BIT(9) +/* Do explicit mode control of cx gdsc */ +#define ADRENO_QUIRK_CX_GDSC BIT(10) + +/* Flags to control command packet settings */ +#define KGSL_CMD_FLAGS_NONE 0 +#define KGSL_CMD_FLAGS_PMODE BIT(0) +#define KGSL_CMD_FLAGS_INTERNAL_ISSUE BIT(1) +#define KGSL_CMD_FLAGS_WFI BIT(2) +#define KGSL_CMD_FLAGS_PROFILE BIT(3) +#define KGSL_CMD_FLAGS_PWRON_FIXUP BIT(4) + +/* Command identifiers */ +#define CONTEXT_TO_MEM_IDENTIFIER 0x2EADBEEF +#define CMD_IDENTIFIER 0x2EEDFACE +#define CMD_INTERNAL_IDENTIFIER 0x2EEDD00D +#define START_IB_IDENTIFIER 0x2EADEABE +#define END_IB_IDENTIFIER 0x2ABEDEAD +#define START_PROFILE_IDENTIFIER 0x2DEFADE1 +#define END_PROFILE_IDENTIFIER 0x2DEFADE2 +#define PWRON_FIXUP_IDENTIFIER 0x2AFAFAFA + +/* Number of times to try hard reset for pre-a6xx GPUs */ +#define NUM_TIMES_RESET_RETRY 4 + +/* Number of times to poll the AHB fence in ISR */ +#define FENCE_RETRY_MAX 100 + +/* One cannot wait forever for the core to idle, so set an upper limit to the + * amount of time to wait for the core to go idle + */ +#define ADRENO_IDLE_TIMEOUT (20 * 1000) + +#define ADRENO_UCHE_GMEM_BASE 0x100000 + +#define ADRENO_FW_PFP 0 +#define ADRENO_FW_SQE 0 +#define ADRENO_FW_PM4 1 + +enum adreno_gpurev { + ADRENO_REV_UNKNOWN = 0, + ADRENO_REV_A304 = 304, + ADRENO_REV_A305 = 305, + ADRENO_REV_A305C = 306, + ADRENO_REV_A306 = 307, + ADRENO_REV_A306A = 308, + ADRENO_REV_A310 = 310, + ADRENO_REV_A320 = 320, + ADRENO_REV_A330 = 330, + ADRENO_REV_A305B = 335, + ADRENO_REV_A405 = 405, + ADRENO_REV_A418 = 418, + ADRENO_REV_A420 = 420, + ADRENO_REV_A430 = 430, + ADRENO_REV_A505 = 505, + ADRENO_REV_A506 = 506, + ADRENO_REV_A508 = 508, + ADRENO_REV_A510 = 510, + ADRENO_REV_A512 = 512, + ADRENO_REV_A530 = 530, + ADRENO_REV_A540 = 540, + ADRENO_REV_A608 = 608, + ADRENO_REV_A615 = 615, + ADRENO_REV_A616 = 616, + ADRENO_REV_A630 = 630, + ADRENO_REV_A640 = 640, + ADRENO_REV_A680 = 680, +}; + +#define ADRENO_START_WARM 0 +#define ADRENO_START_COLD 1 + +#define ADRENO_SOFT_FAULT BIT(0) +#define ADRENO_HARD_FAULT BIT(1) +#define ADRENO_TIMEOUT_FAULT BIT(2) +#define ADRENO_IOMMU_PAGE_FAULT BIT(3) +#define ADRENO_PREEMPT_FAULT BIT(4) +#define ADRENO_GMU_FAULT BIT(5) +#define ADRENO_CTX_DETATCH_TIMEOUT_FAULT BIT(6) + +#define ADRENO_SPTP_PC_CTRL 0 +#define ADRENO_PPD_CTRL 1 +#define ADRENO_LM_CTRL 2 +#define ADRENO_HWCG_CTRL 3 +#define ADRENO_THROTTLING_CTRL 4 +#define ADRENO_ACD_CTRL 5 + +/* VBIF, GBIF halt request and ack mask */ +#define GBIF_HALT_REQUEST 0x1E0 +#define VBIF_RESET_ACK_MASK 0x00f0 +#define VBIF_RESET_ACK_TIMEOUT 100 + +/* number of throttle counters for DCVS adjustment */ +#define ADRENO_GPMU_THROTTLE_COUNTERS 4 +/* base for throttle counters */ +#define ADRENO_GPMU_THROTTLE_COUNTERS_BASE_REG 43 + +struct adreno_gpudev; + +/* Time to allow preemption to complete (in ms) */ +#define ADRENO_PREEMPT_TIMEOUT 10000 + +#define ADRENO_INT_BIT(a, _bit) (((a)->gpucore->gpudev->int_bits) ? \ + (adreno_get_int(a, _bit) < 0 ? 0 : \ + BIT(adreno_get_int(a, _bit))) : 0) + +/** + * enum adreno_preempt_states + * ADRENO_PREEMPT_NONE: No preemption is scheduled + * ADRENO_PREEMPT_START: The S/W has started + * ADRENO_PREEMPT_TRIGGERED: A preeempt has been triggered in the HW + * ADRENO_PREEMPT_FAULTED: The preempt timer has fired + * ADRENO_PREEMPT_PENDING: The H/W has signaled preemption complete + * ADRENO_PREEMPT_COMPLETE: Preemption could not be finished in the IRQ handler, + * worker has been scheduled + */ +enum adreno_preempt_states { + ADRENO_PREEMPT_NONE = 0, + ADRENO_PREEMPT_START, + ADRENO_PREEMPT_TRIGGERED, + ADRENO_PREEMPT_FAULTED, + ADRENO_PREEMPT_PENDING, + ADRENO_PREEMPT_COMPLETE, +}; + +/** + * struct adreno_preemption + * @state: The current state of preemption + * @counters: Memory descriptor for the memory where the GPU writes the + * preemption counters on switch + * @timer: A timer to make sure preemption doesn't stall + * @work: A work struct for the preemption worker (for 5XX) + * @token_submit: Indicates if a preempt token has been submitted in + * current ringbuffer (for 4XX) + * preempt_level: The level of preemption (for 6XX) + * skipsaverestore: To skip saverestore during L1 preemption (for 6XX) + * usesgmem: enable GMEM save/restore across preemption (for 6XX) + * count: Track the number of preemptions triggered + */ +struct adreno_preemption { + atomic_t state; + struct kgsl_memdesc counters; + struct timer_list timer; + struct work_struct work; + bool token_submit; + unsigned int preempt_level; + bool skipsaverestore; + bool usesgmem; + unsigned int count; +}; + + +struct adreno_busy_data { + unsigned int gpu_busy; + unsigned int bif_ram_cycles; + unsigned int bif_ram_cycles_read_ch1; + unsigned int bif_ram_cycles_write_ch0; + unsigned int bif_ram_cycles_write_ch1; + unsigned int bif_starved_ram; + unsigned int bif_starved_ram_ch1; + unsigned int num_ifpc; + unsigned int throttle_cycles[ADRENO_GPMU_THROTTLE_COUNTERS]; +}; + +/** + * struct adreno_firmware - Struct holding fw details + * @fwvirt: Buffer which holds the ucode + * @size: Size of ucode buffer + * @version: Version of ucode + * @memdesc: Memory descriptor which holds ucode buffer info + */ +struct adreno_firmware { + unsigned int *fwvirt; + size_t size; + unsigned int version; + struct kgsl_memdesc memdesc; +}; + +/** + * struct adreno_perfcounter_list_node - struct to store perfcounters + * allocated by a process on a kgsl fd. + * @groupid: groupid of the allocated perfcounter + * @countable: countable assigned to the allocated perfcounter + * @node: list node for perfcounter_list of a process + */ +struct adreno_perfcounter_list_node { + unsigned int groupid; + unsigned int countable; + struct list_head node; +}; + +/** + * struct adreno_device_private - Adreno private structure per fd + * @dev_priv: the kgsl device private structure + * @perfcounter_list: list of perfcounters used by the process + */ +struct adreno_device_private { + struct kgsl_device_private dev_priv; + struct list_head perfcounter_list; +}; + +/** + * struct adreno_gpu_core - A specific GPU core definition + * @gpurev: Unique GPU revision identifier + * @core: Match for the core version of the GPU + * @major: Match for the major version of the GPU + * @minor: Match for the minor version of the GPU + * @patchid: Match for the patch revision of the GPU + * @features: Common adreno features supported by this core + * @pm4fw_name: Filename for th PM4 firmware + * @pfpfw_name: Filename for the PFP firmware + * @zap_name: Filename for the Zap Shader ucode + * @gpudev: Pointer to the GPU family specific functions for this core + * @gmem_size: Amount of binning memory (GMEM/OCMEM) to reserve for the core + * @pm4_jt_idx: Index of the jump table in the PM4 microcode + * @pm4_jt_addr: Address offset to load the jump table for the PM4 microcode + * @pfp_jt_idx: Index of the jump table in the PFP microcode + * @pfp_jt_addr: Address offset to load the jump table for the PFP microcode + * @pm4_bstrp_size: Size of the bootstrap loader for PM4 microcode + * @pfp_bstrp_size: Size of the bootstrap loader for PFP microcde + * @pfp_bstrp_ver: Version of the PFP microcode that supports bootstraping + * @shader_offset: Offset of shader from gpu reg base + * @shader_size: Shader size + * @num_protected_regs: number of protected registers + * @gpmufw_name: Filename for the GPMU firmware + * @gpmu_major: Match for the GPMU & firmware, major revision + * @gpmu_minor: Match for the GPMU & firmware, minor revision + * @gpmu_features: Supported features for any given GPMU version + * @busy_mask: mask to check if GPU is busy in RBBM_STATUS + * @lm_major: Limits Management register sequence, major revision + * @lm_minor: LM register sequence, minor revision + * @regfw_name: Filename for the register sequence firmware + * @gpmu_tsens: ID for the temporature sensor used by the GPMU + * @max_power: Max possible power draw of a core, units elephant tail hairs + * @va_padding: Size to pad allocations to, zero if not required + */ +struct adreno_gpu_core { + enum adreno_gpurev gpurev; + unsigned int core, major, minor, patchid; + unsigned long features; + const char *pm4fw_name; + const char *pfpfw_name; + const char *sqefw_name; + const char *zap_name; + struct adreno_gpudev *gpudev; + size_t gmem_size; + unsigned int pm4_jt_idx; + unsigned int pm4_jt_addr; + unsigned int pfp_jt_idx; + unsigned int pfp_jt_addr; + unsigned int pm4_bstrp_size; + unsigned int pfp_bstrp_size; + unsigned int pfp_bstrp_ver; + unsigned long shader_offset; + unsigned int shader_size; + unsigned int num_protected_regs; + const char *gpmufw_name; + unsigned int gpmu_major; + unsigned int gpmu_minor; + unsigned int gpmu_features; + unsigned int busy_mask; + unsigned int lm_major, lm_minor; + const char *regfw_name; + unsigned int gpmu_tsens; + unsigned int max_power; + uint64_t va_padding; +}; + + +enum gpu_coresight_sources { + GPU_CORESIGHT_GX = 0, + GPU_CORESIGHT_CX = 1, + GPU_CORESIGHT_MAX, +}; + +/** + * struct adreno_device - The mothership structure for all adreno related info + * @dev: Reference to struct kgsl_device + * @priv: Holds the private flags specific to the adreno_device + * @chipid: Chip ID specific to the GPU + * @gmem_base: Base physical address of GMEM + * @gmem_size: GMEM size + * @gpucore: Pointer to the adreno_gpu_core structure + * @pfp_fw: Buffer which holds the pfp ucode + * @pfp_fw_size: Size of pfp ucode buffer + * @pfp_fw_version: Version of pfp ucode + * @pfp: Memory descriptor which holds pfp ucode buffer info + * @pm4_fw: Buffer which holds the pm4 ucode + * @pm4_fw_size: Size of pm4 ucode buffer + * @pm4_fw_version: Version of pm4 ucode + * @pm4: Memory descriptor which holds pm4 ucode buffer info + * @gpmu_cmds_size: Length of gpmu cmd stream + * @gpmu_cmds: gpmu cmd stream + * @ringbuffers: Array of pointers to adreno_ringbuffers + * @num_ringbuffers: Number of ringbuffers for the GPU + * @cur_rb: Pointer to the current ringbuffer + * @next_rb: Ringbuffer we are switching to during preemption + * @prev_rb: Ringbuffer we are switching from during preemption + * @fast_hang_detect: Software fault detection availability + * @ft_policy: Defines the fault tolerance policy + * @long_ib_detect: Long IB detection availability + * @ft_pf_policy: Defines the fault policy for page faults + * @ocmem_hdl: Handle to the ocmem allocated buffer + * @profile: Container for adreno profiler information + * @dispatcher: Container for adreno GPU dispatcher + * @pwron_fixup: Command buffer to run a post-power collapse shader workaround + * @pwron_fixup_dwords: Number of dwords in the command buffer + * @input_work: Work struct for turning on the GPU after a touch event + * @busy_data: Struct holding GPU VBIF busy stats + * @ram_cycles_lo: Number of DDR clock cycles for the monitor session (Only + * DDR channel 0 read cycles in case of GBIF) + * @ram_cycles_lo_ch1_read: Number of DDR channel 1 Read clock cycles for + * the monitor session + * @ram_cycles_lo_ch0_write: Number of DDR channel 0 Write clock cycles for + * the monitor session + * @ram_cycles_lo_ch1_write: Number of DDR channel 0 Write clock cycles for + * the monitor session + * @starved_ram_lo: Number of cycles VBIF/GBIF is stalled by DDR (Only channel 0 + * stall cycles in case of GBIF) + * @starved_ram_lo_ch1: Number of cycles GBIF is stalled by DDR channel 1 + * @perfctr_pwr_lo: GPU busy cycles + * @perfctr_ifpc_lo: IFPC count + * @halt: Atomic variable to check whether the GPU is currently halted + * @pending_irq_refcnt: Atomic variable to keep track of running IRQ handlers + * @ctx_d_debugfs: Context debugfs node + * @pwrctrl_flag: Flag to hold adreno specific power attributes + * @profile_buffer: Memdesc holding the drawobj profiling buffer + * @profile_index: Index to store the start/stop ticks in the profiling + * buffer + * @pwrup_reglist: Memdesc holding the power up register list + * which is used by CP during preemption and IFPC + * @sp_local_gpuaddr: Base GPU virtual address for SP local memory + * @sp_pvt_gpuaddr: Base GPU virtual address for SP private memory + * @lm_fw: The LM firmware handle + * @lm_sequence: Pointer to the start of the register write sequence for LM + * @lm_size: The dword size of the LM sequence + * @lm_limit: limiting value for LM + * @lm_threshold_count: register value for counter for lm threshold breakin + * @lm_threshold_cross: number of current peaks exceeding threshold + * @ifpc_count: Number of times the GPU went into IFPC + * @speed_bin: Indicate which power level set to use + * @csdev: Pointer to a coresight device (if applicable) + * @gpmu_throttle_counters - counteers for number of throttled clocks + * @irq_storm_work: Worker to handle possible interrupt storms + * @active_list: List to track active contexts + * @active_list_lock: Lock to protect active_list + * @gpu_llc_slice: GPU system cache slice descriptor + * @gpu_llc_slice_enable: To enable the GPU system cache slice or not + * @gpuhtw_llc_slice: GPU pagetables system cache slice descriptor + * @gpuhtw_llc_slice_enable: To enable the GPUHTW system cache slice or not + * @zap_loaded: Used to track if zap was successfully loaded or not + * @soc_hw_rev: Indicate which SOC hardware revision to use + */ +struct adreno_device { + struct kgsl_device dev; /* Must be first field in this struct */ + unsigned long priv; + unsigned int chipid; + unsigned long gmem_base; + unsigned long gmem_size; + unsigned long cx_dbgc_base; + unsigned int cx_dbgc_len; + void __iomem *cx_dbgc_virt; + const struct adreno_gpu_core *gpucore; + struct adreno_firmware fw[2]; + size_t gpmu_cmds_size; + unsigned int *gpmu_cmds; + struct adreno_ringbuffer ringbuffers[KGSL_PRIORITY_MAX_RB_LEVELS]; + int num_ringbuffers; + struct adreno_ringbuffer *cur_rb; + struct adreno_ringbuffer *next_rb; + struct adreno_ringbuffer *prev_rb; + unsigned int fast_hang_detect; + unsigned long ft_policy; + unsigned int long_ib_detect; + unsigned long ft_pf_policy; + struct ocmem_buf *ocmem_hdl; + struct adreno_profile profile; + struct adreno_dispatcher dispatcher; + struct kgsl_memdesc pwron_fixup; + unsigned int pwron_fixup_dwords; + struct work_struct input_work; + struct adreno_busy_data busy_data; + unsigned int ram_cycles_lo; + unsigned int ram_cycles_lo_ch1_read; + unsigned int ram_cycles_lo_ch0_write; + unsigned int ram_cycles_lo_ch1_write; + unsigned int starved_ram_lo; + unsigned int starved_ram_lo_ch1; + unsigned int perfctr_pwr_lo; + unsigned int perfctr_ifpc_lo; + atomic_t halt; + atomic_t pending_irq_refcnt; + struct dentry *ctx_d_debugfs; + unsigned long pwrctrl_flag; + + struct kgsl_memdesc profile_buffer; + unsigned int profile_index; + struct kgsl_memdesc pwrup_reglist; + uint64_t sp_local_gpuaddr; + uint64_t sp_pvt_gpuaddr; + const struct firmware *lm_fw; + uint32_t *lm_sequence; + uint32_t lm_size; + struct adreno_preemption preempt; + struct work_struct gpmu_work; + uint32_t lm_leakage; + uint32_t lm_limit; + uint32_t lm_threshold_count; + uint32_t lm_threshold_cross; + uint32_t ifpc_count; + + unsigned int speed_bin; + unsigned int quirks; + + struct coresight_device *csdev[GPU_CORESIGHT_MAX]; + uint32_t gpmu_throttle_counters[ADRENO_GPMU_THROTTLE_COUNTERS]; + struct work_struct irq_storm_work; + + struct list_head active_list; + spinlock_t active_list_lock; + + void *gpu_llc_slice; + bool gpu_llc_slice_enable; + void *gpuhtw_llc_slice; + bool gpuhtw_llc_slice_enable; + unsigned int zap_loaded; + unsigned int soc_hw_rev; +}; + +/** + * enum adreno_device_flags - Private flags for the adreno_device + * @ADRENO_DEVICE_PWRON - Set during init after a power collapse + * @ADRENO_DEVICE_PWRON_FIXUP - Set if the target requires the shader fixup + * after power collapse + * @ADRENO_DEVICE_CORESIGHT - Set if the coresight (trace bus) registers should + * be restored after power collapse + * @ADRENO_DEVICE_HANG_INTR - Set if the hang interrupt should be enabled for + * this target + * @ADRENO_DEVICE_STARTED - Set if the device start sequence is in progress + * @ADRENO_DEVICE_FAULT - Set if the device is currently in fault (and shouldn't + * send any more commands to the ringbuffer) + * @ADRENO_DEVICE_DRAWOBJ_PROFILE - Set if the device supports drawobj + * profiling via the ALWAYSON counter + * @ADRENO_DEVICE_PREEMPTION - Turn on/off preemption + * @ADRENO_DEVICE_SOFT_FAULT_DETECT - Set if soft fault detect is enabled + * @ADRENO_DEVICE_GPMU_INITIALIZED - Set if GPMU firmware initialization succeed + * @ADRENO_DEVICE_ISDB_ENABLED - Set if the Integrated Shader DeBugger is + * attached and enabled + * @ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED - Set if a CACHE_FLUSH_TS irq storm + * is in progress + * @ADRENO_DEVICE_HARD_RESET - Set if soft reset fails and hard reset is needed + */ +enum adreno_device_flags { + ADRENO_DEVICE_PWRON = 0, + ADRENO_DEVICE_PWRON_FIXUP = 1, + ADRENO_DEVICE_INITIALIZED = 2, + ADRENO_DEVICE_CORESIGHT = 3, + ADRENO_DEVICE_HANG_INTR = 4, + ADRENO_DEVICE_STARTED = 5, + ADRENO_DEVICE_FAULT = 6, + ADRENO_DEVICE_DRAWOBJ_PROFILE = 7, + ADRENO_DEVICE_GPU_REGULATOR_ENABLED = 8, + ADRENO_DEVICE_PREEMPTION = 9, + ADRENO_DEVICE_SOFT_FAULT_DETECT = 10, + ADRENO_DEVICE_GPMU_INITIALIZED = 11, + ADRENO_DEVICE_ISDB_ENABLED = 12, + ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED = 13, + ADRENO_DEVICE_HARD_RESET = 14, + ADRENO_DEVICE_CORESIGHT_CX = 16, +}; + +/** + * struct adreno_drawobj_profile_entry - a single drawobj entry in the + * kernel profiling buffer + * @started: Number of GPU ticks at start of the drawobj + * @retired: Number of GPU ticks at the end of the drawobj + */ +struct adreno_drawobj_profile_entry { + uint64_t started; + uint64_t retired; +}; + +#define ADRENO_DRAWOBJ_PROFILE_COUNT \ + (PAGE_SIZE / sizeof(struct adreno_drawobj_profile_entry)) + +#define ADRENO_DRAWOBJ_PROFILE_OFFSET(_index, _member) \ + ((_index) * sizeof(struct adreno_drawobj_profile_entry) \ + + offsetof(struct adreno_drawobj_profile_entry, _member)) + + +/** + * adreno_regs: List of registers that are used in kgsl driver for all + * 3D devices. Each device type has different offset value for the same + * register, so an array of register offsets are declared for every device + * and are indexed by the enumeration values defined in this enum + */ +enum adreno_regs { + ADRENO_REG_CP_ME_RAM_WADDR, + ADRENO_REG_CP_ME_RAM_DATA, + ADRENO_REG_CP_PFP_UCODE_DATA, + ADRENO_REG_CP_PFP_UCODE_ADDR, + ADRENO_REG_CP_WFI_PEND_CTR, + ADRENO_REG_CP_RB_BASE, + ADRENO_REG_CP_RB_BASE_HI, + ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, + ADRENO_REG_CP_RB_RPTR, + ADRENO_REG_CP_RB_WPTR, + ADRENO_REG_CP_CNTL, + ADRENO_REG_CP_ME_CNTL, + ADRENO_REG_CP_RB_CNTL, + ADRENO_REG_CP_IB1_BASE, + ADRENO_REG_CP_IB1_BASE_HI, + ADRENO_REG_CP_IB1_BUFSZ, + ADRENO_REG_CP_IB2_BASE, + ADRENO_REG_CP_IB2_BASE_HI, + ADRENO_REG_CP_IB2_BUFSZ, + ADRENO_REG_CP_TIMESTAMP, + ADRENO_REG_CP_SCRATCH_REG6, + ADRENO_REG_CP_SCRATCH_REG7, + ADRENO_REG_CP_ME_RAM_RADDR, + ADRENO_REG_CP_ROQ_ADDR, + ADRENO_REG_CP_ROQ_DATA, + ADRENO_REG_CP_MERCIU_ADDR, + ADRENO_REG_CP_MERCIU_DATA, + ADRENO_REG_CP_MERCIU_DATA2, + ADRENO_REG_CP_MEQ_ADDR, + ADRENO_REG_CP_MEQ_DATA, + ADRENO_REG_CP_HW_FAULT, + ADRENO_REG_CP_PROTECT_STATUS, + ADRENO_REG_CP_PREEMPT, + ADRENO_REG_CP_PREEMPT_DEBUG, + ADRENO_REG_CP_PREEMPT_DISABLE, + ADRENO_REG_CP_PROTECT_REG_0, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI, + ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, + ADRENO_REG_RBBM_STATUS, + ADRENO_REG_RBBM_STATUS3, + ADRENO_REG_RBBM_PERFCTR_CTL, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD2, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD3, + ADRENO_REG_RBBM_PERFCTR_PWR_1_LO, + ADRENO_REG_RBBM_INT_0_MASK, + ADRENO_REG_RBBM_INT_0_STATUS, + ADRENO_REG_RBBM_PM_OVERRIDE2, + ADRENO_REG_RBBM_INT_CLEAR_CMD, + ADRENO_REG_RBBM_SW_RESET_CMD, + ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD, + ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD2, + ADRENO_REG_RBBM_CLOCK_CTL, + ADRENO_REG_VPC_DEBUG_RAM_SEL, + ADRENO_REG_VPC_DEBUG_RAM_READ, + ADRENO_REG_PA_SC_AA_CONFIG, + ADRENO_REG_SQ_GPR_MANAGEMENT, + ADRENO_REG_SQ_INST_STORE_MANAGEMENT, + ADRENO_REG_TP0_CHICKEN, + ADRENO_REG_RBBM_RBBM_CTL, + ADRENO_REG_UCHE_INVALIDATE0, + ADRENO_REG_UCHE_INVALIDATE1, + ADRENO_REG_RBBM_PERFCTR_RBBM_0_LO, + ADRENO_REG_RBBM_PERFCTR_RBBM_0_HI, + ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + ADRENO_REG_RBBM_SECVID_TRUST_CONTROL, + ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO, + ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI, + ADRENO_REG_RBBM_SECVID_TRUST_CONFIG, + ADRENO_REG_RBBM_SECVID_TSB_CONTROL, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_SIZE, + ADRENO_REG_RBBM_GPR0_CNTL, + ADRENO_REG_RBBM_VBIF_GX_RESET_STATUS, + ADRENO_REG_VBIF_XIN_HALT_CTRL0, + ADRENO_REG_VBIF_XIN_HALT_CTRL1, + ADRENO_REG_VBIF_VERSION, + ADRENO_REG_GBIF_HALT, + ADRENO_REG_GBIF_HALT_ACK, + ADRENO_REG_GMU_AO_AHB_FENCE_CTRL, + ADRENO_REG_GMU_AO_INTERRUPT_EN, + ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, + ADRENO_REG_GMU_AO_HOST_INTERRUPT_STATUS, + ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + ADRENO_REG_GMU_PWR_COL_KEEPALIVE, + ADRENO_REG_GMU_AHB_FENCE_STATUS, + ADRENO_REG_GMU_RPMH_POWER_STATE, + ADRENO_REG_GMU_HFI_CTRL_STATUS, + ADRENO_REG_GMU_HFI_VERSION_INFO, + ADRENO_REG_GMU_HFI_SFR_ADDR, + ADRENO_REG_GMU_GMU2HOST_INTR_CLR, + ADRENO_REG_GMU_GMU2HOST_INTR_INFO, + ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + ADRENO_REG_GMU_HOST2GMU_INTR_SET, + ADRENO_REG_GMU_HOST2GMU_INTR_CLR, + ADRENO_REG_GMU_HOST2GMU_INTR_RAW_INFO, + ADRENO_REG_GMU_NMI_CONTROL_STATUS, + ADRENO_REG_GMU_CM3_CFG, + ADRENO_REG_GMU_RBBM_INT_UNMASKED_STATUS, + ADRENO_REG_GPMU_POWER_COUNTER_ENABLE, + ADRENO_REG_REGISTER_MAX, +}; + +enum adreno_int_bits { + ADRENO_INT_RBBM_AHB_ERROR, + ADRENO_INT_BITS_MAX, +}; + +/** + * adreno_reg_offsets: Holds array of register offsets + * @offsets: Offset array of size defined by enum adreno_regs + * @offset_0: This is the index of the register in offset array whose value + * is 0. 0 is a valid register offset and during initialization of the + * offset array we need to know if an offset value is correctly defined to 0 + */ +struct adreno_reg_offsets { + unsigned int *const offsets; + enum adreno_regs offset_0; +}; + +#define ADRENO_REG_UNUSED 0xFFFFFFFF +#define ADRENO_REG_SKIP 0xFFFFFFFE +#define ADRENO_REG_DEFINE(_offset, _reg)[_offset] = _reg +#define ADRENO_INT_DEFINE(_offset, _val) ADRENO_REG_DEFINE(_offset, _val) + +/* + * struct adreno_vbif_data - Describes vbif register value pair + * @reg: Offset to vbif register + * @val: The value that should be programmed in the register at reg + */ +struct adreno_vbif_data { + unsigned int reg; + unsigned int val; +}; + +/* + * struct adreno_vbif_platform - Holds an array of vbif reg value pairs + * for a particular core + * @devfunc: Pointer to platform/core identification function + * @vbif: Array of reg value pairs for vbif registers + */ +struct adreno_vbif_platform { + int (*devfunc)(struct adreno_device *adreno_dev); + const struct adreno_vbif_data *vbif; +}; + +/* + * struct adreno_vbif_snapshot_registers - Holds an array of vbif registers + * listed for snapshot dump for a particular core + * @version: vbif version + * @mask: vbif revision mask + * @registers: vbif registers listed for snapshot dump + * @count: count of vbif registers listed for snapshot + */ +struct adreno_vbif_snapshot_registers { + const unsigned int version; + const unsigned int mask; + const unsigned int *registers; + const int count; +}; + +/** + * struct adreno_coresight_register - Definition for a coresight (tracebus) + * debug register + * @offset: Offset of the debug register in the KGSL mmio region + * @initial: Default value to write when coresight is enabled + * @value: Current shadow value of the register (to be reprogrammed after power + * collapse) + */ +struct adreno_coresight_register { + unsigned int offset; + unsigned int initial; + unsigned int value; +}; + +struct adreno_coresight_attr { + struct device_attribute attr; + struct adreno_coresight_register *reg; +}; + +ssize_t adreno_coresight_show_register(struct device *device, + struct device_attribute *attr, char *buf); + +ssize_t adreno_coresight_store_register(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size); + +#define ADRENO_CORESIGHT_ATTR(_attrname, _reg) \ + struct adreno_coresight_attr coresight_attr_##_attrname = { \ + __ATTR(_attrname, 0644, \ + adreno_coresight_show_register, \ + adreno_coresight_store_register), \ + (_reg), } + +/** + * struct adreno_coresight - GPU specific coresight definition + * @registers - Array of GPU specific registers to configure trace bus output + * @count - Number of registers in the array + * @groups - Pointer to an attribute list of control files + * @atid - The unique ATID value of the coresight device + */ +struct adreno_coresight { + struct adreno_coresight_register *registers; + unsigned int count; + const struct attribute_group **groups; + unsigned int atid; +}; + + +struct adreno_irq_funcs { + void (*func)(struct adreno_device *adreno_dev, int mask); +}; +#define ADRENO_IRQ_CALLBACK(_c) { .func = _c } + +struct adreno_irq { + unsigned int mask; + struct adreno_irq_funcs *funcs; +}; + +/* + * struct adreno_debugbus_block - Holds info about debug buses of a chip + * @block_id: Bus identifier + * @dwords: Number of dwords of data that this block holds + */ +struct adreno_debugbus_block { + unsigned int block_id; + unsigned int dwords; +}; + +/* + * struct adreno_snapshot_section_sizes - Structure holding the size of + * different sections dumped during device snapshot + * @cp_pfp: CP PFP data section size + * @cp_me: CP ME data section size + * @vpc_mem: VPC memory section size + * @cp_meq: CP MEQ size + * @shader_mem: Size of shader memory of 1 shader section + * @cp_merciu: CP MERCIU size + * @roq: ROQ size + */ +struct adreno_snapshot_sizes { + int cp_pfp; + int cp_me; + int vpc_mem; + int cp_meq; + int shader_mem; + int cp_merciu; + int roq; +}; + +/* + * struct adreno_snapshot_data - Holds data used in snapshot + * @sect_sizes: Has sections sizes + */ +struct adreno_snapshot_data { + struct adreno_snapshot_sizes *sect_sizes; +}; + +enum adreno_cp_marker_type { + IFPC_DISABLE, + IFPC_ENABLE, + IB1LIST_START, + IB1LIST_END, +}; + +struct adreno_gpudev { + /* + * These registers are in a different location on different devices, + * so define them in the structure and use them as variables. + */ + const struct adreno_reg_offsets *reg_offsets; + unsigned int *const int_bits; + const struct adreno_ft_perf_counters *ft_perf_counters; + unsigned int ft_perf_counters_count; + + struct adreno_perfcounters *perfcounters; + const struct adreno_invalid_countables *invalid_countables; + struct adreno_snapshot_data *snapshot_data; + + struct adreno_coresight *coresight[GPU_CORESIGHT_MAX]; + + struct adreno_irq *irq; + int num_prio_levels; + unsigned int vbif_xin_halt_ctrl0_mask; + unsigned int gbif_client_halt_mask; + unsigned int gbif_arb_halt_mask; + /* GPU specific function hooks */ + void (*irq_trace)(struct adreno_device *adreno_dev, + unsigned int status); + void (*snapshot)(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); + void (*platform_setup)(struct adreno_device *adreno_dev); + void (*init)(struct adreno_device *adreno_dev); + void (*remove)(struct adreno_device *adreno_dev); + int (*rb_start)(struct adreno_device *adreno_dev, + unsigned int start_type); + int (*microcode_read)(struct adreno_device *adreno_dev); + void (*perfcounter_init)(struct adreno_device *adreno_dev); + void (*perfcounter_close)(struct adreno_device *adreno_dev); + void (*start)(struct adreno_device *adreno_dev); + bool (*is_sptp_idle)(struct adreno_device *adreno_dev); + int (*regulator_enable)(struct adreno_device *adreno_dev); + void (*regulator_disable)(struct adreno_device *adreno_dev); + void (*pwrlevel_change_settings)(struct adreno_device *adreno_dev, + unsigned int prelevel, unsigned int postlevel, + bool post); + int64_t (*read_throttling_counters)(struct adreno_device *adreno_dev); + void (*count_throttles)(struct adreno_device *adreno_dev, + uint64_t adj); + int (*enable_pwr_counters)(struct adreno_device *adrneo_dev, + unsigned int counter); + unsigned int (*preemption_pre_ibsubmit)( + struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + unsigned int *cmds, + struct kgsl_context *context); + int (*preemption_yield_enable)(unsigned int *cmds); + unsigned int (*set_marker)(unsigned int *cmds, + enum adreno_cp_marker_type type); + unsigned int (*preemption_post_ibsubmit)( + struct adreno_device *adreno_dev, + unsigned int *cmds); + int (*preemption_init)(struct adreno_device *adreno_dev); + void (*preemption_schedule)(struct adreno_device *adreno_dev); + int (*preemption_context_init)(struct kgsl_context *context); + void (*preemption_context_destroy)(struct kgsl_context *context); + void (*enable_64bit)(struct adreno_device *adreno_dev); + void (*clk_set_options)(struct adreno_device *adreno_dev, + const char *name, struct clk *clk, bool on); + void (*llc_configure_gpu_scid)(struct adreno_device *adreno_dev); + void (*llc_configure_gpuhtw_scid)(struct adreno_device *adreno_dev); + void (*llc_enable_overrides)(struct adreno_device *adreno_dev); + void (*pre_reset)(struct adreno_device *adreno_dev); + void (*gpu_keepalive)(struct adreno_device *adreno_dev, + bool state); + bool (*hw_isidle)(struct adreno_device *adreno_dev); + const char *(*iommu_fault_block)(struct adreno_device *adreno_dev, + unsigned int fsynr1); + int (*reset)(struct kgsl_device *device, int fault); + int (*soft_reset)(struct adreno_device *adreno_dev); + bool (*sptprac_is_on)(struct adreno_device *adreno_dev); + unsigned int (*ccu_invalidate)(struct adreno_device *adreno_dev, + unsigned int *cmds); + int (*perfcounter_update)(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg, + bool update_reg); +}; + +/** + * enum kgsl_ft_policy_bits - KGSL fault tolerance policy bits + * @KGSL_FT_OFF: Disable fault detection (not used) + * @KGSL_FT_REPLAY: Replay the faulting command + * @KGSL_FT_SKIPIB: Skip the faulting indirect buffer + * @KGSL_FT_SKIPFRAME: Skip the frame containing the faulting IB + * @KGSL_FT_DISABLE: Tells the dispatcher to disable FT for the command obj + * @KGSL_FT_TEMP_DISABLE: Disables FT for all commands + * @KGSL_FT_THROTTLE: Disable the context if it faults too often + * @KGSL_FT_SKIPCMD: Skip the command containing the faulting IB + */ +enum kgsl_ft_policy_bits { + KGSL_FT_OFF = 0, + KGSL_FT_REPLAY = 1, + KGSL_FT_SKIPIB = 2, + KGSL_FT_SKIPFRAME = 3, + KGSL_FT_DISABLE = 4, + KGSL_FT_TEMP_DISABLE = 5, + KGSL_FT_THROTTLE = 6, + KGSL_FT_SKIPCMD = 7, + /* KGSL_FT_MAX_BITS is used to calculate the mask */ + KGSL_FT_MAX_BITS, + /* Internal bits - set during GFT */ + /* Skip the PM dump on replayed command obj's */ + KGSL_FT_SKIP_PMDUMP = 31, +}; + +#define KGSL_FT_POLICY_MASK GENMASK(KGSL_FT_MAX_BITS - 1, 0) + +#define KGSL_FT_DEFAULT_POLICY \ + (BIT(KGSL_FT_REPLAY) | \ + BIT(KGSL_FT_SKIPCMD) | \ + BIT(KGSL_FT_THROTTLE)) + +#define ADRENO_FT_TYPES \ + { BIT(KGSL_FT_OFF), "off" }, \ + { BIT(KGSL_FT_REPLAY), "replay" }, \ + { BIT(KGSL_FT_SKIPIB), "skipib" }, \ + { BIT(KGSL_FT_SKIPFRAME), "skipframe" }, \ + { BIT(KGSL_FT_DISABLE), "disable" }, \ + { BIT(KGSL_FT_TEMP_DISABLE), "temp" }, \ + { BIT(KGSL_FT_THROTTLE), "throttle"}, \ + { BIT(KGSL_FT_SKIPCMD), "skipcmd" } + +/** + * enum kgsl_ft_pagefault_policy_bits - KGSL pagefault policy bits + * @KGSL_FT_PAGEFAULT_INT_ENABLE: No longer used, but retained for compatibility + * @KGSL_FT_PAGEFAULT_GPUHALT_ENABLE: enable GPU halt on pagefaults + * @KGSL_FT_PAGEFAULT_LOG_ONE_PER_PAGE: log one pagefault per page + * @KGSL_FT_PAGEFAULT_LOG_ONE_PER_INT: log one pagefault per interrupt + */ +enum { + KGSL_FT_PAGEFAULT_INT_ENABLE = 0, + KGSL_FT_PAGEFAULT_GPUHALT_ENABLE = 1, + KGSL_FT_PAGEFAULT_LOG_ONE_PER_PAGE = 2, + KGSL_FT_PAGEFAULT_LOG_ONE_PER_INT = 3, + /* KGSL_FT_PAGEFAULT_MAX_BITS is used to calculate the mask */ + KGSL_FT_PAGEFAULT_MAX_BITS, +}; + +#define KGSL_FT_PAGEFAULT_MASK GENMASK(KGSL_FT_PAGEFAULT_MAX_BITS - 1, 0) + +#define KGSL_FT_PAGEFAULT_DEFAULT_POLICY 0 + +#define FOR_EACH_RINGBUFFER(_dev, _rb, _i) \ + for ((_i) = 0, (_rb) = &((_dev)->ringbuffers[0]); \ + (_i) < (_dev)->num_ringbuffers; \ + (_i)++, (_rb)++) + +struct adreno_ft_perf_counters { + unsigned int counter; + unsigned int countable; +}; + +extern unsigned int *adreno_ft_regs; +extern unsigned int adreno_ft_regs_num; +extern unsigned int *adreno_ft_regs_val; + +extern struct adreno_gpudev adreno_a3xx_gpudev; +extern struct adreno_gpudev adreno_a4xx_gpudev; +extern struct adreno_gpudev adreno_a5xx_gpudev; +extern struct adreno_gpudev adreno_a6xx_gpudev; + +extern int adreno_wake_nice; +extern unsigned int adreno_wake_timeout; + +int adreno_start(struct kgsl_device *device, int priority); +int adreno_soft_reset(struct kgsl_device *device); +long adreno_ioctl(struct kgsl_device_private *dev_priv, + unsigned int cmd, unsigned long arg); + +long adreno_ioctl_helper(struct kgsl_device_private *dev_priv, + unsigned int cmd, unsigned long arg, + const struct kgsl_ioctl *cmds, int len); + +int a5xx_critical_packet_submit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb); +int adreno_set_unsecured_mode(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb); +void adreno_spin_idle_debug(struct adreno_device *adreno_dev, const char *str); +int adreno_spin_idle(struct adreno_device *device, unsigned int timeout); +int adreno_idle(struct kgsl_device *device); +bool adreno_isidle(struct kgsl_device *device); + +int adreno_set_constraint(struct kgsl_device *device, + struct kgsl_context *context, + struct kgsl_device_constraint *constraint); + +void adreno_shadermem_regread(struct kgsl_device *device, + unsigned int offsetwords, + unsigned int *value); + +void adreno_snapshot(struct kgsl_device *device, + struct kgsl_snapshot *snapshot, + struct kgsl_context *context); + +int adreno_reset(struct kgsl_device *device, int fault); + +void adreno_fault_skipcmd_detached(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct kgsl_drawobj *drawobj); + +int adreno_coresight_init(struct adreno_device *adreno_dev); + +void adreno_coresight_start(struct adreno_device *adreno_dev); +void adreno_coresight_stop(struct adreno_device *adreno_dev); + +void adreno_coresight_remove(struct adreno_device *adreno_dev); + +bool adreno_hw_isidle(struct adreno_device *adreno_dev); + +void adreno_fault_detect_start(struct adreno_device *adreno_dev); +void adreno_fault_detect_stop(struct adreno_device *adreno_dev); + +void adreno_hang_int_callback(struct adreno_device *adreno_dev, int bit); +void adreno_cp_callback(struct adreno_device *adreno_dev, int bit); + +int adreno_sysfs_init(struct adreno_device *adreno_dev); +void adreno_sysfs_close(struct adreno_device *adreno_dev); + +void adreno_irqctrl(struct adreno_device *adreno_dev, int state); + +long adreno_ioctl_perfcounter_get(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); + +long adreno_ioctl_perfcounter_put(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); + +int adreno_efuse_map(struct adreno_device *adreno_dev); +int adreno_efuse_read_u32(struct adreno_device *adreno_dev, unsigned int offset, + unsigned int *val); +void adreno_efuse_unmap(struct adreno_device *adreno_dev); + +bool adreno_is_cx_dbgc_register(struct kgsl_device *device, + unsigned int offset); +void adreno_cx_dbgc_regread(struct kgsl_device *adreno_device, + unsigned int offsetwords, unsigned int *value); +void adreno_cx_dbgc_regwrite(struct kgsl_device *device, + unsigned int offsetwords, unsigned int value); + +#define ADRENO_TARGET(_name, _id) \ +static inline int adreno_is_##_name(struct adreno_device *adreno_dev) \ +{ \ + return (ADRENO_GPUREV(adreno_dev) == (_id)); \ +} + +static inline int adreno_is_a3xx(struct adreno_device *adreno_dev) +{ + return ((ADRENO_GPUREV(adreno_dev) >= 300) && + (ADRENO_GPUREV(adreno_dev) < 400)); +} + +ADRENO_TARGET(a304, ADRENO_REV_A304) +ADRENO_TARGET(a305, ADRENO_REV_A305) +ADRENO_TARGET(a305b, ADRENO_REV_A305B) +ADRENO_TARGET(a305c, ADRENO_REV_A305C) +ADRENO_TARGET(a306, ADRENO_REV_A306) +ADRENO_TARGET(a306a, ADRENO_REV_A306A) +ADRENO_TARGET(a310, ADRENO_REV_A310) +ADRENO_TARGET(a320, ADRENO_REV_A320) +ADRENO_TARGET(a330, ADRENO_REV_A330) + +static inline int adreno_is_a330v2(struct adreno_device *adreno_dev) +{ + return ((ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A330) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) > 0)); +} + +static inline int adreno_is_a330v21(struct adreno_device *adreno_dev) +{ + return ((ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A330) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) > 0xF)); +} + +static inline int adreno_is_a4xx(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 400 && + ADRENO_GPUREV(adreno_dev) < 500; +} + +ADRENO_TARGET(a405, ADRENO_REV_A405); + +static inline int adreno_is_a405v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A405) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 0x10); +} + +ADRENO_TARGET(a418, ADRENO_REV_A418) +ADRENO_TARGET(a420, ADRENO_REV_A420) +ADRENO_TARGET(a430, ADRENO_REV_A430) + +static inline int adreno_is_a430v2(struct adreno_device *adreno_dev) +{ + return ((ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A430) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1)); +} + +static inline int adreno_is_a5xx(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 500 && + ADRENO_GPUREV(adreno_dev) < 600; +} + +ADRENO_TARGET(a505, ADRENO_REV_A505) +ADRENO_TARGET(a506, ADRENO_REV_A506) +ADRENO_TARGET(a508, ADRENO_REV_A508) +ADRENO_TARGET(a510, ADRENO_REV_A510) +ADRENO_TARGET(a512, ADRENO_REV_A512) +ADRENO_TARGET(a530, ADRENO_REV_A530) +ADRENO_TARGET(a540, ADRENO_REV_A540) + +static inline int adreno_is_a530v1(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A530) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 0); +} + +static inline int adreno_is_a530v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A530) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +static inline int adreno_is_a530v3(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A530) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 2); +} + +static inline int adreno_is_a505_or_a506(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 505 && + ADRENO_GPUREV(adreno_dev) <= 506; +} + +static inline int adreno_is_a540v1(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A540) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 0); +} + +static inline int adreno_is_a540v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A540) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +static inline int adreno_is_a6xx(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 600 && + ADRENO_GPUREV(adreno_dev) < 700; +} + +ADRENO_TARGET(a608, ADRENO_REV_A608) +ADRENO_TARGET(a615, ADRENO_REV_A615) +ADRENO_TARGET(a616, ADRENO_REV_A616) +ADRENO_TARGET(a630, ADRENO_REV_A630) +ADRENO_TARGET(a640, ADRENO_REV_A640) +ADRENO_TARGET(a680, ADRENO_REV_A680) + +static inline int adreno_is_a630v1(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A630) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 0); +} + +static inline int adreno_is_a630v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A630) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +static inline int adreno_is_a640v1(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A640) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 0); +} + +static inline int adreno_is_a640v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A640) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +/* + * adreno_checkreg_off() - Checks the validity of a register enum + * @adreno_dev: Pointer to adreno device + * @offset_name: The register enum that is checked + */ +static inline bool adreno_checkreg_off(struct adreno_device *adreno_dev, + enum adreno_regs offset_name) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (offset_name >= ADRENO_REG_REGISTER_MAX || + gpudev->reg_offsets->offsets[offset_name] == ADRENO_REG_UNUSED) + return false; + + /* + * GPU register programming is kept common as much as possible + * across the cores, Use ADRENO_REG_SKIP when certain register + * programming needs to be skipped for certain GPU cores. + * Example: Certain registers on a5xx like IB1_BASE are 64 bit. + * Common programming programs 64bit register but upper 32 bits + * are skipped in a4xx and a3xx using ADRENO_REG_SKIP. + */ + if (gpudev->reg_offsets->offsets[offset_name] == ADRENO_REG_SKIP) + return false; + + return true; +} + +/* + * adreno_readreg() - Read a register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @offset_name: The register enum that is to be read + * @val: Register value read is placed here + */ +static inline void adreno_readreg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int *val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + kgsl_regread(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[offset_name], val); + else + *val = 0; +} + +/* + * adreno_writereg() - Write a register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @offset_name: The register enum that is to be written + * @val: Value to write + */ +static inline void adreno_writereg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + kgsl_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[offset_name], val); +} + +/* + * adreno_getreg() - Returns the offset value of a register from the + * register offset array in the gpudev node + * @adreno_dev: Pointer to the the adreno device + * @offset_name: The register enum whore offset is returned + */ +static inline unsigned int adreno_getreg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (!adreno_checkreg_off(adreno_dev, offset_name)) + return ADRENO_REG_REGISTER_MAX; + return gpudev->reg_offsets->offsets[offset_name]; +} + +/* + * adreno_read_gmureg() - Read a GMU register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @offset_name: The register enum that is to be read + * @val: Register value read is placed here + */ +static inline void adreno_read_gmureg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int *val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + gmu_core_regread(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[offset_name], val); + else + *val = 0; +} + +/* + * adreno_write_gmureg() - Write a GMU register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the the adreno device + * @offset_name: The register enum that is to be written + * @val: Value to write + */ +static inline void adreno_write_gmureg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int val) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets->offsets[offset_name], val); +} + +/* + * adreno_get_int() - Returns the offset value of an interrupt bit from + * the interrupt bit array in the gpudev node + * @adreno_dev: Pointer to the the adreno device + * @bit_name: The interrupt bit enum whose bit is returned + */ +static inline unsigned int adreno_get_int(struct adreno_device *adreno_dev, + enum adreno_int_bits bit_name) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (bit_name >= ADRENO_INT_BITS_MAX) + return -ERANGE; + + return gpudev->int_bits[bit_name]; +} + +/** + * adreno_gpu_fault() - Return the current state of the GPU + * @adreno_dev: A pointer to the adreno_device to query + * + * Return 0 if there is no fault or positive with the last type of fault that + * occurred + */ +static inline unsigned int adreno_gpu_fault(struct adreno_device *adreno_dev) +{ + /* make sure we're reading the latest value */ + smp_rmb(); + return atomic_read(&adreno_dev->dispatcher.fault); +} + +/** + * adreno_set_gpu_fault() - Set the current fault status of the GPU + * @adreno_dev: A pointer to the adreno_device to set + * @state: fault state to set + * + */ +static inline void adreno_set_gpu_fault(struct adreno_device *adreno_dev, + int state) +{ + /* only set the fault bit w/o overwriting other bits */ + atomic_add(state, &adreno_dev->dispatcher.fault); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +static inline bool adreno_gmu_gpu_fault(struct adreno_device *adreno_dev) +{ + return adreno_gpu_fault(adreno_dev) & ADRENO_GMU_FAULT; +} + +/** + * adreno_clear_gpu_fault() - Clear the GPU fault register + * @adreno_dev: A pointer to an adreno_device structure + * + * Clear the GPU fault status for the adreno device + */ + +static inline void adreno_clear_gpu_fault(struct adreno_device *adreno_dev) +{ + atomic_set(&adreno_dev->dispatcher.fault, 0); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +/** + * adreno_gpu_halt() - Return the GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline int adreno_gpu_halt(struct adreno_device *adreno_dev) +{ + /* make sure we're reading the latest value */ + smp_rmb(); + return atomic_read(&adreno_dev->halt); +} + + +/** + * adreno_clear_gpu_halt() - Clear the GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_clear_gpu_halt(struct adreno_device *adreno_dev) +{ + atomic_set(&adreno_dev->halt, 0); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +/** + * adreno_get_gpu_halt() - Increment GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_get_gpu_halt(struct adreno_device *adreno_dev) +{ + atomic_inc(&adreno_dev->halt); +} + +/** + * adreno_put_gpu_halt() - Decrement GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_put_gpu_halt(struct adreno_device *adreno_dev) +{ + /* Make sure the refcount is good */ + int ret = atomic_dec_if_positive(&adreno_dev->halt); + + WARN(ret < 0, "GPU halt refcount unbalanced\n"); +} + + +/* + * adreno_vbif_start() - Program VBIF registers, called in device start + * @adreno_dev: Pointer to device whose vbif data is to be programmed + * @vbif_platforms: list register value pair of vbif for a family + * of adreno cores + * @num_platforms: Number of platforms contained in vbif_platforms + */ +static inline void adreno_vbif_start(struct adreno_device *adreno_dev, + const struct adreno_vbif_platform *vbif_platforms, + int num_platforms) +{ + int i; + const struct adreno_vbif_data *vbif = NULL; + + for (i = 0; i < num_platforms; i++) { + if (vbif_platforms[i].devfunc(adreno_dev)) { + vbif = vbif_platforms[i].vbif; + break; + } + } + + while ((vbif != NULL) && (vbif->reg != 0)) { + kgsl_regwrite(KGSL_DEVICE(adreno_dev), vbif->reg, vbif->val); + vbif++; + } +} + +/** + * adreno_set_protected_registers() - Protect the specified range of registers + * from being accessed by the GPU + * @adreno_dev: pointer to the Adreno device + * @index: Pointer to the index of the protect mode register to write to + * @reg: Starting dword register to write + * @mask_len: Size of the mask to protect (# of registers = 2 ** mask_len) + * + * Add the range of registers to the list of protected mode registers that will + * cause an exception if the GPU accesses them. There are 16 available + * protected mode registers. Index is used to specify which register to write + * to - the intent is to call this function multiple times with the same index + * pointer for each range and the registers will be magically programmed in + * incremental fashion + */ +static inline void adreno_set_protected_registers( + struct adreno_device *adreno_dev, unsigned int *index, + unsigned int reg, int mask_len) +{ + unsigned int val; + unsigned int base = + adreno_getreg(adreno_dev, ADRENO_REG_CP_PROTECT_REG_0); + unsigned int offset = *index; + unsigned int max_slots = adreno_dev->gpucore->num_protected_regs ? + adreno_dev->gpucore->num_protected_regs : 16; + + /* Do we have a free slot? */ + if (WARN(*index >= max_slots, "Protected register slots full: %d/%d\n", + *index, max_slots)) + return; + + /* + * On A4XX targets with more than 16 protected mode registers + * the upper registers are not contiguous with the lower 16 + * registers so we have to adjust the base and offset accordingly + */ + + if (adreno_is_a4xx(adreno_dev) && *index >= 0x10) { + base = A4XX_CP_PROTECT_REG_10; + offset = *index - 0x10; + } + + val = 0x60000000 | ((mask_len & 0x1F) << 24) | ((reg << 2) & 0xFFFFF); + + kgsl_regwrite(KGSL_DEVICE(adreno_dev), base + offset, val); + *index = *index + 1; +} + +#ifdef CONFIG_DEBUG_FS +void adreno_debugfs_init(struct adreno_device *adreno_dev); +void adreno_context_debugfs_init(struct adreno_device *adreno_dev, + struct adreno_context *ctx); +#else +static inline void adreno_debugfs_init(struct adreno_device *adreno_dev) { } +static inline void adreno_context_debugfs_init(struct adreno_device *device, + struct adreno_context *context) + { } +#endif + +/** + * adreno_compare_pm4_version() - Compare the PM4 microcode version + * @adreno_dev: Pointer to the adreno_device struct + * @version: Version number to compare again + * + * Compare the current version against the specified version and return -1 if + * the current code is older, 0 if equal or 1 if newer. + */ +static inline int adreno_compare_pm4_version(struct adreno_device *adreno_dev, + unsigned int version) +{ + if (adreno_dev->fw[ADRENO_FW_PM4].version == version) + return 0; + + return (adreno_dev->fw[ADRENO_FW_PM4].version > version) ? 1 : -1; +} + +/** + * adreno_compare_pfp_version() - Compare the PFP microcode version + * @adreno_dev: Pointer to the adreno_device struct + * @version: Version number to compare against + * + * Compare the current version against the specified version and return -1 if + * the current code is older, 0 if equal or 1 if newer. + */ +static inline int adreno_compare_pfp_version(struct adreno_device *adreno_dev, + unsigned int version) +{ + if (adreno_dev->fw[ADRENO_FW_PFP].version == version) + return 0; + + return (adreno_dev->fw[ADRENO_FW_PFP].version > version) ? 1 : -1; +} + +/* + * adreno_bootstrap_ucode() - Checks if Ucode bootstrapping is supported + * @adreno_dev: Pointer to the the adreno device + */ +static inline int adreno_bootstrap_ucode(struct adreno_device *adreno_dev) +{ + return (ADRENO_FEATURE(adreno_dev, ADRENO_USE_BOOTSTRAP) && + adreno_compare_pfp_version(adreno_dev, + adreno_dev->gpucore->pfp_bstrp_ver) >= 0) ? 1 : 0; +} + +/** + * adreno_in_preempt_state() - Check if preemption state is equal to given state + * @adreno_dev: Device whose preemption state is checked + * @state: State to compare against + */ +static inline bool adreno_in_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states state) +{ + return atomic_read(&adreno_dev->preempt.state) == state; +} +/** + * adreno_set_preempt_state() - Set the specified preemption state + * @adreno_dev: Device to change preemption state + * @state: State to set + */ +static inline void adreno_set_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states state) +{ + /* + * atomic_set doesn't use barriers, so we need to do it ourselves. One + * before... + */ + smp_wmb(); + atomic_set(&adreno_dev->preempt.state, state); + + /* ... and one after */ + smp_wmb(); +} + +static inline bool adreno_is_preemption_enabled( + struct adreno_device *adreno_dev) +{ + return test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); +} +/** + * adreno_ctx_get_rb() - Return the ringbuffer that a context should + * use based on priority + * @adreno_dev: The adreno device that context is using + * @drawctxt: The context pointer + */ +static inline struct adreno_ringbuffer *adreno_ctx_get_rb( + struct adreno_device *adreno_dev, + struct adreno_context *drawctxt) +{ + struct kgsl_context *context; + int level; + + if (!drawctxt) + return NULL; + + context = &(drawctxt->base); + + /* + * If preemption is disabled then everybody needs to go on the same + * ringbuffer + */ + + if (!adreno_is_preemption_enabled(adreno_dev)) + return &(adreno_dev->ringbuffers[0]); + + /* + * Math to convert the priority field in context structure to an RB ID. + * Divide up the context priority based on number of ringbuffer levels. + */ + level = context->priority / adreno_dev->num_ringbuffers; + if (level < adreno_dev->num_ringbuffers) + return &(adreno_dev->ringbuffers[level]); + else + return &(adreno_dev->ringbuffers[ + adreno_dev->num_ringbuffers - 1]); +} + +/* + * adreno_compare_prio_level() - Compares 2 priority levels based on enum values + * @p1: First priority level + * @p2: Second priority level + * + * Returns greater than 0 if p1 is higher priority, 0 if levels are equal else + * less than 0 + */ +static inline int adreno_compare_prio_level(int p1, int p2) +{ + return p2 - p1; +} + +void adreno_readreg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t *val); + +void adreno_writereg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t val); + +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb); + +static inline bool adreno_rb_empty(struct adreno_ringbuffer *rb) +{ + return (adreno_get_rptr(rb) == rb->wptr); +} + +static inline bool adreno_soft_fault_detect(struct adreno_device *adreno_dev) +{ + return adreno_dev->fast_hang_detect && + !test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); +} + +static inline bool adreno_long_ib_detect(struct adreno_device *adreno_dev) +{ + return adreno_dev->long_ib_detect && + !test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); +} + +/* + * adreno_support_64bit() - Check the feature flag only if it is in + * 64bit kernel otherwise return false + * adreno_dev: The adreno device + */ +#if BITS_PER_LONG == 64 +static inline bool adreno_support_64bit(struct adreno_device *adreno_dev) +{ + return ADRENO_FEATURE(adreno_dev, ADRENO_64BIT); +} +#else +static inline bool adreno_support_64bit(struct adreno_device *adreno_dev) +{ + return false; +} +#endif /*BITS_PER_LONG*/ + +static inline void adreno_ringbuffer_set_global( + struct adreno_device *adreno_dev, int name) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_sharedmem_writel(device, + &adreno_dev->ringbuffers[0].pagetable_desc, + PT_INFO_OFFSET(current_global_ptname), name); +} + +static inline void adreno_ringbuffer_set_pagetable(struct adreno_ringbuffer *rb, + struct kgsl_pagetable *pt) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned long flags; + + spin_lock_irqsave(&rb->preempt_lock, flags); + + kgsl_sharedmem_writel(device, &rb->pagetable_desc, + PT_INFO_OFFSET(current_rb_ptname), pt->name); + + kgsl_sharedmem_writeq(device, &rb->pagetable_desc, + PT_INFO_OFFSET(ttbr0), kgsl_mmu_pagetable_get_ttbr0(pt)); + + kgsl_sharedmem_writel(device, &rb->pagetable_desc, + PT_INFO_OFFSET(contextidr), + kgsl_mmu_pagetable_get_contextidr(pt)); + + spin_unlock_irqrestore(&rb->preempt_lock, flags); +} + +static inline bool is_power_counter_overflow(struct adreno_device *adreno_dev, + unsigned int reg, unsigned int prev_val, unsigned int *perfctr_pwr_hi) +{ + unsigned int val; + bool ret = false; + + /* + * If prev_val is zero, it is first read after perf counter reset. + * So set perfctr_pwr_hi register to zero. + */ + if (prev_val == 0) { + *perfctr_pwr_hi = 0; + return ret; + } + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_PERFCTR_RBBM_0_HI, &val); + if (val != *perfctr_pwr_hi) { + *perfctr_pwr_hi = val; + ret = true; + } + return ret; +} + +static inline unsigned int counter_delta(struct kgsl_device *device, + unsigned int reg, unsigned int *counter) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int val; + unsigned int ret = 0; + bool overflow = true; + static unsigned int perfctr_pwr_hi; + + /* Read the value */ + kgsl_regread(device, reg, &val); + + if (adreno_is_a5xx(adreno_dev) && reg == adreno_getreg + (adreno_dev, ADRENO_REG_RBBM_PERFCTR_RBBM_0_LO)) + overflow = is_power_counter_overflow(adreno_dev, reg, + *counter, &perfctr_pwr_hi); + + /* Return 0 for the first read */ + if (*counter != 0) { + if (val >= *counter) { + ret = val - *counter; + } else if (overflow) { + ret = (0xFFFFFFFF - *counter) + val; + } else { + /* + * Since KGSL got abnormal value from the counter, + * We will drop the value from being accumulated. + */ + dev_warn_once(device->dev, + "Abnormal value :0x%x (0x%x) from perf counter : 0x%x\n", + val, *counter, reg); + return 0; + } + } + + *counter = val; + return ret; +} + +static inline int adreno_perfcntr_active_oob_get(struct kgsl_device *device) +{ + int ret = kgsl_active_count_get(device); + + if (!ret) { + ret = gmu_core_dev_oob_set(device, oob_perfcntr); + if (ret) { + adreno_set_gpu_fault(ADRENO_DEVICE(device), + ADRENO_GMU_FAULT); + adreno_dispatcher_schedule(device); + kgsl_active_count_put(device); + } + } + + return ret; +} + +static inline void adreno_perfcntr_active_oob_put(struct kgsl_device *device) +{ + gmu_core_dev_oob_clear(device, oob_perfcntr); + kgsl_active_count_put(device); +} + +static inline bool adreno_has_sptprac_gdsc(struct adreno_device *adreno_dev) +{ + if (adreno_is_a615(adreno_dev) || adreno_is_a630(adreno_dev) || + adreno_is_a616(adreno_dev)) + return true; + else + return false; +} + +static inline bool adreno_has_gbif(struct adreno_device *adreno_dev) +{ + if (!adreno_is_a6xx(adreno_dev) || adreno_is_a630(adreno_dev)) + return false; + else + return true; +} + +/** + * adreno_wait_for_halt_ack() - wait for GBIF/VBIF acknowledgment + * for given HALT request. + * @ack_reg: register offset to wait for acknowledge + */ +static inline int adreno_wait_for_halt_ack(struct kgsl_device *device, + int ack_reg, unsigned int mask) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned long wait_for_vbif; + unsigned int val; + int ret = 0; + + /* wait for the transactions to clear */ + wait_for_vbif = jiffies + msecs_to_jiffies(VBIF_RESET_ACK_TIMEOUT); + while (1) { + adreno_readreg(adreno_dev, ack_reg, + &val); + if ((val & mask) == mask) + break; + if (time_after(jiffies, wait_for_vbif)) { + dev_err(device->dev, + "Wait limit reached for GBIF/VBIF Halt\n"); + ret = -ETIMEDOUT; + break; + } + } + + return ret; +} + +static inline void adreno_deassert_gbif_halt(struct adreno_device *adreno_dev) +{ + if (adreno_has_gbif(adreno_dev)) + adreno_writereg(adreno_dev, ADRENO_REG_GBIF_HALT, 0x0); +} +void adreno_gmu_clear_and_unmask_irqs(struct adreno_device *adreno_dev); +void adreno_gmu_mask_and_clear_irqs(struct adreno_device *adreno_dev); +int adreno_gmu_fenced_write(struct adreno_device *adreno_dev, + enum adreno_regs offset, unsigned int val, + unsigned int fence_mask); +int adreno_clear_pending_transactions(struct kgsl_device *device); +#endif /*__ADRENO_H */ diff --git a/drivers/gpu/msm/adreno_a3xx.c b/drivers/gpu/msm/adreno_a3xx.c new file mode 100644 index 0000000000000000000000000000000000000000..d440c8cde8ff94b844a1666d33677f26d89041c4 --- /dev/null +++ b/drivers/gpu/msm/adreno_a3xx.c @@ -0,0 +1,1952 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "kgsl.h" +#include "adreno.h" +#include "kgsl_sharedmem.h" +#include "a3xx_reg.h" +#include "adreno_a3xx.h" +#include "adreno_a4xx.h" +#include "a4xx_reg.h" +#include "adreno_cp_parser.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" +#include "adreno_perfcounter.h" +#include "adreno_snapshot.h" + +/* + * Define registers for a3xx that contain addresses used by the + * cp parser logic + */ +const unsigned int a3xx_cp_addr_regs[ADRENO_CP_ADDR_MAX] = { + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_0, + A3XX_VSC_PIPE_DATA_ADDRESS_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_0, + A3XX_VSC_PIPE_DATA_LENGTH_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_1, + A3XX_VSC_PIPE_DATA_ADDRESS_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_1, + A3XX_VSC_PIPE_DATA_LENGTH_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_2, + A3XX_VSC_PIPE_DATA_ADDRESS_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_2, + A3XX_VSC_PIPE_DATA_LENGTH_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_3, + A3XX_VSC_PIPE_DATA_ADDRESS_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_3, + A3XX_VSC_PIPE_DATA_LENGTH_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_4, + A3XX_VSC_PIPE_DATA_ADDRESS_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_4, + A3XX_VSC_PIPE_DATA_LENGTH_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_5, + A3XX_VSC_PIPE_DATA_ADDRESS_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_5, + A3XX_VSC_PIPE_DATA_LENGTH_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_6, + A3XX_VSC_PIPE_DATA_ADDRESS_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_6, + A3XX_VSC_PIPE_DATA_LENGTH_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_7, + A3XX_VSC_PIPE_DATA_ADDRESS_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_7, + A3XX_VSC_PIPE_DATA_LENGTH_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_0, + A3XX_VFD_FETCH_INSTR_1_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_1, + A3XX_VFD_FETCH_INSTR_1_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_2, + A3XX_VFD_FETCH_INSTR_1_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_3, + A3XX_VFD_FETCH_INSTR_1_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_4, + A3XX_VFD_FETCH_INSTR_1_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_5, + A3XX_VFD_FETCH_INSTR_1_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_6, + A3XX_VFD_FETCH_INSTR_1_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_7, + A3XX_VFD_FETCH_INSTR_1_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_8, + A3XX_VFD_FETCH_INSTR_1_8), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_9, + A3XX_VFD_FETCH_INSTR_1_9), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_10, + A3XX_VFD_FETCH_INSTR_1_A), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_11, + A3XX_VFD_FETCH_INSTR_1_B), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_12, + A3XX_VFD_FETCH_INSTR_1_C), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_13, + A3XX_VFD_FETCH_INSTR_1_D), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_14, + A3XX_VFD_FETCH_INSTR_1_E), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_15, + A3XX_VFD_FETCH_INSTR_1_F), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_SIZE_ADDRESS, + A3XX_VSC_SIZE_ADDRESS), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_PVT_MEM_ADDR, + A3XX_SP_VS_PVT_MEM_ADDR_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_PVT_MEM_ADDR, + A3XX_SP_FS_PVT_MEM_ADDR_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_OBJ_START_REG, + A3XX_SP_VS_OBJ_START_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_OBJ_START_REG, + A3XX_SP_FS_OBJ_START_REG), +}; + +static unsigned int adreno_a3xx_rbbm_clock_ctl_default(struct adreno_device + *adreno_dev) +{ + if (adreno_is_a320(adreno_dev)) + return A320_RBBM_CLOCK_CTL_DEFAULT; + else if (adreno_is_a330v2(adreno_dev)) + return A3XX_RBBM_CLOCK_CTL_DEFAULT; + else if (adreno_is_a330(adreno_dev)) + return A330_RBBM_CLOCK_CTL_DEFAULT; + return A3XX_RBBM_CLOCK_CTL_DEFAULT; +} + +static const unsigned int _a3xx_pwron_fixup_fs_instructions[] = { + 0x00000000, 0x302CC300, 0x00000000, 0x302CC304, + 0x00000000, 0x302CC308, 0x00000000, 0x302CC30C, + 0x00000000, 0x302CC310, 0x00000000, 0x302CC314, + 0x00000000, 0x302CC318, 0x00000000, 0x302CC31C, + 0x00000000, 0x302CC320, 0x00000000, 0x302CC324, + 0x00000000, 0x302CC328, 0x00000000, 0x302CC32C, + 0x00000000, 0x302CC330, 0x00000000, 0x302CC334, + 0x00000000, 0x302CC338, 0x00000000, 0x302CC33C, + 0x00000000, 0x00000400, 0x00020000, 0x63808003, + 0x00060004, 0x63828007, 0x000A0008, 0x6384800B, + 0x000E000C, 0x6386800F, 0x00120010, 0x63888013, + 0x00160014, 0x638A8017, 0x001A0018, 0x638C801B, + 0x001E001C, 0x638E801F, 0x00220020, 0x63908023, + 0x00260024, 0x63928027, 0x002A0028, 0x6394802B, + 0x002E002C, 0x6396802F, 0x00320030, 0x63988033, + 0x00360034, 0x639A8037, 0x003A0038, 0x639C803B, + 0x003E003C, 0x639E803F, 0x00000000, 0x00000400, + 0x00000003, 0x80D60003, 0x00000007, 0x80D60007, + 0x0000000B, 0x80D6000B, 0x0000000F, 0x80D6000F, + 0x00000013, 0x80D60013, 0x00000017, 0x80D60017, + 0x0000001B, 0x80D6001B, 0x0000001F, 0x80D6001F, + 0x00000023, 0x80D60023, 0x00000027, 0x80D60027, + 0x0000002B, 0x80D6002B, 0x0000002F, 0x80D6002F, + 0x00000033, 0x80D60033, 0x00000037, 0x80D60037, + 0x0000003B, 0x80D6003B, 0x0000003F, 0x80D6003F, + 0x00000000, 0x03000000, 0x00000000, 0x00000000, +}; + +static void a3xx_efuse_speed_bin(struct adreno_device *adreno_dev) +{ + unsigned int val; + unsigned int speed_bin[3]; + struct kgsl_device *device = &adreno_dev->dev; + + if (of_property_read_u32_array(device->pdev->dev.of_node, + "qcom,gpu-speed-bin", speed_bin, 3)) + return; + + adreno_efuse_read_u32(adreno_dev, speed_bin[0], &val); + + adreno_dev->speed_bin = (val & speed_bin[1]) >> speed_bin[2]; +} + +static const struct { + int (*check)(struct adreno_device *adreno_dev); + void (*func)(struct adreno_device *adreno_dev); +} a3xx_efuse_funcs[] = { + { adreno_is_a306a, a3xx_efuse_speed_bin }, +}; + +static void a3xx_check_features(struct adreno_device *adreno_dev) +{ + unsigned int i; + + if (adreno_efuse_map(adreno_dev)) + return; + + for (i = 0; i < ARRAY_SIZE(a3xx_efuse_funcs); i++) { + if (a3xx_efuse_funcs[i].check(adreno_dev)) + a3xx_efuse_funcs[i].func(adreno_dev); + } + + adreno_efuse_unmap(adreno_dev); +} + +/** + * _a3xx_pwron_fixup() - Initialize a special command buffer to run a + * post-power collapse shader workaround + * @adreno_dev: Pointer to a adreno_device struct + * + * Some targets require a special workaround shader to be executed after + * power-collapse. Construct the IB once at init time and keep it + * handy + * + * Returns: 0 on success or negative on error + */ +static int _a3xx_pwron_fixup(struct adreno_device *adreno_dev) +{ + unsigned int *cmds; + int count = ARRAY_SIZE(_a3xx_pwron_fixup_fs_instructions); + int ret; + + /* Return if the fixup is already in place */ + if (test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv)) + return 0; + + ret = kgsl_allocate_global(KGSL_DEVICE(adreno_dev), + &adreno_dev->pwron_fixup, PAGE_SIZE, + KGSL_MEMFLAGS_GPUREADONLY, 0, "pwron_fixup"); + + if (ret) + return ret; + + cmds = adreno_dev->pwron_fixup.hostptr; + + *cmds++ = cp_type0_packet(A3XX_UCHE_CACHE_INVALIDATE0_REG, 2); + *cmds++ = 0x00000000; + *cmds++ = 0x90000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_REG_RMW, 3); + *cmds++ = A3XX_RBBM_CLOCK_CTL; + *cmds++ = 0xFFFCFFFF; + *cmds++ = 0x00010000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_SET_CONSTANT, 2); + *cmds++ = CP_REG(A3XX_HLSQ_CONTROL_0_REG); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_1_REG, 1); + *cmds++ = 0x00000040; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_2_REG, 1); + *cmds++ = 0x80000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_3_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_VS_CONTROL_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_FS_CONTROL_REG, 1); + *cmds++ = 0x0D001002; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONST_VSPRESV_RANGE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONST_FSPRESV_RANGE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_0_REG, 1); + *cmds++ = 0x00401101; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_1_REG, 1); + *cmds++ = 0x00000400; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_2_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_3_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_4_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_5_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_6_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_0_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_1_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_CONST_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_X_REG, 1); + *cmds++ = 0x00000010; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_WG_OFFSET_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_SP_CTRL_REG, 1); + *cmds++ = 0x00040000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_CTRL_REG0, 1); + *cmds++ = 0x0000000A; + *cmds++ = cp_type0_packet(A3XX_SP_VS_CTRL_REG1, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PARAM_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_6, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_7, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OBJ_OFFSET_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OBJ_START_REG, 1); + *cmds++ = 0x00000004; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_PARAM_REG, 1); + *cmds++ = 0x04008001; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_ADDR_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_SIZE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_LENGTH_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_CTRL_REG0, 1); + *cmds++ = 0x0DB0400A; + *cmds++ = cp_type0_packet(A3XX_SP_FS_CTRL_REG1, 1); + *cmds++ = 0x00300402; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OBJ_OFFSET_REG, 1); + *cmds++ = 0x00010000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OBJ_START_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_PARAM_REG, 1); + *cmds++ = 0x04008001; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_ADDR_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_SIZE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_FLAT_SHAD_MODE_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_FLAT_SHAD_MODE_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OUTPUT_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_LENGTH_REG, 1); + *cmds++ = 0x0000000D; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_CLIP_CNTL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_GB_CLIP_ADJ, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_XOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_XSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_YOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_YSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_ZOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_ZSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POINT_MINMAX, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POINT_SIZE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POLY_OFFSET_OFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POLY_OFFSET_SCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_MODE_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_SCREEN_SCISSOR_TL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_SCREEN_SCISSOR_BR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_WINDOW_SCISSOR_BR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_WINDOW_SCISSOR_TL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_TSE_DEBUG_ECO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER0_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER1_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER2_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER3_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MODE_CONTROL, 1); + *cmds++ = 0x00008000; + *cmds++ = cp_type0_packet(A3XX_RB_RENDER_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MSAA_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_ALPHA_REFERENCE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_RED, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_GREEN, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_BLUE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_ALPHA, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_BASE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_CLEAR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_BUF_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_BUF_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_CLEAR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_BUF_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_BUF_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_REF_MASK, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_REF_MASK_BF, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_LRZ_VSC_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_WINDOW_OFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_SAMPLE_COUNT_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_SAMPLE_COUNT_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_Z_CLAMP_MIN, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_Z_CLAMP_MAX, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_GMEM_BASE_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEBUG_ECO_CONTROLS_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_PERFCOUNTER0_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_PERFCOUNTER1_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_FRAME_BUFFER_DIMENSION, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (1 << CP_LOADSTATE_DSTOFFSET_SHIFT) | + (0 << CP_LOADSTATE_STATESRC_SHIFT) | + (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT) | + (0 << CP_LOADSTATE_EXTSRCADDR_SHIFT); + *cmds++ = 0x00400000; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (2 << CP_LOADSTATE_DSTOFFSET_SHIFT) | + (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT); + *cmds++ = 0x00400220; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT); + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 2 + count); + *cmds++ = (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (13 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = 0x00000000; + + memcpy(cmds, _a3xx_pwron_fixup_fs_instructions, count << 2); + + cmds += count; + + *cmds++ = cp_type3_packet(CP_EXEC_CL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_0_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_SET_CONSTANT, 2); + *cmds++ = CP_REG(A3XX_HLSQ_CONTROL_0_REG); + *cmds++ = 0x1E000050; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_REG_RMW, 3); + *cmds++ = A3XX_RBBM_CLOCK_CTL; + *cmds++ = 0xFFFCFFFF; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + + /* + * Remember the number of dwords in the command buffer for when we + * program the indirect buffer call in the ringbuffer + */ + adreno_dev->pwron_fixup_dwords = + (cmds - (unsigned int *) adreno_dev->pwron_fixup.hostptr); + + /* Mark the flag in ->priv to show that we have the fix */ + set_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv); + return 0; +} + +static void a3xx_platform_setup(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev; + + if (adreno_is_a306(adreno_dev) || adreno_is_a306a(adreno_dev) + || adreno_is_a304(adreno_dev)) { + gpudev = ADRENO_GPU_DEVICE(adreno_dev); + gpudev->vbif_xin_halt_ctrl0_mask = + A30X_VBIF_XIN_HALT_CTRL0_MASK; + } + + /* Check efuse bits for various capabilties */ + a3xx_check_features(adreno_dev); +} + +static int a3xx_send_me_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 18); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + *cmds++ = cp_type3_packet(CP_ME_INIT, 17); + + *cmds++ = 0x000003f7; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000080; + *cmds++ = 0x00000100; + *cmds++ = 0x00000180; + *cmds++ = 0x00006600; + *cmds++ = 0x00000150; + *cmds++ = 0x0000014e; + *cmds++ = 0x00000154; + *cmds++ = 0x00000001; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + /* Enable protected mode registers for A3XX/A4XX */ + *cmds++ = 0x20000000; + + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + dev_err(device->dev, "CP initialization failed to idle\n"); + kgsl_device_snapshot(device, NULL, false); + } + + return ret; +} + +static int a3xx_rb_start(struct adreno_device *adreno_dev, + unsigned int start_type) +{ + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + int ret; + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + * Also disable the host RPTR shadow register as it might be unreliable + * in certain circumstances. + */ + + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) | + (1 << 27)); + + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_BASE, + rb->buffer_desc.gpuaddr); + + ret = a3xx_microcode_load(adreno_dev, start_type); + if (ret == 0) { + /* clear ME_HALT to start micro engine */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0); + + ret = a3xx_send_me_init(adreno_dev, rb); + } + + return ret; +} + +/* + * a3xx_init() - Initialize gpu specific data + * @adreno_dev: Pointer to adreno device + */ +static void a3xx_init(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + _a3xx_pwron_fixup(adreno_dev); + + /* Adjust snapshot section sizes according to core */ + if ((adreno_is_a330(adreno_dev) || adreno_is_a305b(adreno_dev))) { + gpudev->snapshot_data->sect_sizes->cp_pfp = + A320_SNAPSHOT_CP_STATE_SECTION_SIZE; + gpudev->snapshot_data->sect_sizes->roq = + A320_SNAPSHOT_ROQ_SECTION_SIZE; + gpudev->snapshot_data->sect_sizes->cp_merciu = + A320_SNAPSHOT_CP_MERCIU_SECTION_SIZE; + } +} + +/* + * a3xx_err_callback() - Call back for a3xx error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +static void a3xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + switch (bit) { + case A3XX_INT_RBBM_AHB_ERROR: { + kgsl_regread(device, A3XX_RBBM_AHB_ERROR_STATUS, ®); + + /* + * Return the word address of the erroring register so that it + * matches the register specification + */ + dev_crit_ratelimited(device->dev, + "RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n", + reg & (1 << 28) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2, + (reg >> 20) & 0x3, + (reg >> 24) & 0xF); + + /* Clear the error */ + kgsl_regwrite(device, A3XX_RBBM_AHB_CMD, (1 << 3)); + break; + } + case A3XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus oveflow\n"); + break; + case A3XX_INT_CP_T0_PACKET_IN_IB: + dev_crit_ratelimited(device->dev, + "ringbuffer TO packet in IB interrupt\n"); + break; + case A3XX_INT_CP_OPCODE_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer opcode error interrupt\n"); + break; + case A3XX_INT_CP_RESERVED_BIT_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer reserved bit error interrupt\n"); + break; + case A3XX_INT_CP_HW_FAULT: + kgsl_regread(device, A3XX_CP_HW_FAULT, ®); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", + reg); + break; + case A3XX_INT_CP_REG_PROTECT_FAULT: + kgsl_regread(device, A3XX_CP_PROTECT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error| %s | addr=%x\n", + reg & (1 << 24) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2); + break; + case A3XX_INT_CP_AHB_ERROR_HALT: + dev_crit_ratelimited(device->dev, + "ringbuffer AHB error interrupt\n"); + break; + case A3XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt\n"); + } +} + +#define A3XX_INT_MASK \ + ((1 << A3XX_INT_RBBM_AHB_ERROR) | \ + (1 << A3XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A3XX_INT_CP_T0_PACKET_IN_IB) | \ + (1 << A3XX_INT_CP_OPCODE_ERROR) | \ + (1 << A3XX_INT_CP_RESERVED_BIT_ERROR) | \ + (1 << A3XX_INT_CP_HW_FAULT) | \ + (1 << A3XX_INT_CP_IB1_INT) | \ + (1 << A3XX_INT_CP_IB2_INT) | \ + (1 << A3XX_INT_CP_RB_INT) | \ + (1 << A3XX_INT_CACHE_FLUSH_TS) | \ + (1 << A3XX_INT_CP_REG_PROTECT_FAULT) | \ + (1 << A3XX_INT_CP_AHB_ERROR_HALT) | \ + (1 << A3XX_INT_UCHE_OOB_ACCESS)) + +static struct adreno_irq_funcs a3xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 2 - RBBM_REG_TIMEOUT */ + ADRENO_IRQ_CALLBACK(NULL), /* 3 - RBBM_ME_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(NULL), /* 4 - RBBM_PFP_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 5 - RBBM_ATB_BUS_OVERFLOW */ + ADRENO_IRQ_CALLBACK(NULL), /* 6 - RBBM_VFD_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 7 - CP_SW */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 8 - CP_T0_PACKET_IN_IB */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 9 - CP_OPCODE_ERROR */ + /* 10 - CP_RESERVED_BIT_ERROR */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 11 - CP_HW_FAULT */ + ADRENO_IRQ_CALLBACK(NULL), /* 12 - CP_DMA */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 15 - CP_RB_INT */ + /* 16 - CP_REG_PROTECT_FAULT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_VS_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - CP_PS_DONE_TS */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + /* 21 - CP_AHB_ERROR_FAULT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 22 - Unused */ + ADRENO_IRQ_CALLBACK(NULL), /* 23 - Unused */ + /* 24 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 25 - UCHE_OOB_ACCESS */ +}; + +static struct adreno_irq a3xx_irq = { + .funcs = a3xx_irq_funcs, + .mask = A3XX_INT_MASK, +}; + +/* VBIF registers start after 0x3000 so use 0x0 as end of list marker */ +static const struct adreno_vbif_data a304_vbif[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a305_vbif[] = { + /* Set up 16 deep read/write request queues */ + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_IN_RD_LIM_CONF1, 0x10101010 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303 }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_IN_WR_LIM_CONF1, 0x10101010 }, + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x0000FF }, + /* Set up round robin arbitration between both AXI ports */ + { A3XX_VBIF_ARB_CTL, 0x00000030 }, + /* Set up AOOO */ + { A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003C }, + { A3XX_VBIF_OUT_AXI_AOOO, 0x003C003C }, + {0, 0}, +}; + +static const struct adreno_vbif_data a305b_vbif[] = { + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x00181818 }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x00181818 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000018 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000018 }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x00000303 }, + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a305c_vbif[] = { + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x00101010 }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x00101010 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000010 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000010 }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x00000101 }, + { A3XX_VBIF_ARB_CTL, 0x00000010 }, + /* Set up AOOO */ + { A3XX_VBIF_OUT_AXI_AOOO_EN, 0x00000007 }, + { A3XX_VBIF_OUT_AXI_AOOO, 0x00070007 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a306_vbif[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x0000000A }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x0000000A }, + {0, 0}, +}; + +static const struct adreno_vbif_data a306a_vbif[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000010 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000010 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a310_vbif[] = { + { A3XX_VBIF_ABIT_SORT, 0x0001000F }, + { A3XX_VBIF_ABIT_SORT_CONF, 0x000000A4 }, + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001 }, + /* Set up VBIF_ROUND_ROBIN_QOS_ARB */ + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x3 }, + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x18180C0C }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x1818000C }, + {0, 0}, +}; + +static const struct adreno_vbif_data a320_vbif[] = { + /* Set up 16 deep read/write request queues */ + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_IN_RD_LIM_CONF1, 0x10101010 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303 }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x10101010 }, + { A3XX_VBIF_IN_WR_LIM_CONF1, 0x10101010 }, + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x0000FF }, + /* Set up round robin arbitration between both AXI ports */ + { A3XX_VBIF_ARB_CTL, 0x00000030 }, + /* Set up AOOO */ + { A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003C }, + { A3XX_VBIF_OUT_AXI_AOOO, 0x003C003C }, + /* Enable 1K sort */ + { A3XX_VBIF_ABIT_SORT, 0x000000FF }, + { A3XX_VBIF_ABIT_SORT_CONF, 0x000000A4 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a330_vbif[] = { + /* Set up 16 deep read/write request queues */ + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x18181818 }, + { A3XX_VBIF_IN_RD_LIM_CONF1, 0x00001818 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00001818 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00001818 }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303 }, + { A3XX_VBIF_IN_WR_LIM_CONF0, 0x18181818 }, + { A3XX_VBIF_IN_WR_LIM_CONF1, 0x00001818 }, + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x00003F }, + /* Set up round robin arbitration between both AXI ports */ + { A3XX_VBIF_ARB_CTL, 0x00000030 }, + /* Set up VBIF_ROUND_ROBIN_QOS_ARB */ + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0001 }, + /* Set up AOOO */ + { A3XX_VBIF_OUT_AXI_AOOO_EN, 0x0000003F }, + { A3XX_VBIF_OUT_AXI_AOOO, 0x003F003F }, + /* Enable 1K sort */ + { A3XX_VBIF_ABIT_SORT, 0x0001003F }, + { A3XX_VBIF_ABIT_SORT_CONF, 0x000000A4 }, + /* Disable VBIF clock gating. This is to enable AXI running + * higher frequency than GPU. + */ + { A3XX_VBIF_CLKON, 1 }, + {0, 0}, +}; + +/* + * Most of the VBIF registers on 8974v2 have the correct values at power on, so + * we won't modify those if we don't need to + */ +static const struct adreno_vbif_data a330v2_vbif[] = { + /* Enable 1k sort */ + { A3XX_VBIF_ABIT_SORT, 0x0001003F }, + { A3XX_VBIF_ABIT_SORT_CONF, 0x000000A4 }, + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x00003F }, + { A3XX_VBIF_DDR_OUT_MAX_BURST, 0x0000303 }, + /* Set up VBIF_ROUND_ROBIN_QOS_ARB */ + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + {0, 0}, +}; + +/* + * Most of the VBIF registers on a330v2.1 have the correct values at power on, + * so we won't modify those if we don't need to + */ +static const struct adreno_vbif_data a330v21_vbif[] = { + /* Enable WR-REQ */ + { A3XX_VBIF_GATE_OFF_WRREQ_EN, 0x1 }, + /* Set up VBIF_ROUND_ROBIN_QOS_ARB */ + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + { A3XX_VBIF_IN_RD_LIM_CONF0, 0x18180c0c }, + {0, 0}, +}; + +static const struct adreno_vbif_platform a3xx_vbif_platforms[] = { + { adreno_is_a304, a304_vbif }, + { adreno_is_a305, a305_vbif }, + { adreno_is_a305c, a305c_vbif }, + { adreno_is_a306, a306_vbif }, + { adreno_is_a306a, a306a_vbif }, + { adreno_is_a310, a310_vbif }, + { adreno_is_a320, a320_vbif }, + /* A330v2.1 needs to be ahead of A330v2 so the right device matches */ + { adreno_is_a330v21, a330v21_vbif}, + /* A330v2 needs to be ahead of A330 so the right device matches */ + { adreno_is_a330v2, a330v2_vbif }, + { adreno_is_a330, a330_vbif }, + { adreno_is_a305b, a305b_vbif }, +}; + +/* + * Define the available perfcounter groups - these get used by + * adreno_perfcounter_get and adreno_perfcounter_put + */ + +static struct adreno_perfcount_register a3xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_CP_0_LO, + A3XX_RBBM_PERFCTR_CP_0_HI, 0, A3XX_CP_PERFCOUNTER_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_rbbm[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RBBM_0_LO, + A3XX_RBBM_PERFCTR_RBBM_0_HI, 1, A3XX_RBBM_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RBBM_1_LO, + A3XX_RBBM_PERFCTR_RBBM_1_HI, 2, A3XX_RBBM_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_0_LO, + A3XX_RBBM_PERFCTR_PC_0_HI, 3, A3XX_PC_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_1_LO, + A3XX_RBBM_PERFCTR_PC_1_HI, 4, A3XX_PC_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_2_LO, + A3XX_RBBM_PERFCTR_PC_2_HI, 5, A3XX_PC_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_3_LO, + A3XX_RBBM_PERFCTR_PC_3_HI, 6, A3XX_PC_PERFCOUNTER3_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VFD_0_LO, + A3XX_RBBM_PERFCTR_VFD_0_HI, 7, A3XX_VFD_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VFD_1_LO, + A3XX_RBBM_PERFCTR_VFD_1_HI, 8, A3XX_VFD_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_0_LO, + A3XX_RBBM_PERFCTR_HLSQ_0_HI, 9, + A3XX_HLSQ_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_1_LO, + A3XX_RBBM_PERFCTR_HLSQ_1_HI, 10, + A3XX_HLSQ_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_2_LO, + A3XX_RBBM_PERFCTR_HLSQ_2_HI, 11, + A3XX_HLSQ_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_3_LO, + A3XX_RBBM_PERFCTR_HLSQ_3_HI, 12, + A3XX_HLSQ_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_4_LO, + A3XX_RBBM_PERFCTR_HLSQ_4_HI, 13, + A3XX_HLSQ_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_5_LO, + A3XX_RBBM_PERFCTR_HLSQ_5_HI, 14, + A3XX_HLSQ_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VPC_0_LO, + A3XX_RBBM_PERFCTR_VPC_0_HI, 15, A3XX_VPC_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VPC_1_LO, + A3XX_RBBM_PERFCTR_VPC_1_HI, 16, A3XX_VPC_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TSE_0_LO, + A3XX_RBBM_PERFCTR_TSE_0_HI, 17, A3XX_GRAS_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TSE_1_LO, + A3XX_RBBM_PERFCTR_TSE_1_HI, 18, A3XX_GRAS_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RAS_0_LO, + A3XX_RBBM_PERFCTR_RAS_0_HI, 19, A3XX_GRAS_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RAS_1_LO, + A3XX_RBBM_PERFCTR_RAS_1_HI, 20, A3XX_GRAS_PERFCOUNTER3_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_0_LO, + A3XX_RBBM_PERFCTR_UCHE_0_HI, 21, + A3XX_UCHE_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_1_LO, + A3XX_RBBM_PERFCTR_UCHE_1_HI, 22, + A3XX_UCHE_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_2_LO, + A3XX_RBBM_PERFCTR_UCHE_2_HI, 23, + A3XX_UCHE_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_3_LO, + A3XX_RBBM_PERFCTR_UCHE_3_HI, 24, + A3XX_UCHE_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_4_LO, + A3XX_RBBM_PERFCTR_UCHE_4_HI, 25, + A3XX_UCHE_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_5_LO, + A3XX_RBBM_PERFCTR_UCHE_5_HI, 26, + A3XX_UCHE_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_0_LO, + A3XX_RBBM_PERFCTR_TP_0_HI, 27, A3XX_TP_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_1_LO, + A3XX_RBBM_PERFCTR_TP_1_HI, 28, A3XX_TP_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_2_LO, + A3XX_RBBM_PERFCTR_TP_2_HI, 29, A3XX_TP_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_3_LO, + A3XX_RBBM_PERFCTR_TP_3_HI, 30, A3XX_TP_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_4_LO, + A3XX_RBBM_PERFCTR_TP_4_HI, 31, A3XX_TP_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_5_LO, + A3XX_RBBM_PERFCTR_TP_5_HI, 32, A3XX_TP_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_0_LO, + A3XX_RBBM_PERFCTR_SP_0_HI, 33, A3XX_SP_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_1_LO, + A3XX_RBBM_PERFCTR_SP_1_HI, 34, A3XX_SP_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_2_LO, + A3XX_RBBM_PERFCTR_SP_2_HI, 35, A3XX_SP_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_3_LO, + A3XX_RBBM_PERFCTR_SP_3_HI, 36, A3XX_SP_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_4_LO, + A3XX_RBBM_PERFCTR_SP_4_HI, 37, A3XX_SP_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_5_LO, + A3XX_RBBM_PERFCTR_SP_5_HI, 38, A3XX_SP_PERFCOUNTER5_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_6_LO, + A3XX_RBBM_PERFCTR_SP_6_HI, 39, A3XX_SP_PERFCOUNTER6_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_7_LO, + A3XX_RBBM_PERFCTR_SP_7_HI, 40, A3XX_SP_PERFCOUNTER7_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RB_0_LO, + A3XX_RBBM_PERFCTR_RB_0_HI, 41, A3XX_RB_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RB_1_LO, + A3XX_RBBM_PERFCTR_RB_1_HI, 42, A3XX_RB_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PWR_0_LO, + A3XX_RBBM_PERFCTR_PWR_0_HI, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PWR_1_LO, + A3XX_RBBM_PERFCTR_PWR_1_HI, -1, 0 }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF_PERF_CNT0_LO, + A3XX_VBIF_PERF_CNT0_HI, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF_PERF_CNT1_LO, + A3XX_VBIF_PERF_CNT1_HI, -1, 0 }, +}; +static struct adreno_perfcount_register a3xx_perfcounters_vbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF_PERF_PWR_CNT0_LO, + A3XX_VBIF_PERF_PWR_CNT0_HI, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF_PERF_PWR_CNT1_LO, + A3XX_VBIF_PERF_PWR_CNT1_HI, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF_PERF_PWR_CNT2_LO, + A3XX_VBIF_PERF_PWR_CNT2_HI, -1, 0 }, +}; +static struct adreno_perfcount_register a3xx_perfcounters_vbif2[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW0, + A3XX_VBIF2_PERF_CNT_HIGH0, -1, A3XX_VBIF2_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW1, + A3XX_VBIF2_PERF_CNT_HIGH1, -1, A3XX_VBIF2_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW2, + A3XX_VBIF2_PERF_CNT_HIGH2, -1, A3XX_VBIF2_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW3, + A3XX_VBIF2_PERF_CNT_HIGH3, -1, A3XX_VBIF2_PERF_CNT_SEL3 }, +}; +/* + * Placing EN register in select field since vbif perf counters + * dont have select register to program + */ +static struct adreno_perfcount_register a3xx_perfcounters_vbif2_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW0, + A3XX_VBIF2_PERF_PWR_CNT_HIGH0, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW1, + A3XX_VBIF2_PERF_PWR_CNT_HIGH1, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW2, + A3XX_VBIF2_PERF_PWR_CNT_HIGH2, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN2 }, +}; + +#define A3XX_PERFCOUNTER_GROUP(offset, name) \ + ADRENO_PERFCOUNTER_GROUP(a3xx, offset, name) + +#define A3XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a3xx, offset, name, flags) + +static struct adreno_perfcount_group a3xx_perfcounter_groups[] = { + A3XX_PERFCOUNTER_GROUP(CP, cp), + A3XX_PERFCOUNTER_GROUP(RBBM, rbbm), + A3XX_PERFCOUNTER_GROUP(PC, pc), + A3XX_PERFCOUNTER_GROUP(VFD, vfd), + A3XX_PERFCOUNTER_GROUP(HLSQ, hlsq), + A3XX_PERFCOUNTER_GROUP(VPC, vpc), + A3XX_PERFCOUNTER_GROUP(TSE, tse), + A3XX_PERFCOUNTER_GROUP(RAS, ras), + A3XX_PERFCOUNTER_GROUP(UCHE, uche), + A3XX_PERFCOUNTER_GROUP(TP, tp), + A3XX_PERFCOUNTER_GROUP(SP, sp), + A3XX_PERFCOUNTER_GROUP(RB, rb), + A3XX_PERFCOUNTER_GROUP_FLAGS(PWR, pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A3XX_PERFCOUNTER_GROUP(VBIF, vbif), + A3XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), +}; + +static struct adreno_perfcounters a3xx_perfcounters = { + a3xx_perfcounter_groups, + ARRAY_SIZE(a3xx_perfcounter_groups), +}; + +static struct adreno_ft_perf_counters a3xx_ft_perf_counters[] = { + {KGSL_PERFCOUNTER_GROUP_SP, SP_ALU_ACTIVE_CYCLES}, + {KGSL_PERFCOUNTER_GROUP_SP, SP0_ICL1_MISSES}, + {KGSL_PERFCOUNTER_GROUP_SP, SP_FS_CFLOW_INSTRUCTIONS}, + {KGSL_PERFCOUNTER_GROUP_TSE, TSE_INPUT_PRIM_NUM}, +}; + +static void a3xx_perfcounter_init(struct adreno_device *adreno_dev) +{ + struct adreno_perfcounters *counters = ADRENO_PERFCOUNTERS(adreno_dev); + + /* SP[3] counter is broken on a330 so disable it if a330 device */ + if (adreno_is_a330(adreno_dev)) + a3xx_perfcounters_sp[3].countable = KGSL_PERFCOUNTER_BROKEN; + + if (counters && + (adreno_is_a306(adreno_dev) || adreno_is_a304(adreno_dev) || + adreno_is_a306a(adreno_dev))) { + counters->groups[KGSL_PERFCOUNTER_GROUP_VBIF].regs = + a3xx_perfcounters_vbif2; + counters->groups[KGSL_PERFCOUNTER_GROUP_VBIF_PWR].regs = + a3xx_perfcounters_vbif2_pwr; + } + + /* + * Enable the GPU busy count counter. This is a fixed counter on + * A3XX so we don't need to bother checking the return value + */ + adreno_perfcounter_get(adreno_dev, KGSL_PERFCOUNTER_GROUP_PWR, 1, + NULL, NULL, PERFCOUNTER_FLAG_KERNEL); +} + +static void a3xx_perfcounter_close(struct adreno_device *adreno_dev) +{ + adreno_perfcounter_put(adreno_dev, KGSL_PERFCOUNTER_GROUP_PWR, 1, + PERFCOUNTER_FLAG_KERNEL); +} + +/** + * a3xx_protect_init() - Initializes register protection on a3xx + * @adreno_dev: Pointer to the device structure + * Performs register writes to enable protected access to sensitive + * registers + */ +static void a3xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int index = 0; + struct kgsl_protected_registers *iommu_regs; + + /* enable access protection to privileged registers */ + kgsl_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007); + + /* RBBM registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x18, 0); + adreno_set_protected_registers(adreno_dev, &index, 0x20, 2); + adreno_set_protected_registers(adreno_dev, &index, 0x33, 0); + adreno_set_protected_registers(adreno_dev, &index, 0x42, 0); + adreno_set_protected_registers(adreno_dev, &index, 0x50, 4); + adreno_set_protected_registers(adreno_dev, &index, 0x63, 0); + adreno_set_protected_registers(adreno_dev, &index, 0x100, 4); + + /* CP registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x1C0, 5); + adreno_set_protected_registers(adreno_dev, &index, 0x1EC, 1); + adreno_set_protected_registers(adreno_dev, &index, 0x1F6, 1); + adreno_set_protected_registers(adreno_dev, &index, 0x1F8, 2); + adreno_set_protected_registers(adreno_dev, &index, 0x45E, 2); + adreno_set_protected_registers(adreno_dev, &index, 0x460, 4); + + /* RB registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xCC0, 0); + + /* VBIF registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x3000, 6); + + /* SMMU registers */ + iommu_regs = kgsl_mmu_get_prot_regs(&device->mmu); + if (iommu_regs) + adreno_set_protected_registers(adreno_dev, &index, + iommu_regs->base, ilog2(iommu_regs->range)); +} + +static void a3xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + adreno_vbif_start(adreno_dev, a3xx_vbif_platforms, + ARRAY_SIZE(a3xx_vbif_platforms)); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A3XX_RBBM_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* Tune the hystersis counters for SP and CP idle detection */ + kgsl_regwrite(device, A3XX_RBBM_SP_HYST_CNT, 0x10); + kgsl_regwrite(device, A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL, 0x10); + + /* + * Enable the RBBM error reporting bits. This lets us get + * useful information on failure + */ + + kgsl_regwrite(device, A3XX_RBBM_AHB_CTL0, 0x00000001); + + /* Enable AHB error reporting */ + kgsl_regwrite(device, A3XX_RBBM_AHB_CTL1, 0xA6FFFFFF); + + /* Turn on the power counters */ + kgsl_regwrite(device, A3XX_RBBM_RBBM_CTL, 0x00030000); + + /* + * Turn on hang detection - this spews a lot of useful information + * into the RBBM registers on a hang + */ + if (adreno_is_a330v2(adreno_dev)) { + set_bit(ADRENO_DEVICE_HANG_INTR, &adreno_dev->priv); + gpudev->irq->mask |= (1 << A3XX_INT_MISC_HANG_DETECT); + kgsl_regwrite(device, A3XX_RBBM_INTERFACE_HANG_INT_CTL, + (1 << 31) | 0xFFFF); + } else + kgsl_regwrite(device, A3XX_RBBM_INTERFACE_HANG_INT_CTL, + (1 << 16) | 0xFFF); + + /* Enable 64-byte cacheline size. HW Default is 32-byte (0x000000E0). */ + kgsl_regwrite(device, A3XX_UCHE_CACHE_MODE_CONTROL_REG, 0x00000001); + + /* Enable VFD to access most of the UCHE (7 ways out of 8) */ + kgsl_regwrite(device, A3XX_UCHE_CACHE_WAYS_VFD, 0x07); + + /* Enable Clock gating */ + kgsl_regwrite(device, A3XX_RBBM_CLOCK_CTL, + adreno_a3xx_rbbm_clock_ctl_default(adreno_dev)); + + if (adreno_is_a330v2(adreno_dev)) + kgsl_regwrite(device, A3XX_RBBM_GPR0_CTL, + A330v2_RBBM_GPR0_CTL_DEFAULT); + else if (adreno_is_a330(adreno_dev)) + kgsl_regwrite(device, A3XX_RBBM_GPR0_CTL, + A330_RBBM_GPR0_CTL_DEFAULT); + else if (adreno_is_a310(adreno_dev)) + kgsl_regwrite(device, A3XX_RBBM_GPR0_CTL, + A310_RBBM_GPR0_CTL_DEFAULT); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_USES_OCMEM)) + kgsl_regwrite(device, A3XX_RB_GMEM_BASE_ADDR, + (unsigned int)(adreno_dev->gmem_base >> 14)); + + /* Turn on protection */ + a3xx_protect_init(adreno_dev); + + /* Turn on performance counters */ + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_CTL, 0x01); + + kgsl_regwrite(device, A3XX_CP_DEBUG, A3XX_CP_DEBUG_DEFAULT); + + /* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */ + if (adreno_is_a305b(adreno_dev) || + adreno_is_a310(adreno_dev) || + adreno_is_a330(adreno_dev)) + kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, 0x003E2008); + else + kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, 0x000E0602); + +} + +static struct adreno_coresight_register a3xx_coresight_registers[] = { + { A3XX_RBBM_DEBUG_BUS_CTL, 0x0001093F }, + { A3XX_RBBM_EXT_TRACE_STOP_CNT, 0x00017fff }, + { A3XX_RBBM_EXT_TRACE_START_CNT, 0x0001000f }, + { A3XX_RBBM_EXT_TRACE_PERIOD_CNT, 0x0001ffff }, + { A3XX_RBBM_EXT_TRACE_CMD, 0x00000001 }, + { A3XX_RBBM_EXT_TRACE_BUS_CTL, 0x89100010 }, + { A3XX_RBBM_DEBUG_BUS_STB_CTL0, 0x00000000 }, + { A3XX_RBBM_DEBUG_BUS_STB_CTL1, 0xFFFFFFFE }, + { A3XX_RBBM_INT_TRACE_BUS_CTL, 0x00201111 }, +}; + +static ADRENO_CORESIGHT_ATTR(config_debug_bus, + &a3xx_coresight_registers[0]); +static ADRENO_CORESIGHT_ATTR(config_trace_stop_cnt, + &a3xx_coresight_registers[1]); +static ADRENO_CORESIGHT_ATTR(config_trace_start_cnt, + &a3xx_coresight_registers[2]); +static ADRENO_CORESIGHT_ATTR(config_trace_period_cnt, + &a3xx_coresight_registers[3]); +static ADRENO_CORESIGHT_ATTR(config_trace_cmd, + &a3xx_coresight_registers[4]); +static ADRENO_CORESIGHT_ATTR(config_trace_bus_ctl, + &a3xx_coresight_registers[5]); + +static struct attribute *a3xx_coresight_attrs[] = { + &coresight_attr_config_debug_bus.attr.attr, + &coresight_attr_config_trace_start_cnt.attr.attr, + &coresight_attr_config_trace_stop_cnt.attr.attr, + &coresight_attr_config_trace_period_cnt.attr.attr, + &coresight_attr_config_trace_cmd.attr.attr, + &coresight_attr_config_trace_bus_ctl.attr.attr, + NULL, +}; + +static const struct attribute_group a3xx_coresight_group = { + .attrs = a3xx_coresight_attrs, +}; + +static const struct attribute_group *a3xx_coresight_groups[] = { + &a3xx_coresight_group, + NULL, +}; + +static struct adreno_coresight a3xx_coresight = { + .registers = a3xx_coresight_registers, + .count = ARRAY_SIZE(a3xx_coresight_registers), + .groups = a3xx_coresight_groups, +}; + +static unsigned int a3xx_int_bits[ADRENO_INT_BITS_MAX] = { + ADRENO_INT_DEFINE(ADRENO_INT_RBBM_AHB_ERROR, A3XX_INT_RBBM_AHB_ERROR), +}; + +/* Register offset defines for A3XX */ +static unsigned int a3xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_WADDR, A3XX_CP_ME_RAM_WADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_DATA, A3XX_CP_ME_RAM_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PFP_UCODE_DATA, A3XX_CP_PFP_UCODE_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PFP_UCODE_ADDR, A3XX_CP_PFP_UCODE_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_WFI_PEND_CTR, A3XX_CP_WFI_PEND_CTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A3XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A3XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A3XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A3XX_CP_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A3XX_CP_ME_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A3XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A3XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A3XX_CP_IB1_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A3XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A3XX_CP_IB2_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_TIMESTAMP, A3XX_CP_SCRATCH_REG0), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG6, A3XX_CP_SCRATCH_REG6), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG7, A3XX_CP_SCRATCH_REG7), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_RADDR, A3XX_CP_ME_RAM_RADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_ADDR, A3XX_CP_ROQ_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_DATA, A3XX_CP_ROQ_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_ADDR, A3XX_CP_MERCIU_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA, A3XX_CP_MERCIU_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA2, A3XX_CP_MERCIU_DATA2), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_ADDR, A3XX_CP_MEQ_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_DATA, A3XX_CP_MEQ_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A3XX_CP_PROTECT_REG_0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A3XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_CTL, A3XX_RBBM_PERFCTR_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + A3XX_RBBM_PERFCTR_LOAD_CMD0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + A3XX_RBBM_PERFCTR_LOAD_CMD1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_PWR_1_LO, + A3XX_RBBM_PERFCTR_PWR_1_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A3XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_STATUS, A3XX_RBBM_INT_0_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_CLEAR_CMD, + A3XX_RBBM_INT_CLEAR_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A3XX_RBBM_CLOCK_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_VPC_DEBUG_RAM_SEL, + A3XX_VPC_VPC_DEBUG_RAM_SEL), + ADRENO_REG_DEFINE(ADRENO_REG_VPC_DEBUG_RAM_READ, + A3XX_VPC_VPC_DEBUG_RAM_READ), + ADRENO_REG_DEFINE(ADRENO_REG_PA_SC_AA_CONFIG, A3XX_PA_SC_AA_CONFIG), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PM_OVERRIDE2, A3XX_RBBM_PM_OVERRIDE2), + ADRENO_REG_DEFINE(ADRENO_REG_SQ_GPR_MANAGEMENT, A3XX_SQ_GPR_MANAGEMENT), + ADRENO_REG_DEFINE(ADRENO_REG_SQ_INST_STORE_MANAGEMENT, + A3XX_SQ_INST_STORE_MANAGEMENT), + ADRENO_REG_DEFINE(ADRENO_REG_TP0_CHICKEN, A3XX_TP0_CHICKEN), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_RBBM_CTL, A3XX_RBBM_RBBM_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A3XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE0, + A3XX_UCHE_CACHE_INVALIDATE0_REG), + ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE1, + A3XX_UCHE_CACHE_INVALIDATE1_REG), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_LO, + A3XX_RBBM_PERFCTR_RBBM_0_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_HI, + A3XX_RBBM_PERFCTR_RBBM_0_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + A3XX_RBBM_PERFCTR_LOAD_VALUE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + A3XX_RBBM_PERFCTR_LOAD_VALUE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL0, + A3XX_VBIF_XIN_HALT_CTRL0), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A3XX_VBIF_XIN_HALT_CTRL1), +}; + +static const struct adreno_reg_offsets a3xx_reg_offsets = { + .offsets = a3xx_register_offsets, + .offset_0 = ADRENO_REG_REGISTER_MAX, +}; + +/* + * Defined the size of sections dumped in snapshot, these values + * may change after initialization based on the specific core + */ +static struct adreno_snapshot_sizes a3xx_snap_sizes = { + .cp_pfp = 0x14, + .vpc_mem = 512, + .cp_meq = 16, + .shader_mem = 0x4000, + .cp_merciu = 0, + .roq = 128, +}; + +static struct adreno_snapshot_data a3xx_snapshot_data = { + .sect_sizes = &a3xx_snap_sizes, +}; + +static int _load_firmware(struct kgsl_device *device, const char *fwfile, + void **buf, int *len) +{ + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, fwfile, device->dev); + + if (ret) { + dev_err(device->dev, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + if (fw) + *buf = kmalloc(fw->size, GFP_KERNEL); + else + return -EINVAL; + + if (*buf) { + memcpy(*buf, fw->data, fw->size); + *len = fw->size; + } + + release_firmware(fw); + return (*buf != NULL) ? 0 : -ENOMEM; +} + +int a3xx_microcode_read(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + + if (pm4_fw->fwvirt == NULL) { + int len; + void *ptr; + + int ret = _load_firmware(device, + adreno_dev->gpucore->pm4fw_name, &ptr, &len); + + if (ret) { + dev_err(device->dev, "Failed to read pm4 ucode %s\n", + adreno_dev->gpucore->pm4fw_name); + return ret; + } + + /* PM4 size is 3 dword aligned plus 1 dword of version */ + if (len % ((sizeof(uint32_t) * 3)) != sizeof(uint32_t)) { + dev_err(device->dev, + "Bad pm4 microcode size: %d\n", + len); + kfree(ptr); + return -ENOMEM; + } + + pm4_fw->size = len / sizeof(uint32_t); + pm4_fw->fwvirt = ptr; + pm4_fw->version = pm4_fw->fwvirt[1]; + } + + if (pfp_fw->fwvirt == NULL) { + int len; + void *ptr; + + int ret = _load_firmware(device, + adreno_dev->gpucore->pfpfw_name, &ptr, &len); + if (ret) { + dev_err(device->dev, "Failed to read pfp ucode %s\n", + adreno_dev->gpucore->pfpfw_name); + return ret; + } + + /* PFP size shold be dword aligned */ + if (len % sizeof(uint32_t) != 0) { + dev_err(device->dev, + "Bad PFP microcode size: %d\n", + len); + kfree(ptr); + return -ENOMEM; + } + + pfp_fw->size = len / sizeof(uint32_t); + pfp_fw->fwvirt = ptr; + pfp_fw->version = pfp_fw->fwvirt[1]; + } + + return 0; +} +/** + * load_pm4_ucode() - Load pm4 ucode + * @adreno_dev: Pointer to an adreno device + * @start: Starting index in pm4 ucode to load + * @end: Ending index of pm4 ucode to load + * @addr: Address to load the pm4 ucode + * + * Load the pm4 ucode from @start at @addr. + */ +static inline void load_pm4_ucode(struct adreno_device *adreno_dev, + unsigned int start, unsigned int end, unsigned int addr) +{ + int i; + + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_RAM_WADDR, addr); + for (i = start; i < end; i++) + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_RAM_DATA, + adreno_dev->fw[ADRENO_FW_PM4].fwvirt[i]); +} +/** + * load_pfp_ucode() - Load pfp ucode + * @adreno_dev: Pointer to an adreno device + * @start: Starting index in pfp ucode to load + * @end: Ending index of pfp ucode to load + * @addr: Address to load the pfp ucode + * + * Load the pfp ucode from @start at @addr. + */ +static inline void load_pfp_ucode(struct adreno_device *adreno_dev, + unsigned int start, unsigned int end, unsigned int addr) +{ + int i; + + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR, addr); + for (i = start; i < end; i++) + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA, + adreno_dev->fw[ADRENO_FW_PFP].fwvirt[i]); +} + +/** + * _ringbuffer_bootstrap_ucode() - Bootstrap GPU Ucode + * @adreno_dev: Pointer to an adreno device + * @rb: The ringbuffer to boostrap the code into + * @load_jt: If non zero only load Jump tables + * + * Bootstrap ucode for GPU + * load_jt == 0, bootstrap full microcode + * load_jt == 1, bootstrap jump tables of microcode + * + * For example a bootstrap packet would like below + * Setup a type3 bootstrap packet + * PFP size to bootstrap + * PFP addr to write the PFP data + * PM4 size to bootstrap + * PM4 addr to write the PM4 data + * PFP dwords from microcode to bootstrap + * PM4 size dwords from microcode to bootstrap + */ +static int _ringbuffer_bootstrap_ucode(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int load_jt) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds, bootstrap_size, rb_size; + int i = 0; + int ret; + unsigned int pm4_size, pm4_idx, pm4_addr, pfp_size, pfp_idx, pfp_addr; + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + + /* Only bootstrap jump tables of ucode */ + if (load_jt) { + pm4_idx = adreno_dev->gpucore->pm4_jt_idx; + pm4_addr = adreno_dev->gpucore->pm4_jt_addr; + pfp_idx = adreno_dev->gpucore->pfp_jt_idx; + pfp_addr = adreno_dev->gpucore->pfp_jt_addr; + } else { + /* Bootstrap full ucode */ + pm4_idx = 1; + pm4_addr = 0; + pfp_idx = 1; + pfp_addr = 0; + } + + pm4_size = (pm4_fw->size - pm4_idx); + pfp_size = (pfp_fw->size - pfp_idx); + + bootstrap_size = (pm4_size + pfp_size + 5); + + /* + * Overwrite the first entry in the jump table with the special + * bootstrap opcode + */ + + if (adreno_is_a4xx(adreno_dev)) { + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR, + 0x400); + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA, + 0x6f0009); + /* + * The support packets (the RMW and INTERRUPT) that are sent + * after the bootstrap packet should not be included in the size + * of the bootstrap packet but we do need to reserve enough + * space for those too + */ + rb_size = bootstrap_size + 6; + } else { + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_ADDR, + 0x200); + adreno_writereg(adreno_dev, ADRENO_REG_CP_PFP_UCODE_DATA, + 0x6f0005); + rb_size = bootstrap_size; + } + + /* clear ME_HALT to start micro engine */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0); + + cmds = adreno_ringbuffer_allocspace(rb, rb_size); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + /* Construct the packet that bootsraps the ucode */ + *cmds++ = cp_type3_packet(CP_BOOTSTRAP_UCODE, (bootstrap_size - 1)); + *cmds++ = pfp_size; + *cmds++ = pfp_addr; + *cmds++ = pm4_size; + *cmds++ = pm4_addr; + + /** + * Theory of operation: + * + * In A4x, we cannot have the PFP executing instructions while its + * instruction RAM is loading. We load the PFP's instruction RAM + * using type-0 writes from the ME. + * + * To make sure the PFP is not fetching instructions at the same + * time, we put it in a one-instruction loop: + * mvc (ME), (ringbuffer) + * which executes repeatedly until all of the data has been moved + * from the ring buffer to the ME. + */ + if (adreno_is_a4xx(adreno_dev)) { + for (i = pm4_idx; i < pm4_fw->size; i++) + *cmds++ = pm4_fw->fwvirt[i]; + for (i = pfp_idx; i < pfp_fw->size; i++) + *cmds++ = pfp_fw->fwvirt[i]; + + *cmds++ = cp_type3_packet(CP_REG_RMW, 3); + *cmds++ = 0x20000000 + A4XX_CP_RB_WPTR; + *cmds++ = 0xffffffff; + *cmds++ = 0x00000002; + *cmds++ = cp_type3_packet(CP_INTERRUPT, 1); + *cmds++ = 0; + + rb->_wptr = rb->_wptr - 2; + adreno_ringbuffer_submit(rb, NULL); + rb->_wptr = rb->_wptr + 2; + } else { + for (i = pfp_idx; i < pfp_fw->size; i++) + *cmds++ = pfp_fw->fwvirt[i]; + for (i = pm4_idx; i < pm4_fw->size; i++) + *cmds++ = pm4_fw->fwvirt[i]; + adreno_ringbuffer_submit(rb, NULL); + } + + /* idle device to validate bootstrap */ + ret = adreno_spin_idle(adreno_dev, 2000); + + if (ret) { + dev_err(device->dev, + "microcode bootstrap failed to idle\n"); + kgsl_device_snapshot(device, NULL, false); + } + + /* Clear the chicken bit for speed up on A430 and its derivatives */ + if (!adreno_is_a420(adreno_dev)) + kgsl_regwrite(device, A4XX_CP_DEBUG, + A4XX_CP_DEBUG_DEFAULT & ~(1 << 14)); + + return ret; +} + +int a3xx_microcode_load(struct adreno_device *adreno_dev, + unsigned int start_type) +{ + int status; + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + size_t pm4_size = adreno_dev->fw[ADRENO_FW_PM4].size; + size_t pfp_size = adreno_dev->fw[ADRENO_FW_PFP].size; + + if (start_type == ADRENO_START_COLD) { + /* If bootstrapping if supported to load ucode */ + if (adreno_bootstrap_ucode(adreno_dev)) { + + /* + * load first pm4_bstrp_size + pfp_bstrp_size microcode + * dwords using AHB write, this small microcode has + * dispatcher + booter this initial microcode enables + * CP to understand CP_BOOTSTRAP_UCODE packet in + * function _ringbuffer_bootstrap_ucode. + * CP_BOOTSTRAP_UCODE packet loads rest of the + * microcode. + */ + + load_pm4_ucode(adreno_dev, 1, + adreno_dev->gpucore->pm4_bstrp_size+1, 0); + + load_pfp_ucode(adreno_dev, 1, + adreno_dev->gpucore->pfp_bstrp_size+1, 0); + + /* Bootstrap rest of the ucode here */ + status = _ringbuffer_bootstrap_ucode(adreno_dev, rb, 0); + if (status != 0) + return status; + + } else { + /* load the CP ucode using AHB writes */ + load_pm4_ucode(adreno_dev, 1, pm4_size, 0); + + /* load the prefetch parser ucode using AHB writes */ + load_pfp_ucode(adreno_dev, 1, pfp_size, 0); + } + } else if (start_type == ADRENO_START_WARM) { + /* If bootstrapping if supported to load jump tables */ + if (adreno_bootstrap_ucode(adreno_dev)) { + status = _ringbuffer_bootstrap_ucode(adreno_dev, rb, 1); + if (status != 0) + return status; + + } else { + /* load the CP jump tables using AHB writes */ + load_pm4_ucode(adreno_dev, + adreno_dev->gpucore->pm4_jt_idx, + pm4_size, adreno_dev->gpucore->pm4_jt_addr); + + /* + * load the prefetch parser jump tables using AHB writes + */ + load_pfp_ucode(adreno_dev, + adreno_dev->gpucore->pfp_jt_idx, + pfp_size, adreno_dev->gpucore->pfp_jt_addr); + } + } else + return -EINVAL; + + return 0; +} + +static void a3xx_clk_set_options(struct adreno_device *adreno_dev, + const char *name, struct clk *clk, bool on) +{ + if (!adreno_is_a306a(adreno_dev)) + return; + + /* Handle clock settings for GFX PSCBCs */ + if (on) { + if (!strcmp(name, "mem_iface_clk")) { + clk_set_flags(clk, CLKFLAG_NORETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_NORETAIN_MEM); + } else if (!strcmp(name, "core_clk")) { + clk_set_flags(clk, CLKFLAG_RETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_RETAIN_MEM); + } + } else { + if (!strcmp(name, "core_clk")) { + clk_set_flags(clk, CLKFLAG_NORETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_NORETAIN_MEM); + } + } +} + +struct adreno_gpudev adreno_a3xx_gpudev = { + .reg_offsets = &a3xx_reg_offsets, + .int_bits = a3xx_int_bits, + .ft_perf_counters = a3xx_ft_perf_counters, + .ft_perf_counters_count = ARRAY_SIZE(a3xx_ft_perf_counters), + .perfcounters = &a3xx_perfcounters, + .irq = &a3xx_irq, + .irq_trace = trace_kgsl_a3xx_irq_status, + .snapshot_data = &a3xx_snapshot_data, + .num_prio_levels = 1, + .vbif_xin_halt_ctrl0_mask = A3XX_VBIF_XIN_HALT_CTRL0_MASK, + .platform_setup = a3xx_platform_setup, + .rb_start = a3xx_rb_start, + .init = a3xx_init, + .microcode_read = a3xx_microcode_read, + .perfcounter_init = a3xx_perfcounter_init, + .perfcounter_close = a3xx_perfcounter_close, + .start = a3xx_start, + .snapshot = a3xx_snapshot, + .coresight = {&a3xx_coresight}, + .clk_set_options = a3xx_clk_set_options, +}; diff --git a/drivers/gpu/msm/adreno_a3xx.h b/drivers/gpu/msm/adreno_a3xx.h new file mode 100644 index 0000000000000000000000000000000000000000..a243e9e4df9377c9cf2768e7733b4b0aeeebc4d9 --- /dev/null +++ b/drivers/gpu/msm/adreno_a3xx.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. + */ +#ifndef __A3XX_H +#define __A3XX_H + +#include "a3xx_reg.h" + +#define A3XX_IRQ_FLAGS \ + { BIT(A3XX_INT_RBBM_GPU_IDLE), "RBBM_GPU_IDLE" }, \ + { BIT(A3XX_INT_RBBM_AHB_ERROR), "RBBM_AHB_ERR" }, \ + { BIT(A3XX_INT_RBBM_REG_TIMEOUT), "RBBM_REG_TIMEOUT" }, \ + { BIT(A3XX_INT_RBBM_ME_MS_TIMEOUT), "RBBM_ME_MS_TIMEOUT" }, \ + { BIT(A3XX_INT_RBBM_PFP_MS_TIMEOUT), "RBBM_PFP_MS_TIMEOUT" }, \ + { BIT(A3XX_INT_RBBM_ATB_BUS_OVERFLOW), "RBBM_ATB_BUS_OVERFLOW" }, \ + { BIT(A3XX_INT_VFD_ERROR), "RBBM_VFD_ERROR" }, \ + { BIT(A3XX_INT_CP_SW_INT), "CP_SW" }, \ + { BIT(A3XX_INT_CP_T0_PACKET_IN_IB), "CP_T0_PACKET_IN_IB" }, \ + { BIT(A3XX_INT_CP_OPCODE_ERROR), "CP_OPCODE_ERROR" }, \ + { BIT(A3XX_INT_CP_RESERVED_BIT_ERROR), "CP_RESERVED_BIT_ERROR" }, \ + { BIT(A3XX_INT_CP_HW_FAULT), "CP_HW_FAULT" }, \ + { BIT(A3XX_INT_CP_DMA), "CP_DMA" }, \ + { BIT(A3XX_INT_CP_IB2_INT), "CP_IB2_INT" }, \ + { BIT(A3XX_INT_CP_IB1_INT), "CP_IB1_INT" }, \ + { BIT(A3XX_INT_CP_RB_INT), "CP_RB_INT" }, \ + { BIT(A3XX_INT_CP_REG_PROTECT_FAULT), "CP_REG_PROTECT_FAULT" }, \ + { BIT(A3XX_INT_CP_RB_DONE_TS), "CP_RB_DONE_TS" }, \ + { BIT(A3XX_INT_CP_VS_DONE_TS), "CP_VS_DONE_TS" }, \ + { BIT(A3XX_INT_CP_PS_DONE_TS), "CP_PS_DONE_TS" }, \ + { BIT(A3XX_INT_CACHE_FLUSH_TS), "CACHE_FLUSH_TS" }, \ + { BIT(A3XX_INT_CP_AHB_ERROR_HALT), "CP_AHB_ERROR_HALT" }, \ + { BIT(A3XX_INT_MISC_HANG_DETECT), "MISC_HANG_DETECT" }, \ + { BIT(A3XX_INT_UCHE_OOB_ACCESS), "UCHE_OOB_ACCESS" } + +unsigned int a3xx_irq_pending(struct adreno_device *adreno_dev); + +int a3xx_microcode_read(struct adreno_device *adreno_dev); +int a3xx_microcode_load(struct adreno_device *adreno_dev, + unsigned int start_type); +int a3xx_perfcounter_enable(struct adreno_device *adreno_dev, + unsigned int group, unsigned int counter, unsigned int countable); +uint64_t a3xx_perfcounter_read(struct adreno_device *adreno_dev, + unsigned int group, unsigned int counter); + +void a3xx_a4xx_err_callback(struct adreno_device *adreno_dev, int bit); + +void a3xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); +#endif /*__A3XX_H */ diff --git a/drivers/gpu/msm/adreno_a3xx_snapshot.c b/drivers/gpu/msm/adreno_a3xx_snapshot.c new file mode 100644 index 0000000000000000000000000000000000000000..9473a412a3212dc8441725f8847a58e85b05bd30 --- /dev/null +++ b/drivers/gpu/msm/adreno_a3xx_snapshot.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2012-2017, The Linux Foundation. All rights reserved. + */ + +#include +#include "kgsl.h" +#include "adreno.h" +#include "kgsl_snapshot.h" +#include "a3xx_reg.h" +#include "adreno_snapshot.h" +#include "adreno_a3xx.h" + +/* + * Set of registers to dump for A3XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a3xx_registers[] = { + 0x0000, 0x0002, 0x0010, 0x0012, 0x0018, 0x0018, 0x0020, 0x0027, + 0x0029, 0x002b, 0x002e, 0x0033, 0x0040, 0x0042, 0x0050, 0x005c, + 0x0060, 0x006c, 0x0080, 0x0082, 0x0084, 0x0088, 0x0090, 0x00e5, + 0x00ea, 0x00ed, 0x0100, 0x0100, 0x0110, 0x0123, 0x01c0, 0x01c1, + 0x01c3, 0x01c5, 0x01c7, 0x01c7, 0x01d5, 0x01d9, 0x01dc, 0x01dd, + 0x01ea, 0x01ea, 0x01ee, 0x01f1, 0x01f5, 0x01f6, 0x01f8, 0x01f9, + 0x01fc, 0x01ff, + 0x0440, 0x0440, 0x0443, 0x0443, 0x0445, 0x0445, 0x044d, 0x044f, + 0x0452, 0x0452, 0x0454, 0x046f, 0x047c, 0x047c, 0x047f, 0x047f, + 0x0578, 0x057f, 0x0600, 0x0602, 0x0605, 0x0607, 0x060a, 0x060e, + 0x0612, 0x0614, 0x0c01, 0x0c02, 0x0c06, 0x0c1d, 0x0c3d, 0x0c3f, + 0x0c48, 0x0c4b, 0x0c80, 0x0c80, 0x0c88, 0x0c8b, 0x0ca0, 0x0cb7, + 0x0cc0, 0x0cc1, 0x0cc6, 0x0cc7, 0x0ce4, 0x0ce5, + 0x0e41, 0x0e45, 0x0e64, 0x0e65, + 0x0e80, 0x0e82, 0x0e84, 0x0e89, 0x0ea0, 0x0ea1, 0x0ea4, 0x0ea7, + 0x0ec4, 0x0ecb, 0x0ee0, 0x0ee0, 0x0f00, 0x0f01, 0x0f03, 0x0f09, + 0x2040, 0x2040, 0x2044, 0x2044, 0x2048, 0x204d, 0x2068, 0x2069, + 0x206c, 0x206d, 0x2070, 0x2070, 0x2072, 0x2072, 0x2074, 0x2075, + 0x2079, 0x207a, 0x20c0, 0x20d3, 0x20e4, 0x20ef, 0x2100, 0x2109, + 0x210c, 0x210c, 0x210e, 0x210e, 0x2110, 0x2111, 0x2114, 0x2115, + 0x21e4, 0x21e4, 0x21ea, 0x21ea, 0x21ec, 0x21ed, 0x21f0, 0x21f0, + 0x2240, 0x227e, + 0x2280, 0x228b, 0x22c0, 0x22c0, 0x22c4, 0x22ce, 0x22d0, 0x22d8, + 0x22df, 0x22e6, 0x22e8, 0x22e9, 0x22ec, 0x22ec, 0x22f0, 0x22f7, + 0x22ff, 0x22ff, 0x2340, 0x2343, + 0x2440, 0x2440, 0x2444, 0x2444, 0x2448, 0x244d, + 0x2468, 0x2469, 0x246c, 0x246d, 0x2470, 0x2470, 0x2472, 0x2472, + 0x2474, 0x2475, 0x2479, 0x247a, 0x24c0, 0x24d3, 0x24e4, 0x24ef, + 0x2500, 0x2509, 0x250c, 0x250c, 0x250e, 0x250e, 0x2510, 0x2511, + 0x2514, 0x2515, 0x25e4, 0x25e4, 0x25ea, 0x25ea, 0x25ec, 0x25ed, + 0x25f0, 0x25f0, + 0x2640, 0x267e, 0x2680, 0x268b, 0x26c0, 0x26c0, 0x26c4, 0x26ce, + 0x26d0, 0x26d8, 0x26df, 0x26e6, 0x26e8, 0x26e9, 0x26ec, 0x26ec, + 0x26f0, 0x26f7, 0x26ff, 0x26ff, 0x2740, 0x2743, + 0x300C, 0x300E, 0x301C, 0x301D, + 0x302A, 0x302A, 0x302C, 0x302D, 0x3030, 0x3031, 0x3034, 0x3036, + 0x303C, 0x303C, 0x305E, 0x305F, +}; + +/* Removed the following HLSQ register ranges from being read during + * fault tolerance since reading the registers may cause the device to hang: + */ +static const unsigned int a3xx_hlsq_registers[] = { + 0x0e00, 0x0e05, 0x0e0c, 0x0e0c, 0x0e22, 0x0e23, + 0x2200, 0x2212, 0x2214, 0x2217, 0x221a, 0x221a, + 0x2600, 0x2612, 0x2614, 0x2617, 0x261a, 0x261a, +}; + +/* The set of additional registers to be dumped for A330 */ + +static const unsigned int a330_registers[] = { + 0x1d0, 0x1d0, 0x1d4, 0x1d4, 0x453, 0x453, +}; + +/* Shader memory size in words */ +#define SHADER_MEMORY_SIZE 0x4000 + +/** + * _rbbm_debug_bus_read - Helper function to read data from the RBBM + * debug bus. + * @device - GPU device to read/write registers + * @block_id - Debug bus block to read from + * @index - Index in the debug bus block to read + * @ret - Value of the register read + */ +static void _rbbm_debug_bus_read(struct kgsl_device *device, + unsigned int block_id, unsigned int index, unsigned int *val) +{ + unsigned int block = (block_id << 8) | 1 << 16; + + kgsl_regwrite(device, A3XX_RBBM_DEBUG_BUS_CTL, block | index); + kgsl_regread(device, A3XX_RBBM_DEBUG_BUS_DATA_STATUS, val); +} + +/** + * a3xx_snapshot_shader_memory - Helper function to dump the GPU shader + * memory to the snapshot buffer. + * @device: GPU device whose shader memory is to be dumped + * @buf: Pointer to binary snapshot data blob being made + * @remain: Number of remaining bytes in the snapshot blob + * @priv: Unused parameter + * + */ +static size_t a3xx_snapshot_shader_memory(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int shader_read_len = SHADER_MEMORY_SIZE; + + if (shader_read_len > (device->shader_mem_len >> 2)) + shader_read_len = (device->shader_mem_len >> 2); + + if (remain < DEBUG_SECTION_SZ(shader_read_len)) { + SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_SHADER_MEMORY; + header->size = shader_read_len; + + /* Map shader memory to kernel, for dumping */ + if (device->shader_mem_virt == NULL) + device->shader_mem_virt = devm_ioremap(device->dev, + device->shader_mem_phys, + device->shader_mem_len); + + if (device->shader_mem_virt == NULL) { + dev_err(device->dev, + "Unable to map shader memory region\n"); + return 0; + } + + /* Now, dump shader memory to snapshot */ + for (i = 0; i < shader_read_len; i++) + adreno_shadermem_regread(device, i, &data[i]); + + + return DEBUG_SECTION_SZ(shader_read_len); +} + +static size_t a3xx_snapshot_debugbus_block(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + struct kgsl_snapshot_debugbus *header + = (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int dwords; + size_t size; + + /* + * For A305 and A320 all debug bus regions are the same size (0x40). For + * A330, they can be different sizes - most are still 0x40, but some + * like CP are larger + */ + + dwords = (adreno_is_a330(adreno_dev) || + adreno_is_a305b(adreno_dev)) ? + block->dwords : 0x40; + + size = (dwords * sizeof(unsigned int)) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + + header->id = block->block_id; + header->count = dwords; + + for (i = 0; i < dwords; i++) + _rbbm_debug_bus_read(device, block->block_id, i, &data[i]); + + return size; +} + +static struct adreno_debugbus_block debugbus_blocks[] = { + { RBBM_BLOCK_ID_CP, 0x52, }, + { RBBM_BLOCK_ID_RBBM, 0x40, }, + { RBBM_BLOCK_ID_VBIF, 0x40, }, + { RBBM_BLOCK_ID_HLSQ, 0x40, }, + { RBBM_BLOCK_ID_UCHE, 0x40, }, + { RBBM_BLOCK_ID_PC, 0x40, }, + { RBBM_BLOCK_ID_VFD, 0x40, }, + { RBBM_BLOCK_ID_VPC, 0x40, }, + { RBBM_BLOCK_ID_TSE, 0x40, }, + { RBBM_BLOCK_ID_RAS, 0x40, }, + { RBBM_BLOCK_ID_VSC, 0x40, }, + { RBBM_BLOCK_ID_SP_0, 0x40, }, + { RBBM_BLOCK_ID_SP_1, 0x40, }, + { RBBM_BLOCK_ID_SP_2, 0x40, }, + { RBBM_BLOCK_ID_SP_3, 0x40, }, + { RBBM_BLOCK_ID_TPL1_0, 0x40, }, + { RBBM_BLOCK_ID_TPL1_1, 0x40, }, + { RBBM_BLOCK_ID_TPL1_2, 0x40, }, + { RBBM_BLOCK_ID_TPL1_3, 0x40, }, + { RBBM_BLOCK_ID_RB_0, 0x40, }, + { RBBM_BLOCK_ID_RB_1, 0x40, }, + { RBBM_BLOCK_ID_RB_2, 0x40, }, + { RBBM_BLOCK_ID_RB_3, 0x40, }, + { RBBM_BLOCK_ID_MARB_0, 0x40, }, + { RBBM_BLOCK_ID_MARB_1, 0x40, }, + { RBBM_BLOCK_ID_MARB_2, 0x40, }, + { RBBM_BLOCK_ID_MARB_3, 0x40, }, +}; + +static void a3xx_snapshot_debugbus(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(debugbus_blocks); i++) { + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, snapshot, + a3xx_snapshot_debugbus_block, + (void *) &debugbus_blocks[i]); + } +} + +static void _snapshot_hlsq_regs(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* + * Trying to read HLSQ registers when the HLSQ block is busy + * will cause the device to hang. The RBBM_DEBUG_BUS has information + * that will tell us if the HLSQ block is busy or not. Read values + * from the debug bus to ensure the HLSQ block is not busy (this + * is hardware dependent). If the HLSQ block is busy do not + * dump the registers, otherwise dump the HLSQ registers. + */ + + if (adreno_is_a330(adreno_dev)) { + /* + * stall_ctxt_full status bit: RBBM_BLOCK_ID_HLSQ index 49 [27] + * + * if (!stall_context_full) + * then dump HLSQ registers + */ + unsigned int stall_context_full = 0; + + _rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 49, + &stall_context_full); + stall_context_full &= 0x08000000; + + if (stall_context_full) + return; + } else { + /* + * tpif status bits: RBBM_BLOCK_ID_HLSQ index 4 [4:0] + * spif status bits: RBBM_BLOCK_ID_HLSQ index 7 [5:0] + * + * if ((tpif == 0, 1, 28) && (spif == 0, 1, 10)) + * then dump HLSQ registers + */ + unsigned int next_pif = 0; + + /* check tpif */ + _rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 4, &next_pif); + next_pif &= 0x1f; + if (next_pif != 0 && next_pif != 1 && next_pif != 28) + return; + + /* check spif */ + _rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 7, &next_pif); + next_pif &= 0x3f; + if (next_pif != 0 && next_pif != 1 && next_pif != 10) + return; + } + + SNAPSHOT_REGISTERS(device, snapshot, a3xx_hlsq_registers); +} + +/* + * a3xx_snapshot() - A3XX GPU snapshot function + * @adreno_dev: Device being snapshotted + * @snapshot: Snapshot meta data + * @remain: Amount of space left in snapshot memory + * + * This is where all of the A3XX specific bits and pieces are grabbed + * into the snapshot memory + */ +void a3xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_snapshot_data *snap_data = gpudev->snapshot_data; + unsigned int reg; + + /* Disable Clock gating temporarily for the debug bus to work */ + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_CLOCK_CTL, 0x00); + + SNAPSHOT_REGISTERS(device, snapshot, a3xx_registers); + + _snapshot_hlsq_regs(device, snapshot); + + if (adreno_is_a330(adreno_dev) || adreno_is_a305b(adreno_dev)) + SNAPSHOT_REGISTERS(device, snapshot, a330_registers); + + kgsl_snapshot_indexed_registers(device, snapshot, + A3XX_CP_STATE_DEBUG_INDEX, A3XX_CP_STATE_DEBUG_DATA, + 0x0, snap_data->sect_sizes->cp_pfp); + + /* CP_ME indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A3XX_CP_ME_CNTL, A3XX_CP_ME_STATUS, 64, 44); + + /* VPC memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_vpc_memory, + &snap_data->sect_sizes->vpc_mem); + + /* CP MEQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, snapshot, + adreno_snapshot_cp_meq, &snap_data->sect_sizes->cp_meq); + + /* Shader working/shadow memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_shader_memory, + &snap_data->sect_sizes->shader_mem); + + + /* CP PFP and PM4 */ + + /* + * Reading the microcode while the CP is running will + * basically move the CP instruction pointer to + * whatever address we read. Big badaboom ensues. Stop the CP + * (if it isn't already stopped) to ensure that we are safe. + * We do this here and not earlier to avoid corrupting the RBBM + * status and CP registers - by the time we get here we don't + * care about the contents of the CP anymore. + */ + + adreno_readreg(adreno_dev, ADRENO_REG_CP_ME_CNTL, ®); + reg |= (1 << 27) | (1 << 28); + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, reg); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_pfp_ram, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_pm4_ram, NULL); + + /* CP ROQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_roq, &snap_data->sect_sizes->roq); + + if (snap_data->sect_sizes->cp_merciu) { + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_merciu, + &snap_data->sect_sizes->cp_merciu); + } + + a3xx_snapshot_debugbus(device, snapshot); +} diff --git a/drivers/gpu/msm/adreno_a4xx.c b/drivers/gpu/msm/adreno_a4xx.c new file mode 100644 index 0000000000000000000000000000000000000000..aa951cd240345c0d33e0a3100f07d83412e3a973 --- /dev/null +++ b/drivers/gpu/msm/adreno_a4xx.c @@ -0,0 +1,1805 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include + +#include "adreno.h" +#include "kgsl_sharedmem.h" +#include "a4xx_reg.h" +#include "adreno_a3xx.h" +#include "adreno_a4xx.h" +#include "adreno_cp_parser.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" +#include "adreno_perfcounter.h" + +#define SP_TP_PWR_ON BIT(20) +/* A4XX_RBBM_CLOCK_CTL_IP */ +#define CNTL_IP_SW_COLLAPSE BIT(0) + +/* + * Define registers for a4xx that contain addresses used by the + * cp parser logic + */ +const unsigned int a4xx_cp_addr_regs[ADRENO_CP_ADDR_MAX] = { + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_0, + A4XX_VSC_PIPE_DATA_ADDRESS_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_0, + A4XX_VSC_PIPE_DATA_LENGTH_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_1, + A4XX_VSC_PIPE_DATA_ADDRESS_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_1, + A4XX_VSC_PIPE_DATA_LENGTH_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_2, + A4XX_VSC_PIPE_DATA_ADDRESS_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_2, + A4XX_VSC_PIPE_DATA_LENGTH_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_3, + A4XX_VSC_PIPE_DATA_ADDRESS_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_3, + A4XX_VSC_PIPE_DATA_LENGTH_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_4, + A4XX_VSC_PIPE_DATA_ADDRESS_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_4, + A4XX_VSC_PIPE_DATA_LENGTH_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_5, + A4XX_VSC_PIPE_DATA_ADDRESS_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_5, + A4XX_VSC_PIPE_DATA_LENGTH_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_6, + A4XX_VSC_PIPE_DATA_ADDRESS_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_6, + A4XX_VSC_PIPE_DATA_LENGTH_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_7, + A4XX_VSC_PIPE_DATA_ADDRESS_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_7, + A4XX_VSC_PIPE_DATA_LENGTH_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_0, + A4XX_VFD_FETCH_INSTR_1_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_1, + A4XX_VFD_FETCH_INSTR_1_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_2, + A4XX_VFD_FETCH_INSTR_1_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_3, + A4XX_VFD_FETCH_INSTR_1_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_4, + A4XX_VFD_FETCH_INSTR_1_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_5, + A4XX_VFD_FETCH_INSTR_1_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_6, + A4XX_VFD_FETCH_INSTR_1_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_7, + A4XX_VFD_FETCH_INSTR_1_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_8, + A4XX_VFD_FETCH_INSTR_1_8), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_9, + A4XX_VFD_FETCH_INSTR_1_9), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_10, + A4XX_VFD_FETCH_INSTR_1_10), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_11, + A4XX_VFD_FETCH_INSTR_1_11), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_12, + A4XX_VFD_FETCH_INSTR_1_12), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_13, + A4XX_VFD_FETCH_INSTR_1_13), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_14, + A4XX_VFD_FETCH_INSTR_1_14), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_15, + A4XX_VFD_FETCH_INSTR_1_15), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_16, + A4XX_VFD_FETCH_INSTR_1_16), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_17, + A4XX_VFD_FETCH_INSTR_1_17), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_18, + A4XX_VFD_FETCH_INSTR_1_18), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_19, + A4XX_VFD_FETCH_INSTR_1_19), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_20, + A4XX_VFD_FETCH_INSTR_1_20), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_21, + A4XX_VFD_FETCH_INSTR_1_21), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_22, + A4XX_VFD_FETCH_INSTR_1_22), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_23, + A4XX_VFD_FETCH_INSTR_1_23), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_24, + A4XX_VFD_FETCH_INSTR_1_24), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_25, + A4XX_VFD_FETCH_INSTR_1_25), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_26, + A4XX_VFD_FETCH_INSTR_1_26), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_27, + A4XX_VFD_FETCH_INSTR_1_27), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_28, + A4XX_VFD_FETCH_INSTR_1_28), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_29, + A4XX_VFD_FETCH_INSTR_1_29), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_30, + A4XX_VFD_FETCH_INSTR_1_30), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_31, + A4XX_VFD_FETCH_INSTR_1_31), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_SIZE_ADDRESS, + A4XX_VSC_SIZE_ADDRESS), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_PVT_MEM_ADDR, + A4XX_SP_VS_PVT_MEM_ADDR), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_PVT_MEM_ADDR, + A4XX_SP_FS_PVT_MEM_ADDR), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_OBJ_START_REG, + A4XX_SP_VS_OBJ_START), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_OBJ_START_REG, + A4XX_SP_FS_OBJ_START), + ADRENO_REG_DEFINE(ADRENO_CP_UCHE_INVALIDATE0, + A4XX_UCHE_INVALIDATE0), + ADRENO_REG_DEFINE(ADRENO_CP_UCHE_INVALIDATE1, + A4XX_UCHE_INVALIDATE1), +}; + +static const struct adreno_vbif_data a405_vbif[] = { + { A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a420_vbif[] = { + { A4XX_VBIF_ABIT_SORT, 0x0001001F }, + { A4XX_VBIF_ABIT_SORT_CONF, 0x000000A4 }, + { A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001 }, + { A4XX_VBIF_IN_RD_LIM_CONF0, 0x18181818 }, + { A4XX_VBIF_IN_RD_LIM_CONF1, 0x00000018 }, + { A4XX_VBIF_IN_WR_LIM_CONF0, 0x18181818 }, + { A4XX_VBIF_IN_WR_LIM_CONF1, 0x00000018 }, + { A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003 }, + {0, 0}, +}; + +static const struct adreno_vbif_data a430_vbif[] = { + { A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001 }, + { A4XX_VBIF_IN_RD_LIM_CONF0, 0x18181818 }, + { A4XX_VBIF_IN_RD_LIM_CONF1, 0x00000018 }, + { A4XX_VBIF_IN_WR_LIM_CONF0, 0x18181818 }, + { A4XX_VBIF_IN_WR_LIM_CONF1, 0x00000018 }, + { A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003 }, + {0, 0}, +}; + +static const struct adreno_vbif_platform a4xx_vbif_platforms[] = { + { adreno_is_a405, a405_vbif }, + { adreno_is_a420, a420_vbif }, + { adreno_is_a430, a430_vbif }, + { adreno_is_a418, a430_vbif }, +}; + +/* + * a4xx_is_sptp_idle() - A430 SP/TP should be off to be considered idle + * @adreno_dev: The adreno device pointer + */ +static bool a4xx_is_sptp_idle(struct adreno_device *adreno_dev) +{ + unsigned int reg; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC)) + return true; + + /* If SP/TP pc isn't enabled, don't worry about power */ + kgsl_regread(device, A4XX_CP_POWER_COLLAPSE_CNTL, ®); + if (!(reg & 0x10)) + return true; + + /* Check that SP/TP is off */ + kgsl_regread(device, A4XX_RBBM_POWER_STATUS, ®); + return !(reg & SP_TP_PWR_ON); +} + +/* + * a4xx_enable_hwcg() - Program the clock control registers + * @device: The adreno device pointer + */ +static void a4xx_enable_hwcg(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_TP0, 0x02222202); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_TP1, 0x02222202); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_TP2, 0x02222202); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_TP3, 0x02222202); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_TP0, 0x00002222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_TP1, 0x00002222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_TP2, 0x00002222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_TP3, 0x00002222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_TP0, 0x0E739CE7); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_TP1, 0x0E739CE7); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_TP2, 0x0E739CE7); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_TP3, 0x0E739CE7); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_TP0, 0x00111111); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_TP1, 0x00111111); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_TP2, 0x00111111); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_TP3, 0x00111111); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP0, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP1, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP2, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP3, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP0, 0x00222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP1, 0x00222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP2, 0x00222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP3, 0x00222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_SP0, 0x00000104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_SP1, 0x00000104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_SP2, 0x00000104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_SP3, 0x00000104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_SP0, 0x00000081); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_SP1, 0x00000081); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_SP2, 0x00000081); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_SP3, 0x00000081); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_UCHE, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_UCHE, 0x02222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL3_UCHE, 0x00000000); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL4_UCHE, 0x00000000); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_UCHE, 0x00004444); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_UCHE, 0x00001112); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_RB0, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_RB1, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_RB2, 0x22222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_RB3, 0x22222222); + /* Disable L1 clocking in A420 due to CCU issues with it */ + if (adreno_is_a420(adreno_dev)) { + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB0, 0x00002020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB1, 0x00002020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB2, 0x00002020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB3, 0x00002020); + } else { + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB0, 0x00022020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB1, 0x00022020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB2, 0x00022020); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_RB3, 0x00022020); + } + /* No CCU for A405 */ + if (!adreno_is_a405(adreno_dev)) { + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_CTL_MARB_CCU0, 0x00000922); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_CTL_MARB_CCU1, 0x00000922); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_CTL_MARB_CCU2, 0x00000922); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_CTL_MARB_CCU3, 0x00000922); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU0, 0x00000000); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU1, 0x00000000); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU2, 0x00000000); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU3, 0x00000000); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_0, + 0x00000001); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_1, + 0x00000001); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_2, + 0x00000001); + kgsl_regwrite(device, + A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_3, + 0x00000001); + } + kgsl_regwrite(device, A4XX_RBBM_CLOCK_MODE_GPC, 0x02222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_GPC, 0x04100104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_GPC, 0x00022222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_COM_DCOM, 0x00000022); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_COM_DCOM, 0x0000010F); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_COM_DCOM, 0x00000022); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_TSE_RAS_RBBM, 0x00222222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00004104); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000222); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_HLSQ, 0x00000000); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_HLSQ, 0x00220000); + /* + * Due to a HW timing issue, top level HW clock gating is causing + * register read/writes to be dropped in adreno a430. + * This timing issue started happening because of SP/TP power collapse. + * On targets that do not have SP/TP PC there is no timing issue. + * The HW timing issue could be fixed by + * a) disabling SP/TP power collapse + * b) or disabling HW clock gating. + * Disabling HW clock gating + NAP enabled combination has + * minimal power impact. So this option is chosen over disabling + * SP/TP power collapse. + * Revisions of A430 which chipid 2 and above do not have the issue. + */ + if (adreno_is_a430(adreno_dev) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) < 2)) + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL, 0); + else + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL, 0xAAAAAAAA); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2, 0); +} +/* + * a4xx_regulator_enable() - Enable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly enabled + * on a restart. Clocks must be on during this call. + */ +static int a4xx_regulator_enable(struct adreno_device *adreno_dev) +{ + unsigned int reg; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!(adreno_is_a430(adreno_dev) || adreno_is_a418(adreno_dev))) { + /* Halt the sp_input_clk at HM level */ + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL, 0x00000055); + a4xx_enable_hwcg(device); + return 0; + } + + /* Set the default register values; set SW_COLLAPSE to 0 */ + kgsl_regwrite(device, A4XX_RBBM_POWER_CNTL_IP, 0x778000); + do { + udelay(5); + kgsl_regread(device, A4XX_RBBM_POWER_STATUS, ®); + } while (!(reg & SP_TP_PWR_ON)); + + /* Disable SP clock */ + kgsl_regrmw(device, A4XX_RBBM_CLOCK_CTL_IP, CNTL_IP_SW_COLLAPSE, 0); + /* Enable hardware clockgating */ + a4xx_enable_hwcg(device); + /* Enable SP clock */ + kgsl_regrmw(device, A4XX_RBBM_CLOCK_CTL_IP, CNTL_IP_SW_COLLAPSE, 1); + return 0; +} + +/* + * a4xx_regulator_disable() - Disable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly disabled + * on a power down to prevent current spikes. Clocks must be on + * during this call. + */ +static void a4xx_regulator_disable(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!(adreno_is_a430(adreno_dev) || adreno_is_a418(adreno_dev))) + return; + + /* Set the default register values; set SW_COLLAPSE to 1 */ + kgsl_regwrite(device, A4XX_RBBM_POWER_CNTL_IP, 0x778001); +} + +/* + * a4xx_enable_pc() - Enable the SP/TP block power collapse + * @adreno_dev: The adreno device pointer + */ +static void a4xx_enable_pc(struct adreno_device *adreno_dev) +{ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC) || + !test_bit(ADRENO_SPTP_PC_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + kgsl_regwrite(KGSL_DEVICE(adreno_dev), A4XX_CP_POWER_COLLAPSE_CNTL, + 0x00400010); + trace_adreno_sp_tp((unsigned long) __builtin_return_address(0)); +}; + +/* + * a4xx_enable_ppd() - Enable the Peak power detect logic in the h/w + * @adreno_dev: The adreno device pointer + * + * A430 can detect peak current conditions inside h/w and throttle + * the workload to ALUs to mitigate it. + */ +static void a4xx_enable_ppd(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_PPD) || + !test_bit(ADRENO_PPD_CTRL, &adreno_dev->pwrctrl_flag) || + !adreno_is_a430v2(adreno_dev)) + return; + + /* Program thresholds */ + kgsl_regwrite(device, A4XX_RBBM_PPD_EPOCH_INTER_TH_HIGH_CLEAR_THR, + 0x003F0101); + kgsl_regwrite(device, A4XX_RBBM_PPD_EPOCH_INTER_TH_LOW, 0x00000101); + kgsl_regwrite(device, A4XX_RBBM_PPD_V2_SP_PWR_WEIGHTS, 0x00085014); + kgsl_regwrite(device, A4XX_RBBM_PPD_V2_SP_RB_EPOCH_TH, 0x00000B46); + kgsl_regwrite(device, A4XX_RBBM_PPD_V2_TP_CONFIG, 0xE4525111); + kgsl_regwrite(device, A4XX_RBBM_PPD_RAMP_V2_CONTROL, 0x0000000B); + + /* Enable PPD*/ + kgsl_regwrite(device, A4XX_RBBM_PPD_CTRL, 0x1002E40C); +}; + +/* + * a4xx_pwrlevel_change_settings() - Program the hardware during power level + * transitions + * @adreno_dev: The adreno device pointer + * @prelevel: The previous power level + * @postlevel: The new power level + * @post: True if called after the clock change has taken effect + */ +static void a4xx_pwrlevel_change_settings(struct adreno_device *adreno_dev, + unsigned int prelevel, unsigned int postlevel, + bool post) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + static bool pre; + + /* PPD programming only for A430v2 */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_PPD) || + !test_bit(ADRENO_PPD_CTRL, &adreno_dev->pwrctrl_flag) || + !adreno_is_a430v2(adreno_dev)) + return; + + /* if this is a real pre, or a post without a previous pre, set pre */ + if (!post || (!pre && post)) + pre = true; + else if (post) + pre = false; + + if ((prelevel == 0) && pre) { + /* Going to Non-Turbo mode - mask the throttle and reset */ + kgsl_regwrite(device, A4XX_RBBM_PPD_CTRL, 0x1002E40E); + kgsl_regwrite(device, A4XX_RBBM_PPD_CTRL, 0x1002E40C); + } else if ((postlevel == 0) && post) { + /* Going to Turbo mode - unmask the throttle and reset */ + kgsl_regwrite(device, A4XX_RBBM_PPD_CTRL, 0x1002E40A); + kgsl_regwrite(device, A4XX_RBBM_PPD_CTRL, 0x1002E408); + } + + if (post) + pre = false; +} + +/** + * a4xx_protect_init() - Initializes register protection on a4xx + * @adreno_dev: Pointer to the device structure + * Performs register writes to enable protected access to sensitive + * registers + */ +static void a4xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int index = 0; + struct kgsl_protected_registers *iommu_regs; + + /* enable access protection to privileged registers */ + kgsl_regwrite(device, A4XX_CP_PROTECT_CTRL, 0x00000007); + /* RBBM registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x4, 2); + adreno_set_protected_registers(adreno_dev, &index, 0x8, 3); + adreno_set_protected_registers(adreno_dev, &index, 0x10, 4); + adreno_set_protected_registers(adreno_dev, &index, 0x20, 5); + adreno_set_protected_registers(adreno_dev, &index, 0x40, 6); + adreno_set_protected_registers(adreno_dev, &index, 0x80, 4); + + /* Content protection registers */ + if (kgsl_mmu_is_secured(&device->mmu)) { + adreno_set_protected_registers(adreno_dev, &index, + A4XX_RBBM_SECVID_TSB_TRUSTED_BASE, 3); + adreno_set_protected_registers(adreno_dev, &index, + A4XX_RBBM_SECVID_TRUST_CONTROL, 1); + } + + /* CP registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x200, 7); + adreno_set_protected_registers(adreno_dev, &index, 0x580, 4); + adreno_set_protected_registers(adreno_dev, &index, A4XX_CP_PREEMPT, 1); + /* RB registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xCC0, 0); + + /* HLSQ registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xE00, 0); + + /* VPC registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xE60, 1); + + if (adreno_is_a430(adreno_dev) || adreno_is_a420(adreno_dev) || + adreno_is_a418(adreno_dev)) { + /* + * Protect registers that might cause XPU violation if + * accessed by GPU + */ + adreno_set_protected_registers(adreno_dev, &index, 0x2c00, 10); + adreno_set_protected_registers(adreno_dev, &index, 0x3300, 8); + adreno_set_protected_registers(adreno_dev, &index, 0x3400, 10); + } + + /* SMMU registers */ + iommu_regs = kgsl_mmu_get_prot_regs(&device->mmu); + if (iommu_regs) + adreno_set_protected_registers(adreno_dev, &index, + iommu_regs->base, ilog2(iommu_regs->range)); +} + +static struct adreno_snapshot_sizes a4xx_snap_sizes = { + .cp_pfp = 0x14, + .vpc_mem = 2048, + .cp_meq = 64, + .shader_mem = 0x4000, + .cp_merciu = 64, + .roq = 512, +}; + + +static void a4xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int cp_debug = A4XX_CP_DEBUG_DEFAULT; + + adreno_vbif_start(adreno_dev, a4xx_vbif_platforms, + ARRAY_SIZE(a4xx_vbif_platforms)); + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A4XX_RBBM_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* Tune the hystersis counters for SP and CP idle detection */ + kgsl_regwrite(device, A4XX_RBBM_SP_HYST_CNT, 0x10); + kgsl_regwrite(device, A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL, 0x10); + if (adreno_is_a430(adreno_dev)) + kgsl_regwrite(device, A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL2, 0x30); + + /* + * Enable the RBBM error reporting bits. This lets us get + * useful information on failure + */ + + kgsl_regwrite(device, A4XX_RBBM_AHB_CTL0, 0x00000001); + + /* Enable AHB error reporting */ + kgsl_regwrite(device, A4XX_RBBM_AHB_CTL1, 0xA6FFFFFF); + + /* Turn on the power counters */ + kgsl_regwrite(device, A4XX_RBBM_RBBM_CTL, 0x00000030); + + /* + * Turn on hang detection - this spews a lot of useful information + * into the RBBM registers on a hang + */ + set_bit(ADRENO_DEVICE_HANG_INTR, &adreno_dev->priv); + gpudev->irq->mask |= (1 << A4XX_INT_MISC_HANG_DETECT); + kgsl_regwrite(device, A4XX_RBBM_INTERFACE_HANG_INT_CTL, + (1 << 30) | 0xFFFF); + + /* Set the GMEM/OCMEM base address for A4XX */ + kgsl_regwrite(device, A4XX_RB_GMEM_BASE_ADDR, + (unsigned int)(adreno_dev->gmem_base >> 14)); + + /* Turn on performance counters */ + kgsl_regwrite(device, A4XX_RBBM_PERFCTR_CTL, 0x01); + + /* Enable VFD to access most of the UCHE (7 ways out of 8) */ + kgsl_regwrite(device, A4XX_UCHE_CACHE_WAYS_VFD, 0x07); + + /* Disable L2 bypass to avoid UCHE out of bounds errors */ + kgsl_regwrite(device, UCHE_TRAP_BASE_LO, 0xffff0000); + kgsl_regwrite(device, UCHE_TRAP_BASE_HI, 0xffff0000); + + /* On A420 cores turn on SKIP_IB2_DISABLE in addition to the default */ + if (adreno_is_a420(adreno_dev)) + cp_debug |= (1 << 29); + /* + * Set chicken bit to disable the speed up of bootstrap on A430 + * and its derivatives + */ + else + cp_debug |= (1 << 14); + + kgsl_regwrite(device, A4XX_CP_DEBUG, cp_debug); + + /* On A430 enable SP regfile sleep for power savings */ + if (!adreno_is_a420(adreno_dev)) { + kgsl_regwrite(device, A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_0, + 0x00000441); + kgsl_regwrite(device, A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_1, + 0x00000441); + } + + /* + * For A420 set RBBM_CLOCK_DELAY_HLSQ.CGC_HLSQ_TP_EARLY_CYC >= 2 + * due to timing issue with HLSQ_TP_CLK_EN + */ + if (adreno_is_a420(adreno_dev)) { + unsigned int val; + + kgsl_regread(device, A4XX_RBBM_CLOCK_DELAY_HLSQ, &val); + val &= ~A4XX_CGC_HLSQ_TP_EARLY_CYC_MASK; + val |= 2 << A4XX_CGC_HLSQ_TP_EARLY_CYC_SHIFT; + kgsl_regwrite(device, A4XX_RBBM_CLOCK_DELAY_HLSQ, val); + } + + /* A430 and derivatives offers bigger chunk of CP_STATE_DEBUG regs */ + if (!adreno_is_a420(adreno_dev)) + a4xx_snap_sizes.cp_pfp = 0x34; + + if (adreno_is_a405(adreno_dev)) + gpudev->vbif_xin_halt_ctrl0_mask = + A405_VBIF_XIN_HALT_CTRL0_MASK; + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + a4xx_protect_init(adreno_dev); +} + +/* + * a4xx_err_callback() - Callback for a4xx error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +static void a4xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + switch (bit) { + case A4XX_INT_RBBM_AHB_ERROR: { + kgsl_regread(device, A4XX_RBBM_AHB_ERROR_STATUS, ®); + + /* + * Return the word address of the erroring register so that it + * matches the register specification + */ + dev_crit_ratelimited(device->dev, + "RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n", + reg & (1 << 28) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2, + (reg >> 20) & 0x3, + (reg >> 24) & 0xF); + + /* Clear the error */ + kgsl_regwrite(device, A4XX_RBBM_AHB_CMD, (1 << 4)); + break; + } + case A4XX_INT_RBBM_REG_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: AHB register timeout\n"); + break; + case A4XX_INT_RBBM_ME_MS_TIMEOUT: + kgsl_regread(device, A4XX_RBBM_AHB_ME_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | ME master split timeout | status=%x\n", + reg); + break; + case A4XX_INT_RBBM_PFP_MS_TIMEOUT: + kgsl_regread(device, A4XX_RBBM_AHB_PFP_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | PFP master split timeout | status=%x\n", + reg); + break; + case A4XX_INT_RBBM_ETS_MS_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: ME master split timeout\n"); + break; + case A4XX_INT_RBBM_ASYNC_OVERFLOW: + dev_crit_ratelimited(device->dev, "RBBM: ASYNC overflow\n"); + break; + case A4XX_INT_CP_OPCODE_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer opcode error interrupt\n"); + break; + case A4XX_INT_CP_RESERVED_BIT_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer reserved bit error interrupt\n"); + break; + case A4XX_INT_CP_HW_FAULT: + { + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + kgsl_regread(device, A4XX_CP_HW_FAULT, ®); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", + reg); + /* + * mask off this interrupt since it can spam, it will be + * turned on again when device resets + */ + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_0_MASK, + gpudev->irq->mask & ~(1 << A4XX_INT_CP_HW_FAULT)); + break; + } + case A4XX_INT_CP_REG_PROTECT_FAULT: + kgsl_regread(device, A4XX_CP_PROTECT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error| %s | addr=%x\n", + reg & (1 << 24) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2); + break; + case A4XX_INT_CP_AHB_ERROR_HALT: + dev_crit_ratelimited(device->dev, + "ringbuffer AHB error interrupt\n"); + break; + case A4XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus overflow\n"); + break; + case A4XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + case A4XX_INT_RBBM_DPM_CALC_ERR: + dev_crit_ratelimited(device->dev, "RBBM: dpm calc error\n"); + break; + case A4XX_INT_RBBM_DPM_EPOCH_ERR: + dev_crit_ratelimited(device->dev, + "RBBM: dpm epoch error\n"); + break; + case A4XX_INT_RBBM_DPM_THERMAL_YELLOW_ERR: + dev_crit_ratelimited(device->dev, + "RBBM: dpm thermal yellow\n"); + break; + case A4XX_INT_RBBM_DPM_THERMAL_RED_ERR: + dev_crit_ratelimited(device->dev, + "RBBM: dpm thermal red\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt\n"); + } +} + +static unsigned int a4xx_int_bits[ADRENO_INT_BITS_MAX] = { + ADRENO_INT_DEFINE(ADRENO_INT_RBBM_AHB_ERROR, A4XX_INT_RBBM_AHB_ERROR), +}; + +/* Register offset defines for A4XX, in order of enum adreno_regs */ +static unsigned int a4xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_WADDR, A4XX_CP_ME_RAM_WADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_DATA, A4XX_CP_ME_RAM_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PFP_UCODE_DATA, A4XX_CP_PFP_UCODE_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PFP_UCODE_ADDR, A4XX_CP_PFP_UCODE_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_WFI_PEND_CTR, A4XX_CP_WFI_PEND_CTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A4XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, A4XX_CP_RB_RPTR_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A4XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A4XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A4XX_CP_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A4XX_CP_ME_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A4XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A4XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A4XX_CP_IB1_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A4XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A4XX_CP_IB2_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_RAM_RADDR, A4XX_CP_ME_RAM_RADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_ADDR, A4XX_CP_ROQ_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_DATA, A4XX_CP_ROQ_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_ADDR, A4XX_CP_MERCIU_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA, A4XX_CP_MERCIU_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA2, A4XX_CP_MERCIU_DATA2), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_ADDR, A4XX_CP_MEQ_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_DATA, A4XX_CP_MEQ_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_HW_FAULT, A4XX_CP_HW_FAULT), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_STATUS, A4XX_CP_PROTECT_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG6, A4XX_CP_SCRATCH_REG6), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG7, A4XX_CP_SCRATCH_REG7), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A4XX_CP_PREEMPT), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DEBUG, A4XX_CP_PREEMPT_DEBUG), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DISABLE, + A4XX_CP_PREEMPT_DISABLE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A4XX_CP_PROTECT_REG_0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A4XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_CTL, A4XX_RBBM_PERFCTR_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + A4XX_RBBM_PERFCTR_LOAD_CMD0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + A4XX_RBBM_PERFCTR_LOAD_CMD1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD2, + A4XX_RBBM_PERFCTR_LOAD_CMD2), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD3, + ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_PWR_1_LO, + A4XX_RBBM_PERFCTR_PWR_1_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A4XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_STATUS, A4XX_RBBM_INT_0_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A4XX_RBBM_CLOCK_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_VPC_DEBUG_RAM_SEL, + A4XX_VPC_DEBUG_RAM_SEL), + ADRENO_REG_DEFINE(ADRENO_REG_VPC_DEBUG_RAM_READ, + A4XX_VPC_DEBUG_RAM_READ), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_CLEAR_CMD, + A4XX_RBBM_INT_CLEAR_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_RBBM_CTL, A4XX_RBBM_RBBM_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A4XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE0, A4XX_UCHE_INVALIDATE0), + ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE1, A4XX_UCHE_INVALIDATE1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_LO, + A4XX_RBBM_PERFCTR_RBBM_0_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_HI, + A4XX_RBBM_PERFCTR_RBBM_0_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + A4XX_RBBM_PERFCTR_LOAD_VALUE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + A4XX_RBBM_PERFCTR_LOAD_VALUE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TRUST_CONTROL, + A4XX_RBBM_SECVID_TRUST_CONTROL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO, + A4XX_RBBM_ALWAYSON_COUNTER_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI, + A4XX_RBBM_ALWAYSON_COUNTER_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TRUST_CONFIG, + A4XX_RBBM_SECVID_TRUST_CONFIG), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_CONTROL, + A4XX_RBBM_SECVID_TSB_CONTROL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE, + A4XX_RBBM_SECVID_TSB_TRUSTED_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_SIZE, + A4XX_RBBM_SECVID_TSB_TRUSTED_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL0, + A4XX_VBIF_XIN_HALT_CTRL0), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A4XX_VBIF_XIN_HALT_CTRL1), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_VERSION, + A4XX_VBIF_VERSION), +}; + +static const struct adreno_reg_offsets a4xx_reg_offsets = { + .offsets = a4xx_register_offsets, + .offset_0 = ADRENO_REG_REGISTER_MAX, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_0_LO, + A4XX_RBBM_PERFCTR_CP_0_HI, 0, A4XX_CP_PERFCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_1_LO, + A4XX_RBBM_PERFCTR_CP_1_HI, 1, A4XX_CP_PERFCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_2_LO, + A4XX_RBBM_PERFCTR_CP_2_HI, 2, A4XX_CP_PERFCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_3_LO, + A4XX_RBBM_PERFCTR_CP_3_HI, 3, A4XX_CP_PERFCTR_CP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_4_LO, + A4XX_RBBM_PERFCTR_CP_4_HI, 4, A4XX_CP_PERFCTR_CP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_5_LO, + A4XX_RBBM_PERFCTR_CP_5_HI, 5, A4XX_CP_PERFCTR_CP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_6_LO, + A4XX_RBBM_PERFCTR_CP_6_HI, 6, A4XX_CP_PERFCTR_CP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_7_LO, + A4XX_RBBM_PERFCTR_CP_7_HI, 7, A4XX_CP_PERFCTR_CP_SEL_7 }, +}; + +/* + * Special list of CP registers for 420 to account for flaws. This array is + * inserted into the tables during perfcounter init + */ +static struct adreno_perfcount_register a420_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_0_LO, + A4XX_RBBM_PERFCTR_CP_0_HI, 0, A4XX_CP_PERFCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_1_LO, + A4XX_RBBM_PERFCTR_CP_1_HI, 1, A4XX_CP_PERFCTR_CP_SEL_1 }, + /* + * The selector registers for 3, 5, and 7 are swizzled on the hardware. + * CP_4 and CP_6 are duped to SEL_2 and SEL_3 so we don't enable them + * here + */ + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_3_LO, + A4XX_RBBM_PERFCTR_CP_3_HI, 3, A4XX_CP_PERFCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_5_LO, + A4XX_RBBM_PERFCTR_CP_5_HI, 5, A4XX_CP_PERFCTR_CP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CP_7_LO, + A4XX_RBBM_PERFCTR_CP_7_HI, 7, A4XX_CP_PERFCTR_CP_SEL_4 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_rbbm[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RBBM_0_LO, + A4XX_RBBM_PERFCTR_RBBM_0_HI, 8, A4XX_RBBM_PERFCTR_RBBM_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RBBM_1_LO, + A4XX_RBBM_PERFCTR_RBBM_1_HI, 9, A4XX_RBBM_PERFCTR_RBBM_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RBBM_2_LO, + A4XX_RBBM_PERFCTR_RBBM_2_HI, 10, A4XX_RBBM_PERFCTR_RBBM_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RBBM_3_LO, + A4XX_RBBM_PERFCTR_RBBM_3_HI, 11, A4XX_RBBM_PERFCTR_RBBM_SEL_3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_0_LO, + A4XX_RBBM_PERFCTR_PC_0_HI, 12, A4XX_PC_PERFCTR_PC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_1_LO, + A4XX_RBBM_PERFCTR_PC_1_HI, 13, A4XX_PC_PERFCTR_PC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_2_LO, + A4XX_RBBM_PERFCTR_PC_2_HI, 14, A4XX_PC_PERFCTR_PC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_3_LO, + A4XX_RBBM_PERFCTR_PC_3_HI, 15, A4XX_PC_PERFCTR_PC_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_4_LO, + A4XX_RBBM_PERFCTR_PC_4_HI, 16, A4XX_PC_PERFCTR_PC_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_5_LO, + A4XX_RBBM_PERFCTR_PC_5_HI, 17, A4XX_PC_PERFCTR_PC_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_6_LO, + A4XX_RBBM_PERFCTR_PC_6_HI, 18, A4XX_PC_PERFCTR_PC_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PC_7_LO, + A4XX_RBBM_PERFCTR_PC_7_HI, 19, A4XX_PC_PERFCTR_PC_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_0_LO, + A4XX_RBBM_PERFCTR_VFD_0_HI, 20, A4XX_VFD_PERFCTR_VFD_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_1_LO, + A4XX_RBBM_PERFCTR_VFD_1_HI, 21, A4XX_VFD_PERFCTR_VFD_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_2_LO, + A4XX_RBBM_PERFCTR_VFD_2_HI, 22, A4XX_VFD_PERFCTR_VFD_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_3_LO, + A4XX_RBBM_PERFCTR_VFD_3_HI, 23, A4XX_VFD_PERFCTR_VFD_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_4_LO, + A4XX_RBBM_PERFCTR_VFD_4_HI, 24, A4XX_VFD_PERFCTR_VFD_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_5_LO, + A4XX_RBBM_PERFCTR_VFD_5_HI, 25, A4XX_VFD_PERFCTR_VFD_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_6_LO, + A4XX_RBBM_PERFCTR_VFD_6_HI, 26, A4XX_VFD_PERFCTR_VFD_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VFD_7_LO, + A4XX_RBBM_PERFCTR_VFD_7_HI, 27, A4XX_VFD_PERFCTR_VFD_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_0_LO, + A4XX_RBBM_PERFCTR_HLSQ_0_HI, 28, A4XX_HLSQ_PERFCTR_HLSQ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_1_LO, + A4XX_RBBM_PERFCTR_HLSQ_1_HI, 29, A4XX_HLSQ_PERFCTR_HLSQ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_2_LO, + A4XX_RBBM_PERFCTR_HLSQ_2_HI, 30, A4XX_HLSQ_PERFCTR_HLSQ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_3_LO, + A4XX_RBBM_PERFCTR_HLSQ_3_HI, 31, A4XX_HLSQ_PERFCTR_HLSQ_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_4_LO, + A4XX_RBBM_PERFCTR_HLSQ_4_HI, 32, A4XX_HLSQ_PERFCTR_HLSQ_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_5_LO, + A4XX_RBBM_PERFCTR_HLSQ_5_HI, 33, A4XX_HLSQ_PERFCTR_HLSQ_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_6_LO, + A4XX_RBBM_PERFCTR_HLSQ_6_HI, 34, A4XX_HLSQ_PERFCTR_HLSQ_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_HLSQ_7_LO, + A4XX_RBBM_PERFCTR_HLSQ_7_HI, 35, A4XX_HLSQ_PERFCTR_HLSQ_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VPC_0_LO, + A4XX_RBBM_PERFCTR_VPC_0_HI, 36, A4XX_VPC_PERFCTR_VPC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VPC_1_LO, + A4XX_RBBM_PERFCTR_VPC_1_HI, 37, A4XX_VPC_PERFCTR_VPC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VPC_2_LO, + A4XX_RBBM_PERFCTR_VPC_2_HI, 38, A4XX_VPC_PERFCTR_VPC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VPC_3_LO, + A4XX_RBBM_PERFCTR_VPC_3_HI, 39, A4XX_VPC_PERFCTR_VPC_SEL_3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CCU_0_LO, + A4XX_RBBM_PERFCTR_CCU_0_HI, 40, A4XX_RB_PERFCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CCU_1_LO, + A4XX_RBBM_PERFCTR_CCU_1_HI, 41, A4XX_RB_PERFCTR_CCU_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CCU_2_LO, + A4XX_RBBM_PERFCTR_CCU_2_HI, 42, A4XX_RB_PERFCTR_CCU_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_CCU_3_LO, + A4XX_RBBM_PERFCTR_CCU_3_HI, 43, A4XX_RB_PERFCTR_CCU_SEL_3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TSE_0_LO, + A4XX_RBBM_PERFCTR_TSE_0_HI, 44, A4XX_GRAS_PERFCTR_TSE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TSE_1_LO, + A4XX_RBBM_PERFCTR_TSE_1_HI, 45, A4XX_GRAS_PERFCTR_TSE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TSE_2_LO, + A4XX_RBBM_PERFCTR_TSE_2_HI, 46, A4XX_GRAS_PERFCTR_TSE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TSE_3_LO, + A4XX_RBBM_PERFCTR_TSE_3_HI, 47, A4XX_GRAS_PERFCTR_TSE_SEL_3 }, +}; + + +static struct adreno_perfcount_register a4xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RAS_0_LO, + A4XX_RBBM_PERFCTR_RAS_0_HI, 48, A4XX_GRAS_PERFCTR_RAS_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RAS_1_LO, + A4XX_RBBM_PERFCTR_RAS_1_HI, 49, A4XX_GRAS_PERFCTR_RAS_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RAS_2_LO, + A4XX_RBBM_PERFCTR_RAS_2_HI, 50, A4XX_GRAS_PERFCTR_RAS_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RAS_3_LO, + A4XX_RBBM_PERFCTR_RAS_3_HI, 51, A4XX_GRAS_PERFCTR_RAS_SEL_3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_0_LO, + A4XX_RBBM_PERFCTR_UCHE_0_HI, 52, A4XX_UCHE_PERFCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_1_LO, + A4XX_RBBM_PERFCTR_UCHE_1_HI, 53, A4XX_UCHE_PERFCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_2_LO, + A4XX_RBBM_PERFCTR_UCHE_2_HI, 54, A4XX_UCHE_PERFCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_3_LO, + A4XX_RBBM_PERFCTR_UCHE_3_HI, 55, A4XX_UCHE_PERFCTR_UCHE_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_4_LO, + A4XX_RBBM_PERFCTR_UCHE_4_HI, 56, A4XX_UCHE_PERFCTR_UCHE_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_5_LO, + A4XX_RBBM_PERFCTR_UCHE_5_HI, 57, A4XX_UCHE_PERFCTR_UCHE_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_6_LO, + A4XX_RBBM_PERFCTR_UCHE_6_HI, 58, A4XX_UCHE_PERFCTR_UCHE_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_UCHE_7_LO, + A4XX_RBBM_PERFCTR_UCHE_7_HI, 59, A4XX_UCHE_PERFCTR_UCHE_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_0_LO, + A4XX_RBBM_PERFCTR_TP_0_HI, 60, A4XX_TPL1_PERFCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_1_LO, + A4XX_RBBM_PERFCTR_TP_1_HI, 61, A4XX_TPL1_PERFCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_2_LO, + A4XX_RBBM_PERFCTR_TP_2_HI, 62, A4XX_TPL1_PERFCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_3_LO, + A4XX_RBBM_PERFCTR_TP_3_HI, 63, A4XX_TPL1_PERFCTR_TP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_4_LO, + A4XX_RBBM_PERFCTR_TP_4_HI, 64, A4XX_TPL1_PERFCTR_TP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_5_LO, + A4XX_RBBM_PERFCTR_TP_5_HI, 65, A4XX_TPL1_PERFCTR_TP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_6_LO, + A4XX_RBBM_PERFCTR_TP_6_HI, 66, A4XX_TPL1_PERFCTR_TP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_TP_7_LO, + A4XX_RBBM_PERFCTR_TP_7_HI, 67, A4XX_TPL1_PERFCTR_TP_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_0_LO, + A4XX_RBBM_PERFCTR_SP_0_HI, 68, A4XX_SP_PERFCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_1_LO, + A4XX_RBBM_PERFCTR_SP_1_HI, 69, A4XX_SP_PERFCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_2_LO, + A4XX_RBBM_PERFCTR_SP_2_HI, 70, A4XX_SP_PERFCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_3_LO, + A4XX_RBBM_PERFCTR_SP_3_HI, 71, A4XX_SP_PERFCTR_SP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_4_LO, + A4XX_RBBM_PERFCTR_SP_4_HI, 72, A4XX_SP_PERFCTR_SP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_5_LO, + A4XX_RBBM_PERFCTR_SP_5_HI, 73, A4XX_SP_PERFCTR_SP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_6_LO, + A4XX_RBBM_PERFCTR_SP_6_HI, 74, A4XX_SP_PERFCTR_SP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_7_LO, + A4XX_RBBM_PERFCTR_SP_7_HI, 75, A4XX_SP_PERFCTR_SP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_8_LO, + A4XX_RBBM_PERFCTR_SP_8_HI, 76, A4XX_SP_PERFCTR_SP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_9_LO, + A4XX_RBBM_PERFCTR_SP_9_HI, 77, A4XX_SP_PERFCTR_SP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_10_LO, + A4XX_RBBM_PERFCTR_SP_10_HI, 78, A4XX_SP_PERFCTR_SP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_SP_11_LO, + A4XX_RBBM_PERFCTR_SP_11_HI, 79, A4XX_SP_PERFCTR_SP_SEL_11 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_0_LO, + A4XX_RBBM_PERFCTR_RB_0_HI, 80, A4XX_RB_PERFCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_1_LO, + A4XX_RBBM_PERFCTR_RB_1_HI, 81, A4XX_RB_PERFCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_2_LO, + A4XX_RBBM_PERFCTR_RB_2_HI, 82, A4XX_RB_PERFCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_3_LO, + A4XX_RBBM_PERFCTR_RB_3_HI, 83, A4XX_RB_PERFCTR_RB_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_4_LO, + A4XX_RBBM_PERFCTR_RB_4_HI, 84, A4XX_RB_PERFCTR_RB_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_5_LO, + A4XX_RBBM_PERFCTR_RB_5_HI, 85, A4XX_RB_PERFCTR_RB_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_6_LO, + A4XX_RBBM_PERFCTR_RB_6_HI, 86, A4XX_RB_PERFCTR_RB_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_RB_7_LO, + A4XX_RBBM_PERFCTR_RB_7_HI, 87, A4XX_RB_PERFCTR_RB_SEL_7 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_vsc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VSC_0_LO, + A4XX_RBBM_PERFCTR_VSC_0_HI, 88, A4XX_VSC_PERFCTR_VSC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_VSC_1_LO, + A4XX_RBBM_PERFCTR_VSC_1_HI, 89, A4XX_VSC_PERFCTR_VSC_SEL_1 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PWR_0_LO, + A4XX_RBBM_PERFCTR_PWR_0_HI, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_PERFCTR_PWR_1_LO, + A4XX_RBBM_PERFCTR_PWR_1_HI, -1, 0}, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_vbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_CNT_LOW0, + A4XX_VBIF_PERF_CNT_HIGH0, -1, A4XX_VBIF_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_CNT_LOW1, + A4XX_VBIF_PERF_CNT_HIGH1, -1, A4XX_VBIF_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_CNT_LOW2, + A4XX_VBIF_PERF_CNT_HIGH2, -1, A4XX_VBIF_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_CNT_LOW3, + A4XX_VBIF_PERF_CNT_HIGH3, -1, A4XX_VBIF_PERF_CNT_SEL3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_vbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_PWR_CNT_LOW0, + A4XX_VBIF_PERF_PWR_CNT_HIGH0, -1, A4XX_VBIF_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_PWR_CNT_LOW1, + A4XX_VBIF_PERF_PWR_CNT_HIGH1, -1, A4XX_VBIF_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_PWR_CNT_LOW2, + A4XX_VBIF_PERF_PWR_CNT_HIGH2, -1, A4XX_VBIF_PERF_PWR_CNT_EN2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_VBIF_PERF_PWR_CNT_LOW3, + A4XX_VBIF_PERF_PWR_CNT_HIGH3, -1, A4XX_VBIF_PERF_PWR_CNT_EN3 }, +}; + +static struct adreno_perfcount_register a4xx_perfcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A4XX_RBBM_ALWAYSON_COUNTER_LO, + A4XX_RBBM_ALWAYSON_COUNTER_HI, -1 }, +}; + +#define A4XX_PERFCOUNTER_GROUP(offset, name) \ + ADRENO_PERFCOUNTER_GROUP(a4xx, offset, name) + +#define A4XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a4xx, offset, name, flags) + +static struct adreno_perfcount_group a4xx_perfcounter_groups + [KGSL_PERFCOUNTER_GROUP_MAX] = { + A4XX_PERFCOUNTER_GROUP(CP, cp), + A4XX_PERFCOUNTER_GROUP(RBBM, rbbm), + A4XX_PERFCOUNTER_GROUP(PC, pc), + A4XX_PERFCOUNTER_GROUP(VFD, vfd), + A4XX_PERFCOUNTER_GROUP(HLSQ, hlsq), + A4XX_PERFCOUNTER_GROUP(VPC, vpc), + A4XX_PERFCOUNTER_GROUP(CCU, ccu), + A4XX_PERFCOUNTER_GROUP(TSE, tse), + A4XX_PERFCOUNTER_GROUP(RAS, ras), + A4XX_PERFCOUNTER_GROUP(UCHE, uche), + A4XX_PERFCOUNTER_GROUP(TP, tp), + A4XX_PERFCOUNTER_GROUP(SP, sp), + A4XX_PERFCOUNTER_GROUP(RB, rb), + A4XX_PERFCOUNTER_GROUP(VSC, vsc), + A4XX_PERFCOUNTER_GROUP_FLAGS(PWR, pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A4XX_PERFCOUNTER_GROUP(VBIF, vbif), + A4XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A4XX_PERFCOUNTER_GROUP_FLAGS(ALWAYSON, alwayson, + ADRENO_PERFCOUNTER_GROUP_FIXED), +}; + +static struct adreno_perfcounters a4xx_perfcounters = { + a4xx_perfcounter_groups, + ARRAY_SIZE(a4xx_perfcounter_groups), +}; + +static struct adreno_ft_perf_counters a4xx_ft_perf_counters[] = { + {KGSL_PERFCOUNTER_GROUP_SP, A4XX_SP_ALU_ACTIVE_CYCLES}, + {KGSL_PERFCOUNTER_GROUP_SP, A4XX_SP0_ICL1_MISSES}, + {KGSL_PERFCOUNTER_GROUP_SP, A4XX_SP_FS_CFLOW_INSTRUCTIONS}, + {KGSL_PERFCOUNTER_GROUP_TSE, A4XX_TSE_INPUT_PRIM_NUM}, +}; + +/* + * On A420 a number of perfcounters are un-usable. The following defines the + * array of countables that do not work and should not be used + */ +static const unsigned int a420_pc_invalid_countables[] = { + PC_INSTANCES, PC_VERTEX_HITS, PC_GENERATED_FIBERS, PC_GENERATED_WAVES, +}; + +static const unsigned int a420_vfd_invalid_countables[] = { + VFD_VPC_BYPASS_TRANS, VFD_UPPER_SHADER_FIBERS, VFD_LOWER_SHADER_FIBERS, +}; + +static const unsigned int a420_hlsq_invalid_countables[] = { + HLSQ_SP_VS_STAGE_CONSTANT, HLSQ_SP_VS_STAGE_INSTRUCTIONS, + HLSQ_SP_FS_STAGE_CONSTANT, HLSQ_SP_FS_STAGE_INSTRUCTIONS, + HLSQ_FS_STAGE_16_WAVES, HLSQ_FS_STAGE_32_WAVES, HLSQ_FS_STAGE_64_WAVES, + HLSQ_VS_STAGE_16_WAVES, HLSQ_VS_STAGE_32_WAVES, +}; + +static const unsigned int a420_uche_invalid_countables[] = { + UCHE_READ_REQUESTS_MARB, UCHE_READ_REQUESTS_SP, + UCHE_WRITE_REQUESTS_MARB, UCHE_WRITE_REQUESTS_SP, + UCHE_WRITE_REQUESTS_VPC +}; + +static const unsigned int a420_tp_invalid_countables[] = { + TP_OUTPUT_TEXELS_POINT, TP_OUTPUT_TEXELS_BILINEAR, TP_OUTPUT_TEXELS_MIP, + TP_OUTPUT_TEXELS_ANISO, TP_OUTPUT_TEXELS_OPS16, TP_OUTPUT_TEXELS_OPS32, + TP_ZERO_LOD, TP_LATENCY, TP_LATENCY_TRANS, +}; + +static const unsigned int a420_sp_invalid_countables[] = { + SP_FS_STAGE_BARY_INSTRUCTIONS, +}; + +static const unsigned int a420_rb_invalid_countables[] = { + RB_VALID_SAMPLES, RB_Z_FAIL, RB_S_FAIL, +}; + +static const unsigned int a420_ccu_invalid_countables[] = { + CCU_VBIF_STALL, CCU_VBIF_LATENCY_CYCLES, CCU_VBIF_LATENCY_SAMPLES, + CCU_Z_READ, CCU_Z_WRITE, CCU_C_READ, CCU_C_WRITE, +}; + +static const struct adreno_invalid_countables + a420_perfctr_invalid_countables[KGSL_PERFCOUNTER_GROUP_MAX] = { + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_pc, PC), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_vfd, VFD), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_hlsq, HLSQ), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_tp, TP), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_sp, SP), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_rb, RB), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_ccu, CCU), + ADRENO_PERFCOUNTER_INVALID_COUNTABLE(a420_uche, UCHE), +}; + +static struct adreno_coresight_register a4xx_coresight_registers[] = { + { A4XX_RBBM_CFG_DEBBUS_CTLT }, + { A4XX_RBBM_CFG_DEBBUS_SEL_A }, + { A4XX_RBBM_CFG_DEBBUS_SEL_B }, + { A4XX_RBBM_CFG_DEBBUS_SEL_C }, + { A4XX_RBBM_CFG_DEBBUS_SEL_D }, + { A4XX_RBBM_CFG_DEBBUS_OPL }, + { A4XX_RBBM_CFG_DEBBUS_OPE }, + { A4XX_RBBM_CFG_DEBBUS_IVTL_0 }, + { A4XX_RBBM_CFG_DEBBUS_IVTL_1 }, + { A4XX_RBBM_CFG_DEBBUS_IVTL_2 }, + { A4XX_RBBM_CFG_DEBBUS_IVTL_3 }, + { A4XX_RBBM_CFG_DEBBUS_MASKL_0 }, + { A4XX_RBBM_CFG_DEBBUS_MASKL_1 }, + { A4XX_RBBM_CFG_DEBBUS_MASKL_2 }, + { A4XX_RBBM_CFG_DEBBUS_MASKL_3 }, + { A4XX_RBBM_CFG_DEBBUS_BYTEL_0 }, + { A4XX_RBBM_CFG_DEBBUS_BYTEL_1 }, + { A4XX_RBBM_CFG_DEBBUS_IVTE_0 }, + { A4XX_RBBM_CFG_DEBBUS_IVTE_1 }, + { A4XX_RBBM_CFG_DEBBUS_IVTE_2 }, + { A4XX_RBBM_CFG_DEBBUS_IVTE_3 }, + { A4XX_RBBM_CFG_DEBBUS_MASKE_0 }, + { A4XX_RBBM_CFG_DEBBUS_MASKE_1 }, + { A4XX_RBBM_CFG_DEBBUS_MASKE_2 }, + { A4XX_RBBM_CFG_DEBBUS_MASKE_3 }, + { A4XX_RBBM_CFG_DEBBUS_NIBBLEE }, + { A4XX_RBBM_CFG_DEBBUS_PTRC0 }, + { A4XX_RBBM_CFG_DEBBUS_PTRC1 }, + { A4XX_RBBM_CFG_DEBBUS_CLRC }, + { A4XX_RBBM_CFG_DEBBUS_LOADIVT }, + { A4XX_RBBM_CFG_DEBBUS_IDX }, + { A4XX_RBBM_CFG_DEBBUS_LOADREG }, + { A4XX_RBBM_EXT_TRACE_BUS_CTL }, + { A4XX_RBBM_CFG_DEBBUS_CTLM }, +}; + +static void a4xx_perfcounter_init(struct adreno_device *adreno_dev) +{ + if (adreno_is_a420(adreno_dev)) { + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_perfcounters *counters = gpudev->perfcounters; + + /* + * The CP counters on A420 are... special. Some of the counters + * are swizzled so only a subset of them are usable + */ + + if (counters != NULL) { + counters->groups[KGSL_PERFCOUNTER_GROUP_CP].regs = + a420_perfcounters_cp; + counters->groups[KGSL_PERFCOUNTER_GROUP_CP].reg_count = + ARRAY_SIZE(a420_perfcounters_cp); + } + + /* + * Also on A420 a number of the countables are not functional so + * we maintain a blacklist of countables to protect the user + */ + + gpudev->invalid_countables = a420_perfctr_invalid_countables; + } + + /* + * Enable the GPU busy count counter. This is a fixed counter on + * A4XX so we don't need to bother checking the return value + */ + + adreno_perfcounter_get(adreno_dev, KGSL_PERFCOUNTER_GROUP_PWR, 1, + NULL, NULL, PERFCOUNTER_FLAG_KERNEL); +} + +static void a4xx_perfcounter_close(struct adreno_device *adreno_dev) +{ + adreno_perfcounter_put(adreno_dev, KGSL_PERFCOUNTER_GROUP_PWR, 1, + PERFCOUNTER_FLAG_KERNEL); +} + +static const unsigned int _a4xx_pwron_fixup_fs_instructions[] = { + 0x00000000, 0x304CC300, 0x00000000, 0x304CC304, + 0x00000000, 0x304CC308, 0x00000000, 0x304CC30C, + 0x00000000, 0x304CC310, 0x00000000, 0x304CC314, + 0x00000000, 0x304CC318, 0x00000000, 0x304CC31C, + 0x00000000, 0x304CC320, 0x00000000, 0x304CC324, + 0x00000000, 0x304CC328, 0x00000000, 0x304CC32C, + 0x00000000, 0x304CC330, 0x00000000, 0x304CC334, + 0x00000000, 0x304CC338, 0x00000000, 0x304CC33C, + 0x00000000, 0x00000400, 0x00020000, 0x63808003, + 0x00060004, 0x63828007, 0x000A0008, 0x6384800B, + 0x000E000C, 0x6386800F, 0x00120010, 0x63888013, + 0x00160014, 0x638A8017, 0x001A0018, 0x638C801B, + 0x001E001C, 0x638E801F, 0x00220020, 0x63908023, + 0x00260024, 0x63928027, 0x002A0028, 0x6394802B, + 0x002E002C, 0x6396802F, 0x00320030, 0x63988033, + 0x00360034, 0x639A8037, 0x003A0038, 0x639C803B, + 0x003E003C, 0x639E803F, 0x00000000, 0x00000400, + 0x00000003, 0x80D00003, 0x00000007, 0x80D00007, + 0x0000000B, 0x80D0000B, 0x0000000F, 0x80D0000F, + 0x00000013, 0x80D00013, 0x00000017, 0x80D00017, + 0x0000001B, 0x80D0001B, 0x0000001F, 0x80D0001F, + 0x00000023, 0x80D00023, 0x00000027, 0x80D00027, + 0x0000002B, 0x80D0002B, 0x0000002F, 0x80D0002F, + 0x00000033, 0x80D00033, 0x00000037, 0x80D00037, + 0x0000003B, 0x80D0003B, 0x0000003F, 0x80D0003F, + 0x00000000, 0x00000400, 0xFFFFFFFF, 0x304CC300, + 0xFFFFFFFF, 0x304CC304, 0xFFFFFFFF, 0x304CC308, + 0xFFFFFFFF, 0x304CC30C, 0xFFFFFFFF, 0x304CC310, + 0xFFFFFFFF, 0x304CC314, 0xFFFFFFFF, 0x304CC318, + 0xFFFFFFFF, 0x304CC31C, 0xFFFFFFFF, 0x304CC320, + 0xFFFFFFFF, 0x304CC324, 0xFFFFFFFF, 0x304CC328, + 0xFFFFFFFF, 0x304CC32C, 0xFFFFFFFF, 0x304CC330, + 0xFFFFFFFF, 0x304CC334, 0xFFFFFFFF, 0x304CC338, + 0xFFFFFFFF, 0x304CC33C, 0x00000000, 0x00000400, + 0x00020000, 0x63808003, 0x00060004, 0x63828007, + 0x000A0008, 0x6384800B, 0x000E000C, 0x6386800F, + 0x00120010, 0x63888013, 0x00160014, 0x638A8017, + 0x001A0018, 0x638C801B, 0x001E001C, 0x638E801F, + 0x00220020, 0x63908023, 0x00260024, 0x63928027, + 0x002A0028, 0x6394802B, 0x002E002C, 0x6396802F, + 0x00320030, 0x63988033, 0x00360034, 0x639A8037, + 0x003A0038, 0x639C803B, 0x003E003C, 0x639E803F, + 0x00000000, 0x00000400, 0x00000003, 0x80D00003, + 0x00000007, 0x80D00007, 0x0000000B, 0x80D0000B, + 0x0000000F, 0x80D0000F, 0x00000013, 0x80D00013, + 0x00000017, 0x80D00017, 0x0000001B, 0x80D0001B, + 0x0000001F, 0x80D0001F, 0x00000023, 0x80D00023, + 0x00000027, 0x80D00027, 0x0000002B, 0x80D0002B, + 0x0000002F, 0x80D0002F, 0x00000033, 0x80D00033, + 0x00000037, 0x80D00037, 0x0000003B, 0x80D0003B, + 0x0000003F, 0x80D0003F, 0x00000000, 0x03000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +/** + * _a4xx_pwron_fixup() - Initialize a special command buffer to run a + * post-power collapse shader workaround + * @adreno_dev: Pointer to a adreno_device struct + * + * Some targets require a special workaround shader to be executed after + * power-collapse. Construct the IB once at init time and keep it + * handy + * + * Returns: 0 on success or negative on error + */ +static int _a4xx_pwron_fixup(struct adreno_device *adreno_dev) +{ + unsigned int *cmds; + unsigned int count = ARRAY_SIZE(_a4xx_pwron_fixup_fs_instructions); + unsigned int num_units = count >> 5; + int ret; + + /* Return if the fixup is already in place */ + if (test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv)) + return 0; + + ret = kgsl_allocate_global(KGSL_DEVICE(adreno_dev), + &adreno_dev->pwron_fixup, PAGE_SIZE, + KGSL_MEMFLAGS_GPUREADONLY, 0, "pwron_fixup"); + + if (ret) + return ret; + + cmds = adreno_dev->pwron_fixup.hostptr; + + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4XX_SP_MODE_CONTROL, 1); + *cmds++ = 0x00000018; + *cmds++ = cp_type0_packet(A4XX_TPL1_TP_MODE_CONTROL, 1); + *cmds++ = 0x00000002; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4xx_HLSQ_CONTROL_0, 5); + *cmds++ = 0x800001a0; + *cmds++ = 0xfcfc0000; + *cmds++ = 0xcff3f3f0; + *cmds++ = 0xfcfcfcfc; + *cmds++ = 0xccfcfcfc; + *cmds++ = cp_type0_packet(A4XX_SP_FS_CTRL_1, 1); + *cmds++ = 0x80000000; + *cmds++ = cp_type0_packet(A4XX_HLSQ_UPDATE_CONTROL, 1); + *cmds++ = 0x00000038; + *cmds++ = cp_type0_packet(A4XX_HLSQ_MODE_CONTROL, 1); + *cmds++ = 0x00000003; + *cmds++ = cp_type0_packet(A4XX_HLSQ_UPDATE_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4XX_TPL1_TP_TEX_TSIZE_1, 1); + *cmds++ = 0x00008000; + *cmds++ = cp_type0_packet(A4xx_HLSQ_CONTROL_0, 2); + *cmds++ = 0x800001a0; + *cmds++ = 0xfcfc0000; + *cmds++ = cp_type0_packet(A4XX_HLSQ_CS_CONTROL, 1); + *cmds++ = 0x00018030 | (num_units << 24); + *cmds++ = cp_type0_packet(A4XX_HLSQ_CL_NDRANGE_0, 7); + *cmds++ = 0x000000fd; + *cmds++ = 0x00000040; + *cmds++ = 0x00000000; + *cmds++ = 0x00000001; + *cmds++ = 0x00000000; + *cmds++ = 0x00000001; + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4XX_HLSQ_CL_CONTROL_0, 2); + *cmds++ = 0x0001201f; + *cmds++ = 0x0000f003; + *cmds++ = cp_type0_packet(A4XX_HLSQ_CL_KERNEL_CONST, 1); + *cmds++ = 0x0001800b; + *cmds++ = cp_type0_packet(A4XX_HLSQ_CL_KERNEL_GROUP_X, 3); + *cmds++ = 0x00000001; + *cmds++ = 0x00000001; + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A4XX_HLSQ_CL_WG_OFFSET, 1); + *cmds++ = 0x00000022; + *cmds++ = cp_type0_packet(A4XX_UCHE_INVALIDATE0, 2); + *cmds++ = 0x00000000; + *cmds++ = 0x00000012; + *cmds++ = cp_type0_packet(A4XX_HLSQ_MODE_CONTROL, 1); + *cmds++ = 0x00000003; + *cmds++ = cp_type0_packet(A4XX_SP_SP_CTRL, 1); + *cmds++ = 0x00920000; + *cmds++ = cp_type0_packet(A4XX_SP_INSTR_CACHE_CTRL, 1); + *cmds++ = 0x00000260; + *cmds++ = cp_type0_packet(A4XX_SP_CS_CTRL_0, 1); + *cmds++ = 0x00200400; + *cmds++ = cp_type0_packet(A4XX_SP_CS_OBJ_OFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4XX_SP_CS_OBJ_START, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A4XX_SP_CS_LENGTH, 1); + *cmds++ = num_units; + *cmds++ = cp_type0_packet(A4XX_SP_MODE_CONTROL, 1); + *cmds++ = 0x00000018; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 2 + count); + *cmds++ = 0x00340000 | (num_units << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = 0x00000000; + + memcpy(cmds, _a4xx_pwron_fixup_fs_instructions, count << 2); + cmds += count; + + *cmds++ = cp_type3_packet(CP_EXEC_CL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + + /* + * Remember the number of dwords in the command buffer for when we + * program the indirect buffer call in the ringbuffer + */ + adreno_dev->pwron_fixup_dwords = + (cmds - (unsigned int *) adreno_dev->pwron_fixup.hostptr); + + /* Mark the flag in ->priv to show that we have the fix */ + set_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv); + return 0; +} + +/* + * a4xx_init() - Initialize gpu specific data + * @adreno_dev: Pointer to adreno device + */ +static void a4xx_init(struct adreno_device *adreno_dev) +{ + if ((adreno_is_a405(adreno_dev)) || (adreno_is_a420(adreno_dev))) + _a4xx_pwron_fixup(adreno_dev); +} + +static int a4xx_send_me_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 20); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + *cmds++ = cp_type3_packet(CP_ME_INIT, 17); + + /* + * Ordinal 2 of ME_INIT packet, the bits being set are: + * Ordinal 3, 4, 5-12, 14, 15, 16, 17, 18 are present + * Microcode Default Reset Control = 3 + */ + *cmds++ = 0x000003f7; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000080; + *cmds++ = 0x00000100; + *cmds++ = 0x00000180; + *cmds++ = 0x00006600; + *cmds++ = 0x00000150; + *cmds++ = 0x0000014e; + *cmds++ = 0x00000154; + /* MAX Context */ + *cmds++ = 0x00000001; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + /* Enable protected mode registers for A3XX/A4XX */ + *cmds++ = 0x20000000; + + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + *cmds++ = cp_type3_packet(CP_PREEMPT_ENABLE, 1); + *cmds++ = 1; + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + dev_err(device->dev, "CP initialization failed to idle\n"); + kgsl_device_snapshot(device, NULL, false); + } + + return ret; +} + +/* + * a4xx_rb_start() - Start the ringbuffer + * @adreno_dev: Pointer to adreno device + * @start_type: Warm or cold start + */ +static int a4xx_rb_start(struct adreno_device *adreno_dev, + unsigned int start_type) +{ + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + struct kgsl_device *device = &adreno_dev->dev; + uint64_t addr; + int ret; + + addr = SCRATCH_RPTR_GPU_ADDR(device, rb->id); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, addr); + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + * Also disable the host RPTR shadow register as it might be unreliable + * in certain circumstances. + */ + + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, + ((ilog2(4) << 8) & 0x1F00) | + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)); + + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_BASE, + rb->buffer_desc.gpuaddr); + + ret = a3xx_microcode_load(adreno_dev, start_type); + if (ret) + return ret; + + /* clear ME_HALT to start micro engine */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0); + + ret = a4xx_send_me_init(adreno_dev, rb); + if (ret == 0) { + a4xx_enable_pc(adreno_dev); + a4xx_enable_ppd(adreno_dev); + } + + return ret; +} + +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ctrlt, &a4xx_coresight_registers[0]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_sela, &a4xx_coresight_registers[1]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_selb, &a4xx_coresight_registers[2]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_selc, &a4xx_coresight_registers[3]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_seld, &a4xx_coresight_registers[4]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_opl, &a4xx_coresight_registers[5]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ope, &a4xx_coresight_registers[6]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivtl0, &a4xx_coresight_registers[7]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivtl1, &a4xx_coresight_registers[8]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivtl2, &a4xx_coresight_registers[9]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivtl3, &a4xx_coresight_registers[10]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maskl0, &a4xx_coresight_registers[11]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maskl1, &a4xx_coresight_registers[12]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maskl2, &a4xx_coresight_registers[13]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maskl3, &a4xx_coresight_registers[14]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_bytel0, &a4xx_coresight_registers[15]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_bytel1, &a4xx_coresight_registers[16]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivte0, &a4xx_coresight_registers[17]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivte1, &a4xx_coresight_registers[18]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivte2, &a4xx_coresight_registers[19]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ivte3, &a4xx_coresight_registers[20]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maske0, &a4xx_coresight_registers[21]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maske1, &a4xx_coresight_registers[22]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maske2, &a4xx_coresight_registers[23]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_maske3, &a4xx_coresight_registers[24]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_nibblee, &a4xx_coresight_registers[25]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ptrc0, &a4xx_coresight_registers[26]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ptrc1, &a4xx_coresight_registers[27]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_clrc, &a4xx_coresight_registers[28]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_loadivt, &a4xx_coresight_registers[29]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_idx, &a4xx_coresight_registers[30]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_loadreg, &a4xx_coresight_registers[31]); +static ADRENO_CORESIGHT_ATTR(ext_tracebus_ctl, &a4xx_coresight_registers[32]); +static ADRENO_CORESIGHT_ATTR(cfg_debbus_ctrlm, &a4xx_coresight_registers[33]); + + +static struct attribute *a4xx_coresight_attrs[] = { + &coresight_attr_cfg_debbus_ctrlt.attr.attr, + &coresight_attr_cfg_debbus_sela.attr.attr, + &coresight_attr_cfg_debbus_selb.attr.attr, + &coresight_attr_cfg_debbus_selc.attr.attr, + &coresight_attr_cfg_debbus_seld.attr.attr, + &coresight_attr_cfg_debbus_opl.attr.attr, + &coresight_attr_cfg_debbus_ope.attr.attr, + &coresight_attr_cfg_debbus_ivtl0.attr.attr, + &coresight_attr_cfg_debbus_ivtl1.attr.attr, + &coresight_attr_cfg_debbus_ivtl2.attr.attr, + &coresight_attr_cfg_debbus_ivtl3.attr.attr, + &coresight_attr_cfg_debbus_maskl0.attr.attr, + &coresight_attr_cfg_debbus_maskl1.attr.attr, + &coresight_attr_cfg_debbus_maskl2.attr.attr, + &coresight_attr_cfg_debbus_maskl3.attr.attr, + &coresight_attr_cfg_debbus_bytel0.attr.attr, + &coresight_attr_cfg_debbus_bytel1.attr.attr, + &coresight_attr_cfg_debbus_ivte0.attr.attr, + &coresight_attr_cfg_debbus_ivte1.attr.attr, + &coresight_attr_cfg_debbus_ivte2.attr.attr, + &coresight_attr_cfg_debbus_ivte3.attr.attr, + &coresight_attr_cfg_debbus_maske0.attr.attr, + &coresight_attr_cfg_debbus_maske1.attr.attr, + &coresight_attr_cfg_debbus_maske2.attr.attr, + &coresight_attr_cfg_debbus_maske3.attr.attr, + &coresight_attr_cfg_debbus_nibblee.attr.attr, + &coresight_attr_cfg_debbus_ptrc0.attr.attr, + &coresight_attr_cfg_debbus_ptrc1.attr.attr, + &coresight_attr_cfg_debbus_clrc.attr.attr, + &coresight_attr_cfg_debbus_loadivt.attr.attr, + &coresight_attr_cfg_debbus_idx.attr.attr, + &coresight_attr_cfg_debbus_loadreg.attr.attr, + &coresight_attr_ext_tracebus_ctl.attr.attr, + &coresight_attr_cfg_debbus_ctrlm.attr.attr, + NULL, +}; + +static const struct attribute_group a4xx_coresight_group = { + .attrs = a4xx_coresight_attrs, +}; + +static const struct attribute_group *a4xx_coresight_groups[] = { + &a4xx_coresight_group, + NULL, +}; + +static struct adreno_coresight a4xx_coresight = { + .registers = a4xx_coresight_registers, + .count = ARRAY_SIZE(a4xx_coresight_registers), + .groups = a4xx_coresight_groups, +}; + +static void a4xx_preempt_callback(struct adreno_device *adreno_dev, int bit) +{ + if (atomic_read(&adreno_dev->preempt.state) != ADRENO_PREEMPT_TRIGGERED) + return; + + trace_adreno_hw_preempt_trig_to_comp_int(adreno_dev->cur_rb, + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); + + adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev)); +} + +#define A4XX_INT_MASK \ + ((1 << A4XX_INT_RBBM_AHB_ERROR) | \ + (1 << A4XX_INT_RBBM_REG_TIMEOUT) | \ + (1 << A4XX_INT_RBBM_ME_MS_TIMEOUT) | \ + (1 << A4XX_INT_RBBM_PFP_MS_TIMEOUT) | \ + (1 << A4XX_INT_RBBM_ETS_MS_TIMEOUT) | \ + (1 << A4XX_INT_RBBM_ASYNC_OVERFLOW) | \ + (1 << A4XX_INT_CP_SW) | \ + (1 << A4XX_INT_CP_OPCODE_ERROR) | \ + (1 << A4XX_INT_CP_RESERVED_BIT_ERROR) | \ + (1 << A4XX_INT_CP_HW_FAULT) | \ + (1 << A4XX_INT_CP_IB1_INT) | \ + (1 << A4XX_INT_CP_IB2_INT) | \ + (1 << A4XX_INT_CP_RB_INT) | \ + (1 << A4XX_INT_CACHE_FLUSH_TS) | \ + (1 << A4XX_INT_CP_REG_PROTECT_FAULT) | \ + (1 << A4XX_INT_CP_AHB_ERROR_HALT) | \ + (1 << A4XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A4XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A4XX_INT_RBBM_DPM_CALC_ERR) | \ + (1 << A4XX_INT_RBBM_DPM_EPOCH_ERR) | \ + (1 << A4XX_INT_RBBM_DPM_THERMAL_YELLOW_ERR) |\ + (1 << A4XX_INT_RBBM_DPM_THERMAL_RED_ERR)) + + +static struct adreno_irq_funcs a4xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 2 - RBBM_REG_TIMEOUT */ + /* 3 - RBBM_ME_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + /* 4 - RBBM_PFP_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 5 - RBBM_ETS_MS_TIMEOUT */ + /* 6 - RBBM_ATB_ASYNC_OVERFLOW */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 7 - RBBM_GPC_ERR */ + ADRENO_IRQ_CALLBACK(a4xx_preempt_callback), /* 8 - CP_SW */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 9 - CP_OPCODE_ERROR */ + /* 10 - CP_RESERVED_BIT_ERROR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 11 - CP_HW_FAULT */ + ADRENO_IRQ_CALLBACK(NULL), /* 12 - CP_DMA */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 15 - CP_RB_INT */ + /* 16 - CP_REG_PROTECT_FAULT */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_VS_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - CP_PS_DONE_TS */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + /* 21 - CP_AHB_ERROR_FAULT */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */ + ADRENO_IRQ_CALLBACK(NULL), /* 23 - Unused */ + /* 24 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 25 - UCHE_OOB_ACCESS */ + ADRENO_IRQ_CALLBACK(NULL), /* 26 - Unused */ + ADRENO_IRQ_CALLBACK(NULL), /* 27 - RBBM_TRACE_MISR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 28 - RBBM_DPM_CALC_ERR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), /* 29 - RBBM_DPM_EPOCH_ERR */ + /* 30 - RBBM_DPM_THERMAL_YELLOW_ERR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), + /* 31 - RBBM_DPM_THERMAL_RED_ERR */ + ADRENO_IRQ_CALLBACK(a4xx_err_callback), +}; + +static struct adreno_irq a4xx_irq = { + .funcs = a4xx_irq_funcs, + .mask = A4XX_INT_MASK, +}; + +static struct adreno_snapshot_data a4xx_snapshot_data = { + .sect_sizes = &a4xx_snap_sizes, +}; + +struct adreno_gpudev adreno_a4xx_gpudev = { + .reg_offsets = &a4xx_reg_offsets, + .int_bits = a4xx_int_bits, + .ft_perf_counters = a4xx_ft_perf_counters, + .ft_perf_counters_count = ARRAY_SIZE(a4xx_ft_perf_counters), + .perfcounters = &a4xx_perfcounters, + .irq = &a4xx_irq, + .irq_trace = trace_kgsl_a4xx_irq_status, + .snapshot_data = &a4xx_snapshot_data, + .num_prio_levels = KGSL_PRIORITY_MAX_RB_LEVELS, + .vbif_xin_halt_ctrl0_mask = A4XX_VBIF_XIN_HALT_CTRL0_MASK, + + .perfcounter_init = a4xx_perfcounter_init, + .perfcounter_close = a4xx_perfcounter_close, + .rb_start = a4xx_rb_start, + .init = a4xx_init, + .microcode_read = a3xx_microcode_read, + .coresight = {&a4xx_coresight}, + .start = a4xx_start, + .snapshot = a4xx_snapshot, + .is_sptp_idle = a4xx_is_sptp_idle, + .pwrlevel_change_settings = a4xx_pwrlevel_change_settings, + .regulator_enable = a4xx_regulator_enable, + .regulator_disable = a4xx_regulator_disable, + .preemption_pre_ibsubmit = a4xx_preemption_pre_ibsubmit, + .preemption_schedule = a4xx_preemption_schedule, + .preemption_init = a4xx_preemption_init, +}; diff --git a/drivers/gpu/msm/adreno_a4xx.h b/drivers/gpu/msm/adreno_a4xx.h new file mode 100644 index 0000000000000000000000000000000000000000..8566a9c3c59c37f1b2018ce95ecfc96350a0a460 --- /dev/null +++ b/drivers/gpu/msm/adreno_a4xx.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. + */ + +#ifndef _ADRENO_A4XX_H_ +#define _ADRENO_A4XX_H_ + +#include "a4xx_reg.h" + +#define A4XX_IRQ_FLAGS \ + { BIT(A4XX_INT_RBBM_GPU_IDLE), "RBBM_GPU_IDLE" }, \ + { BIT(A4XX_INT_RBBM_REG_TIMEOUT), "RBBM_REG_TIMEOUT" }, \ + { BIT(A4XX_INT_RBBM_ME_MS_TIMEOUT), "RBBM_ME_MS_TIMEOUT" }, \ + { BIT(A4XX_INT_RBBM_PFP_MS_TIMEOUT), "RBBM_PFP_MS_TIMEOUT" }, \ + { BIT(A4XX_INT_RBBM_ETS_MS_TIMEOUT), "RBBM_ETS_MS_TIMEOUT" }, \ + { BIT(A4XX_INT_RBBM_ASYNC_OVERFLOW), "RBBM_ASYNC_OVERFLOW" }, \ + { BIT(A4XX_INT_RBBM_GPC_ERR), "RBBM_GPC_ERR" }, \ + { BIT(A4XX_INT_CP_SW), "CP_SW" }, \ + { BIT(A4XX_INT_CP_OPCODE_ERROR), "CP_OPCODE_ERROR" }, \ + { BIT(A4XX_INT_CP_RESERVED_BIT_ERROR), "CP_RESERVED_BIT_ERROR" }, \ + { BIT(A4XX_INT_CP_HW_FAULT), "CP_HW_FAULT" }, \ + { BIT(A4XX_INT_CP_DMA), "CP_DMA" }, \ + { BIT(A4XX_INT_CP_IB2_INT), "CP_IB2_INT" }, \ + { BIT(A4XX_INT_CP_IB1_INT), "CP_IB1_INT" }, \ + { BIT(A4XX_INT_CP_RB_INT), "CP_RB_INT" }, \ + { BIT(A4XX_INT_CP_REG_PROTECT_FAULT), "CP_REG_PROTECT_FAULT" }, \ + { BIT(A4XX_INT_CP_RB_DONE_TS), "CP_RB_DONE_TS" }, \ + { BIT(A4XX_INT_CP_VS_DONE_TS), "CP_VS_DONE_TS" }, \ + { BIT(A4XX_INT_CP_PS_DONE_TS), "CP_PS_DONE_TS" }, \ + { BIT(A4XX_INT_CACHE_FLUSH_TS), "CACHE_FLUSH_TS" }, \ + { BIT(A4XX_INT_CP_AHB_ERROR_HALT), "CP_AHB_ERROR_HALT" }, \ + { BIT(A4XX_INT_RBBM_ATB_BUS_OVERFLOW), "RBBM_ATB_BUS_OVERFLOW" }, \ + { BIT(A4XX_INT_MISC_HANG_DETECT), "MISC_HANG_DETECT" }, \ + { BIT(A4XX_INT_UCHE_OOB_ACCESS), "UCHE_OOB_ACCESS" }, \ + { BIT(A4XX_INT_RBBM_DPM_CALC_ERR), "RBBM_DPM_CALC_ERR" }, \ + { BIT(A4XX_INT_RBBM_DPM_EPOCH_ERR), "RBBM_DPM_CALC_ERR" }, \ + { BIT(A4XX_INT_RBBM_DPM_THERMAL_YELLOW_ERR), \ + "RBBM_DPM_THERMAL_YELLOW_ERR" }, \ + { BIT(A4XX_INT_RBBM_DPM_THERMAL_RED_ERR), "RBBM_DPM_THERMAL_RED_ERR" } + +unsigned int a4xx_preemption_pre_ibsubmit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + unsigned int *cmds, + struct kgsl_context *context); + +void a4xx_preemption_schedule(struct adreno_device *adreno_dev); + +int a4xx_preemption_init(struct adreno_device *adreno_dev); + +void a4xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); + +#endif diff --git a/drivers/gpu/msm/adreno_a4xx_preempt.c b/drivers/gpu/msm/adreno_a4xx_preempt.c new file mode 100644 index 0000000000000000000000000000000000000000..21249146978998ba5ec8cf12033aeb5126c96e10 --- /dev/null +++ b/drivers/gpu/msm/adreno_a4xx_preempt.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a4xx.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" + +#define ADRENO_RB_PREEMPT_TOKEN_DWORDS 125 + +static void a4xx_preemption_timer(struct timer_list *t) +{ + struct adreno_preemption *preempt = from_timer(preempt, t, timer); + struct adreno_device *adreno_dev = container_of(preempt, + struct adreno_device, preempt); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int cur_rptr = adreno_get_rptr(adreno_dev->cur_rb); + unsigned int next_rptr = adreno_get_rptr(adreno_dev->next_rb); + + dev_err(device->dev, + "Preemption timed out. cur_rb rptr/wptr %x/%x id %d, next_rb rptr/wptr %x/%x id %d, disp_state: %d\n", + cur_rptr, adreno_dev->cur_rb->wptr, + adreno_dev->cur_rb->id, + next_rptr, adreno_dev->next_rb->wptr, + adreno_dev->next_rb->id, + atomic_read(&adreno_dev->preempt.state)); + + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + adreno_dispatcher_schedule(device); +} + +static unsigned int a4xx_preemption_token(struct adreno_device *adreno_dev, + unsigned int *cmds, uint64_t gpuaddr) +{ + unsigned int *cmds_orig = cmds; + + /* Turn on preemption flag */ + /* preemption token - fill when pt switch command size is known */ + *cmds++ = cp_type3_packet(CP_PREEMPT_TOKEN, 3); + *cmds++ = (uint)gpuaddr; + *cmds++ = 1; + /* generate interrupt on preemption completion */ + *cmds++ = 1 << CP_PREEMPT_ORDINAL_INTERRUPT; + + return (unsigned int) (cmds - cmds_orig); +} + +unsigned int a4xx_preemption_pre_ibsubmit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int *cmds, + struct kgsl_context *context) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds_orig = cmds; + unsigned int cond_addr = device->memstore.gpuaddr + + MEMSTORE_ID_GPU_ADDR(device, context->id, preempted); + + cmds += a4xx_preemption_token(adreno_dev, cmds, cond_addr); + + *cmds++ = cp_type3_packet(CP_COND_EXEC, 4); + *cmds++ = cond_addr; + *cmds++ = cond_addr; + *cmds++ = 1; + *cmds++ = 7; + + /* clear preemption flag */ + *cmds++ = cp_type3_packet(CP_MEM_WRITE, 2); + *cmds++ = cond_addr; + *cmds++ = 0; + *cmds++ = cp_type3_packet(CP_WAIT_MEM_WRITES, 1); + *cmds++ = 0; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_ME, 1); + *cmds++ = 0; + + return (unsigned int) (cmds - cmds_orig); +} + + +static void a4xx_preemption_start(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + uint32_t val; + + /* + * Setup scratch registers from which the GPU will program the + * registers required to start execution of new ringbuffer + * set ringbuffer address + */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG8, + rb->buffer_desc.gpuaddr); + kgsl_regread(device, A4XX_CP_RB_CNTL, &val); + /* scratch REG9 corresponds to CP_RB_CNTL register */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG9, val); + /* scratch REG10 corresponds to rptr address */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG10, + SCRATCH_RPTR_GPU_ADDR(device, rb->id)); + /* scratch REG11 corresponds to rptr */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG11, adreno_get_rptr(rb)); + /* scratch REG12 corresponds to wptr */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG12, rb->wptr); + /* + * scratch REG13 corresponds to IB1_BASE, + * 0 since we do not do switches in between IB's + */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG13, 0); + /* scratch REG14 corresponds to IB1_BUFSZ */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG14, 0); + /* scratch REG15 corresponds to IB2_BASE */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG15, 0); + /* scratch REG16 corresponds to IB2_BUFSZ */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG16, 0); + /* scratch REG17 corresponds to GPR11 */ + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG17, rb->gpr11); +} + +static void a4xx_preemption_save(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_regread(device, A4XX_CP_SCRATCH_REG23, &rb->gpr11); +} + + +static int a4xx_submit_preempt_token(struct adreno_ringbuffer *rb, + struct adreno_ringbuffer *incoming_rb) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *ringcmds, *start; + int ptname; + struct kgsl_pagetable *pt; + int pt_switch_sizedwords = 0, total_sizedwords = 20; + unsigned int link[ADRENO_RB_PREEMPT_TOKEN_DWORDS]; + uint i; + + if (incoming_rb->preempted_midway) { + + kgsl_sharedmem_readl(&incoming_rb->pagetable_desc, + &ptname, PT_INFO_OFFSET(current_rb_ptname)); + pt = kgsl_mmu_get_pt_from_ptname(&(device->mmu), + ptname); + if (IS_ERR_OR_NULL(pt)) + return (pt == NULL) ? -ENOENT : PTR_ERR(pt); + /* set the ringbuffer for incoming RB */ + pt_switch_sizedwords = + adreno_iommu_set_pt_generate_cmds(incoming_rb, + &link[0], pt); + total_sizedwords += pt_switch_sizedwords; + } + + /* + * Allocate total_sizedwords space in RB, this is the max space + * required. + */ + ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords); + + if (IS_ERR(ringcmds)) + return PTR_ERR(ringcmds); + + start = ringcmds; + + ringcmds += cp_protected_mode(adreno_dev, ringcmds, 0); + + if (incoming_rb->preempted_midway) { + for (i = 0; i < pt_switch_sizedwords; i++) + *ringcmds++ = link[i]; + } + + *ringcmds++ = cp_register(adreno_dev, adreno_getreg(adreno_dev, + ADRENO_REG_CP_PREEMPT_DISABLE), 1); + *ringcmds++ = 0; + + ringcmds += cp_protected_mode(adreno_dev, ringcmds, 1); + + ringcmds += a4xx_preemption_token(adreno_dev, ringcmds, + device->memstore.gpuaddr + + MEMSTORE_RB_OFFSET(rb, preempted)); + + if ((uint)(ringcmds - start) > total_sizedwords) + dev_err(device->dev, "Insufficient rb size allocated\n"); + + /* + * If we have commands less than the space reserved in RB + * adjust the wptr accordingly + */ + rb->wptr = rb->wptr - (total_sizedwords - (uint)(ringcmds - start)); + + /* submit just the preempt token */ + mb(); + kgsl_pwrscale_busy(device); + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, rb->wptr); + return 0; +} + +static void a4xx_preempt_trig_state(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int rbbase, val; + int ret; + + /* + * Hardware not yet idle means that preemption interrupt + * may still occur, nothing to do here until interrupt signals + * completion of preemption, just return here + */ + if (!adreno_hw_isidle(adreno_dev)) + return; + + /* + * We just changed states, reschedule dispatcher to change + * preemption states + */ + + if (atomic_read(&adreno_dev->preempt.state) != + ADRENO_PREEMPT_TRIGGERED) { + adreno_dispatcher_schedule(device); + return; + } + + /* + * H/W is idle and we did not get a preemption interrupt, may + * be device went idle w/o encountering any preempt token or + * we already preempted w/o interrupt + */ + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); + /* Did preemption occur, if so then change states and return */ + if (rbbase != adreno_dev->cur_rb->buffer_desc.gpuaddr) { + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val); + if (val && rbbase == adreno_dev->next_rb->buffer_desc.gpuaddr) { + dev_warn(device->dev, + "Preemption completed without interrupt\n"); + trace_adreno_hw_preempt_trig_to_comp(adreno_dev->cur_rb, + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); + adreno_set_preempt_state(adreno_dev, + ADRENO_PREEMPT_COMPLETE); + adreno_dispatcher_schedule(device); + return; + } + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + /* reschedule dispatcher to take care of the fault */ + adreno_dispatcher_schedule(device); + return; + } + /* + * Check if preempt token was submitted after preemption trigger, if so + * then preemption should have occurred, since device is already idle it + * means something went wrong - trigger FT + */ + if (adreno_dev->preempt.token_submit) { + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + /* reschedule dispatcher to take care of the fault */ + adreno_dispatcher_schedule(device); + return; + } + /* + * Preempt token was not submitted after preemption trigger so device + * may have gone idle before preemption could occur, if there are + * commands that got submitted to current RB after triggering preemption + * then submit them as those commands may have a preempt token in them + */ + if (!adreno_rb_empty(adreno_dev->cur_rb)) { + /* + * Memory barrier before informing the + * hardware of new commands + */ + mb(); + kgsl_pwrscale_busy(device); + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, + adreno_dev->cur_rb->wptr); + return; + } + + /* Submit preempt token to make preemption happen */ + ret = adreno_drawctxt_switch(adreno_dev, adreno_dev->cur_rb, + NULL, 0); + if (ret) + dev_err(device->dev, + "Unable to switch context to NULL: %d\n", ret); + + ret = a4xx_submit_preempt_token(adreno_dev->cur_rb, + adreno_dev->next_rb); + if (ret) + dev_err(device->dev, + "Unable to submit preempt token: %d\n", ret); + + adreno_dev->preempt.token_submit = true; + adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; + trace_adreno_hw_preempt_token_submit(adreno_dev->cur_rb, + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); +} + +static struct adreno_ringbuffer *a4xx_next_ringbuffer( + struct adreno_device *adreno_dev) +{ + struct adreno_ringbuffer *rb; + int i; + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + if (!adreno_rb_empty(rb)) + return rb; + } + + return NULL; +} + +static void a4xx_preempt_clear_state(struct adreno_device *adreno_dev) + +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_ringbuffer *highest_busy_rb; + int switch_low_to_high; + int ret; + + /* Device not awake means there is nothing to do */ + if (!kgsl_state_is_awake(device)) + return; + + highest_busy_rb = a4xx_next_ringbuffer(adreno_dev); + if (!highest_busy_rb || highest_busy_rb == adreno_dev->cur_rb) + return; + + switch_low_to_high = adreno_compare_prio_level( + highest_busy_rb->id, + adreno_dev->cur_rb->id); + + if (switch_low_to_high < 0) { + /* + * if switching to lower priority make sure that the rptr and + * wptr are equal, when the lower rb is not starved + */ + if (!adreno_rb_empty(adreno_dev->cur_rb)) + return; + /* + * switch to default context because when we switch back + * to higher context then its not known which pt will + * be current, so by making it default here the next + * commands submitted will set the right pt + */ + ret = adreno_drawctxt_switch(adreno_dev, + adreno_dev->cur_rb, + NULL, 0); + /* + * lower priority RB has to wait until space opens up in + * higher RB + */ + if (ret) { + dev_err(device->dev, + "Unable to switch context to NULL: %d\n", + ret); + + return; + } + + adreno_writereg(adreno_dev, + ADRENO_REG_CP_PREEMPT_DISABLE, 1); + } + + /* + * setup registers to do the switch to highest priority RB + * which is not empty or may be starving away(poor thing) + */ + a4xx_preemption_start(adreno_dev, highest_busy_rb); + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_TRIGGERED); + + adreno_dev->next_rb = highest_busy_rb; + mod_timer(&adreno_dev->preempt.timer, jiffies + + msecs_to_jiffies(ADRENO_PREEMPT_TIMEOUT)); + + trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb, + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); + /* issue PREEMPT trigger */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); + + /* submit preempt token packet to ensure preemption */ + if (switch_low_to_high < 0) { + ret = a4xx_submit_preempt_token( + adreno_dev->cur_rb, adreno_dev->next_rb); + dev_err(device->dev, + "Unable to submit preempt token: %d\n", ret); + adreno_dev->preempt.token_submit = true; + adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; + } else { + adreno_dev->preempt.token_submit = false; + adreno_dispatcher_schedule(device); + adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; + } +} + +static void a4xx_preempt_complete_state(struct adreno_device *adreno_dev) + +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int wptr, rbbase; + unsigned int val, val1; + unsigned int prevrptr; + + del_timer_sync(&adreno_dev->preempt.timer); + + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &val); + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, &val1); + + if (val || !val1) { + dev_err(device->dev, + "Invalid state after preemption CP_PREEMPT: %08x, CP_PREEMPT_DEBUG: %08x\n", + val, val1); + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + adreno_dispatcher_schedule(device); + return; + } + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_BASE, &rbbase); + if (rbbase != adreno_dev->next_rb->buffer_desc.gpuaddr) { + dev_err(device->dev, + "RBBASE incorrect after preemption, expected %x got %016llx\b\n", + rbbase, + adreno_dev->next_rb->buffer_desc.gpuaddr); + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + adreno_dispatcher_schedule(device); + return; + } + + a4xx_preemption_save(adreno_dev, adreno_dev->cur_rb); + + /* new RB is the current RB */ + trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb, + adreno_dev->cur_rb, + adreno_get_rptr(adreno_dev->next_rb), + adreno_get_rptr(adreno_dev->cur_rb)); + + adreno_dev->prev_rb = adreno_dev->cur_rb; + adreno_dev->cur_rb = adreno_dev->next_rb; + adreno_dev->cur_rb->preempted_midway = 0; + adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; + adreno_dev->next_rb = NULL; + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + prevrptr = adreno_get_rptr(adreno_dev->prev_rb); + + if (adreno_compare_prio_level(adreno_dev->prev_rb->id, + adreno_dev->cur_rb->id) < 0) { + if (adreno_dev->prev_rb->wptr_preempt_end != prevrptr) + adreno_dev->prev_rb->preempted_midway = 1; + } + + /* submit wptr if required for new rb */ + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); + if (adreno_dev->cur_rb->wptr != wptr) { + kgsl_pwrscale_busy(device); + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, + adreno_dev->cur_rb->wptr); + } + /* clear preemption register */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT_DEBUG, 0); +} + +void a4xx_preemption_schedule(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_preemption_enabled(adreno_dev)) + return; + + mutex_lock(&device->mutex); + + switch (atomic_read(&adreno_dev->preempt.state)) { + case ADRENO_PREEMPT_NONE: + a4xx_preempt_clear_state(adreno_dev); + break; + case ADRENO_PREEMPT_TRIGGERED: + a4xx_preempt_trig_state(adreno_dev); + /* + * if we transitioned to next state then fall-through + * processing to next state + */ + if (!adreno_in_preempt_state(adreno_dev, + ADRENO_PREEMPT_COMPLETE)) + break; + case ADRENO_PREEMPT_COMPLETE: + a4xx_preempt_complete_state(adreno_dev); + break; + default: + break; + } + + mutex_unlock(&device->mutex); +} + +int a4xx_preemption_init(struct adreno_device *adreno_dev) +{ + timer_setup(&adreno_dev->preempt.timer, a4xx_preemption_timer, 0); + + return 0; +} diff --git a/drivers/gpu/msm/adreno_a4xx_snapshot.c b/drivers/gpu/msm/adreno_a4xx_snapshot.c new file mode 100644 index 0000000000000000000000000000000000000000..4141a53808fec5766ab8bf27421efc3fdf3a5377 --- /dev/null +++ b/drivers/gpu/msm/adreno_a4xx_snapshot.c @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013-2017, The Linux Foundation. All rights reserved. + */ + +#include +#include "kgsl.h" +#include "adreno.h" +#include "kgsl_snapshot.h" +#include "a4xx_reg.h" +#include "adreno_snapshot.h" +#include "adreno_a4xx.h" + +/* + * Set of registers to dump for A4XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a4xx_registers[] = { + /* RBBM */ + 0x0000, 0x0002, 0x0004, 0x0021, 0x0023, 0x0024, 0x0026, 0x0026, + 0x0028, 0x002B, 0x002E, 0x0034, 0x0037, 0x0044, 0x0047, 0x0066, + 0x0068, 0x0095, 0x009C, 0x0170, 0x0174, 0x01AF, + /* CP */ + 0x0200, 0x0226, 0x0228, 0x0233, 0x0240, 0x0258, 0x04C0, 0x04D0, + 0x04D2, 0x04DD, 0x0500, 0x050B, 0x0578, 0x058F, + /* VSC */ + 0x0C00, 0x0C03, 0x0C08, 0x0C41, 0x0C50, 0x0C51, + /* GRAS */ + 0x0C80, 0x0C81, 0x0C88, 0x0C8F, + /* RB */ + 0x0CC0, 0x0CC0, 0x0CC4, 0x0CD2, + /* PC */ + 0x0D00, 0x0D0C, 0x0D10, 0x0D17, 0x0D20, 0x0D23, + /* VFD */ + 0x0E40, 0x0E4A, + /* VPC */ + 0x0E60, 0x0E61, 0x0E63, 0x0E68, + /* UCHE */ + 0x0E80, 0x0E84, 0x0E88, 0x0E95, + /* GRAS CTX 0 */ + 0x2000, 0x2004, 0x2008, 0x2067, 0x2070, 0x2078, 0x207B, 0x216E, + /* PC CTX 0 */ + 0x21C0, 0x21C6, 0x21D0, 0x21D0, 0x21D9, 0x21D9, 0x21E5, 0x21E7, + /* VFD CTX 0 */ + 0x2200, 0x2204, 0x2208, 0x22A9, + /* GRAS CTX 1 */ + 0x2400, 0x2404, 0x2408, 0x2467, 0x2470, 0x2478, 0x247B, 0x256E, + /* PC CTX 1 */ + 0x25C0, 0x25C6, 0x25D0, 0x25D0, 0x25D9, 0x25D9, 0x25E5, 0x25E7, + /* VFD CTX 1 */ + 0x2600, 0x2604, 0x2608, 0x26A9, +}; + +static const unsigned int a4xx_sp_tp_registers[] = { + /* SP */ + 0x0EC0, 0x0ECF, + /* TPL1 */ + 0x0F00, 0x0F0B, + /* SP CTX 0 */ + 0x22C0, 0x22C1, 0x22C4, 0x22E5, 0x22E8, 0x22F8, 0x2300, 0x2306, + 0x230C, 0x2312, 0x2318, 0x2339, 0x2340, 0x2360, + /* TPL1 CTX 0 */ + 0x2380, 0x2382, 0x2384, 0x238F, 0x23A0, 0x23A6, + /* SP CTX 1 */+ + 0x26C0, 0x26C1, 0x26C4, 0x26E5, 0x26E8, 0x26F8, 0x2700, 0x2706, + 0x270C, 0x2712, 0x2718, 0x2739, 0x2740, 0x2760, + /* TPL1 CTX 1 */ + 0x2780, 0x2782, 0x2784, 0x278F, 0x27A0, 0x27A6, +}; + +static const unsigned int a4xx_ppd_registers[] = { + /* V2 Thresholds */ + 0x01B2, 0x01B5, + /* Control and Status */ + 0x01B9, 0x01BE, +}; + +static const unsigned int a4xx_xpu_registers[] = { + /* XPU */ + 0x2C00, 0x2C01, 0x2C10, 0x2C10, 0x2C12, 0x2C16, 0x2C1D, 0x2C20, + 0x2C28, 0x2C28, 0x2C30, 0x2C30, 0x2C32, 0x2C36, 0x2C40, 0x2C40, + 0x2C50, 0x2C50, 0x2C52, 0x2C56, 0x2C80, 0x2C80, 0x2C94, 0x2C95, +}; + +static const unsigned int a4xx_vbif_ver_20000000_registers[] = { + /* VBIF version 0x20000000 & IOMMU V1 */ + 0x3000, 0x3007, 0x300C, 0x3014, 0x3018, 0x301D, 0x3020, 0x3022, + 0x3024, 0x3026, 0x3028, 0x302A, 0x302C, 0x302D, 0x3030, 0x3031, + 0x3034, 0x3036, 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, + 0x3049, 0x3049, 0x3058, 0x3058, 0x305B, 0x3061, 0x3064, 0x3068, + 0x306C, 0x306D, 0x3080, 0x3088, 0x308B, 0x308C, 0x3090, 0x3094, + 0x3098, 0x3098, 0x309C, 0x309C, 0x30C0, 0x30C0, 0x30C8, 0x30C8, + 0x30D0, 0x30D0, 0x30D8, 0x30D8, 0x30E0, 0x30E0, 0x3100, 0x3100, + 0x3108, 0x3108, 0x3110, 0x3110, 0x3118, 0x3118, 0x3120, 0x3120, + 0x3124, 0x3125, 0x3129, 0x3129, 0x3131, 0x3131, 0x330C, 0x330C, + 0x3310, 0x3310, 0x3400, 0x3401, 0x3410, 0x3410, 0x3412, 0x3416, + 0x341D, 0x3420, 0x3428, 0x3428, 0x3430, 0x3430, 0x3432, 0x3436, + 0x3440, 0x3440, 0x3450, 0x3450, 0x3452, 0x3456, 0x3480, 0x3480, + 0x3494, 0x3495, 0x4000, 0x4000, 0x4002, 0x4002, 0x4004, 0x4004, + 0x4008, 0x400A, 0x400C, 0x400D, 0x400F, 0x4012, 0x4014, 0x4016, + 0x401D, 0x401D, 0x4020, 0x4027, 0x4060, 0x4062, 0x4200, 0x4200, + 0x4300, 0x4300, 0x4400, 0x4400, 0x4500, 0x4500, 0x4800, 0x4802, + 0x480F, 0x480F, 0x4811, 0x4811, 0x4813, 0x4813, 0x4815, 0x4816, + 0x482B, 0x482B, 0x4857, 0x4857, 0x4883, 0x4883, 0x48AF, 0x48AF, + 0x48C5, 0x48C5, 0x48E5, 0x48E5, 0x4905, 0x4905, 0x4925, 0x4925, + 0x4945, 0x4945, 0x4950, 0x4950, 0x495B, 0x495B, 0x4980, 0x498E, + 0x4B00, 0x4B00, 0x4C00, 0x4C00, 0x4D00, 0x4D00, 0x4E00, 0x4E00, + 0x4E80, 0x4E80, 0x4F00, 0x4F00, 0x4F08, 0x4F08, 0x4F10, 0x4F10, + 0x4F18, 0x4F18, 0x4F20, 0x4F20, 0x4F30, 0x4F30, 0x4F60, 0x4F60, + 0x4F80, 0x4F81, 0x4F88, 0x4F89, 0x4FEE, 0x4FEE, 0x4FF3, 0x4FF3, + 0x6000, 0x6001, 0x6008, 0x600F, 0x6014, 0x6016, 0x6018, 0x601B, + 0x61FD, 0x61FD, 0x623C, 0x623C, 0x6380, 0x6380, 0x63A0, 0x63A0, + 0x63C0, 0x63C1, 0x63C8, 0x63C9, 0x63D0, 0x63D4, 0x63D6, 0x63D6, + 0x63EE, 0x63EE, 0x6400, 0x6401, 0x6408, 0x640F, 0x6414, 0x6416, + 0x6418, 0x641B, 0x65FD, 0x65FD, 0x663C, 0x663C, 0x6780, 0x6780, + 0x67A0, 0x67A0, 0x67C0, 0x67C1, 0x67C8, 0x67C9, 0x67D0, 0x67D4, + 0x67D6, 0x67D6, 0x67EE, 0x67EE, +}; + +static const unsigned int a4xx_vbif_ver_20020000_registers[] = { + 0x3000, 0x3007, 0x300C, 0x3014, 0x3018, 0x301D, 0x3020, 0x3022, + 0x3024, 0x3026, 0x3028, 0x302A, 0x302C, 0x302D, 0x3030, 0x3031, + 0x3034, 0x3036, 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, + 0x3049, 0x3049, 0x3058, 0x3058, 0x305B, 0x3061, 0x3064, 0x3068, + 0x306C, 0x306D, 0x3080, 0x3088, 0x308B, 0x308C, 0x3090, 0x3094, + 0x3098, 0x3098, 0x309C, 0x309C, 0x30C0, 0x30C0, 0x30C8, 0x30C8, + 0x30D0, 0x30D0, 0x30D8, 0x30D8, 0x30E0, 0x30E0, 0x3100, 0x3100, + 0x3108, 0x3108, 0x3110, 0x3110, 0x3118, 0x3118, 0x3120, 0x3120, + 0x3124, 0x3125, 0x3129, 0x3129, 0x3131, 0x3131, 0x4800, 0x4802, + 0x480F, 0x480F, 0x4811, 0x4811, 0x4813, 0x4813, 0x4815, 0x4816, + 0x482B, 0x482B, 0x4857, 0x4857, 0x4883, 0x4883, 0x48AF, 0x48AF, + 0x48C5, 0x48C5, 0x48E5, 0x48E5, 0x4905, 0x4905, 0x4925, 0x4925, + 0x4945, 0x4945, 0x4950, 0x4950, 0x495B, 0x495B, 0x4980, 0x498E, + 0x4C00, 0x4C00, 0x4D00, 0x4D00, 0x4E00, 0x4E00, 0x4E80, 0x4E80, + 0x4F00, 0x4F00, 0x4F08, 0x4F08, 0x4F10, 0x4F10, 0x4F18, 0x4F18, + 0x4F20, 0x4F20, 0x4F30, 0x4F30, 0x4F60, 0x4F60, 0x4F80, 0x4F81, + 0x4F88, 0x4F89, 0x4FEE, 0x4FEE, 0x4FF3, 0x4FF3, 0x6000, 0x6001, + 0x6008, 0x600F, 0x6014, 0x6016, 0x6018, 0x601B, 0x61FD, 0x61FD, + 0x623C, 0x623C, 0x6380, 0x6380, 0x63A0, 0x63A0, 0x63C0, 0x63C1, + 0x63C8, 0x63C9, 0x63D0, 0x63D4, 0x63D6, 0x63D6, 0x63EE, 0x63EE, + 0x6400, 0x6401, 0x6408, 0x640F, 0x6414, 0x6416, 0x6418, 0x641B, + 0x65FD, 0x65FD, 0x663C, 0x663C, 0x6780, 0x6780, 0x67A0, 0x67A0, + 0x67C0, 0x67C1, 0x67C8, 0x67C9, 0x67D0, 0x67D4, 0x67D6, 0x67D6, + 0x67EE, 0x67EE, +}; + +static const unsigned int a4xx_vbif_ver_20050000_registers[] = { + /* VBIF version 0x20050000 and 0x20090000 */ + 0x3000, 0x3007, 0x302C, 0x302C, 0x3030, 0x3030, 0x3034, 0x3036, + 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, 0x3049, 0x3049, + 0x3058, 0x3058, 0x305B, 0x3061, 0x3064, 0x3068, 0x306C, 0x306D, + 0x3080, 0x3088, 0x308B, 0x308C, 0x3090, 0x3094, 0x3098, 0x3098, + 0x309C, 0x309C, 0x30C0, 0x30C0, 0x30C8, 0x30C8, 0x30D0, 0x30D0, + 0x30D8, 0x30D8, 0x30E0, 0x30E0, 0x3100, 0x3100, 0x3108, 0x3108, + 0x3110, 0x3110, 0x3118, 0x3118, 0x3120, 0x3120, 0x3124, 0x3125, + 0x3129, 0x3129, 0x340C, 0x340C, 0x3410, 0x3410, +}; + +static const struct adreno_vbif_snapshot_registers + a4xx_vbif_snapshot_registers[] = { + { 0x20000000, 0xFFFF0000, a4xx_vbif_ver_20000000_registers, + ARRAY_SIZE(a4xx_vbif_ver_20000000_registers)/2}, + { 0x20020000, 0xFFFF0000, a4xx_vbif_ver_20020000_registers, + ARRAY_SIZE(a4xx_vbif_ver_20020000_registers)/2}, + { 0x20050000, 0xFFFF0000, a4xx_vbif_ver_20050000_registers, + ARRAY_SIZE(a4xx_vbif_ver_20050000_registers)/2}, + { 0x20070000, 0xFFFF0000, a4xx_vbif_ver_20020000_registers, + ARRAY_SIZE(a4xx_vbif_ver_20020000_registers)/2}, + { 0x20090000, 0xFFFF0000, a4xx_vbif_ver_20050000_registers, + ARRAY_SIZE(a4xx_vbif_ver_20050000_registers)/2}, +}; + +#define A4XX_NUM_SHADER_BANKS 4 +#define A405_NUM_SHADER_BANKS 1 +/* Shader memory size in words */ +#define A4XX_SHADER_MEMORY_SIZE 0x4000 + +static const struct adreno_debugbus_block a4xx_debugbus_blocks[] = { + { A4XX_RBBM_DEBBUS_CP_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RBBM_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_VBIF_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_HLSQ_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_UCHE_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_DPM_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TESS_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_PC_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_VFD_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_VPC_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TSE_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RAS_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_VSC_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_COM_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_DCOM_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_SP_0_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TPL1_0_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RB_0_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_MARB_0_ID, 0x100 }, +}; + +static const struct adreno_debugbus_block a420_debugbus_blocks[] = { + { A4XX_RBBM_DEBBUS_SP_1_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_SP_2_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_SP_3_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TPL1_1_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TPL1_2_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_TPL1_3_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RB_1_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RB_2_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_RB_3_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_MARB_1_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_MARB_2_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_MARB_3_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_CCU_0_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_CCU_1_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_CCU_2_ID, 0x100, }, + { A4XX_RBBM_DEBBUS_CCU_3_ID, 0x100, }, +}; + +/** + * a4xx_snapshot_shader_memory - Helper function to dump the GPU shader + * memory to the snapshot buffer. + * @device: GPU device whose shader memory is to be dumped + * @buf: Pointer to binary snapshot data blob being made + * @remain: Number of remaining bytes in the snapshot blob + * @priv: Unused parameter + * + */ +static size_t a4xx_snapshot_shader_memory(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int i, j; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int shader_read_len = A4XX_SHADER_MEMORY_SIZE; + unsigned int shader_banks = A4XX_NUM_SHADER_BANKS; + + if (shader_read_len > (device->shader_mem_len >> 2)) + shader_read_len = (device->shader_mem_len >> 2); + + if (adreno_is_a405(adreno_dev)) + shader_banks = A405_NUM_SHADER_BANKS; + + if (remain < DEBUG_SECTION_SZ(shader_read_len * + shader_banks)) { + SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_SHADER_MEMORY; + header->size = shader_read_len * shader_banks; + + /* Map shader memory to kernel, for dumping */ + if (device->shader_mem_virt == NULL) + device->shader_mem_virt = devm_ioremap(device->dev, + device->shader_mem_phys, + device->shader_mem_len); + + if (device->shader_mem_virt == NULL) { + dev_err(device->dev, + "Unable to map shader memory region\n"); + return 0; + } + + for (j = 0; j < shader_banks; j++) { + unsigned int val; + /* select the SPTP */ + kgsl_regread(device, A4XX_HLSQ_SPTP_RDSEL, &val); + val &= ~0x3; + val |= j; + kgsl_regwrite(device, A4XX_HLSQ_SPTP_RDSEL, val); + /* Now, dump shader memory to snapshot */ + for (i = 0; i < shader_read_len; i++) + adreno_shadermem_regread(device, i, + &data[i + j * shader_read_len]); + } + + + return DEBUG_SECTION_SZ(shader_read_len * shader_banks); +} + +/* + * a4xx_rbbm_debug_bus_read() - Read data from trace bus + * @device: Device whose data bus is read + * @block_id: Trace bus block ID + * @index: Index of data to read + * @val: Output parameter where data is read + */ +static void a4xx_rbbm_debug_bus_read(struct kgsl_device *device, + unsigned int block_id, unsigned int index, unsigned int *val) +{ + unsigned int reg = 0; + + reg |= (block_id << A4XX_RBBM_CFG_DEBBUS_SEL_PING_BLK_SEL_SHIFT); + reg |= (index << A4XX_RBBM_CFG_DEBBUS_SEL_PING_INDEX_SHIFT); + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_SEL_A, reg); + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_SEL_B, reg); + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_SEL_C, reg); + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_SEL_D, reg); + + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_IDX, 0x3020000); + kgsl_regread(device, A4XX_RBBM_CFG_DEBBUS_TRACE_BUF4, val); + val++; + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_IDX, 0x1000000); + kgsl_regread(device, A4XX_RBBM_CFG_DEBBUS_TRACE_BUF4, val); +} + +/* + * a4xx_snapshot_vbif_debugbus() - Dump the VBIF debug data + * @device: Device pointer for which the debug data is dumped + * @buf: Pointer to the memory where the data is dumped + * @remain: Amount of bytes remaining in snapshot + * @priv: Pointer to debug bus block + * + * Returns the number of bytes dumped + */ +static size_t a4xx_snapshot_vbif_debugbus(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i, j; + /* + * Total number of VBIF data words considering 3 sections: + * 2 arbiter blocks of 16 words + * 5 AXI XIN blocks of 4 dwords each + * 5 core clock side XIN blocks of 5 dwords each + */ + unsigned int dwords = (16 * A4XX_NUM_AXI_ARB_BLOCKS) + + (4 * A4XX_NUM_XIN_BLOCKS) + (5 * A4XX_NUM_XIN_BLOCKS); + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + size_t size; + unsigned int reg_clk; + + size = (dwords * sizeof(unsigned int)) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + header->id = block->block_id; + header->count = dwords; + + kgsl_regread(device, A4XX_VBIF_CLKON, ®_clk); + kgsl_regwrite(device, A4XX_VBIF_CLKON, reg_clk | + (A4XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK << + A4XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT)); + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS1_CTRL0, 0); + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS_OUT_CTRL, + (A4XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK << + A4XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT)); + for (i = 0; i < A4XX_NUM_AXI_ARB_BLOCKS; i++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS2_CTRL0, + (1 << (i + 16))); + for (j = 0; j < 16; j++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS2_CTRL1, + ((j & A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A4XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks AXI side */ + for (i = 0; i < A4XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS2_CTRL0, 1 << i); + for (j = 0; j < 4; j++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS2_CTRL1, + ((j & A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A4XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A4XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks core clock side */ + for (i = 0; i < A4XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS1_CTRL0, 1 << i); + for (j = 0; j < 5; j++) { + kgsl_regwrite(device, A4XX_VBIF_TEST_BUS1_CTRL1, + ((j & A4XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK) + << A4XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A4XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + /* restore the clock of VBIF */ + kgsl_regwrite(device, A4XX_VBIF_CLKON, reg_clk); + return size; +} + +/* + * a4xx_snapshot_debugbus_block() - Capture debug data for a gpu block + * @device: Pointer to device + * @buf: Memory where data is captured + * @remain: Number of bytes left in snapshot + * @priv: Pointer to debug bus block + * + * Returns the number of bytes written + */ +static size_t a4xx_snapshot_debugbus_block(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int dwords; + size_t size; + + dwords = block->dwords; + + /* For a4xx each debug bus data unit is 2 DWRODS */ + size = (dwords * sizeof(unsigned int) * 2) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + + header->id = block->block_id; + header->count = dwords * 2; + + for (i = 0; i < dwords; i++) + a4xx_rbbm_debug_bus_read(device, block->block_id, i, + &data[i*2]); + + return size; +} + +/* + * a4xx_snapshot_debugbus() - Capture debug bus data + * @device: The device for which data is captured + * @snapshot: Pointer to the snapshot instance + */ +static void a4xx_snapshot_debugbus(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int i; + + kgsl_regwrite(device, A4XX_RBBM_CFG_DEBBUS_CTLM, + 0xf << A4XX_RBBM_CFG_DEBBUS_CTLT_ENABLE_SHIFT); + + for (i = 0; i < ARRAY_SIZE(a4xx_debugbus_blocks); i++) { + if (A4XX_RBBM_DEBBUS_VBIF_ID == + a4xx_debugbus_blocks[i].block_id) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a4xx_snapshot_vbif_debugbus, + (void *) &a4xx_debugbus_blocks[i]); + else + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a4xx_snapshot_debugbus_block, + (void *) &a4xx_debugbus_blocks[i]); + } + + if (!adreno_is_a405(adreno_dev)) { + for (i = 0; i < ARRAY_SIZE(a420_debugbus_blocks); i++) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a4xx_snapshot_debugbus_block, + (void *) &a420_debugbus_blocks[i]); + + } +} + +static void a4xx_reset_hlsq(struct kgsl_device *device) +{ + unsigned int val, dummy = 0; + + /* reset cp */ + kgsl_regwrite(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, 1 << 20); + kgsl_regread(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, &dummy); + + /* reset hlsq */ + kgsl_regwrite(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, 1 << 25); + kgsl_regread(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, &dummy); + + /* clear reset bits */ + kgsl_regwrite(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, 0); + kgsl_regread(device, A4XX_RBBM_BLOCK_SW_RESET_CMD, &dummy); + + + /* set HLSQ_TIMEOUT_THRESHOLD.cycle_timeout_limit_sp to 26 */ + kgsl_regread(device, A4XX_HLSQ_TIMEOUT_THRESHOLD, &val); + val &= (0x1F << 24); + val |= (26 << 24); + kgsl_regwrite(device, A4XX_HLSQ_TIMEOUT_THRESHOLD, val); +} + +/* + * a4xx_snapshot() - A4XX GPU snapshot function + * @adreno_dev: Device being snapshotted + * @snapshot: Pointer to the snapshot instance + * + * This is where all of the A4XX specific bits and pieces are grabbed + * into the snapshot memory + */ +void a4xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_snapshot_data *snap_data = gpudev->snapshot_data; + + /* Disable SP clock gating for the debug bus to work */ + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP0, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP1, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP2, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL_SP3, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP0, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP1, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP2, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2_SP3, 0); + + /* Disable top level clock gating the debug bus to work */ + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL, 0); + kgsl_regwrite(device, A4XX_RBBM_CLOCK_CTL2, 0); + + /* Master set of (non debug) registers */ + + SNAPSHOT_REGISTERS(device, snapshot, a4xx_registers); + + if (adreno_is_a430(adreno_dev)) + SNAPSHOT_REGISTERS(device, snapshot, a4xx_sp_tp_registers); + + if (adreno_is_a420(adreno_dev)) + SNAPSHOT_REGISTERS(device, snapshot, a4xx_xpu_registers); + + if (adreno_is_a430v2(adreno_dev)) + SNAPSHOT_REGISTERS(device, snapshot, a4xx_ppd_registers); + + adreno_snapshot_vbif_registers(device, snapshot, + a4xx_vbif_snapshot_registers, + ARRAY_SIZE(a4xx_vbif_snapshot_registers)); + + kgsl_snapshot_indexed_registers(device, snapshot, + A4XX_CP_STATE_DEBUG_INDEX, A4XX_CP_STATE_DEBUG_DATA, + 0, snap_data->sect_sizes->cp_pfp); + + /* CP_ME indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A4XX_CP_ME_CNTL, A4XX_CP_ME_STATUS, 64, 44); + + /* VPC memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_vpc_memory, + &snap_data->sect_sizes->vpc_mem); + + /* CP MEQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_meq, + &snap_data->sect_sizes->cp_meq); + + /* CP PFP and PM4 */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_pfp_ram, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_pm4_ram, NULL); + + /* CP ROQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_roq, + &snap_data->sect_sizes->roq); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_merciu, + &snap_data->sect_sizes->cp_merciu); + + /* Debug bus */ + a4xx_snapshot_debugbus(device, snapshot); + + if (!adreno_is_a430(adreno_dev)) { + a4xx_reset_hlsq(device); + SNAPSHOT_REGISTERS(device, snapshot, a4xx_sp_tp_registers); + } + + /* Shader working/shadow memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a4xx_snapshot_shader_memory, + &snap_data->sect_sizes->shader_mem); +} diff --git a/drivers/gpu/msm/adreno_a5xx.c b/drivers/gpu/msm/adreno_a5xx.c new file mode 100644 index 0000000000000000000000000000000000000000..4bfae7b32e814bd8722c9a60c86876c7f9b54bf3 --- /dev/null +++ b/drivers/gpu/msm/adreno_a5xx.c @@ -0,0 +1,3637 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "adreno.h" +#include "a5xx_reg.h" +#include "adreno_a5xx.h" +#include "adreno_cp_parser.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" +#include "adreno_perfcounter.h" +#include "adreno_ringbuffer.h" +#include "kgsl_sharedmem.h" +#include "kgsl.h" +#include "kgsl_trace.h" +#include "adreno_a5xx_packets.h" + +static int critical_packet_constructed; + +static struct kgsl_memdesc crit_pkts; +static unsigned int crit_pkts_dwords; +static struct kgsl_memdesc crit_pkts_refbuf0; +static struct kgsl_memdesc crit_pkts_refbuf1; +static struct kgsl_memdesc crit_pkts_refbuf2; +static struct kgsl_memdesc crit_pkts_refbuf3; + +static const struct adreno_vbif_data a530_vbif[] = { + {A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003}, + {0, 0}, +}; + +static const struct adreno_vbif_data a540_vbif[] = { + {A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003}, + {A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009}, + {0, 0}, +}; + +static const struct adreno_vbif_platform a5xx_vbif_platforms[] = { + { adreno_is_a540, a540_vbif }, + { adreno_is_a530, a530_vbif }, + { adreno_is_a512, a540_vbif }, + { adreno_is_a510, a530_vbif }, + { adreno_is_a508, a530_vbif }, + { adreno_is_a505, a530_vbif }, + { adreno_is_a506, a530_vbif }, +}; + +static void a5xx_irq_storm_worker(struct work_struct *work); +static int _read_fw2_block_header(struct kgsl_device *device, + uint32_t *header, uint32_t remain, + uint32_t id, uint32_t major, uint32_t minor); +static void a5xx_gpmu_reset(struct work_struct *work); +static int a5xx_gpmu_init(struct adreno_device *adreno_dev); + +/** + * Number of times to check if the regulator enabled before + * giving up and returning failure. + */ +#define PWR_RETRY 100 + +/** + * Number of times to check if the GPMU firmware is initialized before + * giving up and returning failure. + */ +#define GPMU_FW_INIT_RETRY 5000 + +#define A530_QFPROM_RAW_PTE_ROW0_MSB 0x134 +#define A530_QFPROM_RAW_PTE_ROW2_MSB 0x144 + +static void a530_efuse_leakage(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int row0, row2; + unsigned int multiplier, gfx_active, leakage_pwr_on, coeff; + + adreno_efuse_read_u32(adreno_dev, + A530_QFPROM_RAW_PTE_ROW0_MSB, &row0); + + adreno_efuse_read_u32(adreno_dev, + A530_QFPROM_RAW_PTE_ROW2_MSB, &row2); + + multiplier = (row0 >> 1) & 0x3; + gfx_active = (row2 >> 2) & 0xFF; + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,base-leakage-coefficient", &coeff)) + return; + + leakage_pwr_on = gfx_active * (1 << multiplier); + + adreno_dev->lm_leakage = (leakage_pwr_on << 16) | + ((leakage_pwr_on * coeff) / 100); +} + +static void a530_efuse_speed_bin(struct adreno_device *adreno_dev) +{ + unsigned int val; + unsigned int speed_bin[3]; + struct kgsl_device *device = &adreno_dev->dev; + + if (of_property_read_u32_array(device->pdev->dev.of_node, + "qcom,gpu-speed-bin", speed_bin, 3)) + return; + + adreno_efuse_read_u32(adreno_dev, speed_bin[0], &val); + + adreno_dev->speed_bin = (val & speed_bin[1]) >> speed_bin[2]; +} + +static const struct { + int (*check)(struct adreno_device *adreno_dev); + void (*func)(struct adreno_device *adreno_dev); +} a5xx_efuse_funcs[] = { + { adreno_is_a530, a530_efuse_leakage }, + { adreno_is_a530, a530_efuse_speed_bin }, + { adreno_is_a505, a530_efuse_speed_bin }, + { adreno_is_a512, a530_efuse_speed_bin }, + { adreno_is_a508, a530_efuse_speed_bin }, +}; + +static void a5xx_check_features(struct adreno_device *adreno_dev) +{ + unsigned int i; + + if (adreno_efuse_map(adreno_dev)) + return; + + for (i = 0; i < ARRAY_SIZE(a5xx_efuse_funcs); i++) { + if (a5xx_efuse_funcs[i].check(adreno_dev)) + a5xx_efuse_funcs[i].func(adreno_dev); + } + + adreno_efuse_unmap(adreno_dev); +} + +static void a5xx_platform_setup(struct adreno_device *adreno_dev) +{ + uint64_t addr; + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) { + gpudev->snapshot_data->sect_sizes->cp_meq = 32; + gpudev->snapshot_data->sect_sizes->cp_merciu = 1024; + gpudev->snapshot_data->sect_sizes->roq = 256; + + /* A505 & A506 having 3 XIN ports in VBIF */ + gpudev->vbif_xin_halt_ctrl0_mask = + A510_VBIF_XIN_HALT_CTRL0_MASK; + } else if (adreno_is_a510(adreno_dev)) { + gpudev->snapshot_data->sect_sizes->cp_meq = 32; + gpudev->snapshot_data->sect_sizes->cp_merciu = 32; + gpudev->snapshot_data->sect_sizes->roq = 256; + + /* A510 has 3 XIN ports in VBIF */ + gpudev->vbif_xin_halt_ctrl0_mask = + A510_VBIF_XIN_HALT_CTRL0_MASK; + } else if (adreno_is_a540(adreno_dev) || + adreno_is_a512(adreno_dev)) { + gpudev->snapshot_data->sect_sizes->cp_merciu = 1024; + } + + /* Calculate SP local and private mem addresses */ + addr = ALIGN(ADRENO_UCHE_GMEM_BASE + adreno_dev->gmem_size, SZ_64K); + adreno_dev->sp_local_gpuaddr = addr; + adreno_dev->sp_pvt_gpuaddr = addr + SZ_64K; + + /* Setup defaults that might get changed by the fuse bits */ + adreno_dev->lm_leakage = A530_DEFAULT_LEAKAGE; + adreno_dev->speed_bin = 0; + + /* Check efuse bits for various capabilties */ + a5xx_check_features(adreno_dev); +} + +static void a5xx_critical_packet_destroy(struct adreno_device *adreno_dev) +{ + kgsl_free_global(&adreno_dev->dev, &crit_pkts); + kgsl_free_global(&adreno_dev->dev, &crit_pkts_refbuf1); + kgsl_free_global(&adreno_dev->dev, &crit_pkts_refbuf2); + kgsl_free_global(&adreno_dev->dev, &crit_pkts_refbuf3); + + kgsl_iommu_unmap_global_secure_pt_entry(KGSL_DEVICE(adreno_dev), + &crit_pkts_refbuf0); + kgsl_sharedmem_free(&crit_pkts_refbuf0); + +} + +static void _do_fixup(const struct adreno_critical_fixup *fixups, int count, + uint64_t *gpuaddrs, unsigned int *buffer) +{ + int i; + + for (i = 0; i < count; i++) { + buffer[fixups[i].lo_offset] = + lower_32_bits(gpuaddrs[fixups[i].buffer]) | + fixups[i].mem_offset; + + buffer[fixups[i].hi_offset] = + upper_32_bits(gpuaddrs[fixups[i].buffer]); + } +} + +static int a5xx_critical_packet_construct(struct adreno_device *adreno_dev) +{ + + unsigned int *cmds; + uint64_t gpuaddrs[CRITICAL_PACKET_MAX]; + int ret; + + ret = kgsl_allocate_global(&adreno_dev->dev, + &crit_pkts, PAGE_SIZE, + KGSL_MEMFLAGS_GPUREADONLY, + 0, "crit_pkts"); + if (ret) + return ret; + + ret = kgsl_allocate_user(&adreno_dev->dev, &crit_pkts_refbuf0, + PAGE_SIZE, KGSL_MEMFLAGS_SECURE); + if (ret) + return ret; + + ret = kgsl_iommu_map_global_secure_pt_entry(&adreno_dev->dev, + &crit_pkts_refbuf0); + if (ret) + return ret; + + ret = kgsl_allocate_global(&adreno_dev->dev, + &crit_pkts_refbuf1, + PAGE_SIZE, 0, 0, "crit_pkts_refbuf1"); + if (ret) + return ret; + + ret = kgsl_allocate_global(&adreno_dev->dev, + &crit_pkts_refbuf2, + PAGE_SIZE, 0, 0, "crit_pkts_refbuf2"); + if (ret) + return ret; + + ret = kgsl_allocate_global(&adreno_dev->dev, + &crit_pkts_refbuf3, + PAGE_SIZE, 0, 0, "crit_pkts_refbuf3"); + if (ret) + return ret; + + cmds = crit_pkts.hostptr; + + gpuaddrs[CRITICAL_PACKET0] = crit_pkts_refbuf0.gpuaddr; + gpuaddrs[CRITICAL_PACKET1] = crit_pkts_refbuf1.gpuaddr; + gpuaddrs[CRITICAL_PACKET2] = crit_pkts_refbuf2.gpuaddr; + gpuaddrs[CRITICAL_PACKET3] = crit_pkts_refbuf3.gpuaddr; + + crit_pkts_dwords = ARRAY_SIZE(_a5xx_critical_pkts); + + memcpy(cmds, _a5xx_critical_pkts, crit_pkts_dwords << 2); + + _do_fixup(critical_pkt_fixups, ARRAY_SIZE(critical_pkt_fixups), + gpuaddrs, cmds); + + cmds = crit_pkts_refbuf1.hostptr; + memcpy(cmds, _a5xx_critical_pkts_mem01, + ARRAY_SIZE(_a5xx_critical_pkts_mem01) << 2); + + cmds = crit_pkts_refbuf2.hostptr; + memcpy(cmds, _a5xx_critical_pkts_mem02, + ARRAY_SIZE(_a5xx_critical_pkts_mem02) << 2); + + cmds = crit_pkts_refbuf3.hostptr; + memcpy(cmds, _a5xx_critical_pkts_mem03, + ARRAY_SIZE(_a5xx_critical_pkts_mem03) << 2); + + _do_fixup(critical_pkt_mem03_fixups, + ARRAY_SIZE(critical_pkt_mem03_fixups), gpuaddrs, cmds); + + critical_packet_constructed = 1; + + return 0; +} + +static void a5xx_init(struct adreno_device *adreno_dev) +{ + if (ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + INIT_WORK(&adreno_dev->gpmu_work, a5xx_gpmu_reset); + + INIT_WORK(&adreno_dev->irq_storm_work, a5xx_irq_storm_worker); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS)) { + int ret; + + ret = a5xx_critical_packet_construct(adreno_dev); + if (ret) + a5xx_critical_packet_destroy(adreno_dev); + } + + a5xx_crashdump_init(adreno_dev); +} + +static void a5xx_remove(struct adreno_device *adreno_dev) +{ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS)) + a5xx_critical_packet_destroy(adreno_dev); +} + +/** + * a5xx_protect_init() - Initializes register protection on a5xx + * @device: Pointer to the device structure + * Performs register writes to enable protected access to sensitive + * registers + */ +static void a5xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int index = 0; + struct kgsl_protected_registers *iommu_regs; + + /* enable access protection to privileged registers */ + kgsl_regwrite(device, A5XX_CP_PROTECT_CNTL, 0x00000007); + + /* RBBM registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x4, 2); + adreno_set_protected_registers(adreno_dev, &index, 0x8, 3); + adreno_set_protected_registers(adreno_dev, &index, 0x10, 4); + adreno_set_protected_registers(adreno_dev, &index, 0x20, 5); + adreno_set_protected_registers(adreno_dev, &index, 0x40, 6); + adreno_set_protected_registers(adreno_dev, &index, 0x80, 6); + + /* Content protection registers */ + adreno_set_protected_registers(adreno_dev, &index, + A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 4); + adreno_set_protected_registers(adreno_dev, &index, + A5XX_RBBM_SECVID_TRUST_CNTL, 1); + + /* CP registers */ + adreno_set_protected_registers(adreno_dev, &index, 0x800, 6); + adreno_set_protected_registers(adreno_dev, &index, 0x840, 3); + adreno_set_protected_registers(adreno_dev, &index, 0x880, 5); + adreno_set_protected_registers(adreno_dev, &index, 0x0AA0, 0); + + /* RB registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xCC0, 0); + adreno_set_protected_registers(adreno_dev, &index, 0xCF0, 1); + + /* VPC registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xE68, 3); + adreno_set_protected_registers(adreno_dev, &index, 0xE70, 4); + + /* UCHE registers */ + adreno_set_protected_registers(adreno_dev, &index, 0xE80, ilog2(16)); + + /* SMMU registers */ + iommu_regs = kgsl_mmu_get_prot_regs(&device->mmu); + if (iommu_regs) + adreno_set_protected_registers(adreno_dev, &index, + iommu_regs->base, ilog2(iommu_regs->range)); +} + +/* + * a5xx_is_sptp_idle() - A530 SP/TP/RAC should be power collapsed to be + * considered idle + * @adreno_dev: The adreno_device pointer + */ +static bool a5xx_is_sptp_idle(struct adreno_device *adreno_dev) +{ + unsigned int reg; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* If feature is not supported or enabled, no worry */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC) || + !test_bit(ADRENO_SPTP_PC_CTRL, &adreno_dev->pwrctrl_flag)) + return true; + kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + return false; + kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, ®); + return !(reg & BIT(20)); +} + +/* + * _poll_gdsc_status() - Poll the GDSC status register + * @adreno_dev: The adreno device pointer + * @status_reg: Offset of the status register + * @status_value: The expected bit value + * + * Poll the status register till the power-on bit is equal to the + * expected value or the max retries are exceeded. + */ +static int _poll_gdsc_status(struct adreno_device *adreno_dev, + unsigned int status_reg, + unsigned int status_value) +{ + unsigned int reg, retry = PWR_RETRY; + + /* Bit 20 is the power on bit of SPTP and RAC GDSC status register */ + do { + udelay(1); + kgsl_regread(KGSL_DEVICE(adreno_dev), status_reg, ®); + } while (((reg & BIT(20)) != (status_value << 20)) && retry--); + if ((reg & BIT(20)) != (status_value << 20)) + return -ETIMEDOUT; + return 0; +} + +static void a5xx_restore_isense_regs(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg, i, ramp = GPMU_ISENSE_SAVE; + static unsigned int isense_regs[6] = {0xFFFF}, isense_reg_addr[] = { + A5XX_GPU_CS_DECIMAL_ALIGN, + A5XX_GPU_CS_SENSOR_PARAM_CORE_1, + A5XX_GPU_CS_SENSOR_PARAM_CORE_2, + A5XX_GPU_CS_SW_OV_FUSE_EN, + A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE, + A5XX_GPMU_TEMP_SENSOR_CONFIG}; + + if (!adreno_is_a540(adreno_dev)) + return; + + /* read signature */ + kgsl_regread(device, ramp++, ®); + + if (reg == 0xBABEFACE) { + /* store memory locations in buffer */ + for (i = 0; i < ARRAY_SIZE(isense_regs); i++) + kgsl_regread(device, ramp + i, isense_regs + i); + + /* clear signature */ + kgsl_regwrite(device, GPMU_ISENSE_SAVE, 0x0); + } + + /* if we never stored memory locations - do nothing */ + if (isense_regs[0] == 0xFFFF) + return; + + /* restore registers from memory */ + for (i = 0; i < ARRAY_SIZE(isense_reg_addr); i++) + kgsl_regwrite(device, isense_reg_addr[i], isense_regs[i]); + +} + +/* + * a5xx_regulator_enable() - Enable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly enabled + * on a restart. Clocks must be on during this call. + */ +static int a5xx_regulator_enable(struct adreno_device *adreno_dev) +{ + unsigned int ret; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!(adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev))) { + /* Halt the sp_input_clk at HM level */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x00000055); + a5xx_hwcg_set(adreno_dev, true); + /* Turn on sp_input_clk at HM level */ + kgsl_regrmw(device, A5XX_RBBM_CLOCK_CNTL, 0xFF, 0); + return 0; + } + + /* + * Turn on smaller power domain first to reduce voltage droop. + * Set the default register values; set SW_COLLAPSE to 0. + */ + kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); + /* Insert a delay between RAC and SPTP GDSC to reduce voltage droop */ + udelay(3); + ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1); + if (ret) { + dev_err(device->dev, "RBCCU GDSC enable failed\n"); + return ret; + } + + kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778000); + ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_SP_PWR_CLK_STATUS, 1); + if (ret) { + dev_err(device->dev, "SPTP GDSC enable failed\n"); + return ret; + } + + /* Disable SP clock */ + kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL, + CNTL_IP_CLK_ENABLE, 0); + /* Enable hardware clockgating */ + a5xx_hwcg_set(adreno_dev, true); + /* Enable SP clock */ + kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL, + CNTL_IP_CLK_ENABLE, 1); + + a5xx_restore_isense_regs(adreno_dev); + return 0; +} + +/* + * a5xx_regulator_disable() - Disable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly disabled + * on a power down to prevent current spikes. Clocks must be on + * during this call. + */ +static void a5xx_regulator_disable(struct adreno_device *adreno_dev) +{ + unsigned int reg; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_is_a512(adreno_dev) || adreno_is_a508(adreno_dev)) + return; + + /* If feature is not supported or not enabled */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC) || + !test_bit(ADRENO_SPTP_PC_CTRL, &adreno_dev->pwrctrl_flag)) { + /* Set the default register values; set SW_COLLAPSE to 1 */ + kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778001); + /* + * Insert a delay between SPTP and RAC GDSC to reduce voltage + * droop. + */ + udelay(3); + if (_poll_gdsc_status(adreno_dev, + A5XX_GPMU_SP_PWR_CLK_STATUS, 0)) + dev_warn(device->dev, "SPTP GDSC disable failed\n"); + + kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778001); + if (_poll_gdsc_status(adreno_dev, + A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 0)) + dev_warn(device->dev, "RBCCU GDSC disable failed\n"); + } else if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED, + &adreno_dev->priv)) { + /* GPMU firmware is supposed to turn off SPTP & RAC GDSCs. */ + kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + dev_warn(device->dev, "SPTP GDSC is not disabled\n"); + kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + dev_warn(device->dev, "RBCCU GDSC is not disabled\n"); + /* + * GPMU firmware is supposed to set GMEM to non-retention. + * Bit 14 is the memory core force on bit. + */ + kgsl_regread(device, A5XX_GPMU_RBCCU_CLOCK_CNTL, ®); + if (reg & BIT(14)) + dev_warn(device->dev, "GMEM is forced on\n"); + } + + if (adreno_is_a530(adreno_dev)) { + /* Reset VBIF before PC to avoid popping bogus FIFO entries */ + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, + 0x003C0000); + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, 0); + } +} + +/* + * a5xx_enable_pc() - Enable the GPMU based power collapse of the SPTP and RAC + * blocks + * @adreno_dev: The adreno device pointer + */ +static void a5xx_enable_pc(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC) || + !test_bit(ADRENO_SPTP_PC_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL, 0x0000007F); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_BINNING_CTRL, 0); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_HYST, 0x000A0080); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_STAGGER_DELAY, 0x00600040); + + trace_adreno_sp_tp((unsigned long) __builtin_return_address(0)); +}; + +/* + * The maximum payload of a type4 packet is the max size minus one for the + * opcode + */ +#define TYPE4_MAX_PAYLOAD (PM4_TYPE4_PKT_SIZE_MAX - 1) + +static int _gpmu_create_load_cmds(struct adreno_device *adreno_dev, + uint32_t *ucode, uint32_t size) +{ + uint32_t *start, *cmds; + uint32_t offset = 0; + uint32_t cmds_size = size; + + /* Add a dword for each PM4 packet */ + cmds_size += (size / TYPE4_MAX_PAYLOAD) + 1; + + /* Add 4 dwords for the protected mode */ + cmds_size += 4; + + if (adreno_dev->gpmu_cmds != NULL) + return 0; + + adreno_dev->gpmu_cmds = kmalloc(cmds_size << 2, GFP_KERNEL); + if (adreno_dev->gpmu_cmds == NULL) + return -ENOMEM; + + cmds = adreno_dev->gpmu_cmds; + start = cmds; + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + /* + * Prebuild the cmd stream to send to the GPU to load + * the GPMU firmware + */ + while (size > 0) { + int tmp_size = size; + + if (size >= TYPE4_MAX_PAYLOAD) + tmp_size = TYPE4_MAX_PAYLOAD; + + *cmds++ = cp_type4_packet( + A5XX_GPMU_INST_RAM_BASE + offset, + tmp_size); + + memcpy(cmds, &ucode[offset], tmp_size << 2); + + cmds += tmp_size; + offset += tmp_size; + size -= tmp_size; + } + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + adreno_dev->gpmu_cmds_size = (size_t) (cmds - start); + + return 0; +} + + +/* + * _load_gpmu_firmware() - Load the ucode into the GPMU RAM + * @adreno_dev: Pointer to adreno device + */ +static int _load_gpmu_firmware(struct adreno_device *adreno_dev) +{ + uint32_t *data; + const struct firmware *fw = NULL; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_gpu_core *gpucore = adreno_dev->gpucore; + uint32_t *cmds, cmd_size; + int ret = -EINVAL; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + return 0; + + /* gpmu fw already saved and verified so do nothing new */ + if (adreno_dev->gpmu_cmds_size != 0) + return 0; + + if (gpucore->gpmufw_name == NULL) + return 0; + + ret = request_firmware(&fw, gpucore->gpmufw_name, device->dev); + if (ret || fw == NULL) { + dev_err(device->dev, "request_firmware (%s) failed: %d\n", + gpucore->gpmufw_name, ret); + return ret; + } + + data = (uint32_t *)fw->data; + + if (data[0] >= (fw->size / sizeof(uint32_t)) || data[0] < 2) + goto err; + + if (data[1] != GPMU_FIRMWARE_ID) + goto err; + ret = _read_fw2_block_header(device, &data[2], + data[0] - 2, + GPMU_FIRMWARE_ID, + adreno_dev->gpucore->gpmu_major, + adreno_dev->gpucore->gpmu_minor); + if (ret) + goto err; + + /* Integer overflow check for cmd_size */ + if (data[2] > (data[0] - 2)) + goto err; + + cmds = data + data[2] + 3; + cmd_size = data[0] - data[2] - 2; + + if (cmd_size > GPMU_INST_RAM_SIZE) { + dev_err(device->dev, + "GPMU firmware block size is larger than RAM size\n"); + goto err; + } + + /* Everything is cool, so create some commands */ + ret = _gpmu_create_load_cmds(adreno_dev, cmds, cmd_size); +err: + if (fw) + release_firmware(fw); + + return ret; +} + +static int _gpmu_send_init_cmds(struct adreno_device *adreno_dev) +{ + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + uint32_t *cmds; + uint32_t size = adreno_dev->gpmu_cmds_size; + int ret; + + if (size == 0 || adreno_dev->gpmu_cmds == NULL) + return -EINVAL; + + cmds = adreno_ringbuffer_allocspace(rb, size); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + /* Copy to the RB the predefined fw sequence cmds */ + memcpy(cmds, adreno_dev->gpmu_cmds, size << 2); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret != 0) + adreno_spin_idle_debug(adreno_dev, + "gpmu initialization failed to idle\n"); + + return ret; +} + +/* + * a5xx_gpmu_start() - Initialize and start the GPMU + * @adreno_dev: Pointer to adreno device + * + * Load the GPMU microcode, set up any features such as hardware clock gating + * or IFPC, and take the GPMU out of reset. + */ +static int a5xx_gpmu_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int reg, retry = GPMU_FW_INIT_RETRY; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + return 0; + + ret = _gpmu_send_init_cmds(adreno_dev); + if (ret) + return ret; + + if (adreno_is_a530(adreno_dev)) { + /* GPMU clock gating setup */ + kgsl_regwrite(device, A5XX_GPMU_WFI_CONFIG, 0x00004014); + } + /* Kick off GPMU firmware */ + kgsl_regwrite(device, A5XX_GPMU_CM3_SYSRESET, 0); + /* + * The hardware team's estimation of GPMU firmware initialization + * latency is about 3000 cycles, that's about 5 to 24 usec. + */ + do { + udelay(1); + kgsl_regread(device, A5XX_GPMU_GENERAL_0, ®); + } while ((reg != 0xBABEFACE) && retry--); + + if (reg != 0xBABEFACE) { + dev_err(device->dev, + "GPMU firmware initialization timed out\n"); + return -ETIMEDOUT; + } + + if (!adreno_is_a530(adreno_dev)) { + kgsl_regread(device, A5XX_GPMU_GENERAL_1, ®); + + if (reg) { + dev_err(device->dev, + "GPMU firmware initialization failed: %d\n", + reg); + return -EIO; + } + } + set_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv); + /* + * We are in AWARE state and IRQ line from GPU to host is + * disabled. + * Read pending GPMU interrupts and clear GPMU_RBBM_INTR_INFO. + */ + kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, ®); + /* + * Clear RBBM interrupt mask if any of GPMU interrupts + * are pending. + */ + if (reg) + kgsl_regwrite(device, + A5XX_RBBM_INT_CLEAR_CMD, + 1 << A5XX_INT_GPMU_FIRMWARE); + return ret; +} + +struct kgsl_hwcg_reg { + unsigned int off; + unsigned int val; +}; + +static const struct kgsl_hwcg_reg a50x_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} +}; + +static const struct kgsl_hwcg_reg a510_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} +}; + +static const struct kgsl_hwcg_reg a530_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} +}; + + +static const struct kgsl_hwcg_reg a540_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_GPMU, 0x00000222}, + {A5XX_RBBM_CLOCK_DELAY_GPMU, 0x00000770}, + {A5XX_RBBM_CLOCK_HYST_GPMU, 0x00000004} +}; + +static const struct kgsl_hwcg_reg a512_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, +}; + +static const struct { + int (*devfunc)(struct adreno_device *adreno_dev); + const struct kgsl_hwcg_reg *regs; + unsigned int count; +} a5xx_hwcg_registers[] = { + { adreno_is_a540, a540_hwcg_regs, ARRAY_SIZE(a540_hwcg_regs) }, + { adreno_is_a530, a530_hwcg_regs, ARRAY_SIZE(a530_hwcg_regs) }, + { adreno_is_a512, a512_hwcg_regs, ARRAY_SIZE(a512_hwcg_regs) }, + { adreno_is_a510, a510_hwcg_regs, ARRAY_SIZE(a510_hwcg_regs) }, + { adreno_is_a505, a50x_hwcg_regs, ARRAY_SIZE(a50x_hwcg_regs) }, + { adreno_is_a506, a50x_hwcg_regs, ARRAY_SIZE(a50x_hwcg_regs) }, + { adreno_is_a508, a50x_hwcg_regs, ARRAY_SIZE(a50x_hwcg_regs) }, +}; + +void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct kgsl_hwcg_reg *regs; + int i, j; + + if (!test_bit(ADRENO_HWCG_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + for (i = 0; i < ARRAY_SIZE(a5xx_hwcg_registers); i++) { + if (a5xx_hwcg_registers[i].devfunc(adreno_dev)) + break; + } + + if (i == ARRAY_SIZE(a5xx_hwcg_registers)) + return; + + regs = a5xx_hwcg_registers[i].regs; + + for (j = 0; j < a5xx_hwcg_registers[i].count; j++) + kgsl_regwrite(device, regs[j].off, on ? regs[j].val : 0); + + /* enable top level HWCG */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, on ? 0xAAA8AA00 : 0); + kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, on ? 0x00000182 : 0x00000180); +} + +static int _read_fw2_block_header(struct kgsl_device *device, + uint32_t *header, uint32_t remain, + uint32_t id, uint32_t major, uint32_t minor) +{ + uint32_t header_size; + int i = 1; + + if (header == NULL) + return -ENOMEM; + + header_size = header[0]; + /* Headers have limited size and always occur as pairs of words */ + if (header_size > MAX_HEADER_SIZE || header_size >= remain || + header_size % 2 || header_size == 0) + return -EINVAL; + /* Sequences must have an identifying id first thing in their header */ + if (id == GPMU_SEQUENCE_ID) { + if (header[i] != HEADER_SEQUENCE || + (header[i + 1] >= MAX_SEQUENCE_ID)) + return -EINVAL; + i += 2; + } + for (; i < header_size; i += 2) { + switch (header[i]) { + /* Major Version */ + case HEADER_MAJOR: + if ((major > header[i + 1]) && + header[i + 1]) { + dev_err(device->dev, + "GPMU major version mis-match %d, %d\n", + major, header[i + 1]); + return -EINVAL; + } + break; + case HEADER_MINOR: + if (minor > header[i + 1]) + dev_err(device->dev, + "GPMU minor version mis-match %d %d\n", + minor, header[i + 1]); + break; + case HEADER_DATE: + case HEADER_TIME: + break; + default: + dev_err(device->dev, "GPMU unknown header ID %d\n", + header[i]); + } + } + return 0; +} + +/* + * Read in the register sequence file and save pointers to the + * necessary sequences. + * + * GPU sequence file format (one dword per field unless noted): + * Block 1 length (length dword field not inclusive) + * Block 1 type = Sequence = 3 + * Block Header length (length dword field not inclusive) + * BH field ID = Sequence field ID + * BH field data = Sequence ID + * BH field ID + * BH field data + * ... + * Opcode 0 ID + * Opcode 0 data M words + * Opcode 1 ID + * Opcode 1 data N words + * ... + * Opcode X ID + * Opcode X data O words + * Block 2 length... + */ +static void _load_regfile(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct firmware *fw; + uint64_t block_size = 0, block_total = 0; + uint32_t fw_size, *block; + int ret = -EINVAL; + + if (!adreno_dev->gpucore->regfw_name) + return; + + ret = request_firmware(&fw, adreno_dev->gpucore->regfw_name, + device->dev); + if (ret) { + dev_err(device->dev, "request firmware failed %d, %s\n", + ret, adreno_dev->gpucore->regfw_name); + return; + } + + fw_size = fw->size / sizeof(uint32_t); + /* Min valid file of size 6, see file description */ + if (fw_size < 6) + goto err; + block = (uint32_t *)fw->data; + /* All offset numbers calculated from file description */ + while (block_total < fw_size) { + block_size = block[0]; + if (((block_total + block_size) >= fw_size) + || block_size < 5) + goto err; + if (block[1] != GPMU_SEQUENCE_ID) + goto err; + + /* For now ignore blocks other than the LM sequence */ + if (block[4] == LM_SEQUENCE_ID) { + ret = _read_fw2_block_header(device, &block[2], + block_size - 2, + GPMU_SEQUENCE_ID, + adreno_dev->gpucore->lm_major, + adreno_dev->gpucore->lm_minor); + if (ret) + goto err; + + adreno_dev->lm_fw = fw; + + if (block[2] > (block_size - 2)) + goto err; + adreno_dev->lm_sequence = block + block[2] + 3; + adreno_dev->lm_size = block_size - block[2] - 2; + } + block_total += (block_size + 1); + block += (block_size + 1); + } + if (adreno_dev->lm_sequence) + return; + +err: + release_firmware(fw); + dev_err(device->dev, + "Register file failed to load sz=%d bsz=%llu header=%d\n", + fw_size, block_size, ret); +} + +static int _execute_reg_sequence(struct adreno_device *adreno_dev, + uint32_t *opcode, uint32_t length) +{ + uint32_t *cur = opcode; + uint64_t reg, val; + + /* todo double check the reg writes */ + while ((cur - opcode) < length) { + if (cur[0] == 1 && (length - (cur - opcode) >= 4)) { + /* Write a 32 bit value to a 64 bit reg */ + reg = cur[2]; + reg = (reg << 32) | cur[1]; + kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, cur[3]); + cur += 4; + } else if (cur[0] == 2 && (length - (cur - opcode) >= 5)) { + /* Write a 64 bit value to a 64 bit reg */ + reg = cur[2]; + reg = (reg << 32) | cur[1]; + val = cur[4]; + val = (val << 32) | cur[3]; + kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, val); + cur += 5; + } else if (cur[0] == 3 && (length - (cur - opcode) >= 2)) { + /* Delay for X usec */ + udelay(cur[1]); + cur += 2; + } else + return -EINVAL; + } + return 0; +} + +static uint32_t _write_voltage_table(struct adreno_device *adreno_dev, + unsigned int addr) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int i; + struct dev_pm_opp *opp; + int levels = pwr->num_pwrlevels - 1; + unsigned int mvolt = 0; + + kgsl_regwrite(device, addr++, adreno_dev->gpucore->max_power); + kgsl_regwrite(device, addr++, levels); + + /* Write voltage in mV and frequency in MHz */ + for (i = 0; i < levels; i++) { + opp = dev_pm_opp_find_freq_exact(&device->pdev->dev, + pwr->pwrlevels[i].gpu_freq, true); + /* _opp_get returns uV, convert to mV */ + if (!IS_ERR(opp)) { + mvolt = dev_pm_opp_get_voltage(opp) / 1000; + dev_pm_opp_put(opp); + } + kgsl_regwrite(device, addr++, mvolt); + kgsl_regwrite(device, addr++, + pwr->pwrlevels[i].gpu_freq / 1000000); + } + return (levels * 2 + 2); +} + +static uint32_t lm_limit(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_dev->lm_limit) + return adreno_dev->lm_limit; + + if (of_property_read_u32(device->pdev->dev.of_node, "qcom,lm-limit", + &adreno_dev->lm_limit)) + adreno_dev->lm_limit = LM_DEFAULT_LIMIT; + + return adreno_dev->lm_limit; +} +/* + * a5xx_lm_init() - Initialize LM/DPM on the GPMU + * @adreno_dev: The adreno device pointer + */ +static void a530_lm_init(struct adreno_device *adreno_dev) +{ + uint32_t length; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM) || + !test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + /* If something was wrong with the sequence file, return */ + if (adreno_dev->lm_sequence == NULL) + return; + + /* Write LM registers including DPM ucode, coefficients, and config */ + if (_execute_reg_sequence(adreno_dev, adreno_dev->lm_sequence, + adreno_dev->lm_size)) { + /* If the sequence is invalid, it's not getting better */ + adreno_dev->lm_sequence = NULL; + dev_warn(device->dev, + "Invalid LM sequence\n"); + return; + } + + kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_ID, + adreno_dev->gpucore->gpmu_tsens); + kgsl_regwrite(device, A5XX_GPMU_DELTA_TEMP_THRESHOLD, 0x1); + kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, 0x1); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE, + (0x80000000 | device->pwrctrl.active_pwrlevel)); + /* use the leakage to set this value at runtime */ + kgsl_regwrite(device, A5XX_GPMU_BASE_LEAKAGE, + adreno_dev->lm_leakage); + + /* Enable the power threshold and set it to 6000m */ + kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD, + 0x80000000 | lm_limit(adreno_dev)); + + kgsl_regwrite(device, A5XX_GPMU_BEC_ENABLE, 0x10001FFF); + kgsl_regwrite(device, A5XX_GDPM_CONFIG1, 0x00201FF1); + + /* Send an initial message to the GPMU with the LM voltage table */ + kgsl_regwrite(device, AGC_MSG_STATE, 1); + kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID); + length = _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD); + kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE, length * sizeof(uint32_t)); + kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE); +} + +/* + * a5xx_lm_enable() - Enable the LM/DPM feature on the GPMU + * @adreno_dev: The adreno device pointer + */ +static void a530_lm_enable(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM) || + !test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + /* If no sequence properly initialized, return */ + if (adreno_dev->lm_sequence == NULL) + return; + + kgsl_regwrite(device, A5XX_GDPM_INT_MASK, 0x00000000); + kgsl_regwrite(device, A5XX_GDPM_INT_EN, 0x0000000A); + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, 0x00000001); + kgsl_regwrite(device, A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK, + 0x00050000); + kgsl_regwrite(device, A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL, + 0x00030000); + + if (adreno_is_a530(adreno_dev)) + /* Program throttle control, do not enable idle DCS on v3+ */ + kgsl_regwrite(device, A5XX_GPMU_CLOCK_THROTTLE_CTRL, + adreno_is_a530v2(adreno_dev) ? 0x00060011 : 0x00000011); +} + +static void a540_lm_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + uint32_t agc_lm_config = AGC_BCL_DISABLED | + ((ADRENO_CHIPID_PATCH(adreno_dev->chipid) & 0x3) + << AGC_GPU_VERSION_SHIFT); + unsigned int r; + + if (!test_bit(ADRENO_THROTTLING_CTRL, &adreno_dev->pwrctrl_flag)) + agc_lm_config |= AGC_THROTTLE_DISABLE; + + if (lm_on(adreno_dev)) { + agc_lm_config |= + AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE | + AGC_LM_CONFIG_ISENSE_ENABLE; + + kgsl_regread(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, &r); + + if ((r & GPMU_ISENSE_STATUS) == GPMU_ISENSE_END_POINT_CAL_ERR) { + dev_err(device->dev, + "GPMU: ISENSE end point calibration failure\n"); + agc_lm_config |= AGC_LM_CONFIG_ENABLE_ERROR; + } + } + + kgsl_regwrite(device, AGC_MSG_STATE, 0x80000001); + kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID); + (void) _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD); + kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LM_CONFIG, agc_lm_config); + kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LEVEL_CONFIG, + (unsigned int) ~(GENMASK(LM_DCVS_LIMIT, 0) | + GENMASK(16+LM_DCVS_LIMIT, 16))); + + kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE, + (AGC_LEVEL_CONFIG + 1) * sizeof(uint32_t)); + kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE, + (0x80000000 | device->pwrctrl.active_pwrlevel)); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD, + PWR_THRESHOLD_VALID | lm_limit(adreno_dev)); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, + VOLTAGE_INTR_EN); +} + + +static void a5xx_lm_enable(struct adreno_device *adreno_dev) +{ + if (adreno_is_a530(adreno_dev)) + a530_lm_enable(adreno_dev); +} + +static void a5xx_lm_init(struct adreno_device *adreno_dev) +{ + if (adreno_is_a530(adreno_dev)) + a530_lm_init(adreno_dev); + else if (adreno_is_a540(adreno_dev)) + a540_lm_init(adreno_dev); +} + +static int gpmu_set_level(struct adreno_device *adreno_dev, unsigned int val) +{ + unsigned int reg; + int retry = 100; + + kgsl_regwrite(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE, val); + + do { + kgsl_regread(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE, + ®); + } while ((reg & 0x80000000) && retry--); + + return (reg & 0x80000000) ? -ETIMEDOUT : 0; +} + +/* + * a5xx_pwrlevel_change_settings() - Program the hardware during power level + * transitions + * @adreno_dev: The adreno device pointer + * @prelevel: The previous power level + * @postlevel: The new power level + * @post: True if called after the clock change has taken effect + */ +static void a5xx_pwrlevel_change_settings(struct adreno_device *adreno_dev, + unsigned int prelevel, unsigned int postlevel, + bool post) +{ + int on = 0; + + /* + * On pre A540 HW only call through if PPD or LMx + * is supported and enabled + */ + if (ADRENO_FEATURE(adreno_dev, ADRENO_PPD) && + test_bit(ADRENO_PPD_CTRL, &adreno_dev->pwrctrl_flag)) + on = ADRENO_PPD; + + if (ADRENO_FEATURE(adreno_dev, ADRENO_LM) && + test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag)) + on = ADRENO_LM; + + /* On 540+ HW call through unconditionally as long as GPMU is enabled */ + if (ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) { + if (adreno_is_a540(adreno_dev)) + on = ADRENO_GPMU; + } + + if (!on) + return; + + if (!post) { + if (gpmu_set_level(adreno_dev, (0x80000010 | postlevel))) + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPMU pre powerlevel did not stabilize\n"); + } else { + if (gpmu_set_level(adreno_dev, (0x80000000 | postlevel))) + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPMU post powerlevel did not stabilize\n"); + } +} + +static void a5xx_clk_set_options(struct adreno_device *adreno_dev, + const char *name, struct clk *clk, bool on) +{ + + if (!adreno_is_a540(adreno_dev) && !adreno_is_a512(adreno_dev) && + !adreno_is_a508(adreno_dev)) + return; + + /* Handle clock settings for GFX PSCBCs */ + if (on) { + if (!strcmp(name, "mem_iface_clk")) { + clk_set_flags(clk, CLKFLAG_NORETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_NORETAIN_MEM); + } else if (!strcmp(name, "core_clk")) { + clk_set_flags(clk, CLKFLAG_RETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_RETAIN_MEM); + } + } else { + if (!strcmp(name, "core_clk")) { + clk_set_flags(clk, CLKFLAG_NORETAIN_PERIPH); + clk_set_flags(clk, CLKFLAG_NORETAIN_MEM); + } + } +} + +static void a5xx_count_throttles(struct adreno_device *adreno_dev, + uint64_t adj) +{ + if (adreno_is_a530(adreno_dev)) + kgsl_regread(KGSL_DEVICE(adreno_dev), + adreno_dev->lm_threshold_count, + &adreno_dev->lm_threshold_cross); + else if (adreno_is_a540(adreno_dev)) + adreno_dev->lm_threshold_cross = adj; +} + +static int a5xx_enable_pwr_counters(struct adreno_device *adreno_dev, + unsigned int counter) +{ + /* + * On 5XX we have to emulate the PWR counters which are physically + * missing. Program countable 6 on RBBM_PERFCTR_RBBM_0 as a substitute + * for PWR:1. Don't emulate PWR:0 as nobody uses it and we don't want + * to take away too many of the generic RBBM counters. + */ + + if (counter == 0) + return -EINVAL; + + kgsl_regwrite(KGSL_DEVICE(adreno_dev), A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); + + return 0; +} + +/* FW driven idle 10% throttle */ +#define IDLE_10PCT 0 +/* number of cycles when clock is throttled by 50% (CRC) */ +#define CRC_50PCT 1 +/* number of cycles when clock is throttled by more than 50% (CRC) */ +#define CRC_MORE50PCT 2 +/* number of cycles when clock is throttle by less than 50% (CRC) */ +#define CRC_LESS50PCT 3 + +static int64_t a5xx_read_throttling_counters(struct adreno_device *adreno_dev) +{ + int i; + int64_t adj; + uint32_t th[ADRENO_GPMU_THROTTLE_COUNTERS]; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + + if (!adreno_is_a540(adreno_dev)) + return 0; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + return 0; + + if (!test_bit(ADRENO_THROTTLING_CTRL, &adreno_dev->pwrctrl_flag)) + return 0; + + for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) { + if (!adreno_dev->gpmu_throttle_counters[i]) + return 0; + + th[i] = counter_delta(KGSL_DEVICE(adreno_dev), + adreno_dev->gpmu_throttle_counters[i], + &busy->throttle_cycles[i]); + } + adj = th[CRC_MORE50PCT] - th[IDLE_10PCT]; + adj = th[CRC_50PCT] + th[CRC_LESS50PCT] / 3 + (adj < 0 ? 0 : adj) * 3; + + trace_kgsl_clock_throttling( + th[IDLE_10PCT], th[CRC_50PCT], + th[CRC_MORE50PCT], th[CRC_LESS50PCT], + adj); + return adj; +} + +static void a5xx_enable_64bit(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_regwrite(device, A5XX_CP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VSC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_GRAS_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_RB_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_PC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VFD_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VPC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_UCHE_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_SP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_TPL1_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); +} + +/* + * a5xx_gpmu_reset() - Re-enable GPMU based power features and restart GPMU + * @work: Pointer to the work struct for gpmu reset + * + * Load the GPMU microcode, set up any features such as hardware clock gating + * or IFPC, and take the GPMU out of reset. + */ +static void a5xx_gpmu_reset(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, gpmu_work); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv)) + return; + + /* + * If GPMU has already experienced a restart or is in the process of it + * after the watchdog timeout, then there is no need to reset GPMU + * again. + */ + if (device->state != KGSL_STATE_NAP && + device->state != KGSL_STATE_AWARE && + device->state != KGSL_STATE_ACTIVE) + return; + + mutex_lock(&device->mutex); + + if (device->state == KGSL_STATE_NAP) + kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + + if (a5xx_regulator_enable(adreno_dev)) + goto out; + + /* Soft reset of the GPMU block */ + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, BIT(16)); + + /* GPU comes up in secured mode, make it unsecured by default */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_CONTENT_PROTECTION)) + kgsl_regwrite(device, A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); + + + a5xx_gpmu_init(adreno_dev); + +out: + mutex_unlock(&device->mutex); +} + +static void _setup_throttling_counters(struct adreno_device *adreno_dev) +{ + int i, ret; + + if (!adreno_is_a540(adreno_dev)) + return; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) + return; + + for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) { + /* reset throttled cycles ivalue */ + adreno_dev->busy_data.throttle_cycles[i] = 0; + + if (adreno_dev->gpmu_throttle_counters[i] != 0) + continue; + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_GPMU_PWR, + ADRENO_GPMU_THROTTLE_COUNTERS_BASE_REG + i, + &adreno_dev->gpmu_throttle_counters[i], + NULL, + PERFCOUNTER_FLAG_KERNEL); + WARN_ONCE(ret, "Unable to get clock throttling counter %x\n", + ADRENO_GPMU_THROTTLE_COUNTERS_BASE_REG + i); + } +} + +/* + * a5xx_start() - Device start + * @adreno_dev: Pointer to adreno device + * + * a5xx device start + */ +static void a5xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int bit; + int ret; + + if (adreno_is_a530(adreno_dev) && ADRENO_FEATURE(adreno_dev, ADRENO_LM) + && adreno_dev->lm_threshold_count == 0) { + + ret = adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 27, + &adreno_dev->lm_threshold_count, NULL, + PERFCOUNTER_FLAG_KERNEL); + /* Ignore noncritical ret - used for debugfs */ + if (ret) + adreno_dev->lm_threshold_count = 0; + } + + _setup_throttling_counters(adreno_dev); + + adreno_vbif_start(adreno_dev, a5xx_vbif_platforms, + ARRAY_SIZE(a5xx_vbif_platforms)); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* + * Enable the RBBM error reporting bits. This lets us get + * useful information on failure + */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL0, 0x00000001); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_FAULT_DETECT_MASK)) { + /* + * We have 4 RB units, and only RB0 activity signals are + * working correctly. Mask out RB1-3 activity signals + * from the HW hang detection logic as per + * recommendation of hardware team. + */ + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, + 0xF0000000); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, + 0xFFFFFFFF); + } + + /* + * Turn on hang detection for a530 v2 and beyond. This spews a + * lot of useful information into the RBBM registers on a hang. + */ + if (!adreno_is_a530v1(adreno_dev)) { + + set_bit(ADRENO_DEVICE_HANG_INTR, &adreno_dev->priv); + gpudev->irq->mask |= (1 << A5XX_INT_MISC_HANG_DETECT); + /* + * Set hang detection threshold to 4 million cycles + * (0x3FFFF*16) + */ + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_INT_CNTL, + (1 << 30) | 0x3FFFF); + } + + + /* Turn on performance counters */ + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_CNTL, 0x01); + + /* + * This is to increase performance by restricting VFD's cache access, + * so that LRZ and other data get evicted less. + */ + kgsl_regwrite(device, A5XX_UCHE_CACHE_WAYS, 0x02); + + /* + * Set UCHE_WRITE_THRU_BASE to the UCHE_TRAP_BASE effectively + * disabling L2 bypass + */ + kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_LO, 0xffff0000); + kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_HI, 0x0001ffff); + kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_LO, 0xffff0000); + kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); + + /* Program the GMEM VA range for the UCHE path */ + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_LO, + ADRENO_UCHE_GMEM_BASE); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x0); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_LO, + ADRENO_UCHE_GMEM_BASE + + adreno_dev->gmem_size - 1); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x0); + + /* + * Below CP registers are 0x0 by default, program init + * values based on a5xx flavor. + */ + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); + } else if (adreno_is_a510(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x20); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); + } else if (adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); + } else { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x40); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); + } + + /* + * vtxFifo and primFifo thresholds default values + * are different. + */ + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x100 << 11 | 0x100 << 22)); + else if (adreno_is_a510(adreno_dev) || adreno_is_a512(adreno_dev)) + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x200 << 11 | 0x200 << 22)); + else + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x400 << 11 | 0x300 << 22)); + + /* + * A5x USP LDST non valid pixel wrongly update read combine offset + * In A5xx we added optimization for read combine. There could be cases + * on a530 v1 there is no valid pixel but the active masks is not + * cleared and the offset can be wrongly updated if the invalid address + * can be combined. The wrongly latched value will make the returning + * data got shifted at wrong offset. workaround this issue by disabling + * LD combine, bit[25] of SP_DBG_ECO_CNTL (sp chicken bit[17]) need to + * be set to 1, default is 0(enable) + */ + if (adreno_is_a530v1(adreno_dev)) + kgsl_regrmw(device, A5XX_SP_DBG_ECO_CNTL, 0, (1 << 25)); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_TWO_PASS_USE_WFI)) { + /* + * Set TWOPASSUSEWFI in A5XX_PC_DBG_ECO_CNTL for + * microcodes after v77 + */ + if ((adreno_compare_pfp_version(adreno_dev, 0x5FF077) >= 0)) + kgsl_regrmw(device, A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); + } + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING)) { + /* + * Disable RB sampler datapath DP2 clock gating + * optimization for 1-SP GPU's, by default it is enabled. + */ + kgsl_regrmw(device, A5XX_RB_DBG_ECO_CNT, 0, (1 << 9)); + } + /* + * Disable UCHE global filter as SP can invalidate/flush + * independently + */ + kgsl_regwrite(device, A5XX_UCHE_MODE_CNTL, BIT(29)); + /* Set the USE_RETENTION_FLOPS chicken bit */ + kgsl_regwrite(device, A5XX_CP_CHICKEN_DBG, 0x02000000); + + /* Enable ISDB mode if requested */ + if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv)) { + if (!kgsl_active_count_get(device)) { + /* + * Disable ME/PFP split timeouts when the debugger is + * enabled because the CP doesn't know when a shader is + * in active debug + */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0x06FFFFFF); + + /* Force the SP0/SP1 clocks on to enable ISDB */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP0, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP1, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP2, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP3, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP0, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP1, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP2, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP3, 0x0); + + /* disable HWCG */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x0); + kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, 0x0); + } else + dev_err(device->dev, + "Active count failed while turning on ISDB\n"); + } else { + /* if not in ISDB mode enable ME/PFP split notification */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); + } + + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL2, 0x0000003F); + + if (!of_property_read_u32(device->pdev->dev.of_node, + "qcom,highest-bank-bit", &bit)) { + if (bit >= 13 && bit <= 16) { + bit = (bit - 13) & 0x03; + + /* + * Program the highest DDR bank bit that was passed in + * from the DT in a handful of registers. Some of these + * registers will also be written by the UMD, but we + * want to program them in case we happen to use the + * UCHE before the UMD does + */ + + kgsl_regwrite(device, A5XX_TPL1_MODE_CNTL, bit << 7); + kgsl_regwrite(device, A5XX_RB_MODE_CNTL, bit << 1); + if (adreno_is_a540(adreno_dev) || + adreno_is_a512(adreno_dev)) + kgsl_regwrite(device, A5XX_UCHE_DBG_ECO_CNTL_2, + bit); + } + + } + + /* Disable All flat shading optimization */ + kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 10); + + /* + * VPC corner case with local memory load kill leads to corrupt + * internal state. Normal Disable does not work for all a5x chips. + * So do the following setting to disable it. + */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_LMLOADKILL)) { + kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 23); + kgsl_regrmw(device, A5XX_HLSQ_DBG_ECO_CNTL, 0x1 << 18, 0); + } + + a5xx_preemption_start(adreno_dev); + a5xx_protect_init(adreno_dev); +} + +/* + * Follow the ME_INIT sequence with a preemption yield to allow the GPU to move + * to a different ringbuffer, if desired + */ +static int _preemption_init( + struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int *cmds, + struct kgsl_context *context) +{ + unsigned int *cmds_orig = cmds; + uint64_t gpuaddr = rb->preemption_desc.gpuaddr; + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + /* + * CP during context switch will save context switch info to + * a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR + */ + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1); + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1); + *cmds++ = upper_32_bits(gpuaddr); + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1); + *cmds++ = 0; + + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 1; + + /* Enable yield in RB only */ + *cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1); + *cmds++ = 1; + + *cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + cmds += cp_gpuaddr(adreno_dev, cmds, 0x0); + *cmds++ = 0; + /* generate interrupt on preemption completion */ + *cmds++ = 1; + + return cmds - cmds_orig; +} + +static int a5xx_post_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int *cmds, *start; + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + + if (!adreno_is_a530(adreno_dev) && + !adreno_is_preemption_enabled(adreno_dev)) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 42); + if (IS_ERR(cmds)) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + dev_err(device->dev, + "error allocating preemtion init cmds\n"); + return PTR_ERR(cmds); + } + start = cmds; + + /* + * Send a pipeline stat event whenever the GPU gets powered up + * to cause misbehaving perf counters to start ticking + */ + if (adreno_is_a530(adreno_dev)) { + *cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1); + *cmds++ = 0xF; + } + + if (adreno_is_preemption_enabled(adreno_dev)) + cmds += _preemption_init(adreno_dev, rb, cmds, NULL); + + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "hw initialization failed to idle\n"); + + return ret; +} + +static int a5xx_gpmu_init(struct adreno_device *adreno_dev) +{ + int ret; + + /* Set up LM before initializing the GPMU */ + a5xx_lm_init(adreno_dev); + + /* Enable SPTP based power collapse before enabling GPMU */ + a5xx_enable_pc(adreno_dev); + + ret = a5xx_gpmu_start(adreno_dev); + if (ret) + return ret; + + /* Enable limits management */ + a5xx_lm_enable(adreno_dev); + return 0; +} + +/* + * a5xx_microcode_load() - Load microcode + * @adreno_dev: Pointer to adreno device + */ +static int a5xx_microcode_load(struct adreno_device *adreno_dev) +{ + void *ptr; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + uint64_t gpuaddr; + + gpuaddr = pm4_fw->memdesc.gpuaddr; + kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_LO, + lower_32_bits(gpuaddr)); + kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_HI, + upper_32_bits(gpuaddr)); + + gpuaddr = pfp_fw->memdesc.gpuaddr; + kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_LO, + lower_32_bits(gpuaddr)); + kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_HI, + upper_32_bits(gpuaddr)); + + /* + * Do not invoke to load zap shader if MMU does + * not support secure mode. + */ + if (!device->mmu.secured) + return 0; + + /* + * Resume call to write the zap shader base address into the + * appropriate register, + * skip if retention is supported for the CPZ register + */ + if (adreno_dev->zap_loaded && !(ADRENO_FEATURE(adreno_dev, + ADRENO_CPZ_RETENTION))) { + int ret; + struct scm_desc desc = {0}; + + desc.args[0] = 0; + desc.args[1] = 13; + desc.arginfo = SCM_ARGS(2); + + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_BOOT, 0xA), &desc); + if (ret) { + dev_err(device->dev, + "SCM resume call failed with error %d\n", ret); + return ret; + } + + } + + /* Load the zap shader firmware through PIL if its available */ + if (adreno_dev->gpucore->zap_name && !adreno_dev->zap_loaded) { + ptr = subsystem_get(adreno_dev->gpucore->zap_name); + + /* Return error if the zap shader cannot be loaded */ + if (IS_ERR_OR_NULL(ptr)) + return (ptr == NULL) ? -ENODEV : PTR_ERR(ptr); + adreno_dev->zap_loaded = 1; + } + + return 0; +} + +static int _me_init_ucode_workarounds(struct adreno_device *adreno_dev) +{ + switch (ADRENO_GPUREV(adreno_dev)) { + case ADRENO_REV_A510: + return 0x00000001; /* Ucode workaround for token end syncs */ + case ADRENO_REV_A505: + case ADRENO_REV_A506: + case ADRENO_REV_A530: + /* + * Ucode workarounds for token end syncs, + * WFI after every direct-render 3D mode draw and + * WFI after every 2D Mode 3 draw. + */ + return 0x0000000B; + case ADRENO_REV_A540: + /* + * WFI after every direct-render 3D mode draw and + * WFI after every 2D Mode 3 draw. This is needed + * only on a540v1. + */ + if (adreno_is_a540v1(adreno_dev)) + return 0x0000000A; + default: + return 0x00000000; /* No ucode workarounds enabled */ + } +} + +/* + * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can + * be used at once of if they should be serialized + */ +#define CP_INIT_MAX_CONTEXT BIT(0) + +/* Enables register protection mode */ +#define CP_INIT_ERROR_DETECTION_CONTROL BIT(1) + +/* Header dump information */ +#define CP_INIT_HEADER_DUMP BIT(2) /* Reserved */ + +/* Default Reset states enabled for PFP and ME */ +#define CP_INIT_DEFAULT_RESET_STATE BIT(3) + +/* Drawcall filter range */ +#define CP_INIT_DRAWCALL_FILTER_RANGE BIT(4) + +/* Ucode workaround masks */ +#define CP_INIT_UCODE_WORKAROUND_MASK BIT(5) + +#define CP_INIT_MASK (CP_INIT_MAX_CONTEXT | \ + CP_INIT_ERROR_DETECTION_CONTROL | \ + CP_INIT_HEADER_DUMP | \ + CP_INIT_DEFAULT_RESET_STATE | \ + CP_INIT_UCODE_WORKAROUND_MASK) + +static void _set_ordinals(struct adreno_device *adreno_dev, + unsigned int *cmds, unsigned int count) +{ + unsigned int *start = cmds; + + /* Enabled ordinal mask */ + *cmds++ = CP_INIT_MASK; + + if (CP_INIT_MASK & CP_INIT_MAX_CONTEXT) { + /* + * Multiple HW ctxs are unreliable on a530v1, + * use single hw context. + * Use multiple contexts if bit set, otherwise serialize: + * 3D (bit 0) 2D (bit 1) + */ + if (adreno_is_a530v1(adreno_dev)) + *cmds++ = 0x00000000; + else + *cmds++ = 0x00000003; + } + + if (CP_INIT_MASK & CP_INIT_ERROR_DETECTION_CONTROL) + *cmds++ = 0x20000000; + + if (CP_INIT_MASK & CP_INIT_HEADER_DUMP) { + /* Header dump address */ + *cmds++ = 0x00000000; + /* Header dump enable and dump size */ + *cmds++ = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_DRAWCALL_FILTER_RANGE) { + /* Start range */ + *cmds++ = 0x00000000; + /* End range (inclusive) */ + *cmds++ = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_UCODE_WORKAROUND_MASK) + *cmds++ = _me_init_ucode_workarounds(adreno_dev); + + /* Pad rest of the cmds with 0's */ + while ((unsigned int)(cmds - start) < count) + *cmds++ = 0x0; +} + +int a5xx_critical_packet_submit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + if (!critical_packet_constructed) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 4); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_mem_packet(adreno_dev, CP_INDIRECT_BUFFER_PFE, 2, 1); + cmds += cp_gpuaddr(adreno_dev, cmds, crit_pkts.gpuaddr); + *cmds++ = crit_pkts_dwords; + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 20); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "Critical packet submission failed to idle\n"); + + return ret; +} + +/* + * a5xx_send_me_init() - Initialize ringbuffer + * @adreno_dev: Pointer to adreno device + * @rb: Pointer to the ringbuffer of device + * + * Submit commands for ME initialization, + */ +static int a5xx_send_me_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 9); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + *cmds++ = cp_type7_packet(CP_ME_INIT, 8); + + _set_ordinals(adreno_dev, cmds, 8); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "CP initialization failed to idle\n"); + + return ret; +} + +/* + * a5xx_rb_start() - Start the ringbuffer + * @adreno_dev: Pointer to adreno device + * @start_type: Warm or cold start + */ +static int a5xx_rb_start(struct adreno_device *adreno_dev, + unsigned int start_type) +{ + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + struct kgsl_device *device = &adreno_dev->dev; + uint64_t addr; + int ret; + + addr = SCRATCH_RPTR_GPU_ADDR(device, rb->id); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, addr); + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + * Also disable the host RPTR shadow register as it might be unreliable + * in certain circumstances. + */ + + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, + A5XX_CP_RB_CNTL_DEFAULT); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_BASE, + ADRENO_REG_CP_RB_BASE_HI, rb->buffer_desc.gpuaddr); + + ret = a5xx_microcode_load(adreno_dev); + if (ret) + return ret; + + /* clear ME_HALT to start micro engine */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0); + + ret = a5xx_send_me_init(adreno_dev, rb); + if (ret) + return ret; + + /* GPU comes up in secured mode, make it unsecured by default */ + ret = adreno_set_unsecured_mode(adreno_dev, rb); + if (ret) + return ret; + + ret = a5xx_gpmu_init(adreno_dev); + if (ret) + return ret; + + a5xx_post_start(adreno_dev); + + return 0; +} + +static int _load_firmware(struct kgsl_device *device, const char *fwfile, + struct adreno_firmware *firmware) +{ + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, fwfile, device->dev); + + if (ret) { + dev_err(device->dev, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + ret = kgsl_allocate_global(device, &firmware->memdesc, fw->size - 4, + KGSL_MEMFLAGS_GPUREADONLY, 0, "ucode"); + + if (ret) + goto done; + + memcpy(firmware->memdesc.hostptr, &fw->data[4], fw->size - 4); + firmware->size = (fw->size - 4) / sizeof(uint32_t); + firmware->version = *(unsigned int *)&fw->data[4]; + +done: + release_firmware(fw); + + return ret; +} + +/* + * a5xx_microcode_read() - Read microcode + * @adreno_dev: Pointer to adreno device + */ +static int a5xx_microcode_read(struct adreno_device *adreno_dev) +{ + int ret; + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + + if (pm4_fw->memdesc.hostptr == NULL) { + ret = _load_firmware(KGSL_DEVICE(adreno_dev), + adreno_dev->gpucore->pm4fw_name, pm4_fw); + if (ret) + return ret; + } + + if (pfp_fw->memdesc.hostptr == NULL) { + ret = _load_firmware(KGSL_DEVICE(adreno_dev), + adreno_dev->gpucore->pfpfw_name, pfp_fw); + if (ret) + return ret; + } + + ret = _load_gpmu_firmware(adreno_dev); + if (ret) + return ret; + + _load_regfile(adreno_dev); + + return ret; +} + +static struct adreno_perfcount_register a5xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_0_LO, + A5XX_RBBM_PERFCTR_CP_0_HI, 0, A5XX_CP_PERFCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_1_LO, + A5XX_RBBM_PERFCTR_CP_1_HI, 1, A5XX_CP_PERFCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_2_LO, + A5XX_RBBM_PERFCTR_CP_2_HI, 2, A5XX_CP_PERFCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_3_LO, + A5XX_RBBM_PERFCTR_CP_3_HI, 3, A5XX_CP_PERFCTR_CP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_4_LO, + A5XX_RBBM_PERFCTR_CP_4_HI, 4, A5XX_CP_PERFCTR_CP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_5_LO, + A5XX_RBBM_PERFCTR_CP_5_HI, 5, A5XX_CP_PERFCTR_CP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_6_LO, + A5XX_RBBM_PERFCTR_CP_6_HI, 6, A5XX_CP_PERFCTR_CP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_7_LO, + A5XX_RBBM_PERFCTR_CP_7_HI, 7, A5XX_CP_PERFCTR_CP_SEL_7 }, +}; + +/* + * Note that PERFCTR_RBBM_0 is missing - it is used to emulate the PWR counters. + * See below. + */ +static struct adreno_perfcount_register a5xx_perfcounters_rbbm[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_1_LO, + A5XX_RBBM_PERFCTR_RBBM_1_HI, 9, A5XX_RBBM_PERFCTR_RBBM_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_2_LO, + A5XX_RBBM_PERFCTR_RBBM_2_HI, 10, A5XX_RBBM_PERFCTR_RBBM_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_3_LO, + A5XX_RBBM_PERFCTR_RBBM_3_HI, 11, A5XX_RBBM_PERFCTR_RBBM_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_0_LO, + A5XX_RBBM_PERFCTR_PC_0_HI, 12, A5XX_PC_PERFCTR_PC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_1_LO, + A5XX_RBBM_PERFCTR_PC_1_HI, 13, A5XX_PC_PERFCTR_PC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_2_LO, + A5XX_RBBM_PERFCTR_PC_2_HI, 14, A5XX_PC_PERFCTR_PC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_3_LO, + A5XX_RBBM_PERFCTR_PC_3_HI, 15, A5XX_PC_PERFCTR_PC_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_4_LO, + A5XX_RBBM_PERFCTR_PC_4_HI, 16, A5XX_PC_PERFCTR_PC_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_5_LO, + A5XX_RBBM_PERFCTR_PC_5_HI, 17, A5XX_PC_PERFCTR_PC_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_6_LO, + A5XX_RBBM_PERFCTR_PC_6_HI, 18, A5XX_PC_PERFCTR_PC_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_7_LO, + A5XX_RBBM_PERFCTR_PC_7_HI, 19, A5XX_PC_PERFCTR_PC_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_0_LO, + A5XX_RBBM_PERFCTR_VFD_0_HI, 20, A5XX_VFD_PERFCTR_VFD_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_1_LO, + A5XX_RBBM_PERFCTR_VFD_1_HI, 21, A5XX_VFD_PERFCTR_VFD_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_2_LO, + A5XX_RBBM_PERFCTR_VFD_2_HI, 22, A5XX_VFD_PERFCTR_VFD_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_3_LO, + A5XX_RBBM_PERFCTR_VFD_3_HI, 23, A5XX_VFD_PERFCTR_VFD_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_4_LO, + A5XX_RBBM_PERFCTR_VFD_4_HI, 24, A5XX_VFD_PERFCTR_VFD_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_5_LO, + A5XX_RBBM_PERFCTR_VFD_5_HI, 25, A5XX_VFD_PERFCTR_VFD_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_6_LO, + A5XX_RBBM_PERFCTR_VFD_6_HI, 26, A5XX_VFD_PERFCTR_VFD_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_7_LO, + A5XX_RBBM_PERFCTR_VFD_7_HI, 27, A5XX_VFD_PERFCTR_VFD_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_0_LO, + A5XX_RBBM_PERFCTR_HLSQ_0_HI, 28, A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_1_LO, + A5XX_RBBM_PERFCTR_HLSQ_1_HI, 29, A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_2_LO, + A5XX_RBBM_PERFCTR_HLSQ_2_HI, 30, A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_3_LO, + A5XX_RBBM_PERFCTR_HLSQ_3_HI, 31, A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_4_LO, + A5XX_RBBM_PERFCTR_HLSQ_4_HI, 32, A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_5_LO, + A5XX_RBBM_PERFCTR_HLSQ_5_HI, 33, A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_6_LO, + A5XX_RBBM_PERFCTR_HLSQ_6_HI, 34, A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_7_LO, + A5XX_RBBM_PERFCTR_HLSQ_7_HI, 35, A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_0_LO, + A5XX_RBBM_PERFCTR_VPC_0_HI, 36, A5XX_VPC_PERFCTR_VPC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_1_LO, + A5XX_RBBM_PERFCTR_VPC_1_HI, 37, A5XX_VPC_PERFCTR_VPC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_2_LO, + A5XX_RBBM_PERFCTR_VPC_2_HI, 38, A5XX_VPC_PERFCTR_VPC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_3_LO, + A5XX_RBBM_PERFCTR_VPC_3_HI, 39, A5XX_VPC_PERFCTR_VPC_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_0_LO, + A5XX_RBBM_PERFCTR_CCU_0_HI, 40, A5XX_RB_PERFCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_1_LO, + A5XX_RBBM_PERFCTR_CCU_1_HI, 41, A5XX_RB_PERFCTR_CCU_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_2_LO, + A5XX_RBBM_PERFCTR_CCU_2_HI, 42, A5XX_RB_PERFCTR_CCU_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_3_LO, + A5XX_RBBM_PERFCTR_CCU_3_HI, 43, A5XX_RB_PERFCTR_CCU_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_0_LO, + A5XX_RBBM_PERFCTR_TSE_0_HI, 44, A5XX_GRAS_PERFCTR_TSE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_1_LO, + A5XX_RBBM_PERFCTR_TSE_1_HI, 45, A5XX_GRAS_PERFCTR_TSE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_2_LO, + A5XX_RBBM_PERFCTR_TSE_2_HI, 46, A5XX_GRAS_PERFCTR_TSE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_3_LO, + A5XX_RBBM_PERFCTR_TSE_3_HI, 47, A5XX_GRAS_PERFCTR_TSE_SEL_3 }, +}; + + +static struct adreno_perfcount_register a5xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_0_LO, + A5XX_RBBM_PERFCTR_RAS_0_HI, 48, A5XX_GRAS_PERFCTR_RAS_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_1_LO, + A5XX_RBBM_PERFCTR_RAS_1_HI, 49, A5XX_GRAS_PERFCTR_RAS_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_2_LO, + A5XX_RBBM_PERFCTR_RAS_2_HI, 50, A5XX_GRAS_PERFCTR_RAS_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_3_LO, + A5XX_RBBM_PERFCTR_RAS_3_HI, 51, A5XX_GRAS_PERFCTR_RAS_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_0_LO, + A5XX_RBBM_PERFCTR_UCHE_0_HI, 52, A5XX_UCHE_PERFCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_1_LO, + A5XX_RBBM_PERFCTR_UCHE_1_HI, 53, A5XX_UCHE_PERFCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_2_LO, + A5XX_RBBM_PERFCTR_UCHE_2_HI, 54, A5XX_UCHE_PERFCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_3_LO, + A5XX_RBBM_PERFCTR_UCHE_3_HI, 55, A5XX_UCHE_PERFCTR_UCHE_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_4_LO, + A5XX_RBBM_PERFCTR_UCHE_4_HI, 56, A5XX_UCHE_PERFCTR_UCHE_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_5_LO, + A5XX_RBBM_PERFCTR_UCHE_5_HI, 57, A5XX_UCHE_PERFCTR_UCHE_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_6_LO, + A5XX_RBBM_PERFCTR_UCHE_6_HI, 58, A5XX_UCHE_PERFCTR_UCHE_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_7_LO, + A5XX_RBBM_PERFCTR_UCHE_7_HI, 59, A5XX_UCHE_PERFCTR_UCHE_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_0_LO, + A5XX_RBBM_PERFCTR_TP_0_HI, 60, A5XX_TPL1_PERFCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_1_LO, + A5XX_RBBM_PERFCTR_TP_1_HI, 61, A5XX_TPL1_PERFCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_2_LO, + A5XX_RBBM_PERFCTR_TP_2_HI, 62, A5XX_TPL1_PERFCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_3_LO, + A5XX_RBBM_PERFCTR_TP_3_HI, 63, A5XX_TPL1_PERFCTR_TP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_4_LO, + A5XX_RBBM_PERFCTR_TP_4_HI, 64, A5XX_TPL1_PERFCTR_TP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_5_LO, + A5XX_RBBM_PERFCTR_TP_5_HI, 65, A5XX_TPL1_PERFCTR_TP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_6_LO, + A5XX_RBBM_PERFCTR_TP_6_HI, 66, A5XX_TPL1_PERFCTR_TP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_7_LO, + A5XX_RBBM_PERFCTR_TP_7_HI, 67, A5XX_TPL1_PERFCTR_TP_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_0_LO, + A5XX_RBBM_PERFCTR_SP_0_HI, 68, A5XX_SP_PERFCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_1_LO, + A5XX_RBBM_PERFCTR_SP_1_HI, 69, A5XX_SP_PERFCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_2_LO, + A5XX_RBBM_PERFCTR_SP_2_HI, 70, A5XX_SP_PERFCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_3_LO, + A5XX_RBBM_PERFCTR_SP_3_HI, 71, A5XX_SP_PERFCTR_SP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_4_LO, + A5XX_RBBM_PERFCTR_SP_4_HI, 72, A5XX_SP_PERFCTR_SP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_5_LO, + A5XX_RBBM_PERFCTR_SP_5_HI, 73, A5XX_SP_PERFCTR_SP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_6_LO, + A5XX_RBBM_PERFCTR_SP_6_HI, 74, A5XX_SP_PERFCTR_SP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_7_LO, + A5XX_RBBM_PERFCTR_SP_7_HI, 75, A5XX_SP_PERFCTR_SP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_8_LO, + A5XX_RBBM_PERFCTR_SP_8_HI, 76, A5XX_SP_PERFCTR_SP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_9_LO, + A5XX_RBBM_PERFCTR_SP_9_HI, 77, A5XX_SP_PERFCTR_SP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_10_LO, + A5XX_RBBM_PERFCTR_SP_10_HI, 78, A5XX_SP_PERFCTR_SP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_11_LO, + A5XX_RBBM_PERFCTR_SP_11_HI, 79, A5XX_SP_PERFCTR_SP_SEL_11 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_0_LO, + A5XX_RBBM_PERFCTR_RB_0_HI, 80, A5XX_RB_PERFCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_1_LO, + A5XX_RBBM_PERFCTR_RB_1_HI, 81, A5XX_RB_PERFCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_2_LO, + A5XX_RBBM_PERFCTR_RB_2_HI, 82, A5XX_RB_PERFCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_3_LO, + A5XX_RBBM_PERFCTR_RB_3_HI, 83, A5XX_RB_PERFCTR_RB_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_4_LO, + A5XX_RBBM_PERFCTR_RB_4_HI, 84, A5XX_RB_PERFCTR_RB_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_5_LO, + A5XX_RBBM_PERFCTR_RB_5_HI, 85, A5XX_RB_PERFCTR_RB_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_6_LO, + A5XX_RBBM_PERFCTR_RB_6_HI, 86, A5XX_RB_PERFCTR_RB_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_7_LO, + A5XX_RBBM_PERFCTR_RB_7_HI, 87, A5XX_RB_PERFCTR_RB_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vsc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VSC_0_LO, + A5XX_RBBM_PERFCTR_VSC_0_HI, 88, A5XX_VSC_PERFCTR_VSC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VSC_1_LO, + A5XX_RBBM_PERFCTR_VSC_1_HI, 89, A5XX_VSC_PERFCTR_VSC_SEL_1 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_lrz[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_0_LO, + A5XX_RBBM_PERFCTR_LRZ_0_HI, 90, A5XX_GRAS_PERFCTR_LRZ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_1_LO, + A5XX_RBBM_PERFCTR_LRZ_1_HI, 91, A5XX_GRAS_PERFCTR_LRZ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_2_LO, + A5XX_RBBM_PERFCTR_LRZ_2_HI, 92, A5XX_GRAS_PERFCTR_LRZ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_3_LO, + A5XX_RBBM_PERFCTR_LRZ_3_HI, 93, A5XX_GRAS_PERFCTR_LRZ_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_cmp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_0_LO, + A5XX_RBBM_PERFCTR_CMP_0_HI, 94, A5XX_RB_PERFCTR_CMP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_1_LO, + A5XX_RBBM_PERFCTR_CMP_1_HI, 95, A5XX_RB_PERFCTR_CMP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_2_LO, + A5XX_RBBM_PERFCTR_CMP_2_HI, 96, A5XX_RB_PERFCTR_CMP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_3_LO, + A5XX_RBBM_PERFCTR_CMP_3_HI, 97, A5XX_RB_PERFCTR_CMP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW0, + A5XX_VBIF_PERF_CNT_HIGH0, -1, A5XX_VBIF_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW1, + A5XX_VBIF_PERF_CNT_HIGH1, -1, A5XX_VBIF_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW2, + A5XX_VBIF_PERF_CNT_HIGH2, -1, A5XX_VBIF_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW3, + A5XX_VBIF_PERF_CNT_HIGH3, -1, A5XX_VBIF_PERF_CNT_SEL3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW0, + A5XX_VBIF_PERF_PWR_CNT_HIGH0, -1, A5XX_VBIF_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW1, + A5XX_VBIF_PERF_PWR_CNT_HIGH1, -1, A5XX_VBIF_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW2, + A5XX_VBIF_PERF_PWR_CNT_HIGH2, -1, A5XX_VBIF_PERF_PWR_CNT_EN2 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_ALWAYSON_COUNTER_LO, + A5XX_RBBM_ALWAYSON_COUNTER_HI, -1 }, +}; + +/* + * 5XX targets don't really have physical PERFCTR_PWR registers - we emulate + * them using similar performance counters from the RBBM block. The difference + * between using this group and the RBBM group is that the RBBM counters are + * reloaded after a power collapse which is not how the PWR counters behaved on + * legacy hardware. In order to limit the disruption on the rest of the system + * we go out of our way to ensure backwards compatibility. Since RBBM counters + * are in short supply, we don't emulate PWR:0 which nobody uses - mark it as + * broken. + */ +static struct adreno_perfcount_register a5xx_perfcounters_pwr[] = { + { KGSL_PERFCOUNTER_BROKEN, 0, 0, 0, 0, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_0_LO, + A5XX_RBBM_PERFCTR_RBBM_0_HI, -1, 0}, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_0_LO, + A5XX_SP_POWER_COUNTER_0_HI, -1, A5XX_SP_POWERCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_1_LO, + A5XX_SP_POWER_COUNTER_1_HI, -1, A5XX_SP_POWERCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_2_LO, + A5XX_SP_POWER_COUNTER_2_HI, -1, A5XX_SP_POWERCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_3_LO, + A5XX_SP_POWER_COUNTER_3_HI, -1, A5XX_SP_POWERCTR_SP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_0_LO, + A5XX_TP_POWER_COUNTER_0_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_1_LO, + A5XX_TP_POWER_COUNTER_1_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_2_LO, + A5XX_TP_POWER_COUNTER_2_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_3_LO, + A5XX_TP_POWER_COUNTER_3_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_0_LO, + A5XX_RB_POWER_COUNTER_0_HI, -1, A5XX_RB_POWERCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_1_LO, + A5XX_RB_POWER_COUNTER_1_HI, -1, A5XX_RB_POWERCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_2_LO, + A5XX_RB_POWER_COUNTER_2_HI, -1, A5XX_RB_POWERCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_3_LO, + A5XX_RB_POWER_COUNTER_3_HI, -1, A5XX_RB_POWERCTR_RB_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CCU_POWER_COUNTER_0_LO, + A5XX_CCU_POWER_COUNTER_0_HI, -1, A5XX_RB_POWERCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CCU_POWER_COUNTER_1_LO, + A5XX_CCU_POWER_COUNTER_1_HI, -1, A5XX_RB_POWERCTR_CCU_SEL_1 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_0_LO, + A5XX_UCHE_POWER_COUNTER_0_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_1_LO, + A5XX_UCHE_POWER_COUNTER_1_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_2_LO, + A5XX_UCHE_POWER_COUNTER_2_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_3_LO, + A5XX_UCHE_POWER_COUNTER_3_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_0_LO, + A5XX_CP_POWER_COUNTER_0_HI, -1, A5XX_CP_POWERCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_1_LO, + A5XX_CP_POWER_COUNTER_1_HI, -1, A5XX_CP_POWERCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_2_LO, + A5XX_CP_POWER_COUNTER_2_HI, -1, A5XX_CP_POWERCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_3_LO, + A5XX_CP_POWER_COUNTER_3_HI, -1, A5XX_CP_POWERCTR_CP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_gpmu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_0_LO, + A5XX_GPMU_POWER_COUNTER_0_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_1_LO, + A5XX_GPMU_POWER_COUNTER_1_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_2_LO, + A5XX_GPMU_POWER_COUNTER_2_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_3_LO, + A5XX_GPMU_POWER_COUNTER_3_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_4_LO, + A5XX_GPMU_POWER_COUNTER_4_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_5_LO, + A5XX_GPMU_POWER_COUNTER_5_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_1 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_ALWAYS_ON_COUNTER_LO, + A5XX_GPMU_ALWAYS_ON_COUNTER_HI, -1 }, +}; + +#define A5XX_PERFCOUNTER_GROUP(offset, name) \ + ADRENO_PERFCOUNTER_GROUP(a5xx, offset, name) + +#define A5XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a5xx, offset, name, flags) + +#define A5XX_POWER_COUNTER_GROUP(offset, name) \ + ADRENO_POWER_COUNTER_GROUP(a5xx, offset, name) + +static struct adreno_perfcount_group a5xx_perfcounter_groups + [KGSL_PERFCOUNTER_GROUP_MAX] = { + A5XX_PERFCOUNTER_GROUP(CP, cp), + A5XX_PERFCOUNTER_GROUP(RBBM, rbbm), + A5XX_PERFCOUNTER_GROUP(PC, pc), + A5XX_PERFCOUNTER_GROUP(VFD, vfd), + A5XX_PERFCOUNTER_GROUP(HLSQ, hlsq), + A5XX_PERFCOUNTER_GROUP(VPC, vpc), + A5XX_PERFCOUNTER_GROUP(CCU, ccu), + A5XX_PERFCOUNTER_GROUP(CMP, cmp), + A5XX_PERFCOUNTER_GROUP(TSE, tse), + A5XX_PERFCOUNTER_GROUP(RAS, ras), + A5XX_PERFCOUNTER_GROUP(LRZ, lrz), + A5XX_PERFCOUNTER_GROUP(UCHE, uche), + A5XX_PERFCOUNTER_GROUP(TP, tp), + A5XX_PERFCOUNTER_GROUP(SP, sp), + A5XX_PERFCOUNTER_GROUP(RB, rb), + A5XX_PERFCOUNTER_GROUP(VSC, vsc), + A5XX_PERFCOUNTER_GROUP_FLAGS(PWR, pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A5XX_PERFCOUNTER_GROUP(VBIF, vbif), + A5XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A5XX_PERFCOUNTER_GROUP_FLAGS(ALWAYSON, alwayson, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A5XX_POWER_COUNTER_GROUP(SP, sp), + A5XX_POWER_COUNTER_GROUP(TP, tp), + A5XX_POWER_COUNTER_GROUP(RB, rb), + A5XX_POWER_COUNTER_GROUP(CCU, ccu), + A5XX_POWER_COUNTER_GROUP(UCHE, uche), + A5XX_POWER_COUNTER_GROUP(CP, cp), + A5XX_POWER_COUNTER_GROUP(GPMU, gpmu), + A5XX_POWER_COUNTER_GROUP(ALWAYSON, alwayson), +}; + +static struct adreno_perfcounters a5xx_perfcounters = { + a5xx_perfcounter_groups, + ARRAY_SIZE(a5xx_perfcounter_groups), +}; + +static struct adreno_ft_perf_counters a5xx_ft_perf_counters[] = { + {KGSL_PERFCOUNTER_GROUP_SP, A5XX_SP_ALU_ACTIVE_CYCLES}, + {KGSL_PERFCOUNTER_GROUP_SP, A5XX_SP0_ICL1_MISSES}, + {KGSL_PERFCOUNTER_GROUP_SP, A5XX_SP_FS_CFLOW_INSTRUCTIONS}, + {KGSL_PERFCOUNTER_GROUP_TSE, A5XX_TSE_INPUT_PRIM_NUM}, +}; + +static unsigned int a5xx_int_bits[ADRENO_INT_BITS_MAX] = { + ADRENO_INT_DEFINE(ADRENO_INT_RBBM_AHB_ERROR, A5XX_INT_RBBM_AHB_ERROR), +}; + +/* Register offset defines for A5XX, in order of enum adreno_regs */ +static unsigned int a5xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + ADRENO_REG_DEFINE(ADRENO_REG_CP_WFI_PEND_CTR, A5XX_CP_WFI_PEND_CTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A5XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A5XX_CP_RB_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, + A5XX_CP_RB_RPTR_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI, + A5XX_CP_RB_RPTR_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A5XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A5XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A5XX_CP_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A5XX_CP_ME_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A5XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A5XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, A5XX_CP_IB1_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A5XX_CP_IB1_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A5XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, A5XX_CP_IB2_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A5XX_CP_IB2_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_ADDR, A5XX_CP_ROQ_DBG_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_DATA, A5XX_CP_ROQ_DBG_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_ADDR, A5XX_CP_MERCIU_DBG_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA, A5XX_CP_MERCIU_DBG_DATA_1), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MERCIU_DATA2, + A5XX_CP_MERCIU_DBG_DATA_2), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_ADDR, A5XX_CP_MEQ_DBG_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_MEQ_DATA, A5XX_CP_MEQ_DBG_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A5XX_CP_PROTECT_REG_0), + ADRENO_REG_DEFINE(ADRENO_REG_CP_HW_FAULT, A5XX_CP_HW_FAULT), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A5XX_CP_CONTEXT_SWITCH_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DEBUG, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DISABLE, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A5XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS3, A5XX_RBBM_STATUS3), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_CTL, A5XX_RBBM_PERFCTR_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + A5XX_RBBM_PERFCTR_LOAD_CMD0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + A5XX_RBBM_PERFCTR_LOAD_CMD1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD2, + A5XX_RBBM_PERFCTR_LOAD_CMD2), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD3, + A5XX_RBBM_PERFCTR_LOAD_CMD3), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A5XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_STATUS, A5XX_RBBM_INT_0_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A5XX_RBBM_CLOCK_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_CLEAR_CMD, + A5XX_RBBM_INT_CLEAR_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A5XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD, + A5XX_RBBM_BLOCK_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD2, + A5XX_RBBM_BLOCK_SW_RESET_CMD2), + ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE0, A5XX_UCHE_INVALIDATE0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_LO, + A5XX_RBBM_PERFCTR_RBBM_0_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_RBBM_0_HI, + A5XX_RBBM_PERFCTR_RBBM_0_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + A5XX_RBBM_PERFCTR_LOAD_VALUE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + A5XX_RBBM_PERFCTR_LOAD_VALUE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TRUST_CONTROL, + A5XX_RBBM_SECVID_TRUST_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TRUST_CONFIG, + A5XX_RBBM_SECVID_TRUST_CONFIG), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_CONTROL, + A5XX_RBBM_SECVID_TSB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE, + A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_SIZE, + A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO, + A5XX_RBBM_ALWAYSON_COUNTER_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI, + A5XX_RBBM_ALWAYSON_COUNTER_HI), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL0, + A5XX_VBIF_XIN_HALT_CTRL0), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A5XX_VBIF_XIN_HALT_CTRL1), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_VERSION, + A5XX_VBIF_VERSION), + ADRENO_REG_DEFINE(ADRENO_REG_GPMU_POWER_COUNTER_ENABLE, + A5XX_GPMU_POWER_COUNTER_ENABLE), +}; + +static const struct adreno_reg_offsets a5xx_reg_offsets = { + .offsets = a5xx_register_offsets, + .offset_0 = ADRENO_REG_REGISTER_MAX, +}; + +static void a5xx_cp_hw_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status1, status2; + + kgsl_regread(device, A5XX_CP_INTERRUPT_STATUS, &status1); + + if (status1 & BIT(A5XX_CP_OPCODE_ERROR)) { + unsigned int val; + + kgsl_regwrite(device, A5XX_CP_PFP_STAT_ADDR, 0); + + /* + * A5XX_CP_PFP_STAT_DATA is indexed, so read it twice to get the + * value we want + */ + kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val); + kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val); + + dev_crit_ratelimited(device->dev, + "ringbuffer opcode error | possible opcode=0x%8.8X\n", + val); + } + if (status1 & BIT(A5XX_CP_RESERVED_BIT_ERROR)) + dev_crit_ratelimited(device->dev, + "ringbuffer reserved bit error interrupt\n"); + if (status1 & BIT(A5XX_CP_HW_FAULT_ERROR)) { + kgsl_regread(device, A5XX_CP_HW_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", + status2); + } + if (status1 & BIT(A5XX_CP_DMA_ERROR)) + dev_crit_ratelimited(device->dev, "CP | DMA error\n"); + if (status1 & BIT(A5XX_CP_REGISTER_PROTECTION_ERROR)) { + kgsl_regread(device, A5XX_CP_PROTECT_STATUS, &status2); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error| %s | addr=%x | status=%x\n", + status2 & (1 << 24) ? "WRITE" : "READ", + (status2 & 0xFFFFF) >> 2, status2); + } + if (status1 & BIT(A5XX_CP_AHB_ERROR)) { + kgsl_regread(device, A5XX_CP_AHB_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "ringbuffer AHB error interrupt | status=%x\n", + status2); + } +} + +static void a5xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + switch (bit) { + case A5XX_INT_RBBM_AHB_ERROR: { + kgsl_regread(device, A5XX_RBBM_AHB_ERROR_STATUS, ®); + + /* + * Return the word address of the erroring register so that it + * matches the register specification + */ + dev_crit_ratelimited(device->dev, + "RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n", + reg & (1 << 28) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2, + (reg >> 20) & 0x3, + (reg >> 24) & 0xF); + + /* Clear the error */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CMD, (1 << 4)); + break; + } + case A5XX_INT_RBBM_TRANSFER_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: AHB transfer timeout\n"); + break; + case A5XX_INT_RBBM_ME_MS_TIMEOUT: + kgsl_regread(device, A5XX_RBBM_AHB_ME_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | ME master split timeout | status=%x\n", + reg); + break; + case A5XX_INT_RBBM_PFP_MS_TIMEOUT: + kgsl_regread(device, A5XX_RBBM_AHB_PFP_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | PFP master split timeout | status=%x\n", + reg); + break; + case A5XX_INT_RBBM_ETS_MS_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: ME master split timeout\n"); + break; + case A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB ASYNC overflow\n"); + break; + case A5XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus overflow\n"); + break; + case A5XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + case A5XX_INT_UCHE_TRAP_INTR: + dev_crit_ratelimited(device->dev, "UCHE: Trap interrupt\n"); + break; + case A5XX_INT_GPMU_VOLTAGE_DROOP: + dev_crit_ratelimited(device->dev, "GPMU: Voltage droop\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt %d\n", + bit); + } +} + +static void a5xx_irq_storm_worker(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, irq_storm_work); + struct kgsl_device *device = &adreno_dev->dev; + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int status; + + mutex_lock(&device->mutex); + + /* Wait for the storm to clear up */ + do { + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_CLEAR_CMD, + BIT(A5XX_INT_CP_CACHE_FLUSH_TS)); + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_INT_0_STATUS, + &status); + } while (status & BIT(A5XX_INT_CP_CACHE_FLUSH_TS)); + + /* Re-enable the interrupt bit in the mask */ + gpudev->irq->mask |= BIT(A5XX_INT_CP_CACHE_FLUSH_TS); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_0_MASK, + gpudev->irq->mask); + clear_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv); + + dev_warn(device->dev, "Re-enabled A5XX_INT_CP_CACHE_FLUSH_TS\n"); + mutex_unlock(&device->mutex); + + /* Reschedule just to make sure everything retires */ + adreno_dispatcher_schedule(device); +} + +static void a5xx_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = &adreno_dev->dev; + unsigned int cur; + static unsigned int count; + static unsigned int prev; + + if (test_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv)) + return; + + kgsl_sharedmem_readl(&device->memstore, &cur, + KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, + ref_wait_ts)); + + /* + * prev holds a previously read value + * from memory. It should be changed by the GPU with every + * interrupt. If the value we know about and the value we just + * read are the same, then we are likely in a storm. + * If this happens twice, disable the interrupt in the mask + * so the dispatcher can take care of the issue. It is then + * up to the dispatcher to re-enable the mask once all work + * is done and the storm has ended. + */ + if (prev == cur) { + count++; + if (count == 2) { + struct adreno_gpudev *gpudev = + ADRENO_GPU_DEVICE(adreno_dev); + + /* disable interrupt from the mask */ + set_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, + &adreno_dev->priv); + gpudev->irq->mask &= ~BIT(A5XX_INT_CP_CACHE_FLUSH_TS); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_0_MASK, + gpudev->irq->mask); + + kgsl_schedule_work(&adreno_dev->irq_storm_work); + + return; + } + } else { + count = 0; + prev = cur; + } + + a5xx_preemption_trigger(adreno_dev); + adreno_dispatcher_schedule(device); +} + +static const char *gpmu_int_msg[32] = { + [FW_INTR_INFO] = "FW_INTR_INFO", + [LLM_ACK_ERR_INTR] = "LLM_ACK_ERR_INTR", + [ISENS_TRIM_ERR_INTR] = "ISENS_TRIM_ERR_INTR", + [ISENS_ERR_INTR] = "ISENS_ERR_INTR", + [ISENS_IDLE_ERR_INTR] = "ISENS_IDLE_ERR_INTR", + [ISENS_PWR_ON_ERR_INTR] = "ISENS_PWR_ON_ERR_INTR", + [6 ... 30] = "", + [WDOG_EXPITED] = "WDOG_EXPITED"}; + +static void a5xx_gpmu_int_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg, i; + + kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, ®); + + if (reg & (~VALID_GPMU_IRQ)) { + dev_crit_ratelimited(device->dev, + "GPMU: Unknown IRQ mask 0x%08lx in 0x%08x\n", + reg & (~VALID_GPMU_IRQ), reg); + } + + for (i = 0; i < 32; i++) + switch (reg & BIT(i)) { + case BIT(WDOG_EXPITED): + if (test_and_clear_bit(ADRENO_DEVICE_GPMU_INITIALIZED, + &adreno_dev->priv)) { + /* Stop GPMU */ + kgsl_regwrite(device, + A5XX_GPMU_CM3_SYSRESET, 1); + kgsl_schedule_work(&adreno_dev->gpmu_work); + } + /* fallthrough */ + case BIT(FW_INTR_INFO): + case BIT(LLM_ACK_ERR_INTR): + case BIT(ISENS_TRIM_ERR_INTR): + case BIT(ISENS_ERR_INTR): + case BIT(ISENS_IDLE_ERR_INTR): + case BIT(ISENS_PWR_ON_ERR_INTR): + dev_crit_ratelimited(device->dev, + "GPMU: interrupt %s(%08lx)\n", + gpmu_int_msg[i], + BIT(i)); + break; + } +} + +/* + * a5x_gpc_err_int_callback() - Isr for GPC error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +void a5x_gpc_err_int_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * GPC error is typically the result of mistake SW programming. + * Force GPU fault for this interrupt so that we can debug it + * with help of register dump. + */ + + dev_crit(device->dev, "RBBM: GPC error\n"); + adreno_irqctrl(adreno_dev, 0); + + /* Trigger a fault in the dispatcher - this will effect a restart */ + adreno_set_gpu_fault(adreno_dev, ADRENO_SOFT_FAULT); + adreno_dispatcher_schedule(device); +} + +#define A5XX_INT_MASK \ + ((1 << A5XX_INT_RBBM_AHB_ERROR) | \ + (1 << A5XX_INT_RBBM_TRANSFER_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ME_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_PFP_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ETS_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW) | \ + (1 << A5XX_INT_RBBM_GPC_ERROR) | \ + (1 << A5XX_INT_CP_HW_ERROR) | \ + (1 << A5XX_INT_CP_CACHE_FLUSH_TS) | \ + (1 << A5XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A5XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A5XX_INT_UCHE_TRAP_INTR) | \ + (1 << A5XX_INT_CP_SW) | \ + (1 << A5XX_INT_GPMU_FIRMWARE) | \ + (1 << A5XX_INT_GPMU_VOLTAGE_DROOP)) + + +static struct adreno_irq_funcs a5xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 2 - RBBM_TRANSFER_TIMEOUT */ + /* 3 - RBBM_ME_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 4 - RBBM_PFP_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 5 - RBBM_ETS_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 6 - RBBM_ATB_ASYNC_OVERFLOW */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + ADRENO_IRQ_CALLBACK(a5x_gpc_err_int_callback), /* 7 - GPC_ERR */ + ADRENO_IRQ_CALLBACK(a5xx_preempt_callback),/* 8 - CP_SW */ + ADRENO_IRQ_CALLBACK(a5xx_cp_hw_err_callback), /* 9 - CP_HW_ERROR */ + /* 10 - CP_CCU_FLUSH_DEPTH_TS */ + ADRENO_IRQ_CALLBACK(NULL), + /* 11 - CP_CCU_FLUSH_COLOR_TS */ + ADRENO_IRQ_CALLBACK(NULL), + /* 12 - CP_CCU_RESOLVE_TS */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(NULL), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 15 - CP_RB_INT */ + /* 16 - CCP_UNUSED_1 */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_WT_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - UNKNOWN_1 */ + ADRENO_IRQ_CALLBACK(a5xx_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + /* 21 - UNUSED_2 */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */ + /* 23 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 24 - UCHE_OOB_ACCESS */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 25 - UCHE_TRAP_INTR */ + ADRENO_IRQ_CALLBACK(NULL), /* 26 - DEBBUS_INTR_0 */ + ADRENO_IRQ_CALLBACK(NULL), /* 27 - DEBBUS_INTR_1 */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 28 - GPMU_VOLTAGE_DROOP */ + ADRENO_IRQ_CALLBACK(a5xx_gpmu_int_callback), /* 29 - GPMU_FIRMWARE */ + ADRENO_IRQ_CALLBACK(NULL), /* 30 - ISDB_CPU_IRQ */ + ADRENO_IRQ_CALLBACK(NULL), /* 31 - ISDB_UNDER_DEBUG */ +}; + +static struct adreno_irq a5xx_irq = { + .funcs = a5xx_irq_funcs, + .mask = A5XX_INT_MASK, +}; + +/* + * Default size for CP queues for A5xx targets. You must + * overwrite these value in platform_setup function for + * A5xx derivatives if size differs. + */ +static struct adreno_snapshot_sizes a5xx_snap_sizes = { + .cp_pfp = 36, + .cp_me = 29, + .cp_meq = 64, + .cp_merciu = 64, + .roq = 512, +}; + +static struct adreno_snapshot_data a5xx_snapshot_data = { + .sect_sizes = &a5xx_snap_sizes, +}; + +static struct adreno_coresight_register a5xx_coresight_registers[] = { + { A5XX_RBBM_CFG_DBGBUS_SEL_A }, + { A5XX_RBBM_CFG_DBGBUS_SEL_B }, + { A5XX_RBBM_CFG_DBGBUS_SEL_C }, + { A5XX_RBBM_CFG_DBGBUS_SEL_D }, + { A5XX_RBBM_CFG_DBGBUS_CNTLT }, + { A5XX_RBBM_CFG_DBGBUS_CNTLM }, + { A5XX_RBBM_CFG_DBGBUS_OPL }, + { A5XX_RBBM_CFG_DBGBUS_OPE }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_0 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_2 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_3 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_0 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_1 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_2 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_3 }, + { A5XX_RBBM_CFG_DBGBUS_BYTEL_0 }, + { A5XX_RBBM_CFG_DBGBUS_BYTEL_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_0 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_2 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_3 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_0 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_1 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_2 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_3 }, + { A5XX_RBBM_CFG_DBGBUS_NIBBLEE }, + { A5XX_RBBM_CFG_DBGBUS_PTRC0 }, + { A5XX_RBBM_CFG_DBGBUS_PTRC1 }, + { A5XX_RBBM_CFG_DBGBUS_LOADREG }, + { A5XX_RBBM_CFG_DBGBUS_IDX }, + { A5XX_RBBM_CFG_DBGBUS_CLRC }, + { A5XX_RBBM_CFG_DBGBUS_LOADIVT }, + { A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC }, + { A5XX_RBBM_CFG_DBGBUS_OVER }, + { A5XX_RBBM_CFG_DBGBUS_COUNT0 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT1 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT2 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT3 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT4 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT5 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4 }, + { A5XX_RBBM_CFG_DBGBUS_MISR0 }, + { A5XX_RBBM_CFG_DBGBUS_MISR1 }, + { A5XX_RBBM_AHB_DBG_CNTL }, + { A5XX_RBBM_READ_AHB_THROUGH_DBG }, + { A5XX_RBBM_DBG_LO_HI_GPIO }, + { A5XX_RBBM_EXT_TRACE_BUS_CNTL }, + { A5XX_RBBM_EXT_VBIF_DBG_CNTL }, +}; + +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_a, &a5xx_coresight_registers[0]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_b, &a5xx_coresight_registers[1]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_c, &a5xx_coresight_registers[2]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_d, &a5xx_coresight_registers[3]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlt, &a5xx_coresight_registers[4]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlm, &a5xx_coresight_registers[5]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_opl, &a5xx_coresight_registers[6]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ope, &a5xx_coresight_registers[7]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_0, &a5xx_coresight_registers[8]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_1, &a5xx_coresight_registers[9]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_2, &a5xx_coresight_registers[10]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_3, &a5xx_coresight_registers[11]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_0, &a5xx_coresight_registers[12]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_1, &a5xx_coresight_registers[13]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_2, &a5xx_coresight_registers[14]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_3, &a5xx_coresight_registers[15]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_0, &a5xx_coresight_registers[16]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_1, &a5xx_coresight_registers[17]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_0, &a5xx_coresight_registers[18]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_1, &a5xx_coresight_registers[19]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_2, &a5xx_coresight_registers[20]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_3, &a5xx_coresight_registers[21]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_0, &a5xx_coresight_registers[22]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_1, &a5xx_coresight_registers[23]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_2, &a5xx_coresight_registers[24]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_3, &a5xx_coresight_registers[25]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_nibblee, &a5xx_coresight_registers[26]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc0, &a5xx_coresight_registers[27]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc1, &a5xx_coresight_registers[28]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadreg, &a5xx_coresight_registers[29]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_idx, &a5xx_coresight_registers[30]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_clrc, &a5xx_coresight_registers[31]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadivt, &a5xx_coresight_registers[32]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_event_logic, + &a5xx_coresight_registers[33]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_over, &a5xx_coresight_registers[34]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count0, &a5xx_coresight_registers[35]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count1, &a5xx_coresight_registers[36]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count2, &a5xx_coresight_registers[37]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count3, &a5xx_coresight_registers[38]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count4, &a5xx_coresight_registers[39]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count5, &a5xx_coresight_registers[40]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_addr, + &a5xx_coresight_registers[41]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf0, + &a5xx_coresight_registers[42]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf1, + &a5xx_coresight_registers[43]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf2, + &a5xx_coresight_registers[44]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf3, + &a5xx_coresight_registers[45]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf4, + &a5xx_coresight_registers[46]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_misr0, &a5xx_coresight_registers[47]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_misr1, &a5xx_coresight_registers[48]); +static ADRENO_CORESIGHT_ATTR(ahb_dbg_cntl, &a5xx_coresight_registers[49]); +static ADRENO_CORESIGHT_ATTR(read_ahb_through_dbg, + &a5xx_coresight_registers[50]); +static ADRENO_CORESIGHT_ATTR(dbg_lo_hi_gpio, &a5xx_coresight_registers[51]); +static ADRENO_CORESIGHT_ATTR(ext_trace_bus_cntl, &a5xx_coresight_registers[52]); +static ADRENO_CORESIGHT_ATTR(ext_vbif_dbg_cntl, &a5xx_coresight_registers[53]); + +static struct attribute *a5xx_coresight_attrs[] = { + &coresight_attr_cfg_dbgbus_sel_a.attr.attr, + &coresight_attr_cfg_dbgbus_sel_b.attr.attr, + &coresight_attr_cfg_dbgbus_sel_c.attr.attr, + &coresight_attr_cfg_dbgbus_sel_d.attr.attr, + &coresight_attr_cfg_dbgbus_cntlt.attr.attr, + &coresight_attr_cfg_dbgbus_cntlm.attr.attr, + &coresight_attr_cfg_dbgbus_opl.attr.attr, + &coresight_attr_cfg_dbgbus_ope.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_3.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_0.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_1.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_2.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_3.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_0.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_3.attr.attr, + &coresight_attr_cfg_dbgbus_maske_0.attr.attr, + &coresight_attr_cfg_dbgbus_maske_1.attr.attr, + &coresight_attr_cfg_dbgbus_maske_2.attr.attr, + &coresight_attr_cfg_dbgbus_maske_3.attr.attr, + &coresight_attr_cfg_dbgbus_nibblee.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc0.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc1.attr.attr, + &coresight_attr_cfg_dbgbus_loadreg.attr.attr, + &coresight_attr_cfg_dbgbus_idx.attr.attr, + &coresight_attr_cfg_dbgbus_clrc.attr.attr, + &coresight_attr_cfg_dbgbus_loadivt.attr.attr, + &coresight_attr_cfg_dbgbus_event_logic.attr.attr, + &coresight_attr_cfg_dbgbus_over.attr.attr, + &coresight_attr_cfg_dbgbus_count0.attr.attr, + &coresight_attr_cfg_dbgbus_count1.attr.attr, + &coresight_attr_cfg_dbgbus_count2.attr.attr, + &coresight_attr_cfg_dbgbus_count3.attr.attr, + &coresight_attr_cfg_dbgbus_count4.attr.attr, + &coresight_attr_cfg_dbgbus_count5.attr.attr, + &coresight_attr_cfg_dbgbus_trace_addr.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf0.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf1.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf2.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf3.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf4.attr.attr, + &coresight_attr_cfg_dbgbus_misr0.attr.attr, + &coresight_attr_cfg_dbgbus_misr1.attr.attr, + &coresight_attr_ahb_dbg_cntl.attr.attr, + &coresight_attr_read_ahb_through_dbg.attr.attr, + &coresight_attr_dbg_lo_hi_gpio.attr.attr, + &coresight_attr_ext_trace_bus_cntl.attr.attr, + &coresight_attr_ext_vbif_dbg_cntl.attr.attr, + NULL, +}; + +static const struct attribute_group a5xx_coresight_group = { + .attrs = a5xx_coresight_attrs, +}; + +static const struct attribute_group *a5xx_coresight_groups[] = { + &a5xx_coresight_group, + NULL, +}; + +static struct adreno_coresight a5xx_coresight = { + .registers = a5xx_coresight_registers, + .count = ARRAY_SIZE(a5xx_coresight_registers), + .groups = a5xx_coresight_groups, +}; + +struct adreno_gpudev adreno_a5xx_gpudev = { + .reg_offsets = &a5xx_reg_offsets, + .int_bits = a5xx_int_bits, + .ft_perf_counters = a5xx_ft_perf_counters, + .ft_perf_counters_count = ARRAY_SIZE(a5xx_ft_perf_counters), + .coresight = {&a5xx_coresight}, + .start = a5xx_start, + .snapshot = a5xx_snapshot, + .irq = &a5xx_irq, + .snapshot_data = &a5xx_snapshot_data, + .irq_trace = trace_kgsl_a5xx_irq_status, + .num_prio_levels = KGSL_PRIORITY_MAX_RB_LEVELS, + .platform_setup = a5xx_platform_setup, + .init = a5xx_init, + .remove = a5xx_remove, + .rb_start = a5xx_rb_start, + .microcode_read = a5xx_microcode_read, + .perfcounters = &a5xx_perfcounters, + .vbif_xin_halt_ctrl0_mask = A5XX_VBIF_XIN_HALT_CTRL0_MASK, + .is_sptp_idle = a5xx_is_sptp_idle, + .regulator_enable = a5xx_regulator_enable, + .regulator_disable = a5xx_regulator_disable, + .pwrlevel_change_settings = a5xx_pwrlevel_change_settings, + .read_throttling_counters = a5xx_read_throttling_counters, + .count_throttles = a5xx_count_throttles, + .enable_pwr_counters = a5xx_enable_pwr_counters, + .preemption_pre_ibsubmit = a5xx_preemption_pre_ibsubmit, + .preemption_yield_enable = + a5xx_preemption_yield_enable, + .preemption_post_ibsubmit = + a5xx_preemption_post_ibsubmit, + .preemption_init = a5xx_preemption_init, + .preemption_schedule = a5xx_preemption_schedule, + .enable_64bit = a5xx_enable_64bit, + .clk_set_options = a5xx_clk_set_options, +}; diff --git a/drivers/gpu/msm/adreno_a5xx.h b/drivers/gpu/msm/adreno_a5xx.h new file mode 100644 index 0000000000000000000000000000000000000000..b2f4e4efd2c7115cbc8d66490d23047e4b92b253 --- /dev/null +++ b/drivers/gpu/msm/adreno_a5xx.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved. + */ + +#ifndef _ADRENO_A5XX_H_ +#define _ADRENO_A5XX_H_ + +#include "a5xx_reg.h" + +#define A5XX_IRQ_FLAGS \ + { BIT(A5XX_INT_RBBM_GPU_IDLE), "RBBM_GPU_IDLE" }, \ + { BIT(A5XX_INT_RBBM_AHB_ERROR), "RBBM_AHB_ERR" }, \ + { BIT(A5XX_INT_RBBM_TRANSFER_TIMEOUT), "RBBM_TRANSFER_TIMEOUT" }, \ + { BIT(A5XX_INT_RBBM_ME_MS_TIMEOUT), "RBBM_ME_MS_TIMEOUT" }, \ + { BIT(A5XX_INT_RBBM_PFP_MS_TIMEOUT), "RBBM_PFP_MS_TIMEOUT" }, \ + { BIT(A5XX_INT_RBBM_ETS_MS_TIMEOUT), "RBBM_ETS_MS_TIMEOUT" }, \ + { BIT(A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW), "RBBM_ATB_ASYNC_OVERFLOW" }, \ + { BIT(A5XX_INT_RBBM_GPC_ERROR), "RBBM_GPC_ERR" }, \ + { BIT(A5XX_INT_CP_SW), "CP_SW" }, \ + { BIT(A5XX_INT_CP_HW_ERROR), "CP_OPCODE_ERROR" }, \ + { BIT(A5XX_INT_CP_CCU_FLUSH_DEPTH_TS), "CP_CCU_FLUSH_DEPTH_TS" }, \ + { BIT(A5XX_INT_CP_CCU_FLUSH_COLOR_TS), "CP_CCU_FLUSH_COLOR_TS" }, \ + { BIT(A5XX_INT_CP_CCU_RESOLVE_TS), "CP_CCU_RESOLVE_TS" }, \ + { BIT(A5XX_INT_CP_IB2), "CP_IB2_INT" }, \ + { BIT(A5XX_INT_CP_IB1), "CP_IB1_INT" }, \ + { BIT(A5XX_INT_CP_RB), "CP_RB_INT" }, \ + { BIT(A5XX_INT_CP_UNUSED_1), "CP_UNUSED_1" }, \ + { BIT(A5XX_INT_CP_RB_DONE_TS), "CP_RB_DONE_TS" }, \ + { BIT(A5XX_INT_CP_WT_DONE_TS), "CP_WT_DONE_TS" }, \ + { BIT(A5XX_INT_UNKNOWN_1), "UNKNOWN_1" }, \ + { BIT(A5XX_INT_CP_CACHE_FLUSH_TS), "CP_CACHE_FLUSH_TS" }, \ + { BIT(A5XX_INT_UNUSED_2), "UNUSED_2" }, \ + { BIT(A5XX_INT_RBBM_ATB_BUS_OVERFLOW), "RBBM_ATB_BUS_OVERFLOW" }, \ + { BIT(A5XX_INT_MISC_HANG_DETECT), "MISC_HANG_DETECT" }, \ + { BIT(A5XX_INT_UCHE_OOB_ACCESS), "UCHE_OOB_ACCESS" }, \ + { BIT(A5XX_INT_UCHE_TRAP_INTR), "UCHE_TRAP_INTR" }, \ + { BIT(A5XX_INT_DEBBUS_INTR_0), "DEBBUS_INTR_0" }, \ + { BIT(A5XX_INT_DEBBUS_INTR_1), "DEBBUS_INTR_1" }, \ + { BIT(A5XX_INT_GPMU_VOLTAGE_DROOP), "GPMU_VOLTAGE_DROOP" }, \ + { BIT(A5XX_INT_GPMU_FIRMWARE), "GPMU_FIRMWARE" }, \ + { BIT(A5XX_INT_ISDB_CPU_IRQ), "ISDB_CPU_IRQ" }, \ + { BIT(A5XX_INT_ISDB_UNDER_DEBUG), "ISDB_UNDER_DEBUG" } + +#define A5XX_CP_CTXRECORD_MAGIC_REF 0x27C4BAFCUL +/* Size of each CP preemption record */ +#define A5XX_CP_CTXRECORD_SIZE_IN_BYTES 0x10000 +/* Size of the preemption counter block (in bytes) */ +#define A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE (16 * 4) + +/** + * struct a5xx_cp_preemption_record - CP context record for + * preemption. + * @magic: (00) Value at this offset must be equal to + * A5XX_CP_CTXRECORD_MAGIC_REF. + * @info: (04) Type of record. Written non-zero (usually) by CP. + * we must set to zero for all ringbuffers. + * @data: (08) DATA field in SET_RENDER_MODE or checkpoint packets. + * Written by CP when switching out. Not used on switch-in. + * we must initialize to zero. + * @cntl: (12) RB_CNTL, saved and restored by CP. + * @rptr: (16) RB_RPTR, saved and restored by CP. + * @wptr: (20) RB_WPTR, saved and restored by CP. + * @rptr_addr: (24) RB_RPTR_ADDR_LO|HI saved and restored. + * rbase: (32) RB_BASE_LO|HI saved and restored. + * counter: (40) Pointer to preemption counter + */ +struct a5xx_cp_preemption_record { + uint32_t magic; + uint32_t info; + uint32_t data; + uint32_t cntl; + uint32_t rptr; + uint32_t wptr; + uint64_t rptr_addr; + uint64_t rbase; + uint64_t counter; +}; + +#define A5XX_CP_SMMU_INFO_MAGIC_REF 0x3618CDA3UL + +/** + * struct a5xx_cp_smmu_info - CP preemption SMMU info. + * @magic: (00) The value at this offset must be equal to + * A5XX_CP_SMMU_INFO_MAGIC_REF. + * @_pad4: (04) Reserved/padding + * @ttbr0: (08) Base address of the page table for the + * incoming context. + * @context_idr: (16) Context Identification Register value. + */ +struct a5xx_cp_smmu_info { + uint32_t magic; + uint32_t _pad4; + uint64_t ttbr0; + uint32_t asid; + uint32_t context_idr; +}; + +void a5xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); +unsigned int a5xx_num_registers(void); + +void a5xx_crashdump_init(struct adreno_device *adreno_dev); + +void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on); + +#define A5XX_CP_RB_CNTL_DEFAULT (((ilog2(4) << 8) & 0x1F00) | \ + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)) +/* GPMU interrupt multiplexor */ +#define FW_INTR_INFO (0) +#define LLM_ACK_ERR_INTR (1) +#define ISENS_TRIM_ERR_INTR (2) +#define ISENS_ERR_INTR (3) +#define ISENS_IDLE_ERR_INTR (4) +#define ISENS_PWR_ON_ERR_INTR (5) +#define WDOG_EXPITED (31) + +#define VALID_GPMU_IRQ (\ + BIT(FW_INTR_INFO) | \ + BIT(LLM_ACK_ERR_INTR) | \ + BIT(ISENS_TRIM_ERR_INTR) | \ + BIT(ISENS_ERR_INTR) | \ + BIT(ISENS_IDLE_ERR_INTR) | \ + BIT(ISENS_PWR_ON_ERR_INTR) | \ + BIT(WDOG_EXPITED)) + +/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL */ +#define STATE_OF_CHILD GENMASK(5, 4) +#define STATE_OF_CHILD_01 BIT(4) +#define STATE_OF_CHILD_11 (BIT(4) | BIT(5)) +#define IDLE_FULL_LM_SLEEP BIT(0) + +/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS */ +#define WAKEUP_ACK BIT(1) +#define IDLE_FULL_ACK BIT(0) + +/* A5XX_GPMU_GPMU_ISENSE_CTRL */ +#define ISENSE_CGC_EN_DISABLE BIT(0) + +/* A5XX_GPMU_TEMP_SENSOR_CONFIG */ +#define GPMU_BCL_ENABLED BIT(4) +#define GPMU_LLM_ENABLED BIT(9) +#define GPMU_ISENSE_STATUS GENMASK(3, 0) +#define GPMU_ISENSE_END_POINT_CAL_ERR BIT(0) + +#define AMP_CALIBRATION_RETRY_CNT 3 +#define AMP_CALIBRATION_TIMEOUT 6 + +/* A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK */ +#define VOLTAGE_INTR_EN BIT(0) + +/* A5XX_GPMU_GPMU_PWR_THRESHOLD */ +#define PWR_THRESHOLD_VALID 0x80000000 + +/* A5XX_GPMU_GPMU_SP_CLOCK_CONTROL */ +#define CNTL_IP_CLK_ENABLE BIT(0) +/* AGC */ +#define AGC_INIT_BASE A5XX_GPMU_DATA_RAM_BASE +#define AGC_INIT_MSG_MAGIC (AGC_INIT_BASE + 5) +#define AGC_MSG_BASE (AGC_INIT_BASE + 7) + +#define AGC_MSG_STATE (AGC_MSG_BASE + 0) +#define AGC_MSG_COMMAND (AGC_MSG_BASE + 1) +#define AGC_MSG_PAYLOAD_SIZE (AGC_MSG_BASE + 3) +#define AGC_MSG_PAYLOAD (AGC_MSG_BASE + 5) + +#define AGC_INIT_MSG_VALUE 0xBABEFACE +#define AGC_POWER_CONFIG_PRODUCTION_ID 1 + +#define AGC_LM_CONFIG (136/4) +#define AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE (1) + +#define AGC_LM_CONFIG_ENABLE_ERROR (3 << 4) +#define AGC_LM_CONFIG_ISENSE_ENABLE (1 << 4) + +#define AGC_THROTTLE_SEL_DCS (1 << 8) +#define AGC_THROTTLE_DISABLE (2 << 8) + + +#define AGC_LLM_ENABLED (1 << 16) +#define AGC_GPU_VERSION_MASK GENMASK(18, 17) +#define AGC_GPU_VERSION_SHIFT 17 +#define AGC_BCL_DISABLED (1 << 24) + + +#define AGC_LEVEL_CONFIG (140/4) + +#define LM_DCVS_LIMIT 1 +/* FW file tages */ +#define GPMU_FIRMWARE_ID 2 +#define GPMU_SEQUENCE_ID 3 +#define GPMU_INST_RAM_SIZE 0xFFF + +#define HEADER_MAJOR 1 +#define HEADER_MINOR 2 +#define HEADER_DATE 3 +#define HEADER_TIME 4 +#define HEADER_SEQUENCE 5 + +#define MAX_HEADER_SIZE 10 + +#define LM_SEQUENCE_ID 1 +#define MAX_SEQUENCE_ID 3 + +#define GPMU_ISENSE_SAVE (A5XX_GPMU_DATA_RAM_BASE + 200/4) +/* LM defaults */ +#define LM_DEFAULT_LIMIT 6000 +#define A530_DEFAULT_LEAKAGE 0x004E001A + +static inline bool lm_on(struct adreno_device *adreno_dev) +{ + return ADRENO_FEATURE(adreno_dev, ADRENO_LM) && + test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag); +} + +/* Preemption functions */ +void a5xx_preemption_trigger(struct adreno_device *adreno_dev); +void a5xx_preemption_schedule(struct adreno_device *adreno_dev); +void a5xx_preemption_start(struct adreno_device *adreno_dev); +int a5xx_preemption_init(struct adreno_device *adreno_dev); +int a5xx_preemption_yield_enable(unsigned int *cmds); + +unsigned int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, + unsigned int *cmds); +unsigned int a5xx_preemption_pre_ibsubmit( + struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + unsigned int *cmds, struct kgsl_context *context); + + +void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit); + +#endif diff --git a/drivers/gpu/msm/adreno_a5xx_packets.h b/drivers/gpu/msm/adreno_a5xx_packets.h new file mode 100644 index 0000000000000000000000000000000000000000..16fe81e17ce10f87abfb5374e9741845e0eff9e4 --- /dev/null +++ b/drivers/gpu/msm/adreno_a5xx_packets.h @@ -0,0 +1,1414 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2016, The Linux Foundation. All rights reserved. + */ + +enum adreno_critical_fixup_buffers { + CRITICAL_PACKET0 = 0, + CRITICAL_PACKET1, + CRITICAL_PACKET2, + CRITICAL_PACKET3, + CRITICAL_PACKET_MAX, +}; + +struct adreno_critical_fixup { + unsigned int lo_offset; + unsigned int hi_offset; + enum adreno_critical_fixup_buffers buffer; + uint64_t mem_offset; +}; + +static unsigned int _a5xx_critical_pkts[] = { + 0x400E0601, /* [0x0000] == TYPE4 == */ + 0x00000002, /* [0x0001] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x40E78A01, /* [0x0002] == TYPE4 == */ + 0x000FFFFF, /* [0x0003] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x48E78401, /* [0x0004] == TYPE4 == */ + 0x00000005, /* [0x0005] A5X_HLSQ_CNTL_0_CTX_0 (0xE784)*/ + 0x40E78501, /* [0x0006] == TYPE4 == */ + 0x00000009, /* [0x0007] A5X_HLSQ_CNTL_1_CTX_0 (0xE785)*/ + 0x48E78B85, /* [0x0008] == TYPE4 == */ + 0x00000001, /* [0x0009] A5X_HLSQ_VS_CONFIG_CTX_0 (0xE78B)*/ + 0x00002085, /* [0x000A] A5X_HLSQ_PS_CONFIG_CTX_0 (0xE78C)*/ + 0x00002084, /* [0x000B] A5X_HLSQ_HS_CONFIG_CTX_0 (0xE78D)*/ + 0x00002084, /* [0x000C] A5X_HLSQ_DS_CONFIG_CTX_0 (0xE78E)*/ + 0x00002084, /* [0x000D] A5X_HLSQ_GS_CONFIG_CTX_0 (0xE78F)*/ + 0x40E58485, /* [0x000E] == TYPE4 == */ + 0x00000001, /* [0x000F] A5X_SP_VS_CONFIG_CTX_0 (0xE584)*/ + 0x00002085, /* [0x0010] A5X_SP_PS_CONFIG_CTX_0 (0xE585)*/ + 0x00002084, /* [0x0011] A5X_SP_HS_CONFIG_CTX_0 (0xE586)*/ + 0x00002084, /* [0x0012] A5X_SP_DS_CONFIG_CTX_0 (0xE587)*/ + 0x00002084, /* [0x0013] A5X_SP_GS_CONFIG_CTX_0 (0xE588)*/ + 0x40E79101, /* [0x0014] == TYPE4 == */ + 0x00000004, /* [0x0015] A5X_HLSQ_VS_CNTL_CTX_0 (0xE791)*/ + 0x40E79201, /* [0x0016] == TYPE4 == */ + 0x00000002, /* [0x0017] A5X_HLSQ_PS_CNTL_CTX_0 (0xE792)*/ + 0x48E58001, /* [0x0018] == TYPE4 == */ + 0x00000010, /* [0x0019] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x70B00043, /* [0x001A] == TYPE7: LOAD_STATE (30) == */ + 0x00A00000, /* [0x001B] */ + 0x00000000, /* [0x001C] */ + 0x00000000, /* [0x001D] */ + 0x20020003, /* [0x001E] */ + 0x56D81803, /* [0x001F] */ + 0x00000003, /* [0x0020] */ + 0x20150000, /* [0x0021] */ + 0x00000000, /* [0x0022] */ + 0x00000200, /* [0x0023] */ + 0x00000000, /* [0x0024] */ + 0x201100F4, /* [0x0025] */ + 0x00000000, /* [0x0026] */ + 0x00000500, /* [0x0027] */ + 0x00000C21, /* [0x0028] */ + 0x20154004, /* [0x0029] */ + 0x00000C20, /* [0x002A] */ + 0x20154003, /* [0x002B] */ + 0x00000C23, /* [0x002C] */ + 0x20154008, /* [0x002D] */ + 0x00000C22, /* [0x002E] */ + 0x20156007, /* [0x002F] */ + 0x00000000, /* [0x0030] */ + 0x20554005, /* [0x0031] */ + 0x3F800000, /* [0x0032] */ + 0x20554006, /* [0x0033] */ + 0x00000000, /* [0x0034] */ + 0x03000000, /* [0x0035] */ + 0x20050000, /* [0x0036] */ + 0x46F00009, /* [0x0037] */ + 0x201F0000, /* [0x0038] */ + 0x4398000A, /* [0x0039] */ + 0x201F0009, /* [0x003A] */ + 0x43980809, /* [0x003B] */ + 0x20180009, /* [0x003C] */ + 0x46100809, /* [0x003D] */ + 0x00091014, /* [0x003E] */ + 0x62050009, /* [0x003F] */ + 0x00000000, /* [0x0040] */ + 0x00000500, /* [0x0041] */ + 0x04800006, /* [0x0042] */ + 0xC2C61300, /* [0x0043] */ + 0x0280000E, /* [0x0044] */ + 0xC2C61310, /* [0x0045] */ + 0x00000000, /* [0x0046] */ + 0x04800000, /* [0x0047] */ + 0x00000000, /* [0x0048] */ + 0x05000000, /* [0x0049] */ + 0x00000000, /* [0x004A] */ + 0x00000000, /* [0x004B] */ + 0x00000000, /* [0x004C] */ + 0x00000000, /* [0x004D] */ + 0x00000000, /* [0x004E] */ + 0x00000000, /* [0x004F] */ + 0x00000000, /* [0x0050] */ + 0x00000000, /* [0x0051] */ + 0x00000000, /* [0x0052] */ + 0x00000000, /* [0x0053] */ + 0x00000000, /* [0x0054] */ + 0x00000000, /* [0x0055] */ + 0x00000000, /* [0x0056] */ + 0x00000000, /* [0x0057] */ + 0x00000000, /* [0x0058] */ + 0x00000000, /* [0x0059] */ + 0x00000000, /* [0x005A] */ + 0x00000000, /* [0x005B] */ + 0x00000000, /* [0x005C] */ + 0x00000000, /* [0x005D] */ + 0x70B00023, /* [0x005E] == TYPE7: LOAD_STATE (30) == */ + 0x00700000, /* [0x005F] */ + 0x00000000, /* [0x0060] */ + 0x00000000, /* [0x0061] */ + 0x00000000, /* [0x0062] */ + 0x03000000, /* [0x0063] */ + 0x00000000, /* [0x0064] */ + 0x00000000, /* [0x0065] */ + 0x00000000, /* [0x0066] */ + 0x00000000, /* [0x0067] */ + 0x00000000, /* [0x0068] */ + 0x00000000, /* [0x0069] */ + 0x00000000, /* [0x006A] */ + 0x00000000, /* [0x006B] */ + 0x00000000, /* [0x006C] */ + 0x00000000, /* [0x006D] */ + 0x00000000, /* [0x006E] */ + 0x00000000, /* [0x006F] */ + 0x00000000, /* [0x0070] */ + 0x00000000, /* [0x0071] */ + 0x00000000, /* [0x0072] */ + 0x00000000, /* [0x0073] */ + 0x00000000, /* [0x0074] */ + 0x00000000, /* [0x0075] */ + 0x00000000, /* [0x0076] */ + 0x00000000, /* [0x0077] */ + 0x00000000, /* [0x0078] */ + 0x00000000, /* [0x0079] */ + 0x00000000, /* [0x007A] */ + 0x00000000, /* [0x007B] */ + 0x00000000, /* [0x007C] */ + 0x00000000, /* [0x007D] */ + 0x00000000, /* [0x007E] */ + 0x00000000, /* [0x007F] */ + 0x00000000, /* [0x0080] */ + 0x00000000, /* [0x0081] */ + 0x70B08003, /* [0x0082] == TYPE7: LOAD_STATE (30) == */ + 0x00620000, /* [0x0083] */ + 0x00000000, /* [0x0084] */ + 0x00000000, /* [0x0085] */ + 0x70B08003, /* [0x0086] == TYPE7: LOAD_STATE (30) == */ + 0x01220008, /* [0x0087] */ + 0x00000000, /* [0x0088] */ + 0x00000000, /* [0x0089] */ + 0x70B0000B, /* [0x008A] == TYPE7: LOAD_STATE (30) == */ + 0x01180000, /* [0x008B] */ + 0x00000001, /* [0x008C] */ + 0x00000000, /* [0x008D] */ + 0x00000000, /* [0x008E] */ + 0x00000000, /* [0x008F] */ + 0x00000000, /* [0x0090] */ + 0x00000000, /* [0x0091] */ + 0x00000000, /* [0x0092] */ + 0x00000000, /* [0x0093] */ + 0x00000000, /* [0x0094] */ + 0x01400000, /* [0x0095] */ + 0x70460001, /* [0x0096] == TYPE7: EVENT_WRITE (46) == */ + 0x00000019, /* [0x0097] */ + 0x70460004, /* [0x0098] == TYPE7: EVENT_WRITE (46) == */ + 0x0000001D, /* [0x0099] */ + 0x00000000, /* [0x009A] */ + 0x00000000, /* [0x009B] */ + 0x00000001, /* [0x009C] */ + 0x70460004, /* [0x009D] == TYPE7: EVENT_WRITE (46) == */ + 0x0000001C, /* [0x009E] */ + 0x00000000, /* [0x009F] */ + 0x00000000, /* [0x00A0] */ + 0x00000001, /* [0x00A1] */ + 0x480E9185, /* [0x00A2] == TYPE4 == */ + 0x00000000, /* [0x00A3] A5X_UCHE_CACHE_INVALIDATE_MIN_LO (0x0E91)*/ + 0x00000000, /* [0x00A4] A5X_UCHE_CACHE_INVALIDATE_MIN_HI (0x0E92)*/ + 0x00000000, /* [0x00A5] A5X_UCHE_CACHE_INVALIDATE_MAX_LO (0x0E93)*/ + 0x00000000, /* [0x00A6] A5X_UCHE_CACHE_INVALIDATE_MAX_HI (0x0E94)*/ + 0x00000012, /* [0x00A7] A5X_UCHE_CACHE_INVALIDATE (0x0E95)*/ + 0x70268000, /* [0x00A8] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x40E78A01, /* [0x00A9] == TYPE4 == */ + 0x000FFFFF, /* [0x00AA] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70D08003, /* [0x00AB] == TYPE7: PERFCOUNTER_ACTION (50) == */ + 0x00000000, /* [0x00AC] */ + 0x00000000, /* [0x00AD] */ + 0x00000000, /* [0x00AE] */ + 0x70D08003, /* [0x00AF] == TYPE7: PERFCOUNTER_ACTION (50) == */ + 0x00000010, /* [0x00B0] */ + 0x00000000, /* [0x00B1] */ + 0x00000000, /* [0x00B2] */ + 0x70268000, /* [0x00B3] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x48E38C01, /* [0x00B4] == TYPE4 == */ + 0xFFFFFFFF, /* [0x00B5] A5X_PC_RESTART_INDEX_CTX_0 (0xE38C)*/ + 0x40E38801, /* [0x00B6] == TYPE4 == */ + 0x00000012, /* [0x00B7] A5X_PC_RASTER_CNTL_CTX_0 (0xE388)*/ + 0x48E09102, /* [0x00B8] == TYPE4 == */ + 0xFFC00010, /* [0x00B9] A5X_GRAS_SU_POINT_MINMAX_CTX_0 (0xE091)*/ + 0x00000008, /* [0x00BA] A5X_GRAS_SU_POINT_SIZE_CTX_0 (0xE092)*/ + 0x40E09901, /* [0x00BB] == TYPE4 == */ + 0x00000000, /* [0x00BC] A5X_GRAS_SU_CONSERVATIVE_RAS_CNTL_CTX_0 + * (0xE099) + */ + 0x48E0A401, /* [0x00BD] == TYPE4 == */ + 0x00000000, /* [0x00BE] A5X_GRAS_SC_SCREEN_SCISSOR_CNTL_CTX_0 (0xE0A4)*/ + 0x48E58A01, /* [0x00BF] == TYPE4 == */ + 0x00000000, /* [0x00C0] A5X_SP_VS_CONFIG_MAX_CONST_CTX_0 (0xE58A)*/ + 0x40E58B01, /* [0x00C1] == TYPE4 == */ + 0x00000000, /* [0x00C2] A5X_SP_PS_CONFIG_MAX_CONST_CTX_0 (0xE58B)*/ + 0x480CC601, /* [0x00C3] == TYPE4 == */ + 0x00000044, /* [0x00C4] A5X_RB_MODE_CNTL (0x0CC6)*/ + 0x400CC401, /* [0x00C5] == TYPE4 == */ + 0x00100000, /* [0x00C6] A5X_RB_DBG_ECO_CNTL (0x0CC4)*/ + 0x400E4201, /* [0x00C7] == TYPE4 == */ + 0x00000000, /* [0x00C8] A5X_VFD_MODE_CNTL (0x0E42)*/ + 0x480D0201, /* [0x00C9] == TYPE4 == */ + 0x0000001F, /* [0x00CA] A5X_PC_MODE_CNTL (0x0D02)*/ + 0x480EC201, /* [0x00CB] == TYPE4 == */ + 0x0000001E, /* [0x00CC] A5X_SP_MODE_CNTL (0x0EC2)*/ + 0x400EC001, /* [0x00CD] == TYPE4 == */ + 0x40000800, /* [0x00CE] A5X_SP_DBG_ECO_CNTL (0x0EC0)*/ + 0x400F0201, /* [0x00CF] == TYPE4 == */ + 0x00000544, /* [0x00D0] A5X_TPL1_MODE_CNTL (0x0F02)*/ + 0x400E0002, /* [0x00D1] == TYPE4 == */ + 0x00000080, /* [0x00D2] A5X_HLSQ_TIMEOUT_THRESHOLD_0 (0x0E00)*/ + 0x00000000, /* [0x00D3] A5X_HLSQ_TIMEOUT_THRESHOLD_1 (0x0E01)*/ + 0x400E6001, /* [0x00D4] == TYPE4 == */ + 0x00000400, /* [0x00D5] A5X_VPC_DBG_ECO_CNTL (0x0E60)*/ + 0x400E0601, /* [0x00D6] == TYPE4 == */ + 0x00000001, /* [0x00D7] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x480E6201, /* [0x00D8] == TYPE4 == */ + 0x00000000, /* [0x00D9] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x70EC8005, /* [0x00DA] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000002, /* [0x00DB] */ + 0x00000000, /* [0x00DC] */ + 0x00000000, /* [0x00DD] */ + 0x00000008, /* [0x00DE] */ + 0x00000001, /* [0x00DF] */ + 0x40E14001, /* [0x00E0] == TYPE4 == */ + 0x00000204, /* [0x00E1] A5X_RB_CNTL_CTX_0 (0xE140)*/ + 0x709D0001, /* [0x00E2] == TYPE7: SKIP_IB2_ENABLE_GLOBAL (1D) == */ + 0x00000000, /* [0x00E3] */ + 0x48E0EA02, /* [0x00E4] == TYPE4 == */ + 0x00000000, /* [0x00E5] A5X_GRAS_SC_WINDOW_SCISSOR_TL_CTX_0 (0xE0EA)*/ + 0x001F0073, /* [0x00E6] A5X_GRAS_SC_WINDOW_SCISSOR_BR_CTX_0 (0xE0EB)*/ + 0x48E21102, /* [0x00E7] == TYPE4 == */ + 0x00000000, /* [0x00E8] A5X_RB_RESOLVE_CNTL_1_CTX_0 (0xE211)*/ + 0x00000000, /* [0x00E9] A5X_RB_RESOLVE_CNTL_2_CTX_0 (0xE212)*/ + 0x480BC283, /* [0x00EA] == TYPE4 == */ + 0x00000204, /* [0x00EB] UNKNOWN (0x0BC2)*/ + 0x00000000, /* [0x00EC] UNKNOWN (0x0BC3)*/ + 0x00000000, /* [0x00ED] UNKNOWN (0x0BC4)*/ + 0x400BC502, /* [0x00EE] == TYPE4 == */ + 0x00000000, /* [0x00EF] UNKNOWN (0x0BC5)*/ + 0x00000000, /* [0x00F0] UNKNOWN (0x0BC6)*/ + 0x480BD001, /* [0x00F1] == TYPE4 == */ + 0x01100000, /* [0x00F2] UNKNOWN (0x0BD0)*/ + 0x480BE002, /* [0x00F3] == TYPE4 == */ + 0x00000000, /* [0x00F4] UNKNOWN (0x0BE0)*/ + 0x00000000, /* [0x00F5] UNKNOWN (0x0BE1)*/ + 0x480C0001, /* [0x00F6] == TYPE4 == */ + 0x00000020, /* [0x00F7] A5X_VSC_PIPE_DATA_LENGTH_0 (0x0C00)*/ + 0x48E3B001, /* [0x00F8] == TYPE4 == */ + 0x00000003, /* [0x00F9] A5X_PC_POWER_CNTL_CTX_0 (0xE3B0)*/ + 0x48E4F001, /* [0x00FA] == TYPE4 == */ + 0x00000003, /* [0x00FB] A5X_VFD_POWER_CNTL_CTX_0 (0xE4F0)*/ + 0x480E6201, /* [0x00FC] == TYPE4 == */ + 0x00000001, /* [0x00FD] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x70460001, /* [0x00FE] == TYPE7: EVENT_WRITE (46) == */ + 0x0000002C, /* [0x00FF] */ + 0x40E1D001, /* [0x0100] == TYPE4 == */ + 0x00000000, /* [0x0101] A5X_RB_WINDOW_OFFSET_CTX_0 (0xE1D0)*/ + 0x70BF8003, /* [0x0102] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x0103] */ + 0x00000000, /* [0x0104] */ + 0x000000A0, /* [0x0105] */ + 0x70460001, /* [0x0106] == TYPE7: EVENT_WRITE (46) == */ + 0x0000002D, /* [0x0107] */ + 0x70460004, /* [0x0108] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x0109] */ + 0x00000000, /* [0x010A] */ + 0x00000000, /* [0x010B] */ + 0x00000000, /* [0x010C] */ + 0x70268000, /* [0x010D] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x480E6201, /* [0x010E] == TYPE4 == */ + 0x00000000, /* [0x010F] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x48E3B001, /* [0x0110] == TYPE4 == */ + 0x00000003, /* [0x0111] A5X_PC_POWER_CNTL_CTX_0 (0xE3B0)*/ + 0x48E4F001, /* [0x0112] == TYPE4 == */ + 0x00000003, /* [0x0113] A5X_VFD_POWER_CNTL_CTX_0 (0xE4F0)*/ + 0x70268000, /* [0x0114] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x400CC701, /* [0x0115] == TYPE4 == */ + 0x7C13C080, /* [0x0116] A5X_RB_CCU_CNTL (0x0CC7)*/ + 0x70EC8005, /* [0x0117] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000001, /* [0x0118] */ + 0x00000000, /* [0x0119] */ + 0x00000000, /* [0x011A] */ + 0x00000010, /* [0x011B] */ + 0x00000001, /* [0x011C] */ + 0x70EA0001, /* [0x011D] == TYPE7: PREEMPT_ENABLE_LOCAL (6A) == */ + 0x00000000, /* [0x011E] */ + 0x48E0EA02, /* [0x011F] == TYPE4 == */ + 0x00000000, /* [0x0120] A5X_GRAS_SC_WINDOW_SCISSOR_TL_CTX_0 (0xE0EA)*/ + 0x001F0073, /* [0x0121] A5X_GRAS_SC_WINDOW_SCISSOR_BR_CTX_0 (0xE0EB)*/ + 0x48E21102, /* [0x0122] == TYPE4 == */ + 0x00000000, /* [0x0123] A5X_RB_RESOLVE_CNTL_1_CTX_0 (0xE211)*/ + 0x00030007, /* [0x0124] A5X_RB_RESOLVE_CNTL_2_CTX_0 (0xE212)*/ + 0x70138000, /* [0x0125] == TYPE7: WAIT_FOR_ME (13) == */ + 0x70640001, /* [0x0126] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000000, /* [0x0127] */ + 0x702F8005, /* [0x0128] == TYPE7: SET_BIN_DATA (2F) == */ + 0x00010000, /* [0x0129] */ + 0x00000000, /* [0x012A] */ + 0x00000000, /* [0x012B] */ + 0x00000000, /* [0x012C] */ + 0x00000000, /* [0x012D] */ + 0x40E1D001, /* [0x012E] == TYPE4 == */ + 0x00000000, /* [0x012F] A5X_RB_WINDOW_OFFSET_CTX_0 (0xE1D0)*/ + 0x40E2A201, /* [0x0130] == TYPE4 == */ + 0x00000001, /* [0x0131] A5X_VPC_SO_OVERRIDE_CTX_0 (0xE2A2)*/ + 0x70640001, /* [0x0132] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000000, /* [0x0133] */ + 0x48E1B285, /* [0x0134] == TYPE4 == */ + 0x00000001, /* [0x0135] A5X_RB_DEPTH_BUFFER_INFO_CTX_0 (0xE1B2)*/ + 0x00004000, /* [0x0136] A5X_RB_DEPTH_BUFFER_BASE_LO_CTX_0 (0xE1B3)*/ + 0x00000000, /* [0x0137] A5X_RB_DEPTH_BUFFER_BASE_HI_CTX_0 (0xE1B4)*/ + 0x00000004, /* [0x0138] A5X_RB_DEPTH_BUFFER_PITCH_CTX_0 (0xE1B5)*/ + 0x000000C0, /* [0x0139] A5X_RB_DEPTH_BUFFER_ARRAY_PITCH_CTX_0 (0xE1B6)*/ + 0x48E09801, /* [0x013A] == TYPE4 == */ + 0x00000001, /* [0x013B] A5X_GRAS_SU_DEPTH_BUFFER_INFO_CTX_0 (0xE098)*/ + 0x40E24083, /* [0x013C] == TYPE4 == */ + 0x00000000, /* [0x013D] A5X_RB_DEPTH_FLAG_BUFFER_BASE_LO_CTX_0 + * (0xE240) + */ + 0x00000000, /* [0x013E] A5X_RB_DEPTH_FLAG_BUFFER_BASE_HI_CTX_0 + * (0xE241) + */ + 0x00000000, /* [0x013F] A5X_RB_DEPTH_FLAG_BUFFER_PITCH_CTX_0 (0xE242)*/ + 0x40E15285, /* [0x0140] == TYPE4 == */ + 0x00001230, /* [0x0141] A5X_RB_MRT_BUFFER_INFO_0_CTX_0 (0xE152)*/ + 0x00000008, /* [0x0142] A5X_RB_MRT_BUFFER_PITCH_0_CTX_0 (0xE153)*/ + 0x00000100, /* [0x0143] A5X_RB_MRT_BUFFER_ARRAY_PITCH_0_CTX_0 (0xE154)*/ + 0x00000000, /* [0x0144] A5X_RB_MRT_BUFFER_BASE_LO_0_CTX_0 (0xE155)*/ + 0x00000000, /* [0x0145] A5X_RB_MRT_BUFFER_BASE_HI_0_CTX_0 (0xE156)*/ + 0x40E40801, /* [0x0146] == TYPE4 == */ + 0x00000000, /* [0x0147] A5X_VFD_INDEX_OFFSET_CTX_0 (0xE408)*/ + 0x48E40901, /* [0x0148] == TYPE4 == */ + 0x00000000, /* [0x0149] A5X_VFD_INSTANCE_START_OFFSET_CTX_0 (0xE409)*/ + 0x70BF8003, /* [0x014A] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x014B] */ + 0x00000000, /* [0x014C] */ + 0x00000112, /* [0x014D] */ + 0x70230001, /* [0x014E] == TYPE7: SKIP_IB2_ENABLE_LOCAL (23) == */ + 0x00000000, /* [0x014F] */ + 0x70BF8003, /* [0x0150] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x0151] */ + 0x00000000, /* [0x0152] */ + 0x0000001B, /* [0x0153] */ + 0x70EC8005, /* [0x0154] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000001, /* [0x0155] */ + 0x00000000, /* [0x0156] */ + 0x00000000, /* [0x0157] */ + 0x00000000, /* [0x0158] */ + 0x00000001, /* [0x0159] */ + 0x70438003, /* [0x015A] == TYPE7: SET_DRAW_STATE (43) == */ + 0x00080059, /* [0x015B] */ + 0x00000000, /* [0x015C] */ + 0x00000000, /* [0x015D] */ + 0x70388003, /* [0x015E] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00000888, /* [0x015F] */ + 0x00000000, /* [0x0160] */ + 0x00000002, /* [0x0161] */ + 0x70A88003, /* [0x0162] == TYPE7: DRAW_INDIRECT (28) == */ + 0x00200884, /* [0x0163] */ + 0x00000000, /* [0x0164] */ + 0x00000000, /* [0x0165] */ + 0x70298006, /* [0x0166] == TYPE7: DRAW_INDX_INDIRECT (29) == */ + 0x00200404, /* [0x0167] */ + 0x00000000, /* [0x0168] */ + 0x00000000, /* [0x0169] */ + 0x00000006, /* [0x016A] */ + 0x00000000, /* [0x016B] */ + 0x00000000, /* [0x016C] */ + 0x40E2A783, /* [0x016D] == TYPE4 == */ + 0x00000000, /* [0x016E] A5X_VPC_SO_BUFFER_BASE_LO_0_CTX_0 (0xE2A7)*/ + 0x00000000, /* [0x016F] A5X_VPC_SO_BUFFER_BASE_HI_0_CTX_0 (0xE2A8)*/ + 0x00000004, /* [0x0170] A5X_VPC_SO_BUFFER_SIZE_0_CTX_0 (0xE2A9)*/ + 0x48E2AC02, /* [0x0171] == TYPE4 == */ + 0x00000000, /* [0x0172] A5X_VPC_SO_FLUSH_BASE_LO_0_CTX_0 (0xE2AC)*/ + 0x00000000, /* [0x0173] A5X_VPC_SO_FLUSH_BASE_HI_0_CTX_0 (0xE2AD)*/ + 0x70460001, /* [0x0174] == TYPE7: EVENT_WRITE (46) == */ + 0x00000011, /* [0x0175] */ + 0x48E10001, /* [0x0176] == TYPE4 == */ + 0x00000009, /* [0x0177] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x70460001, /* [0x0178] == TYPE7: EVENT_WRITE (46) == */ + 0x00000026, /* [0x0179] */ + 0x48E10001, /* [0x017A] == TYPE4 == */ + 0x00000008, /* [0x017B] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x40E10185, /* [0x017C] == TYPE4 == */ + 0x00000000, /* [0x017D] A5X_GRAS_LRZ_BUFFER_BASE_LO_CTX_0 (0xE101)*/ + 0x00000000, /* [0x017E] A5X_GRAS_LRZ_BUFFER_BASE_HI_CTX_0 (0xE102)*/ + 0x00000001, /* [0x017F] A5X_GRAS_LRZ_BUFFER_PITCH_CTX_0 (0xE103)*/ + 0x00000000, /* [0x0180] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO_CTX_0 + * (0xE104) + */ + 0x00000000, /* [0x0181] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI_CTX_0 + * (0xE105) + */ + 0x70460001, /* [0x0182] == TYPE7: EVENT_WRITE (46) == */ + 0x00000025, /* [0x0183] */ + 0x70460001, /* [0x0184] == TYPE7: EVENT_WRITE (46) == */ + 0x00000019, /* [0x0185] */ + 0x70460001, /* [0x0186] == TYPE7: EVENT_WRITE (46) == */ + 0x00000018, /* [0x0187] */ + 0x70EA0001, /* [0x0188] == TYPE7: PREEMPT_ENABLE_LOCAL (6A) == */ + 0x00000000, /* [0x0189] */ + 0x70EC0001, /* [0x018A] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000006, /* [0x018B] */ + 0x70438003, /* [0x018C] == TYPE7: SET_DRAW_STATE (43) == */ + 0x00080059, /* [0x018D] */ + 0x00000000, /* [0x018E] */ + 0x00000000, /* [0x018F] */ + 0x70DC0002, /* [0x0190] == TYPE7: CONTEXT_REG_BUNCH (5C) == */ + 0x0000E2A1, /* [0x0191] */ + 0x00008001, /* [0x0192] */ + 0x709D0001, /* [0x0193] == TYPE7: SKIP_IB2_ENABLE_GLOBAL (1D) == */ + 0x00000000, /* [0x0194] */ + 0x70138000, /* [0x0195] == TYPE7: WAIT_FOR_ME (13) == */ + 0x70640001, /* [0x0196] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000001, /* [0x0197] */ + 0x70380007, /* [0x0198] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200506, /* [0x0199] */ + 0x00000000, /* [0x019A] */ + 0x00000004, /* [0x019B] */ + 0x00000000, /* [0x019C] */ + 0x00000000, /* [0x019D] */ + 0x00000000, /* [0x019E] */ + 0x00000004, /* [0x019F] */ + 0x703D8005, /* [0x01A0] == TYPE7: MEM_WRITE (3D) == */ + 0x00000000, /* [0x01A1] */ + 0x00000000, /* [0x01A2] */ + 0x00000001, /* [0x01A3] */ + 0x00000001, /* [0x01A4] */ + 0x00000001, /* [0x01A5] */ + 0x70928000, /* [0x01A6] == TYPE7: WAIT_MEM_WRITES (12) == */ + 0x70BF8003, /* [0x01A7] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x01A8] */ + 0x00000000, /* [0x01A9] */ + 0x00000028, /* [0x01AA] */ + 0x70C48006, /* [0x01AB] == TYPE7: COND_EXEC (44) == */ + 0x00000000, /* [0x01AC] */ + 0x00000000, /* [0x01AD] */ + 0x00000000, /* [0x01AE] */ + 0x00000000, /* [0x01AF] */ + 0x00000001, /* [0x01B0] */ + 0x00000002, /* [0x01B1] */ + 0x70100001, /* [0x01B2] == TYPE7: NOP (10) == */ + 0x00000000, /* [0x01B3] */ + 0x70C28003, /* [0x01B4] == TYPE7: MEM_TO_REG (42) == */ + 0xC000E2AB, /* [0x01B5] */ + 0x00000000, /* [0x01B6] */ + 0x00000000, /* [0x01B7] */ + 0x70230001, /* [0x01B8] == TYPE7: SKIP_IB2_ENABLE_LOCAL (23) == */ + 0x00000000, /* [0x01B9] */ + 0x70E90001, /* [0x01BA] == TYPE7: PREEMPT_ENABLE_GLOBAL (69) == */ + 0x00000000, /* [0x01BB] */ + 0x70BC8006, /* [0x01BC] == TYPE7: WAIT_REG_MEM (3C) == */ + 0x00000010, /* [0x01BD] */ + 0x00000000, /* [0x01BE] */ + 0x00000000, /* [0x01BF] */ + 0x00000001, /* [0x01C0] */ + 0xFFFFFFFF, /* [0x01C1] */ + 0x00000001, /* [0x01C2] */ + 0x70738009, /* [0x01C3] == TYPE7: MEM_TO_MEM (73) == */ + 0x20000004, /* [0x01C4] */ + 0x00000000, /* [0x01C5] */ + 0x00000000, /* [0x01C6] */ + 0x00000000, /* [0x01C7] */ + 0x00000000, /* [0x01C8] */ + 0x00000000, /* [0x01C9] */ + 0x00000000, /* [0x01CA] */ + 0x00000000, /* [0x01CB] */ + 0x00000000, /* [0x01CC] */ + 0x70738009, /* [0x01CD] == TYPE7: MEM_TO_MEM (73) == */ + 0xE0000004, /* [0x01CE] */ + 0x00000000, /* [0x01CF] */ + 0x00000000, /* [0x01D0] */ + 0x00000000, /* [0x01D1] */ + 0x00000000, /* [0x01D2] */ + 0x00000000, /* [0x01D3] */ + 0x00000000, /* [0x01D4] */ + 0x00000000, /* [0x01D5] */ + 0x00000000, /* [0x01D6] */ + 0x70B50001, /* [0x01D7] == TYPE7: SET_SUBDRAW_SIZE (35) == */ + 0x00000001, /* [0x01D8] */ + 0x40E78A01, /* [0x01D9] == TYPE4 == */ + 0x000FFFFF, /* [0x01DA] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70268000, /* [0x01DB] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x400E0601, /* [0x01DC] == TYPE4 == */ + 0x00000001, /* [0x01DD] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x706E0004, /* [0x01DE] == TYPE7: COMPUTE_CHECKPOINT (6E) == */ + 0x00000000, /* [0x01DF] */ + 0x00000000, /* [0x01E0] */ + 0x00000018, /* [0x01E1] */ + 0x00000001, /* [0x01E2] */ + 0x40E14001, /* [0x01E3] == TYPE4 == */ + 0x00020000, /* [0x01E4] A5X_RB_CNTL_CTX_0 (0xE140)*/ + 0x40E78A01, /* [0x01E5] == TYPE4 == */ + 0x01F00000, /* [0x01E6] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70268000, /* [0x01E7] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x48E38C01, /* [0x01E8] == TYPE4 == */ + 0xFFFFFFFF, /* [0x01E9] A5X_PC_RESTART_INDEX_CTX_0 (0xE38C)*/ + 0x480D0201, /* [0x01EA] == TYPE4 == */ + 0x0000001F, /* [0x01EB] A5X_PC_MODE_CNTL (0x0D02)*/ + 0x480EC201, /* [0x01EC] == TYPE4 == */ + 0x0000001E, /* [0x01ED] A5X_SP_MODE_CNTL (0x0EC2)*/ + 0x48E58001, /* [0x01EE] == TYPE4 == */ + 0x00000000, /* [0x01EF] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x40E2A201, /* [0x01F0] == TYPE4 == */ + 0x00000001, /* [0x01F1] A5X_VPC_SO_OVERRIDE_CTX_0 (0xE2A2)*/ + 0x70640001, /* [0x01F2] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000001, /* [0x01F3] */ + 0x48E78401, /* [0x01F4] == TYPE4 == */ + 0x00000881, /* [0x01F5] A5X_HLSQ_CNTL_0_CTX_0 (0xE784)*/ + 0x40E5F001, /* [0x01F6] == TYPE4 == */ + 0x00000C06, /* [0x01F7] A5X_SP_CS_CNTL_0_CTX_0 (0xE5F0)*/ + 0x48E79001, /* [0x01F8] == TYPE4 == */ + 0x00000001, /* [0x01F9] A5X_HLSQ_CS_CONFIG_CTX_0 (0xE790)*/ + 0x48E79601, /* [0x01FA] == TYPE4 == */ + 0x00000005, /* [0x01FB] A5X_HLSQ_CS_CNTL_CTX_0 (0xE796)*/ + 0x48E58901, /* [0x01FC] == TYPE4 == */ + 0x00000001, /* [0x01FD] A5X_SP_CS_CONFIG_CTX_0 (0xE589)*/ + 0x40E7DC01, /* [0x01FE] == TYPE4 == */ + 0x00000030, /* [0x01FF] A5X_HLSQ_CONTEXT_SWITCH_CS_SW_3_CTX_0 (0xE7DC)*/ + 0x48E7DD01, /* [0x0200] == TYPE4 == */ + 0x00000002, /* [0x0201] A5X_HLSQ_CONTEXT_SWITCH_CS_SW_4_CTX_0 (0xE7DD)*/ + 0x40E7B001, /* [0x0202] == TYPE4 == */ + 0x00000003, /* [0x0203] A5X_HLSQ_CS_NDRANGE_0_CTX_0 (0xE7B0)*/ + 0x48E7B702, /* [0x0204] == TYPE4 == */ + 0x00FCC0CF, /* [0x0205] A5X_HLSQ_CS_CNTL_0_CTX_0 (0xE7B7)*/ + 0x00000000, /* [0x0206] A5X_HLSQ_CS_CNTL_1_CTX_0 (0xE7B8)*/ + 0x40E7B983, /* [0x0207] == TYPE4 == */ + 0x00000001, /* [0x0208] A5X_HLSQ_CS_KERNEL_GROUP_X_CTX_0 (0xE7B9)*/ + 0x00000001, /* [0x0209] A5X_HLSQ_CS_KERNEL_GROUP_Y_CTX_0 (0xE7BA)*/ + 0x00000001, /* [0x020A] A5X_HLSQ_CS_KERNEL_GROUP_Z_CTX_0 (0xE7BB)*/ + 0x70B08003, /* [0x020B] == TYPE7: LOAD_STATE (30) == */ + 0x00B60000, /* [0x020C] */ + 0x00000000, /* [0x020D] */ + 0x00000000, /* [0x020E] */ + 0x70B08003, /* [0x020F] == TYPE7: LOAD_STATE (30) == */ + 0x01360008, /* [0x0210] */ + 0x00000000, /* [0x0211] */ + 0x00000000, /* [0x0212] */ + 0x70B0000B, /* [0x0213] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0214] */ + 0x00000000, /* [0x0215] */ + 0x00000000, /* [0x0216] */ + 0x00000000, /* [0x0217] */ + 0x00000000, /* [0x0218] */ + 0x00000000, /* [0x0219] */ + 0x00000000, /* [0x021A] */ + 0x00000000, /* [0x021B] */ + 0x00000000, /* [0x021C] */ + 0x00000000, /* [0x021D] */ + 0x00000000, /* [0x021E] */ + 0x70B00007, /* [0x021F] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0220] */ + 0x00000001, /* [0x0221] */ + 0x00000000, /* [0x0222] */ + 0x00040000, /* [0x0223] */ + 0x00000000, /* [0x0224] */ + 0x00040000, /* [0x0225] */ + 0x00000000, /* [0x0226] */ + 0x70B00007, /* [0x0227] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0228] */ + 0x00000002, /* [0x0229] */ + 0x00000000, /* [0x022A] */ + 0x00000000, /* [0x022B] */ + 0x00000000, /* [0x022C] */ + 0x00000000, /* [0x022D] */ + 0x00000000, /* [0x022E] */ + 0x48E7B186, /* [0x022F] == TYPE4 == */ + 0x00000001, /* [0x0230] A5X_HLSQ_CS_NDRANGE_1_CTX_0 (0xE7B1)*/ + 0x00000000, /* [0x0231] A5X_HLSQ_CS_NDRANGE_2_CTX_0 (0xE7B2)*/ + 0x00000001, /* [0x0232] A5X_HLSQ_CS_NDRANGE_3_CTX_0 (0xE7B3)*/ + 0x00000000, /* [0x0233] A5X_HLSQ_CS_NDRANGE_4_CTX_0 (0xE7B4)*/ + 0x00000001, /* [0x0234] A5X_HLSQ_CS_NDRANGE_5_CTX_0 (0xE7B5)*/ + 0x00000000, /* [0x0235] A5X_HLSQ_CS_NDRANGE_6_CTX_0 (0xE7B6)*/ + 0x70B30004, /* [0x0236] == TYPE7: EXEC_CS (33) == */ + 0x00000000, /* [0x0237] */ + 0x00000001, /* [0x0238] */ + 0x00000001, /* [0x0239] */ + 0x00000001, /* [0x023A] */ + 0x480E6201, /* [0x023B] == TYPE4 == */ + 0x00000001, /* [0x023C] A5X_VPC_MODE_CNTL (0x0E62)*/ +}; + +/* + * These are fixups for the addresses _a5xx_critical_pkts[]. The first two + * numbers are the dword offsets into the buffer above. The third enum is a + * clue as to which buffer is being patched in and the final number is an offset + * in said buffer. + */ +static const struct adreno_critical_fixup critical_pkt_fixups[] = { + { 132, 133, CRITICAL_PACKET2, 0x0000 }, + { 136, 137, CRITICAL_PACKET2, 0x0001 }, + { 154, 155, CRITICAL_PACKET2, 0x0100 }, + { 159, 160, CRITICAL_PACKET2, 0x0104 }, + { 173, 174, CRITICAL_PACKET2, 0x0200 }, + { 177, 178, CRITICAL_PACKET2, 0x0300 }, + { 236, 237, CRITICAL_PACKET0, 0x0000 }, + { 244, 245, CRITICAL_PACKET0, 0x0040 }, + { 259, 260, CRITICAL_PACKET3, 0x0000 }, + { 266, 267, CRITICAL_PACKET2, 0x0108 }, + { 298, 299, CRITICAL_PACKET0, 0x0040 }, + { 300, 301, CRITICAL_PACKET2, 0x0080 }, + { 331, 332, CRITICAL_PACKET3, 0x02A0 }, + { 337, 338, CRITICAL_PACKET3, 0x0700 }, + { 348, 349, CRITICAL_PACKET3, 0x0920 }, + { 356, 357, CRITICAL_PACKET1, 0x008C }, + { 360, 361, CRITICAL_PACKET1, 0x0080 }, + { 363, 364, CRITICAL_PACKET1, 0x008C }, + { 366, 367, CRITICAL_PACKET0, 0x0100 }, + { 370, 371, CRITICAL_PACKET0, 0x0120 }, + { 381, 382, CRITICAL_PACKET1, 0x0480 }, + { 384, 385, CRITICAL_PACKET1, 0x0400 }, + { 398, 399, CRITICAL_PACKET3, 0x0920 }, + { 413, 414, CRITICAL_PACKET1, 0x0080 }, + { 417, 418, CRITICAL_PACKET1, 0x0300 }, + { 424, 425, CRITICAL_PACKET3, 0x0880 }, + { 428, 429, CRITICAL_PACKET1, 0x0300 }, + { 430, 431, CRITICAL_PACKET1, 0x0300 }, + { 438, 439, CRITICAL_PACKET1, 0x0300 }, + { 446, 447, CRITICAL_PACKET1, 0x0300 }, + { 453, 454, CRITICAL_PACKET1, 0x0320 }, + { 455, 456, CRITICAL_PACKET1, 0x0300 }, + { 457, 458, CRITICAL_PACKET1, 0x0304 }, + { 459, 460, CRITICAL_PACKET1, 0x0308 }, + { 463, 464, CRITICAL_PACKET1, 0x0320 }, + { 465, 466, CRITICAL_PACKET1, 0x0300 }, + { 467, 468, CRITICAL_PACKET1, 0x0304 }, + { 469, 470, CRITICAL_PACKET1, 0x0308 }, + { 525, 526, CRITICAL_PACKET1, 0x0160 }, + { 529, 530, CRITICAL_PACKET1, 0x0101 }, + { 535, 536, CRITICAL_PACKET1, 0x0140 }, + { 539, 540, CRITICAL_PACKET0, 0x0800 }, + { 555, 556, CRITICAL_PACKET1, 0x0140 }, + { 557, 558, CRITICAL_PACKET0, 0x0800 }, +}; + +static unsigned int _a5xx_critical_pkts_mem01[] = { + 0xBECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x3ECCCCCD, + 0xBECCCCCD, 0x00000000, 0xBECCCCCD, 0x3ECCCCCD, 0x3ECCCCCD, 0x00000000, + 0xBECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0xBECCCCCD, + 0xBECCCCCD, 0x00000000, 0xBECCCCCD, 0xBECCCCCD, 0x3ECCCCCD, 0x00000000, + 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, + 0x00000000, 0x00000000, 0x00040003, 0x00090005, 0x000B000A, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000003, 0x00000001, + 0x00000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000003, 0x00000001, 0x00000003, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3EF5C28F, 0x3ED1EB85, 0x3E6147AE, 0x3F800000, + 0x3E947AE1, 0x3E6147AE, 0x3D4CCCCD, 0x3F800000, 0x00000000, 0x20554002, + 0x3F800000, 0x20444003, 0x000000CF, 0x20044904, 0x00000000, 0x00000200, + 0x00050001, 0x42300001, 0x00000002, 0x20154005, 0x00000020, 0x20244006, + 0x00000000, 0x00000000, 0x10200001, 0x46500007, 0x20030004, 0x46D00004, + 0x00000000, 0x20554008, 0x00070001, 0x61830806, 0x00061020, 0x61808001, + 0x00040000, 0x42380800, 0x00010000, 0x42380800, 0x20040000, 0x46D80800, + 0x00000000, 0x20154007, 0x20020000, 0x46F80000, 0x00000007, 0x20154001, + 0x00000000, 0x00000200, 0x60030001, 0x43900004, 0x60030001, 0x43900001, + 0x00000000, 0x00000400, 0x00013600, 0xC6E20004, 0x40040003, 0x50180104, + 0x40060003, 0x40180803, 0x00000003, 0x20044006, 0x00000000, 0x00000500, + 0x00003609, 0xC7260201, 0x00000000, 0x03000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, +}; + +static unsigned int _a5xx_critical_pkts_mem02[] = { + 0x00000000, 0x03000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0000000C, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x8ACFE7F3, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +static unsigned int _a5xx_critical_pkts_mem03[] = { + 0x70438003, /* [0x0000] == TYPE7: SET_DRAW_STATE (43) == */ + 0x0008003A, /* [0x0001] */ + 0x00000000, /* [0x0002] */ + 0x00000000, /* [0x0003] */ + 0x70B08003, /* [0x0004] == TYPE7: LOAD_STATE (30) == */ + 0x00620000, /* [0x0005] */ + 0x00000000, /* [0x0006] */ + 0x00000000, /* [0x0007] */ + 0x40E29801, /* [0x0008] == TYPE4 == */ + 0x0000FFFF, /* [0x0009] A5X_VPC_GS_SIV_CNTL_CTX_0 (0xE298)*/ + 0x48E2A001, /* [0x000A] == TYPE4 == */ + 0x000000FF, /* [0x000B] A5X_VPC_PS_PRIMITIVEID_CNTL_CTX_0 (0xE2A0)*/ + 0x40E40185, /* [0x000C] == TYPE4 == */ + 0x00FCFCFC, /* [0x000D] A5X_VFD_CNTL_1_CTX_0 (0xE401)*/ + 0x0000FCFC, /* [0x000E] A5X_VFD_CNTL_2_CTX_0 (0xE402)*/ + 0x0000FCFC, /* [0x000F] A5X_VFD_CNTL_3_CTX_0 (0xE403)*/ + 0x000000FC, /* [0x0010] A5X_VFD_CNTL_4_CTX_0 (0xE404)*/ + 0x00000000, /* [0x0011] A5X_VFD_CNTL_5_CTX_0 (0xE405)*/ + 0x48E38F01, /* [0x0012] == TYPE4 == */ + 0x00000000, /* [0x0013] A5X_PC_HS_PARAM_CTX_0 (0xE38F)*/ + 0x48E58001, /* [0x0014] == TYPE4 == */ + 0x00000010, /* [0x0015] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x40E00001, /* [0x0016] == TYPE4 == */ + 0x00000080, /* [0x0017] A5X_GRAS_CL_CNTL_CTX_0 (0xE000)*/ + 0x40E09583, /* [0x0018] == TYPE4 == */ + 0x00000000, /* [0x0019] A5X_GRAS_SU_POLY_OFFSET_SCALE_CTX_0 (0xE095)*/ + 0x00000000, /* [0x001A] A5X_GRAS_SU_POLY_OFFSET_OFFSET_CTX_0 (0xE096)*/ + 0x00000000, /* [0x001B] A5X_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP_CTX_0 + * (0xE097) + */ + 0x40E09001, /* [0x001C] == TYPE4 == */ + 0x00000010, /* [0x001D] A5X_GRAS_SU_CNTL_CTX_0 (0xE090)*/ + 0x40E0AA02, /* [0x001E] == TYPE4 == */ + 0x00000000, /* [0x001F] A5X_GRAS_SC_SCREEN_SCISSOR_TL_0_CTX_0 (0xE0AA)*/ + 0x001F0073, /* [0x0020] A5X_GRAS_SC_SCREEN_SCISSOR_BR_0_CTX_0 (0xE0AB)*/ + 0x48E01086, /* [0x0021] == TYPE4 == */ + 0x42680000, /* [0x0022] A5X_GRAS_CL_VIEWPORT_XOFFSET_0_CTX_0 (0xE010)*/ + 0x42680000, /* [0x0023] A5X_GRAS_CL_VIEWPORT_XSCALE_0_CTX_0 (0xE011)*/ + 0x41800000, /* [0x0024] A5X_GRAS_CL_VIEWPORT_YOFFSET_0_CTX_0 (0xE012)*/ + 0xC1800000, /* [0x0025] A5X_GRAS_CL_VIEWPORT_YSCALE_0_CTX_0 (0xE013)*/ + 0x3EFFFEE0, /* [0x0026] A5X_GRAS_CL_VIEWPORT_ZOFFSET_0_CTX_0 (0xE014)*/ + 0x3EFFFEE0, /* [0x0027] A5X_GRAS_CL_VIEWPORT_ZSCALE_0_CTX_0 (0xE015)*/ + 0x40E0CA02, /* [0x0028] == TYPE4 == */ + 0x00000000, /* [0x0029] A5X_GRAS_SC_VIEWPORT_SCISSOR_TL_0_CTX_0 + * (0xE0CA) + */ + 0x001F0073, /* [0x002A] A5X_GRAS_SC_VIEWPORT_SCISSOR_BR_0_CTX_0 + * (0xE0CB) + */ + 0x40E00601, /* [0x002B] == TYPE4 == */ + 0x0007FDFF, /* [0x002C] A5X_GRAS_CL_GUARDBAND_CLIP_ADJ_CTX_0 (0xE006)*/ + 0x40E70401, /* [0x002D] == TYPE4 == */ + 0x00000000, /* [0x002E] A5X_TPL1_TP_RAS_MSAA_CNTL_CTX_0 (0xE704)*/ + 0x48E70501, /* [0x002F] == TYPE4 == */ + 0x00000004, /* [0x0030] A5X_TPL1_TP_DEST_MSAA_CNTL_CTX_0 (0xE705)*/ + 0x48E14201, /* [0x0031] == TYPE4 == */ + 0x00000000, /* [0x0032] A5X_RB_RAS_MSAA_CNTL_CTX_0 (0xE142)*/ + 0x40E14301, /* [0x0033] == TYPE4 == */ + 0x00000004, /* [0x0034] A5X_RB_DEST_MSAA_CNTL_CTX_0 (0xE143)*/ + 0x40E78683, /* [0x0035] == TYPE4 == */ + 0xFCFCFCFC, /* [0x0036] A5X_HLSQ_CNTL_2_CTX_0 (0xE786)*/ + 0xFCFCFCFC, /* [0x0037] A5X_HLSQ_CNTL_3_CTX_0 (0xE787)*/ + 0xFCFCFCFC, /* [0x0038] A5X_HLSQ_CNTL_4_CTX_0 (0xE788)*/ + 0x48E0A201, /* [0x0039] == TYPE4 == */ + 0x00000000, /* [0x003A] A5X_GRAS_SC_RAS_MSAA_CNTL_CTX_0 (0xE0A2)*/ + 0x40E0A301, /* [0x003B] == TYPE4 == */ + 0x00000004, /* [0x003C] A5X_GRAS_SC_DEST_MSAA_CNTL_CTX_0 (0xE0A3)*/ + 0x48E14101, /* [0x003D] == TYPE4 == */ + 0x0000C089, /* [0x003E] A5X_RB_RENDER_CNTL_CTX_0 (0xE141)*/ + 0x40E0A001, /* [0x003F] == TYPE4 == */ + 0x00000009, /* [0x0040] A5X_GRAS_SC_CNTL_CTX_0 (0xE0A0)*/ + 0x40E28001, /* [0x0041] == TYPE4 == */ + 0x00010004, /* [0x0042] A5X_VPC_CNTL_0_CTX_0 (0xE280)*/ + 0x40E38401, /* [0x0043] == TYPE4 == */ + 0x00000404, /* [0x0044] A5X_PC_PRIMITIVE_CNTL_CTX_0 (0xE384)*/ + 0x40E78501, /* [0x0045] == TYPE4 == */ + 0x0000003F, /* [0x0046] A5X_HLSQ_CNTL_1_CTX_0 (0xE785)*/ + 0x48E5D301, /* [0x0047] == TYPE4 == */ + 0x00000030, /* [0x0048] A5X_SP_PS_MRT_0_CTX_0 (0xE5D3)*/ + 0x48E5CB01, /* [0x0049] == TYPE4 == */ + 0x00000100, /* [0x004A] A5X_SP_PS_OUTPUT_0_CTX_0 (0xE5CB)*/ + 0x40E5CA01, /* [0x004B] == TYPE4 == */ + 0x001F9F81, /* [0x004C] A5X_SP_PS_OUTPUT_CNTL_CTX_0 (0xE5CA)*/ + 0x40E14601, /* [0x004D] == TYPE4 == */ + 0x00000001, /* [0x004E] A5X_RB_PS_OUTPUT_CNTL_CTX_0 (0xE146)*/ + 0x40E38E01, /* [0x004F] == TYPE4 == */ + 0x00000000, /* [0x0050] A5X_PC_GS_PARAM_CTX_0 (0xE38E)*/ + 0x40E28A01, /* [0x0051] == TYPE4 == */ + 0x00000000, /* [0x0052] A5X_VPC_VARYING_REPLACE_MODE_0_CTX_0 (0xE28A)*/ + 0x48E1A901, /* [0x0053] == TYPE4 == */ + 0xFFFF0100, /* [0x0054] A5X_RB_BLEND_CNTL_CTX_0 (0xE1A9)*/ + 0x40E5C901, /* [0x0055] == TYPE4 == */ + 0x00000100, /* [0x0056] A5X_SP_BLEND_CNTL_CTX_0 (0xE5C9)*/ + 0x40E76401, /* [0x0057] == TYPE4 == */ + 0x00000000, /* [0x0058] A5X_TPL1_TP_PS_ROTATION_CNTL_CTX_0 (0xE764)*/ + 0x48E09401, /* [0x0059] == TYPE4 == */ + 0x00000000, /* [0x005A] A5X_GRAS_SU_DEPTH_PLANE_CNTL_CTX_0 (0xE094)*/ + 0x40E1B001, /* [0x005B] == TYPE4 == */ + 0x00000000, /* [0x005C] A5X_RB_DEPTH_PLANE_CNTL_CTX_0 (0xE1B0)*/ + 0x48E1B101, /* [0x005D] == TYPE4 == */ + 0x00000000, /* [0x005E] A5X_RB_DEPTH_CNTL_CTX_0 (0xE1B1)*/ + 0x48E40001, /* [0x005F] == TYPE4 == */ + 0x00000001, /* [0x0060] A5X_VFD_CNTL_0_CTX_0 (0xE400)*/ + 0x48E40A04, /* [0x0061] == TYPE4 == */ + 0x00000000, /* [0x0062] A5X_VFD_VERTEX_BUFFER_BASE_LO_0_CTX_0 (0xE40A)*/ + 0x00000000, /* [0x0063] A5X_VFD_VERTEX_BUFFER_BASE_HI_0_CTX_0 (0xE40B)*/ + 0x00000078, /* [0x0064] A5X_VFD_VERTEX_BUFFER_SIZE_0_CTX_0 (0xE40C)*/ + 0x00000008, /* [0x0065] A5X_VFD_VERTEX_BUFFER_STRIDE_0_CTX_0 (0xE40D)*/ + 0x40E48A02, /* [0x0066] == TYPE4 == */ + 0xC6700000, /* [0x0067] A5X_VFD_FETCH_INSTR_0_CTX_0 (0xE48A)*/ + 0x00000001, /* [0x0068] A5X_VFD_FETCH_INSTR_STEP_RATE_0_CTX_0 (0xE48B)*/ + 0x48E4CA01, /* [0x0069] == TYPE4 == */ + 0x0000000F, /* [0x006A] A5X_VFD_DEST_CNTL_0_CTX_0 (0xE4CA)*/ + 0x48E10001, /* [0x006B] == TYPE4 == */ + 0x00000008, /* [0x006C] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x48E0A101, /* [0x006D] == TYPE4 == */ + 0x00000004, /* [0x006E] A5X_GRAS_SC_BIN_CNTL_CTX_0 (0xE0A1)*/ + 0x40E10185, /* [0x006F] == TYPE4 == */ + 0x00000000, /* [0x0070] A5X_GRAS_LRZ_BUFFER_BASE_LO_CTX_0 (0xE101)*/ + 0x00000000, /* [0x0071] A5X_GRAS_LRZ_BUFFER_BASE_HI_CTX_0 (0xE102)*/ + 0x00000001, /* [0x0072] A5X_GRAS_LRZ_BUFFER_PITCH_CTX_0 (0xE103)*/ + 0x00000000, /* [0x0073] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO_CTX_0 + * (0xE104) + */ + 0x00000000, /* [0x0074] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI_CTX_0 + * (0xE105) + */ + 0x70388003, /* [0x0075] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200884, /* [0x0076] */ + 0x00000001, /* [0x0077] */ + 0x00000003, /* [0x0078] */ + 0x70380007, /* [0x0079] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200404, /* [0x007A] */ + 0x00000001, /* [0x007B] */ + 0x00000003, /* [0x007C] */ + 0x00000000, /* [0x007D] */ + 0x00000000, /* [0x007E] */ + 0x00000000, /* [0x007F] */ + 0x00000006, /* [0x0080] */ + 0x70460004, /* [0x0081] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x0082] */ + 0x00000000, /* [0x0083] */ + 0x00000000, /* [0x0084] */ + 0x00000001, /* [0x0085] */ + 0x70268000, /* [0x0086] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x70A88003, /* [0x0087] == TYPE7: DRAW_INDIRECT (28) == */ + 0x00200884, /* [0x0088] */ + 0x00000000, /* [0x0089] */ + 0x00000000, /* [0x008A] */ + 0x70460004, /* [0x008B] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x008C] */ + 0x00000000, /* [0x008D] */ + 0x00000000, /* [0x008E] */ + 0x00000001, /* [0x008F] */ + 0x70268000, /* [0x0090] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x70298006, /* [0x0091] == TYPE7: DRAW_INDX_INDIRECT (29) == */ + 0x00200404, /* [0x0092] */ + 0x00000000, /* [0x0093] */ + 0x00000000, /* [0x0094] */ + 0x00000006, /* [0x0095] */ + 0x00000000, /* [0x0096] */ + 0x00000000, /* [0x0097] */ + 0x40E40801, /* [0x0098] == TYPE4 == */ + 0x0000000D, /* [0x0099] A5X_VFD_INDEX_OFFSET_CTX_0 (0xE408)*/ + 0x48E40901, /* [0x009A] == TYPE4 == */ + 0x00000000, /* [0x009B] A5X_VFD_INSTANCE_START_OFFSET_CTX_0 (0xE409)*/ + 0x70388003, /* [0x009C] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200884, /* [0x009D] */ + 0x00000001, /* [0x009E] */ + 0x00000003, /* [0x009F] */ + 0x00000000, /* [0x00A0] */ + 0x00000000, /* [0x00A1] */ + 0x00000000, /* [0x00A2] */ + 0x00000000, /* [0x00A3] */ + 0x00000000, /* [0x00A4] */ + 0x00000000, /* [0x00A5] */ + 0x00000000, /* [0x00A6] */ + 0x00000000, /* [0x00A7] */ + 0x48E78401, /* [0x00A8] */ + 0x00000881, /* [0x00A9] */ + 0x40E5C001, /* [0x00AA] */ + 0x0004001E, /* [0x00AB] */ + 0x70438003, /* [0x00AC] */ + 0x0000003A, /* [0x00AD] */ + 0x00000000, /* [0x00AE] */ + 0x00000000, /* [0x00AF] */ + 0x70B00023, /* [0x00B0] */ + 0x00600000, /* [0x00B1] */ + 0x00000000, /* [0x00B2] */ + 0x00000000, /* [0x00B3] */ + 0x00000000, /* [0x00B4] */ + 0x03000000, /* [0x00B5] */ + 0x00000000, /* [0x00B6] */ + 0x00000000, /* [0x00B7] */ + 0x00000000, /* [0x00B8] */ + 0x00000000, /* [0x00B9] */ + 0x00000000, /* [0x00BA] */ + 0x00000000, /* [0x00BB] */ + 0x00000000, /* [0x00BC] */ + 0x00000000, /* [0x00BD] */ + 0x00000000, /* [0x00BE] */ + 0x00000000, /* [0x00BF] */ + 0x00000000, /* [0x00C0] */ + 0x00000000, /* [0x00C1] */ + 0x00000000, /* [0x00C2] */ + 0x00000000, /* [0x00C3] */ + 0x00000000, /* [0x00C4] */ + 0x00000000, /* [0x00C5] */ + 0x00000000, /* [0x00C6] */ + 0x00000000, /* [0x00C7] */ + 0x00000000, /* [0x00C8] */ + 0x00000000, /* [0x00C9] */ + 0x00000000, /* [0x00CA] */ + 0x00000000, /* [0x00CB] */ + 0x00000000, /* [0x00CC] */ + 0x00000000, /* [0x00CD] */ + 0x00000000, /* [0x00CE] */ + 0x00000000, /* [0x00CF] */ + 0x00000000, /* [0x00D0] */ + 0x00000000, /* [0x00D1] */ + 0x00000000, /* [0x00D2] */ + 0x00000000, /* [0x00D3] */ + 0x40E09301, /* [0x00D4] */ + 0x00000000, /* [0x00D5] */ + 0x40E38D01, /* [0x00D6] */ + 0x00000000, /* [0x00D7] */ + 0x40E29801, /* [0x00D8] */ + 0x0000FFFF, /* [0x00D9] */ + 0x48E28201, /* [0x00DA] */ + 0xEAEAEAEA, /* [0x00DB] */ + 0x40E29404, /* [0x00DC] */ + 0xFFFFFFFF, /* [0x00DD] */ + 0xFFFFFFFF, /* [0x00DE] */ + 0xFFFFFFFF, /* [0x00DF] */ + 0xFFFFFFFF, /* [0x00E0] */ + 0x40E5DB01, /* [0x00E1] */ + 0x00000000, /* [0x00E2] */ + 0x48E14701, /* [0x00E3] */ + 0x0000000F, /* [0x00E4] */ + 0x70B00023, /* [0x00E5] */ + 0x00700000, /* [0x00E6] */ + 0x00000000, /* [0x00E7] */ + 0x00000000, /* [0x00E8] */ + 0x00003C00, /* [0x00E9] */ + 0x20400000, /* [0x00EA] */ + 0x00000000, /* [0x00EB] */ + 0x20400001, /* [0x00EC] */ + 0x00000000, /* [0x00ED] */ + 0x20400002, /* [0x00EE] */ + 0x00003C00, /* [0x00EF] */ + 0x20400003, /* [0x00F0] */ + 0x00000000, /* [0x00F1] */ + 0x03000000, /* [0x00F2] */ + 0x00000000, /* [0x00F3] */ + 0x00000000, /* [0x00F4] */ + 0x00000000, /* [0x00F5] */ + 0x00000000, /* [0x00F6] */ + 0x00000000, /* [0x00F7] */ + 0x00000000, /* [0x00F8] */ + 0x00000000, /* [0x00F9] */ + 0x00000000, /* [0x00FA] */ + 0x00000000, /* [0x00FB] */ + 0x00000000, /* [0x00FC] */ + 0x00000000, /* [0x00FD] */ + 0x00000000, /* [0x00FE] */ + 0x00000000, /* [0x00FF] */ + 0x00000000, /* [0x0100] */ + 0x00000000, /* [0x0101] */ + 0x00000000, /* [0x0102] */ + 0x00000000, /* [0x0103] */ + 0x00000000, /* [0x0104] */ + 0x00000000, /* [0x0105] */ + 0x00000000, /* [0x0106] */ + 0x00000000, /* [0x0107] */ + 0x00000000, /* [0x0108] */ + 0x48E2A001, /* [0x0109] */ + 0x000000FF, /* [0x010A] */ + 0x40E40185, /* [0x010B] */ + 0x00FCFCFC, /* [0x010C] */ + 0x0000FCFC, /* [0x010D] */ + 0x0000FCFC, /* [0x010E] */ + 0x000000FC, /* [0x010F] */ + 0x00000000, /* [0x0110] */ + 0x48E38F01, /* [0x0111] */ + 0x00000000, /* [0x0112] */ + 0x48E58001, /* [0x0113] */ + 0x00000010, /* [0x0114] */ + 0x40E1A801, /* [0x0115] */ + 0x00000E00, /* [0x0116] */ + 0x48E15001, /* [0x0117] */ + 0x000007E0, /* [0x0118] */ + 0x40E15101, /* [0x0119] */ + 0x00000000, /* [0x011A] */ + 0x40E00001, /* [0x011B] */ + 0x00000080, /* [0x011C] */ + 0x40E09583, /* [0x011D] */ + 0x00000000, /* [0x011E] */ + 0x00000000, /* [0x011F] */ + 0x00000000, /* [0x0120] */ + 0x40E09001, /* [0x0121] */ + 0x00000010, /* [0x0122] */ + 0x40E0AA02, /* [0x0123] */ + 0x00000000, /* [0x0124] */ + 0x001F0073, /* [0x0125] */ + 0x48E01086, /* [0x0126] */ + 0x42680000, /* [0x0127] */ + 0x42680000, /* [0x0128] */ + 0x41800000, /* [0x0129] */ + 0xC1800000, /* [0x012A] */ + 0x3EFFFEE0, /* [0x012B] */ + 0x3EFFFEE0, /* [0x012C] */ + 0x40E0CA02, /* [0x012D] */ + 0x00000000, /* [0x012E] */ + 0x001F0073, /* [0x012F] */ + 0x40E00601, /* [0x0130] */ + 0x0007FDFF, /* [0x0131] */ + 0x40E70401, /* [0x0132] */ + 0x00000000, /* [0x0133] */ + 0x48E70501, /* [0x0134] */ + 0x00000004, /* [0x0135] */ + 0x48E14201, /* [0x0136] */ + 0x00000000, /* [0x0137] */ + 0x40E14301, /* [0x0138] */ + 0x00000004, /* [0x0139] */ + 0x40E78683, /* [0x013A] */ + 0xFCFCFCFC, /* [0x013B] */ + 0xFCFCFCFC, /* [0x013C] */ + 0xFCFCFCFC, /* [0x013D] */ + 0x48E0A201, /* [0x013E] */ + 0x00000000, /* [0x013F] */ + 0x40E0A301, /* [0x0140] */ + 0x00000004, /* [0x0141] */ + 0x48E1B285, /* [0x0142] */ + 0x00000001, /* [0x0143] */ + 0x00004000, /* [0x0144] */ + 0x00000000, /* [0x0145] */ + 0x00000004, /* [0x0146] */ + 0x000000C0, /* [0x0147] */ + 0x48E09801, /* [0x0148] */ + 0x00000001, /* [0x0149] */ + 0x48E00401, /* [0x014A] */ + 0x00000000, /* [0x014B] */ + 0x480CDD02, /* [0x014C] */ + 0x00200074, /* [0x014D] */ + 0x00000000, /* [0x014E] */ + 0x40E15285, /* [0x014F] */ + 0x00000A30, /* [0x0150] */ + 0x00000008, /* [0x0151] */ + 0x00000100, /* [0x0152] */ + 0x00000000, /* [0x0153] */ + 0x00000000, /* [0x0154] */ + 0x48E14101, /* [0x0155] */ + 0x0000C008, /* [0x0156] */ + 0x40E0A001, /* [0x0157] */ + 0x00000008, /* [0x0158] */ + 0x40E28001, /* [0x0159] */ + 0x00010004, /* [0x015A] */ + 0x40E38401, /* [0x015B] */ + 0x00000404, /* [0x015C] */ + 0x40E78501, /* [0x015D] */ + 0x0000003F, /* [0x015E] */ + 0x48E5D301, /* [0x015F] */ + 0x00000030, /* [0x0160] */ + 0x48E5CB01, /* [0x0161] */ + 0x00000100, /* [0x0162] */ + 0x40E5CA01, /* [0x0163] */ + 0x001F9F81, /* [0x0164] */ + 0x40E14601, /* [0x0165] */ + 0x00000001, /* [0x0166] */ + 0x40E38E01, /* [0x0167] */ + 0x00000000, /* [0x0168] */ + 0x40E28A01, /* [0x0169] */ + 0x00000000, /* [0x016A] */ + 0x48E1A901, /* [0x016B] */ + 0xFFFF0100, /* [0x016C] */ + 0x40E5C901, /* [0x016D] */ + 0x00000100, /* [0x016E] */ + 0x40E76401, /* [0x016F] */ + 0x00000000, /* [0x0170] */ + 0x48E09401, /* [0x0171] */ + 0x00000000, /* [0x0172] */ + 0x40E1B001, /* [0x0173] */ + 0x00000000, /* [0x0174] */ + 0x48E1B101, /* [0x0175] */ + 0x00000006, /* [0x0176] */ + 0x48E40001, /* [0x0177] */ + 0x00000001, /* [0x0178] */ + 0x48E40A04, /* [0x0179] */ + 0x00000000, /* [0x017A] */ + 0x00000000, /* [0x017B] */ + 0x00000078, /* [0x017C] */ + 0x00000008, /* [0x017D] */ + 0x40E48A02, /* [0x017E] */ + 0xC6700000, /* [0x017F] */ + 0x00000001, /* [0x0180] */ + 0x48E4CA01, /* [0x0181] */ + 0x0000000F, /* [0x0182] */ + 0x48E10001, /* [0x0183] */ + 0x00000008, /* [0x0184] */ + 0x48E0A101, /* [0x0185] */ + 0x00000000, /* [0x0186] */ + 0x40E10185, /* [0x0187] */ + 0x00000000, /* [0x0188] */ + 0x00000000, /* [0x0189] */ + 0x00000001, /* [0x018A] */ + 0x00000000, /* [0x018B] */ + 0x00000000, /* [0x018C] */ + 0x70230001, /* [0x018D] */ + 0x00000000, /* [0x018E] */ + 0x70388003, /* [0x018F] */ + 0x00200984, /* [0x0190] */ + 0x00000001, /* [0x0191] */ + 0x00000003, /* [0x0192] */ + 0x70380007, /* [0x0193] */ + 0x00200504, /* [0x0194] */ + 0x00000001, /* [0x0195] */ + 0x00000003, /* [0x0196] */ + 0x00000000, /* [0x0197] */ + 0x00000000, /* [0x0198] */ + 0x00000000, /* [0x0199] */ + 0x00000006, /* [0x019A] */ + 0x70460004, /* [0x019B] */ + 0x00000004, /* [0x019C] */ + 0x00000000, /* [0x019D] */ + 0x00000000, /* [0x019E] */ + 0x00000000, /* [0x019F] */ + 0x70268000, /* [0x01A0] */ + 0x70A88003, /* [0x01A1] */ + 0x00200984, /* [0x01A2] */ + 0x00000000, /* [0x01A3] */ + 0x00000000, /* [0x01A4] */ + 0x70460004, /* [0x01A5] */ + 0x00000004, /* [0x01A6] */ + 0x00000000, /* [0x01A7] */ + 0x00000000, /* [0x01A8] */ + 0x00000001, /* [0x01A9] */ + 0x70268000, /* [0x01AA] */ + 0x70298006, /* [0x01AB] */ + 0x00200504, /* [0x01AC] */ + 0x00000000, /* [0x01AD] */ + 0x00000000, /* [0x01AE] */ + 0x00000006, /* [0x01AF] */ + 0x00000000, /* [0x01B0] */ + 0x00000000, /* [0x01B1] */ + 0x40E40801, /* [0x01B2] */ + 0x0000000D, /* [0x01B3] */ + 0x48E40901, /* [0x01B4] */ + 0x00000000, /* [0x01B5] */ + 0x70388003, /* [0x01B6] */ + 0x00200984, /* [0x01B7] */ + 0x00000001, /* [0x01B8] */ + 0x00000003, /* [0x01B9] */ + 0x00000000, /* [0x01BA] */ + 0x00000000, /* [0x01BB] */ + 0x00000000, /* [0x01BC] */ + 0x00000000, /* [0x01BD] */ + 0x00000000, /* [0x01BE] */ + 0x00000000, /* [0x01BF] */ + 0x70EA0001, /* [0x01C0] */ + 0x00000000, /* [0x01C1] */ + 0x40E78A01, /* [0x01C2] */ + 0x000FFFFF, /* [0x01C3] */ + 0x40E09001, /* [0x01C4] */ + 0x00000000, /* [0x01C5] */ + 0x40E00501, /* [0x01C6] */ + 0x00000000, /* [0x01C7] */ + 0x40E00001, /* [0x01C8] */ + 0x00000181, /* [0x01C9] */ + 0x48E10001, /* [0x01CA] */ + 0x00000000, /* [0x01CB] */ + 0x40E21385, /* [0x01CC] */ + 0x00000004, /* [0x01CD] */ + 0x00000000, /* [0x01CE] */ + 0x00000000, /* [0x01CF] */ + 0x00000001, /* [0x01D0] */ + 0x00000001, /* [0x01D1] */ + 0x40E21C01, /* [0x01D2] */ + 0x00000000, /* [0x01D3] */ + 0x40E21001, /* [0x01D4] */ + 0x00000000, /* [0x01D5] */ + 0x70460004, /* [0x01D6] */ + 0x0000001E, /* [0x01D7] */ + 0x00000000, /* [0x01D8] */ + 0x00000000, /* [0x01D9] */ + 0x00000001, /* [0x01DA] */ + 0x00000000, /* [0x01DB] */ + 0x00000000, /* [0x01DC] */ + 0x00000000, /* [0x01DD] */ + 0x00000000, /* [0x01DE] */ + 0x00000000, /* [0x01DF] */ + 0x40E78A01, /* [0x01E0] */ + 0x020FFFFF, /* [0x01E1] */ + 0x48E78B85, /* [0x01E2] */ + 0x00000001, /* [0x01E3] */ + 0x00003F05, /* [0x01E4] */ + 0x00003F04, /* [0x01E5] */ + 0x00003F04, /* [0x01E6] */ + 0x00003F04, /* [0x01E7] */ + 0x48E79001, /* [0x01E8] */ + 0x00000000, /* [0x01E9] */ + 0x40E79101, /* [0x01EA] */ + 0x00000002, /* [0x01EB] */ + 0x40E79201, /* [0x01EC] */ + 0x00000002, /* [0x01ED] */ + 0x40E58485, /* [0x01EE] */ + 0x00000001, /* [0x01EF] */ + 0x00003F05, /* [0x01F0] */ + 0x00003F04, /* [0x01F1] */ + 0x00003F04, /* [0x01F2] */ + 0x00003F04, /* [0x01F3] */ + 0x48E58901, /* [0x01F4] */ + 0x00000000, /* [0x01F5] */ + 0x48E7C302, /* [0x01F6] */ + 0x00000002, /* [0x01F7] */ + 0x00000001, /* [0x01F8] */ + 0x48E7D702, /* [0x01F9] */ + 0x00000002, /* [0x01FA] */ + 0x00000001, /* [0x01FB] */ + 0x40E7C802, /* [0x01FC] */ + 0x00000000, /* [0x01FD] */ + 0x00000000, /* [0x01FE] */ + 0x40E7CD02, /* [0x01FF] */ + 0x00000000, /* [0x0200] */ + 0x00000000, /* [0x0201] */ + 0x48E7D202, /* [0x0202] */ + 0x00000000, /* [0x0203] */ + 0x00000000, /* [0x0204] */ + 0x40E7DC02, /* [0x0205] */ + 0x00000000, /* [0x0206] */ + 0x00000000, /* [0x0207] */ + 0x48E38901, /* [0x0208] */ + 0x00000000, /* [0x0209] */ + 0x48E29A01, /* [0x020A] */ + 0x00FFFF00, /* [0x020B] */ + 0x48E00101, /* [0x020C] */ + 0x00000000, /* [0x020D] */ + 0x40E29D01, /* [0x020E] */ + 0x0000FF00, /* [0x020F] */ + 0x40E59001, /* [0x0210] */ + 0x00000406, /* [0x0211] */ + 0x48E59201, /* [0x0212] */ + 0x00000001, /* [0x0213] */ + 0x40E59301, /* [0x0214] */ + 0x00000F00, /* [0x0215] */ + 0x40E5A301, /* [0x0216] */ + 0x00000000, /* [0x0217] */ + 0x48E38501, /* [0x0218] */ + 0x00000000, /* [0x0219] */ + 0x00000000, /* [0x021A] */ + 0x00000000, /* [0x021B] */ + 0x00000000, /* [0x021C] */ + 0x00000000, /* [0x021D] */ + 0x00000000, /* [0x021E] */ + 0x00000000, /* [0x021F] */ + 0x48210001, /* [0x0220] */ + 0x86000000, /* [0x0221] */ + 0x40218001, /* [0x0222] */ + 0x86000000, /* [0x0223] */ + 0x40211089, /* [0x0224] */ + 0x00001331, /* [0x0225] */ + 0x00000000, /* [0x0226] */ + 0x00000000, /* [0x0227] */ + 0x00020001, /* [0x0228] */ + 0x00000000, /* [0x0229] */ + 0x00000000, /* [0x022A] */ + 0x00000000, /* [0x022B] */ + 0x00000000, /* [0x022C] */ + 0x00000000, /* [0x022D] */ + 0x48218201, /* [0x022E] */ + 0x00001331, /* [0x022F] */ + 0x40214383, /* [0x0230] */ + 0x00000000, /* [0x0231] */ + 0x00000000, /* [0x0232] */ + 0x00000001, /* [0x0233] */ + 0x40210789, /* [0x0234] */ + 0x00000021, /* [0x0235] */ + 0x00000000, /* [0x0236] */ + 0x00000000, /* [0x0237] */ + 0x00020001, /* [0x0238] */ + 0x00000000, /* [0x0239] */ + 0x00000000, /* [0x023A] */ + 0x00000000, /* [0x023B] */ + 0x00000000, /* [0x023C] */ + 0x00000000, /* [0x023D] */ + 0x48218101, /* [0x023E] */ + 0x00000021, /* [0x023F] */ + 0x48218401, /* [0x0240] */ + 0x00000001, /* [0x0241] */ + 0x702C8005, /* [0x0242] */ + 0x00000002, /* [0x0243] */ + 0x00000000, /* [0x0244] */ + 0x00010001, /* [0x0245] */ + 0x00000000, /* [0x0246] */ + 0x00010001, /* [0x0247] */ + 0x70B00023, /* [0x0248] */ + 0x00600000, /* [0x0249] */ + 0x00000000, /* [0x024A] */ + 0x00000000, /* [0x024B] */ + 0x00000000, /* [0x024C] */ + 0x03000000, /* [0x024D] */ + 0x00000000, /* [0x024E] */ + 0x00000000, /* [0x024F] */ + 0x00000000, /* [0x0250] */ + 0x00000000, /* [0x0251] */ + 0x00000000, /* [0x0252] */ + 0x00000000, /* [0x0253] */ + 0x00000000, /* [0x0254] */ + 0x00000000, /* [0x0255] */ + 0x00000000, /* [0x0256] */ + 0x00000000, /* [0x0257] */ + 0x00000000, /* [0x0258] */ + 0x00000000, /* [0x0259] */ + 0x00000000, /* [0x025A] */ + 0x00000000, /* [0x025B] */ + 0x00000000, /* [0x025C] */ + 0x00000000, /* [0x025D] */ + 0x00000000, /* [0x025E] */ + 0x00000000, /* [0x025F] */ + 0x00000000, /* [0x0260] */ + 0x00000000, /* [0x0261] */ + 0x00000000, /* [0x0262] */ + 0x00000000, /* [0x0263] */ + 0x00000000, /* [0x0264] */ + 0x00000000, /* [0x0265] */ + 0x00000000, /* [0x0266] */ + 0x00000000, /* [0x0267] */ + 0x00000000, /* [0x0268] */ + 0x00000000, /* [0x0269] */ + 0x00000000, /* [0x026A] */ + 0x00000000, /* [0x026B] */ + 0x40E09301, /* [0x026C] */ + 0x00000000, /* [0x026D] */ + 0x40E38D01, /* [0x026E] */ + 0x00000000, /* [0x026F] */ + 0x40E29801, /* [0x0270] */ + 0x0000FFFF, /* [0x0271] */ + 0x48E28201, /* [0x0272] */ + 0xEAEAEAEA, /* [0x0273] */ + 0x40E29404, /* [0x0274] */ + 0xFFFFFFFF, /* [0x0275] */ + 0xFFFFFFFF, /* [0x0276] */ + 0xFFFFFFFF, /* [0x0277] */ + 0xFFFFFFFF, /* [0x0278] */ + 0x40E5DB01, /* [0x0279] */ + 0x00000000, /* [0x027A] */ + 0x48E14701, /* [0x027B] */ + 0x0000000F, /* [0x027C] */ + 0x70B00023, /* [0x027D] */ + 0x00700000, /* [0x027E] */ + 0x00000000, /* [0x027F] */ + 0x00000000, /* [0x0280] */ + 0x00003C00, /* [0x0281] */ + 0x20400000, /* [0x0282] */ + 0x00000000, /* [0x0283] */ + 0x20400001, /* [0x0284] */ + 0x00000000, /* [0x0285] */ + 0x20400002, /* [0x0286] */ + 0x00003C00, /* [0x0287] */ + 0x20400003, /* [0x0288] */ + 0x00000000, /* [0x0289] */ + 0x03000000, /* [0x028A] */ + 0x00000000, /* [0x028B] */ + 0x00000000, /* [0x028C] */ + 0x00000000, /* [0x028D] */ + 0x00000000, /* [0x028E] */ + 0x00000000, /* [0x028F] */ + 0x00000000, /* [0x0290] */ + 0x00000000, /* [0x0291] */ + 0x00000000, /* [0x0292] */ + 0x00000000, /* [0x0293] */ + 0x00000000, /* [0x0294] */ + 0x00000000, /* [0x0295] */ + 0x00000000, /* [0x0296] */ + 0x00000000, /* [0x0297] */ + 0x00000000, /* [0x0298] */ + 0x00000000, /* [0x0299] */ + 0x00000000, /* [0x029A] */ + 0x00000000, /* [0x029B] */ + 0x00000000, /* [0x029C] */ + 0x00000000, /* [0x029D] */ + 0x00000000, /* [0x029E] */ + 0x00000000, /* [0x029F] */ + 0x00000000, /* [0x02A0] */ +}; + +/* Fixups for the IBs in _a5xx_critical_pkts_mem03 */ +static const struct adreno_critical_fixup critical_pkt_mem03_fixups[] = { + { 2, 3, CRITICAL_PACKET3, 0x0780 }, + { 6, 7, CRITICAL_PACKET2, 0x0000 }, + { 98, 99, CRITICAL_PACKET1, 0x0000 }, + { 112, 113, CRITICAL_PACKET1, 0x0480 }, + { 115, 116, CRITICAL_PACKET1, 0x0400 }, + { 126, 127, CRITICAL_PACKET1, 0x0080 }, + { 131, 132, CRITICAL_PACKET2, 0x0108 }, + { 137, 138, CRITICAL_PACKET1, 0x00A0 }, + { 141, 142, CRITICAL_PACKET2, 0x0108 }, + { 147, 148, CRITICAL_PACKET1, 0x0080 }, + { 150, 151, CRITICAL_PACKET1, 0x00C0 }, + { 174, 175, CRITICAL_PACKET3, 0x0780 }, + { 378, 379, CRITICAL_PACKET1, 0x0000 }, + { 392, 393, CRITICAL_PACKET1, 0x0480 }, + { 395, 396, CRITICAL_PACKET1, 0x0400 }, + { 408, 409, CRITICAL_PACKET1, 0x0080 }, + { 413, 414, CRITICAL_PACKET2, 0x0108 }, + { 419, 420, CRITICAL_PACKET1, 0x00A0 }, + { 423, 424, CRITICAL_PACKET2, 0x0108 }, + { 429, 430, CRITICAL_PACKET1, 0x0080 }, + { 432, 433, CRITICAL_PACKET1, 0x00C0 }, + { 462, 463, CRITICAL_PACKET0, 0x0700 }, + { 472, 473, CRITICAL_PACKET2, 0x0110 }, + { 550, 551, CRITICAL_PACKET1, 0x0500 }, + { 561, 562, CRITICAL_PACKET1, 0x0600 }, + { 566, 567, CRITICAL_PACKET1, 0x0700 }, +}; diff --git a/drivers/gpu/msm/adreno_a5xx_preempt.c b/drivers/gpu/msm/adreno_a5xx_preempt.c new file mode 100644 index 0000000000000000000000000000000000000000..93dbde1be60803c507f3a3a8159719a16ba1a704 --- /dev/null +++ b/drivers/gpu/msm/adreno_a5xx_preempt.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2017, The Linux Foundation. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "a5xx_reg.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" + +#define PREEMPT_RECORD(_field) \ + offsetof(struct a5xx_cp_preemption_record, _field) + +#define PREEMPT_SMMU_RECORD(_field) \ + offsetof(struct a5xx_cp_smmu_info, _field) + +static void _update_wptr(struct adreno_device *adreno_dev, bool reset_timer) +{ + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + unsigned int wptr; + unsigned long flags; + + spin_lock_irqsave(&rb->preempt_lock, flags); + + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); + + if (wptr != rb->wptr) { + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, + rb->wptr); + /* + * In case something got submitted while preemption was on + * going, reset the timer. + */ + reset_timer = true; + } + + if (reset_timer) + rb->dispatch_q.expires = jiffies + + msecs_to_jiffies(adreno_drawobj_timeout); + + spin_unlock_irqrestore(&rb->preempt_lock, flags); +} + +static inline bool adreno_move_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states old, enum adreno_preempt_states new) +{ + return (atomic_cmpxchg(&adreno_dev->preempt.state, old, new) == old); +} + +static void _a5xx_preemption_done(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status; + + /* + * In the very unlikely case that the power is off, do nothing - the + * state will be reset on power up and everybody will be happy + */ + + if (!kgsl_state_is_awake(device)) + return; + + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status); + + if (status != 0) { + dev_err(device->dev, + "Preemption not complete: status=%X cur=%d R/W=%X/%X next=%d R/W=%X/%X\n", + status, adreno_dev->cur_rb->id, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_dev->cur_rb->wptr, + adreno_dev->next_rb->id, + adreno_get_rptr(adreno_dev->next_rb), + adreno_dev->next_rb->wptr); + + /* Set a fault and restart */ + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + adreno_dispatcher_schedule(device); + + return; + } + + del_timer_sync(&adreno_dev->preempt.timer); + + trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb, 0); + + /* Clean up all the bits */ + adreno_dev->prev_rb = adreno_dev->cur_rb; + adreno_dev->cur_rb = adreno_dev->next_rb; + adreno_dev->next_rb = NULL; + + /* Update the wptr for the new command queue */ + _update_wptr(adreno_dev, true); + + /* Update the dispatcher timer for the new command queue */ + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + + /* Clear the preempt state */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); +} + +static void _a5xx_preemption_fault(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status; + + /* + * If the power is on check the preemption status one more time - if it + * was successful then just transition to the complete state + */ + if (kgsl_state_is_awake(device)) { + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status); + + if (status == 0) { + adreno_set_preempt_state(adreno_dev, + ADRENO_PREEMPT_COMPLETE); + + adreno_dispatcher_schedule(device); + return; + } + } + + dev_err(device->dev, + "Preemption timed out: cur=%d R/W=%X/%X, next=%d R/W=%X/%X\n", + adreno_dev->cur_rb->id, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_dev->cur_rb->wptr, + adreno_dev->next_rb->id, + adreno_get_rptr(adreno_dev->next_rb), + adreno_dev->next_rb->wptr); + + adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + adreno_dispatcher_schedule(device); +} + +static void _a5xx_preemption_worker(struct work_struct *work) +{ + struct adreno_preemption *preempt = container_of(work, + struct adreno_preemption, work); + struct adreno_device *adreno_dev = container_of(preempt, + struct adreno_device, preempt); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* Need to take the mutex to make sure that the power stays on */ + mutex_lock(&device->mutex); + + if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_FAULTED)) + _a5xx_preemption_fault(adreno_dev); + + mutex_unlock(&device->mutex); +} + +static void _a5xx_preemption_timer(struct timer_list *t) +{ + struct adreno_preemption *preempt = from_timer(preempt, t, timer); + struct adreno_device *adreno_dev = container_of(preempt, + struct adreno_device, preempt); + + /* We should only be here from a triggered state */ + if (!adreno_move_preempt_state(adreno_dev, + ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_FAULTED)) + return; + + /* Schedule the worker to take care of the details */ + queue_work(system_unbound_wq, &adreno_dev->preempt.work); +} + +/* Find the highest priority active ringbuffer */ +static struct adreno_ringbuffer *a5xx_next_ringbuffer( + struct adreno_device *adreno_dev) +{ + struct adreno_ringbuffer *rb; + unsigned long flags; + unsigned int i; + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + bool empty; + + spin_lock_irqsave(&rb->preempt_lock, flags); + empty = adreno_rb_empty(rb); + spin_unlock_irqrestore(&rb->preempt_lock, flags); + + if (!empty) + return rb; + } + + return NULL; +} + +void a5xx_preemption_trigger(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device); + struct adreno_ringbuffer *next; + uint64_t ttbr0; + unsigned int contextidr; + unsigned long flags; + + /* Put ourselves into a possible trigger state */ + if (!adreno_move_preempt_state(adreno_dev, + ADRENO_PREEMPT_NONE, ADRENO_PREEMPT_START)) + return; + + /* Get the next ringbuffer to preempt in */ + next = a5xx_next_ringbuffer(adreno_dev); + + /* + * Nothing to do if every ringbuffer is empty or if the current + * ringbuffer is the only active one + */ + if (next == NULL || next == adreno_dev->cur_rb) { + /* + * Update any critical things that might have been skipped while + * we were looking for a new ringbuffer + */ + + if (next != NULL) { + _update_wptr(adreno_dev, false); + + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + } + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + return; + } + + /* Turn off the dispatcher timer */ + del_timer(&adreno_dev->dispatcher.timer); + + /* + * This is the most critical section - we need to take care not to race + * until we have programmed the CP for the switch + */ + + spin_lock_irqsave(&next->preempt_lock, flags); + + /* + * Get the pagetable from the pagetable info. + * The pagetable_desc is allocated and mapped at probe time, and + * preemption_desc at init time, so no need to check if + * sharedmem accesses to these memdescs succeed. + */ + kgsl_sharedmem_readq(&next->pagetable_desc, &ttbr0, + PT_INFO_OFFSET(ttbr0)); + kgsl_sharedmem_readl(&next->pagetable_desc, &contextidr, + PT_INFO_OFFSET(contextidr)); + + kgsl_sharedmem_writel(device, &next->preemption_desc, + PREEMPT_RECORD(wptr), next->wptr); + + spin_unlock_irqrestore(&next->preempt_lock, flags); + + /* And write it to the smmu info */ + kgsl_sharedmem_writeq(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(ttbr0), ttbr0); + kgsl_sharedmem_writel(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(context_idr), contextidr); + + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, + lower_32_bits(next->preemption_desc.gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI, + upper_32_bits(next->preemption_desc.gpuaddr)); + + adreno_dev->next_rb = next; + + /* Start the timer to detect a stuck preemption */ + mod_timer(&adreno_dev->preempt.timer, + jiffies + msecs_to_jiffies(ADRENO_PREEMPT_TIMEOUT)); + + trace_adreno_preempt_trigger(adreno_dev->cur_rb, adreno_dev->next_rb, + 1); + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_TRIGGERED); + + /* Trigger the preemption */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); +} + +void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit) +{ + unsigned int status; + + if (!adreno_move_preempt_state(adreno_dev, + ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_PENDING)) + return; + + adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status); + + if (status != 0) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "preempt interrupt with non-zero status: %X\n", + status); + + /* + * Under the assumption that this is a race between the + * interrupt and the register, schedule the worker to clean up. + * If the status still hasn't resolved itself by the time we get + * there then we have to assume something bad happened + */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE); + adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev)); + return; + } + + del_timer(&adreno_dev->preempt.timer); + + trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb, 0); + + adreno_dev->prev_rb = adreno_dev->cur_rb; + adreno_dev->cur_rb = adreno_dev->next_rb; + adreno_dev->next_rb = NULL; + + /* Update the wptr if it changed while preemption was ongoing */ + _update_wptr(adreno_dev, true); + + /* Update the dispatcher timer for the new command queue */ + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + a5xx_preemption_trigger(adreno_dev); +} + +void a5xx_preemption_schedule(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_preemption_enabled(adreno_dev)) + return; + + mutex_lock(&device->mutex); + + if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE)) + _a5xx_preemption_done(adreno_dev); + + a5xx_preemption_trigger(adreno_dev); + + mutex_unlock(&device->mutex); +} + +unsigned int a5xx_preemption_pre_ibsubmit( + struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + unsigned int *cmds, struct kgsl_context *context) +{ + unsigned int *cmds_orig = cmds; + uint64_t gpuaddr = rb->preemption_desc.gpuaddr; + unsigned int preempt_style = 0; + + if (context) { + /* + * Preemption from secure to unsecure needs Zap shader to be + * run to clear all secure content. CP does not know during + * preemption if it is switching between secure and unsecure + * contexts so restrict Secure contexts to be preempted at + * ringbuffer level. + */ + if (context->flags & KGSL_CONTEXT_SECURE) + preempt_style = KGSL_CONTEXT_PREEMPT_STYLE_RINGBUFFER; + else + preempt_style = ADRENO_PREEMPT_STYLE(context->flags); + } + + /* + * CP_PREEMPT_ENABLE_GLOBAL(global preemption) can only be set by KMD + * in ringbuffer. + * 1) set global preemption to 0x0 to disable global preemption. + * Only RB level preemption is allowed in this mode + * 2) Set global preemption to defer(0x2) for finegrain preemption. + * when global preemption is set to defer(0x2), + * CP_PREEMPT_ENABLE_LOCAL(local preemption) determines the + * preemption point. Local preemption + * can be enabled by both UMD(within IB) and KMD. + */ + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1); + *cmds++ = ((preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN) + ? 2 : 0); + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + /* + * CP during context switch will save context switch info to + * a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR + */ + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1); + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1); + *cmds++ = upper_32_bits(gpuaddr); + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + /* + * Enable local preemption for finegrain preemption in case of + * a misbehaving IB + */ + if (preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN) { + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 1; + } else { + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 0; + } + + /* Enable CP_CONTEXT_SWITCH_YIELD packets in the IB2s */ + *cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1); + *cmds++ = 2; + + return (unsigned int) (cmds - cmds_orig); +} + +int a5xx_preemption_yield_enable(unsigned int *cmds) +{ + /* + * SRM -- set render mode (ex binning, direct render etc) + * SRM is set by UMD usually at start of IB to tell CP the type of + * preemption. + * KMD needs to set SRM to NULL to indicate CP that rendering is + * done by IB. + */ + *cmds++ = cp_type7_packet(CP_SET_RENDER_MODE, 5); + *cmds++ = 0; + *cmds++ = 0; + *cmds++ = 0; + *cmds++ = 0; + *cmds++ = 0; + + *cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1); + *cmds++ = 1; + + return 8; +} + +unsigned int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, + unsigned int *cmds) +{ + int dwords = 0; + + cmds[dwords++] = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + /* Write NULL to the address to skip the data write */ + dwords += cp_gpuaddr(adreno_dev, &cmds[dwords], 0x0); + cmds[dwords++] = 1; + /* generate interrupt on preemption completion */ + cmds[dwords++] = 1; + + return dwords; +} + +void a5xx_preemption_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device); + struct adreno_ringbuffer *rb; + unsigned int i; + + if (!adreno_is_preemption_enabled(adreno_dev)) + return; + + /* Force the state to be clear */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + /* smmu_info is allocated and mapped in a5xx_preemption_iommu_init */ + kgsl_sharedmem_writel(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(magic), A5XX_CP_SMMU_INFO_MAGIC_REF); + kgsl_sharedmem_writeq(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(ttbr0), MMU_DEFAULT_TTBR0(device)); + + /* The CP doesn't use the asid record, so poison it */ + kgsl_sharedmem_writel(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(asid), 0xDECAFBAD); + kgsl_sharedmem_writel(device, &iommu->smmu_info, + PREEMPT_SMMU_RECORD(context_idr), + MMU_DEFAULT_CONTEXTIDR(device)); + + adreno_writereg64(adreno_dev, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + iommu->smmu_info.gpuaddr); + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + /* + * preemption_desc is allocated and mapped at init time, + * so no need to check sharedmem_writel return value + */ + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(rptr), 0); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(wptr), 0); + + adreno_ringbuffer_set_pagetable(rb, + device->mmu.defaultpagetable); + } + +} + +static int a5xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, uint64_t counteraddr) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = kgsl_allocate_global(device, &rb->preemption_desc, + A5XX_CP_CTXRECORD_SIZE_IN_BYTES, 0, KGSL_MEMDESC_PRIVILEGED, + "preemption_desc"); + if (ret) + return ret; + + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(magic), A5XX_CP_CTXRECORD_MAGIC_REF); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(info), 0); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(data), 0); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(cntl), A5XX_CP_RB_CNTL_DEFAULT); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(rptr), 0); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(wptr), 0); + kgsl_sharedmem_writeq(device, &rb->preemption_desc, + PREEMPT_RECORD(rptr_addr), SCRATCH_RPTR_GPU_ADDR(device, + rb->id)); + kgsl_sharedmem_writeq(device, &rb->preemption_desc, + PREEMPT_RECORD(rbase), rb->buffer_desc.gpuaddr); + kgsl_sharedmem_writeq(device, &rb->preemption_desc, + PREEMPT_RECORD(counter), counteraddr); + + return 0; +} + +#ifdef CONFIG_QCOM_KGSL_IOMMU +static int a5xx_preemption_iommu_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device); + + /* Allocate mem for storing preemption smmu record */ + return kgsl_allocate_global(device, &iommu->smmu_info, PAGE_SIZE, + KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_PRIVILEGED, + "smmu_info"); +} + +static void a5xx_preemption_iommu_close(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device); + + kgsl_free_global(device, &iommu->smmu_info); +} + +#else +static int a5xx_preemption_iommu_init(struct adreno_device *adreno_dev) +{ + return -ENODEV; +} + +static void a5xx_preemption_iommu_close(struct adreno_device *adreno_dev) +{ +} +#endif + +static void a5xx_preemption_close(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_preemption *preempt = &adreno_dev->preempt; + struct adreno_ringbuffer *rb; + unsigned int i; + + del_timer(&preempt->timer); + kgsl_free_global(device, &preempt->counters); + a5xx_preemption_iommu_close(adreno_dev); + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + kgsl_free_global(device, &rb->preemption_desc); + } +} + +int a5xx_preemption_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_preemption *preempt = &adreno_dev->preempt; + struct adreno_ringbuffer *rb; + int ret; + unsigned int i; + uint64_t addr; + + /* We are dependent on IOMMU to make preemption go on the CP side */ + if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_IOMMU) + return -ENODEV; + + INIT_WORK(&preempt->work, _a5xx_preemption_worker); + + timer_setup(&preempt->timer, _a5xx_preemption_timer, 0); + + /* Allocate mem for storing preemption counters */ + ret = kgsl_allocate_global(device, &preempt->counters, + adreno_dev->num_ringbuffers * + A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE, 0, 0, + "preemption_counters"); + if (ret) + goto err; + + addr = preempt->counters.gpuaddr; + + /* Allocate mem for storing preemption switch record */ + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + ret = a5xx_preemption_ringbuffer_init(adreno_dev, rb, addr); + if (ret) + goto err; + + addr += A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE; + } + + ret = a5xx_preemption_iommu_init(adreno_dev); + +err: + if (ret) + a5xx_preemption_close(device); + + return ret; +} diff --git a/drivers/gpu/msm/adreno_a5xx_snapshot.c b/drivers/gpu/msm/adreno_a5xx_snapshot.c new file mode 100644 index 0000000000000000000000000000000000000000..fdd0523342e7c967db239eaad8b6166014bef01b --- /dev/null +++ b/drivers/gpu/msm/adreno_a5xx_snapshot.c @@ -0,0 +1,1134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include "kgsl.h" +#include "adreno.h" +#include "kgsl_snapshot.h" +#include "adreno_snapshot.h" +#include "a5xx_reg.h" +#include "adreno_a5xx.h" + +enum a5xx_rbbm_debbus_id { + A5XX_RBBM_DBGBUS_CP = 0x1, + A5XX_RBBM_DBGBUS_RBBM = 0x2, + A5XX_RBBM_DBGBUS_VBIF = 0x3, + A5XX_RBBM_DBGBUS_HLSQ = 0x4, + A5XX_RBBM_DBGBUS_UCHE = 0x5, + A5XX_RBBM_DBGBUS_DPM = 0x6, + A5XX_RBBM_DBGBUS_TESS = 0x7, + A5XX_RBBM_DBGBUS_PC = 0x8, + A5XX_RBBM_DBGBUS_VFDP = 0x9, + A5XX_RBBM_DBGBUS_VPC = 0xa, + A5XX_RBBM_DBGBUS_TSE = 0xb, + A5XX_RBBM_DBGBUS_RAS = 0xc, + A5XX_RBBM_DBGBUS_VSC = 0xd, + A5XX_RBBM_DBGBUS_COM = 0xe, + A5XX_RBBM_DBGBUS_DCOM = 0xf, + A5XX_RBBM_DBGBUS_LRZ = 0x10, + A5XX_RBBM_DBGBUS_A2D_DSP = 0x11, + A5XX_RBBM_DBGBUS_CCUFCHE = 0x12, + A5XX_RBBM_DBGBUS_GPMU = 0x13, + A5XX_RBBM_DBGBUS_RBP = 0x14, + A5XX_RBBM_DBGBUS_HM = 0x15, + A5XX_RBBM_DBGBUS_RBBM_CFG = 0x16, + A5XX_RBBM_DBGBUS_VBIF_CX = 0x17, + A5XX_RBBM_DBGBUS_GPC = 0x1d, + A5XX_RBBM_DBGBUS_LARC = 0x1e, + A5XX_RBBM_DBGBUS_HLSQ_SPTP = 0x1f, + A5XX_RBBM_DBGBUS_RB_0 = 0x20, + A5XX_RBBM_DBGBUS_RB_1 = 0x21, + A5XX_RBBM_DBGBUS_RB_2 = 0x22, + A5XX_RBBM_DBGBUS_RB_3 = 0x23, + A5XX_RBBM_DBGBUS_CCU_0 = 0x28, + A5XX_RBBM_DBGBUS_CCU_1 = 0x29, + A5XX_RBBM_DBGBUS_CCU_2 = 0x2a, + A5XX_RBBM_DBGBUS_CCU_3 = 0x2b, + A5XX_RBBM_DBGBUS_A2D_RAS_0 = 0x30, + A5XX_RBBM_DBGBUS_A2D_RAS_1 = 0x31, + A5XX_RBBM_DBGBUS_A2D_RAS_2 = 0x32, + A5XX_RBBM_DBGBUS_A2D_RAS_3 = 0x33, + A5XX_RBBM_DBGBUS_VFD_0 = 0x38, + A5XX_RBBM_DBGBUS_VFD_1 = 0x39, + A5XX_RBBM_DBGBUS_VFD_2 = 0x3a, + A5XX_RBBM_DBGBUS_VFD_3 = 0x3b, + A5XX_RBBM_DBGBUS_SP_0 = 0x40, + A5XX_RBBM_DBGBUS_SP_1 = 0x41, + A5XX_RBBM_DBGBUS_SP_2 = 0x42, + A5XX_RBBM_DBGBUS_SP_3 = 0x43, + A5XX_RBBM_DBGBUS_TPL1_0 = 0x48, + A5XX_RBBM_DBGBUS_TPL1_1 = 0x49, + A5XX_RBBM_DBGBUS_TPL1_2 = 0x4a, + A5XX_RBBM_DBGBUS_TPL1_3 = 0x4b +}; + +static const struct adreno_debugbus_block a5xx_debugbus_blocks[] = { + { A5XX_RBBM_DBGBUS_CP, 0x100, }, + { A5XX_RBBM_DBGBUS_RBBM, 0x100, }, + { A5XX_RBBM_DBGBUS_VBIF, 0x100, }, + { A5XX_RBBM_DBGBUS_HLSQ, 0x100, }, + { A5XX_RBBM_DBGBUS_UCHE, 0x100, }, + { A5XX_RBBM_DBGBUS_DPM, 0x100, }, + { A5XX_RBBM_DBGBUS_TESS, 0x100, }, + { A5XX_RBBM_DBGBUS_PC, 0x100, }, + { A5XX_RBBM_DBGBUS_VFDP, 0x100, }, + { A5XX_RBBM_DBGBUS_VPC, 0x100, }, + { A5XX_RBBM_DBGBUS_TSE, 0x100, }, + { A5XX_RBBM_DBGBUS_RAS, 0x100, }, + { A5XX_RBBM_DBGBUS_VSC, 0x100, }, + { A5XX_RBBM_DBGBUS_COM, 0x100, }, + { A5XX_RBBM_DBGBUS_DCOM, 0x100, }, + { A5XX_RBBM_DBGBUS_LRZ, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_DSP, 0x100, }, + { A5XX_RBBM_DBGBUS_CCUFCHE, 0x100, }, + { A5XX_RBBM_DBGBUS_GPMU, 0x100, }, + { A5XX_RBBM_DBGBUS_RBP, 0x100, }, + { A5XX_RBBM_DBGBUS_HM, 0x100, }, + { A5XX_RBBM_DBGBUS_RBBM_CFG, 0x100, }, + { A5XX_RBBM_DBGBUS_VBIF_CX, 0x100, }, + { A5XX_RBBM_DBGBUS_GPC, 0x100, }, + { A5XX_RBBM_DBGBUS_LARC, 0x100, }, + { A5XX_RBBM_DBGBUS_HLSQ_SPTP, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_0, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_1, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_2, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_3, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_0, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_1, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_2, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_3, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_0, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_1, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_2, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_3, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_0, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_1, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_2, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_3, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_0, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_1, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_2, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_3, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_0, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_1, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_2, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_3, 0x100, }, +}; + +#define A5XX_NUM_AXI_ARB_BLOCKS 2 +#define A5XX_NUM_XIN_BLOCKS 4 + +/* Width of A5XX_CP_DRAW_STATE_ADDR is 8 bits */ +#define A5XX_CP_DRAW_STATE_ADDR_WIDTH 8 + +/* a5xx_snapshot_cp_pm4() - Dump PM4 data in snapshot */ +static size_t a5xx_snapshot_cp_pm4(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + size_t size = fw->size; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PM4 RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PM4_RAM; + header->size = size; + + memcpy(data, fw->memdesc.hostptr, size * sizeof(uint32_t)); + + return DEBUG_SECTION_SZ(size); +} + +/* a5xx_snapshot_cp_pfp() - Dump the PFP data on snapshot */ +static size_t a5xx_snapshot_cp_pfp(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + int size = fw->size; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PFP RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PFP_RAM; + header->size = size; + + memcpy(data, fw->memdesc.hostptr, size * sizeof(uint32_t)); + + return DEBUG_SECTION_SZ(size); +} + +/* a5xx_rbbm_debug_bus_read() - Read data from trace bus */ +static void a5xx_rbbm_debug_bus_read(struct kgsl_device *device, + unsigned int block_id, unsigned int index, unsigned int *val) +{ + unsigned int reg; + + reg = (block_id << A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT) | + (index << A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_A, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_B, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_C, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_D, reg); + + kgsl_regread(device, A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2, val); + val++; + kgsl_regread(device, A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1, val); + +} + +/* a5xx_snapshot_vbif_debugbus() - Dump the VBIF debug data */ +static size_t a5xx_snapshot_vbif_debugbus(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i, j; + /* + * Total number of VBIF data words considering 3 sections: + * 2 arbiter blocks of 16 words + * 4 AXI XIN blocks of 18 dwords each + * 4 core clock side XIN blocks of 12 dwords each + */ + unsigned int dwords = (16 * A5XX_NUM_AXI_ARB_BLOCKS) + + (18 * A5XX_NUM_XIN_BLOCKS) + (12 * A5XX_NUM_XIN_BLOCKS); + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + size_t size; + unsigned int reg_clk; + + size = (dwords * sizeof(unsigned int)) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + header->id = block->block_id; + header->count = dwords; + + kgsl_regread(device, A5XX_VBIF_CLKON, ®_clk); + kgsl_regwrite(device, A5XX_VBIF_CLKON, reg_clk | + (A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK << + A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT)); + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL0, 0); + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS_OUT_CTRL, + (A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK << + A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT)); + for (i = 0; i < A5XX_NUM_AXI_ARB_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL0, + (1 << (i + 16))); + for (j = 0; j < 16; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL1, + ((j & A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks AXI side */ + for (i = 0; i < A5XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL0, 1 << i); + for (j = 0; j < 18; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL1, + ((j & A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks core clock side */ + for (i = 0; i < A5XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL0, 1 << i); + for (j = 0; j < 12; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL1, + ((j & A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + /* restore the clock of VBIF */ + kgsl_regwrite(device, A5XX_VBIF_CLKON, reg_clk); + return size; +} + +/* a5xx_snapshot_debugbus_block() - Capture debug data for a gpu block */ +static size_t a5xx_snapshot_debugbus_block(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int dwords; + size_t size; + + dwords = block->dwords; + + /* For a5xx each debug bus data unit is 2 DWRODS */ + size = (dwords * sizeof(unsigned int) * 2) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + + header->id = block->block_id; + header->count = dwords * 2; + + for (i = 0; i < dwords; i++) + a5xx_rbbm_debug_bus_read(device, block->block_id, i, + &data[i*2]); + + return size; +} + +/* a5xx_snapshot_debugbus() - Capture debug bus data */ +static void a5xx_snapshot_debugbus(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + int i; + + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_CNTLM, + 0xf << A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT); + + for (i = 0; i < ARRAY_SIZE(a5xx_debugbus_blocks); i++) { + if (a5xx_debugbus_blocks[i].block_id == A5XX_RBBM_DBGBUS_VBIF) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a5xx_snapshot_vbif_debugbus, + (void *) &a5xx_debugbus_blocks[i]); + else + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a5xx_snapshot_debugbus_block, + (void *) &a5xx_debugbus_blocks[i]); + } +} + +static const unsigned int a5xx_vbif_ver_20xxxxxx_registers[] = { + 0x3000, 0x3007, 0x300C, 0x3014, 0x3018, 0x302C, 0x3030, 0x3030, + 0x3034, 0x3036, 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, + 0x3042, 0x3042, 0x3049, 0x3049, 0x3058, 0x3058, 0x305A, 0x3061, + 0x3064, 0x3068, 0x306C, 0x306D, 0x3080, 0x3088, 0x308C, 0x308C, + 0x3090, 0x3094, 0x3098, 0x3098, 0x309C, 0x309C, 0x30C0, 0x30C0, + 0x30C8, 0x30C8, 0x30D0, 0x30D0, 0x30D8, 0x30D8, 0x30E0, 0x30E0, + 0x3100, 0x3100, 0x3108, 0x3108, 0x3110, 0x3110, 0x3118, 0x3118, + 0x3120, 0x3120, 0x3124, 0x3125, 0x3129, 0x3129, 0x3131, 0x3131, + 0x340C, 0x340C, 0x3410, 0x3410, 0x3800, 0x3801, +}; + +static const struct adreno_vbif_snapshot_registers +a5xx_vbif_snapshot_registers[] = { + { 0x20000000, 0xFF000000, a5xx_vbif_ver_20xxxxxx_registers, + ARRAY_SIZE(a5xx_vbif_ver_20xxxxxx_registers)/2}, +}; + +/* + * Set of registers to dump for A5XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a5xx_registers[] = { + /* RBBM */ + 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, + 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, + 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, + 0x04E0, 0x04F4, 0X04F8, 0x0529, 0x0531, 0x0533, 0x0540, 0x0555, + 0xF400, 0xF400, 0xF800, 0xF807, + /* CP */ + 0x0800, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, 0x0880, 0x08A0, + 0x0B00, 0x0B12, 0x0B15, 0X0B1C, 0X0B1E, 0x0B28, 0x0B78, 0x0B7F, + 0x0BB0, 0x0BBD, + /* VSC */ + 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, + /* GRAS */ + 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, + 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, + /* RB */ + 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, + 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, + 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, + /* PC */ + 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, + 0x24C0, 0x24C0, + /* VFD */ + 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, + /* VPC */ + 0x0E60, 0x0E7C, + /* UCHE */ + 0x0E80, 0x0E8F, 0x0E90, 0x0E96, 0xEA0, 0xEA8, 0xEB0, 0xEB2, + + /* RB CTX 0 */ + 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, + 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, + 0xE240, 0xE268, + /* GRAS CTX 0 */ + 0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, + 0xE100, 0xE105, + /* PC CTX 0 */ + 0xE380, 0xE38F, 0xE3B0, 0xE3B0, + /* VFD CTX 0 */ + 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, + /* VPC CTX 0 */ + 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, + + /* RB CTX 1 */ + 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, + 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, + 0xEA40, 0xEA68, + /* GRAS CTX 1 */ + 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, + 0xE900, 0xE905, + /* PC CTX 1 */ + 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, + /* VFD CTX 1 */ + 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0, + /* VPC CTX 1 */ + 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, +}; + +/* + * GPMU registers to dump for A5XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a5xx_gpmu_registers[] = { + /* GPMU */ + 0xA800, 0xA8FF, 0xAC60, 0xAC60, +}; + +/* + * Set of registers to dump for A5XX before actually triggering crash dumper. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ +static const unsigned int a5xx_pre_crashdumper_registers[] = { + /* RBBM: RBBM_STATUS - RBBM_STATUS3 */ + 0x04F5, 0x04F7, 0x0530, 0x0530, + /* CP: CP_STATUS_1 */ + 0x0B1D, 0x0B1D, +}; + + +struct a5xx_hlsq_sp_tp_regs { + unsigned int statetype; + unsigned int ahbaddr; + unsigned int size; + uint64_t offset; +}; + +static struct a5xx_hlsq_sp_tp_regs a5xx_hlsq_sp_tp_registers[] = { + /* HSLQ non context. 0xe32 - 0xe3f are holes so don't include them */ + { 0x35, 0xE00, 0x32 }, + /* HLSQ CTX 0 2D */ + { 0x31, 0x2080, 0x1 }, + /* HLSQ CTX 1 2D */ + { 0x33, 0x2480, 0x1 }, + /* HLSQ CTX 0 3D. 0xe7e2 - 0xe7ff are holes so don't inculde them */ + { 0x32, 0xE780, 0x62 }, + /* HLSQ CTX 1 3D. 0xefe2 - 0xefff are holes so don't include them */ + { 0x34, 0xEF80, 0x62 }, + + /* SP non context */ + { 0x3f, 0x0EC0, 0x40 }, + /* SP CTX 0 2D */ + { 0x3d, 0x2040, 0x1 }, + /* SP CTX 1 2D */ + { 0x3b, 0x2440, 0x1 }, + /* SP CTX 0 3D */ + { 0x3e, 0xE580, 0x180 }, + /* SP CTX 1 3D */ + { 0x3c, 0xED80, 0x180 }, + + /* TP non context. 0x0f1c - 0x0f3f are holes so don't include them */ + { 0x3a, 0x0F00, 0x1c }, + /* TP CTX 0 2D. 0x200a - 0x200f are holes so don't include them */ + { 0x38, 0x2000, 0xa }, + /* TP CTX 1 2D. 0x240a - 0x240f are holes so don't include them */ + { 0x36, 0x2400, 0xa }, + /* TP CTX 0 3D */ + { 0x39, 0xE700, 0x80 }, + /* TP CTX 1 3D */ + { 0x37, 0xEF00, 0x80 }, +}; + + +#define A5XX_NUM_SHADER_BANKS 4 +#define A5XX_SHADER_STATETYPE_SHIFT 8 + +enum a5xx_shader_obj { + A5XX_TP_W_MEMOBJ = 1, + A5XX_TP_W_SAMPLER = 2, + A5XX_TP_W_MIPMAP_BASE = 3, + A5XX_TP_W_MEMOBJ_TAG = 4, + A5XX_TP_W_SAMPLER_TAG = 5, + A5XX_TP_S_3D_MEMOBJ = 6, + A5XX_TP_S_3D_SAMPLER = 0x7, + A5XX_TP_S_3D_MEMOBJ_TAG = 0x8, + A5XX_TP_S_3D_SAMPLER_TAG = 0x9, + A5XX_TP_S_CS_MEMOBJ = 0xA, + A5XX_TP_S_CS_SAMPLER = 0xB, + A5XX_TP_S_CS_MEMOBJ_TAG = 0xC, + A5XX_TP_S_CS_SAMPLER_TAG = 0xD, + A5XX_SP_W_INSTR = 0xE, + A5XX_SP_W_CONST = 0xF, + A5XX_SP_W_UAV_SIZE = 0x10, + A5XX_SP_W_CB_SIZE = 0x11, + A5XX_SP_W_UAV_BASE = 0x12, + A5XX_SP_W_CB_BASE = 0x13, + A5XX_SP_W_INST_TAG = 0x14, + A5XX_SP_W_STATE = 0x15, + A5XX_SP_S_3D_INSTR = 0x16, + A5XX_SP_S_3D_CONST = 0x17, + A5XX_SP_S_3D_CB_BASE = 0x18, + A5XX_SP_S_3D_CB_SIZE = 0x19, + A5XX_SP_S_3D_UAV_BASE = 0x1A, + A5XX_SP_S_3D_UAV_SIZE = 0x1B, + A5XX_SP_S_CS_INSTR = 0x1C, + A5XX_SP_S_CS_CONST = 0x1D, + A5XX_SP_S_CS_CB_BASE = 0x1E, + A5XX_SP_S_CS_CB_SIZE = 0x1F, + A5XX_SP_S_CS_UAV_BASE = 0x20, + A5XX_SP_S_CS_UAV_SIZE = 0x21, + A5XX_SP_S_3D_INSTR_DIRTY = 0x22, + A5XX_SP_S_3D_CONST_DIRTY = 0x23, + A5XX_SP_S_3D_CB_BASE_DIRTY = 0x24, + A5XX_SP_S_3D_CB_SIZE_DIRTY = 0x25, + A5XX_SP_S_3D_UAV_BASE_DIRTY = 0x26, + A5XX_SP_S_3D_UAV_SIZE_DIRTY = 0x27, + A5XX_SP_S_CS_INSTR_DIRTY = 0x28, + A5XX_SP_S_CS_CONST_DIRTY = 0x29, + A5XX_SP_S_CS_CB_BASE_DIRTY = 0x2A, + A5XX_SP_S_CS_CB_SIZE_DIRTY = 0x2B, + A5XX_SP_S_CS_UAV_BASE_DIRTY = 0x2C, + A5XX_SP_S_CS_UAV_SIZE_DIRTY = 0x2D, + A5XX_HLSQ_ICB = 0x2E, + A5XX_HLSQ_ICB_DIRTY = 0x2F, + A5XX_HLSQ_ICB_CB_BASE_DIRTY = 0x30, + A5XX_SP_POWER_RESTORE_RAM = 0x40, + A5XX_SP_POWER_RESTORE_RAM_TAG = 0x41, + A5XX_TP_POWER_RESTORE_RAM = 0x42, + A5XX_TP_POWER_RESTORE_RAM_TAG = 0x43, + +}; + +struct a5xx_shader_block { + unsigned int statetype; + unsigned int sz; + uint64_t offset; +}; + +struct a5xx_shader_block_info { + struct a5xx_shader_block *block; + unsigned int bank; + uint64_t offset; +}; + +static struct a5xx_shader_block a5xx_shader_blocks[] = { + {A5XX_TP_W_MEMOBJ, 0x200}, + {A5XX_TP_W_MIPMAP_BASE, 0x3C0}, + {A5XX_TP_W_SAMPLER_TAG, 0x40}, + {A5XX_TP_S_3D_SAMPLER, 0x80}, + {A5XX_TP_S_3D_SAMPLER_TAG, 0x20}, + {A5XX_TP_S_CS_SAMPLER, 0x40}, + {A5XX_TP_S_CS_SAMPLER_TAG, 0x10}, + {A5XX_SP_W_CONST, 0x800}, + {A5XX_SP_W_CB_SIZE, 0x30}, + {A5XX_SP_W_CB_BASE, 0xF0}, + {A5XX_SP_W_STATE, 0x1}, + {A5XX_SP_S_3D_CONST, 0x800}, + {A5XX_SP_S_3D_CB_SIZE, 0x28}, + {A5XX_SP_S_3D_UAV_SIZE, 0x80}, + {A5XX_SP_S_CS_CONST, 0x400}, + {A5XX_SP_S_CS_CB_SIZE, 0x8}, + {A5XX_SP_S_CS_UAV_SIZE, 0x80}, + {A5XX_SP_S_3D_CONST_DIRTY, 0x12}, + {A5XX_SP_S_3D_CB_SIZE_DIRTY, 0x1}, + {A5XX_SP_S_3D_UAV_SIZE_DIRTY, 0x2}, + {A5XX_SP_S_CS_CONST_DIRTY, 0xA}, + {A5XX_SP_S_CS_CB_SIZE_DIRTY, 0x1}, + {A5XX_SP_S_CS_UAV_SIZE_DIRTY, 0x2}, + {A5XX_HLSQ_ICB_DIRTY, 0xB}, + {A5XX_SP_POWER_RESTORE_RAM_TAG, 0xA}, + {A5XX_TP_POWER_RESTORE_RAM_TAG, 0xA}, + {A5XX_TP_W_SAMPLER, 0x80}, + {A5XX_TP_W_MEMOBJ_TAG, 0x40}, + {A5XX_TP_S_3D_MEMOBJ, 0x200}, + {A5XX_TP_S_3D_MEMOBJ_TAG, 0x20}, + {A5XX_TP_S_CS_MEMOBJ, 0x100}, + {A5XX_TP_S_CS_MEMOBJ_TAG, 0x10}, + {A5XX_SP_W_INSTR, 0x800}, + {A5XX_SP_W_UAV_SIZE, 0x80}, + {A5XX_SP_W_UAV_BASE, 0x80}, + {A5XX_SP_W_INST_TAG, 0x40}, + {A5XX_SP_S_3D_INSTR, 0x800}, + {A5XX_SP_S_3D_CB_BASE, 0xC8}, + {A5XX_SP_S_3D_UAV_BASE, 0x80}, + {A5XX_SP_S_CS_INSTR, 0x400}, + {A5XX_SP_S_CS_CB_BASE, 0x28}, + {A5XX_SP_S_CS_UAV_BASE, 0x80}, + {A5XX_SP_S_3D_INSTR_DIRTY, 0x1}, + {A5XX_SP_S_3D_CB_BASE_DIRTY, 0x5}, + {A5XX_SP_S_3D_UAV_BASE_DIRTY, 0x2}, + {A5XX_SP_S_CS_INSTR_DIRTY, 0x1}, + {A5XX_SP_S_CS_CB_BASE_DIRTY, 0x1}, + {A5XX_SP_S_CS_UAV_BASE_DIRTY, 0x2}, + {A5XX_HLSQ_ICB, 0x200}, + {A5XX_HLSQ_ICB_CB_BASE_DIRTY, 0x4}, + {A5XX_SP_POWER_RESTORE_RAM, 0x140}, + {A5XX_TP_POWER_RESTORE_RAM, 0x40}, +}; + +static struct kgsl_memdesc capturescript; +static struct kgsl_memdesc registers; +static bool crash_dump_valid; + +static size_t a5xx_snapshot_shader_memory(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_shader *header = + (struct kgsl_snapshot_shader *) buf; + struct a5xx_shader_block_info *info = + (struct a5xx_shader_block_info *) priv; + struct a5xx_shader_block *block = info->block; + unsigned int *data = (unsigned int *) (buf + sizeof(*header)); + + if (remain < SHADER_SECTION_SZ(block->sz)) { + SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY"); + return 0; + } + + header->type = block->statetype; + header->index = info->bank; + header->size = block->sz; + + memcpy(data, registers.hostptr + info->offset, + block->sz * sizeof(unsigned int)); + + return SHADER_SECTION_SZ(block->sz); +} + +static void a5xx_snapshot_shader(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + unsigned int i, j; + struct a5xx_shader_block_info info; + + /* Shader blocks can only be read by the crash dumper */ + if (!crash_dump_valid) + return; + + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + for (j = 0; j < A5XX_NUM_SHADER_BANKS; j++) { + info.block = &a5xx_shader_blocks[i]; + info.bank = j; + info.offset = a5xx_shader_blocks[i].offset + + (j * a5xx_shader_blocks[i].sz); + + /* Shader working/shadow memory */ + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_SHADER, + snapshot, a5xx_snapshot_shader_memory, &info); + } + } +} + +/* Dump registers which get affected by crash dumper trigger */ +static size_t a5xx_snapshot_pre_crashdump_regs(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_registers pre_cdregs = { + .regs = a5xx_pre_crashdumper_registers, + .count = ARRAY_SIZE(a5xx_pre_crashdumper_registers)/2, + }; + + return kgsl_snapshot_dump_registers(device, buf, remain, &pre_cdregs); +} + +struct registers { + const unsigned int *regs; + size_t size; +}; + +static size_t a5xx_legacy_snapshot_registers(struct kgsl_device *device, + u8 *buf, size_t remain, const unsigned int *regs, size_t size) +{ + struct kgsl_snapshot_registers snapshot_regs = { + .regs = regs, + .count = size / 2, + }; + + return kgsl_snapshot_dump_registers(device, buf, remain, + &snapshot_regs); +} + +#define REG_PAIR_COUNT(_a, _i) \ + (((_a)[(2 * (_i)) + 1] - (_a)[2 * (_i)]) + 1) + +static size_t a5xx_snapshot_registers(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int *src = (unsigned int *) registers.hostptr; + struct registers *regs = (struct registers *)priv; + unsigned int j, k; + unsigned int count = 0; + + if (!crash_dump_valid) + return a5xx_legacy_snapshot_registers(device, buf, remain, + regs->regs, regs->size); + + if (remain < sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + return 0; + } + + remain -= sizeof(*header); + + for (j = 0; j < regs->size / 2; j++) { + unsigned int start = regs->regs[2 * j]; + unsigned int end = regs->regs[(2 * j) + 1]; + + if (remain < ((end - start) + 1) * 8) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + goto out; + } + + remain -= ((end - start) + 1) * 8; + + for (k = start; k <= end; k++, count++) { + *data++ = k; + *data++ = *src++; + } + } + +out: + header->count = count; + + /* Return the size of the section */ + return (count * 8) + sizeof(*header); +} + +/* Snapshot a preemption record buffer */ +static size_t snapshot_preemption_record(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_memdesc *memdesc = priv; + + struct kgsl_snapshot_gpu_object_v2 *header = + (struct kgsl_snapshot_gpu_object_v2 *)buf; + + u8 *ptr = buf + sizeof(*header); + + if (remain < (SZ_64K + sizeof(*header))) { + SNAPSHOT_ERR_NOMEM(device, "PREEMPTION RECORD"); + return 0; + } + + header->size = SZ_64K >> 2; + header->gpuaddr = memdesc->gpuaddr; + header->ptbase = + kgsl_mmu_pagetable_get_ttbr0(device->mmu.defaultpagetable); + header->type = SNAPSHOT_GPU_OBJECT_GLOBAL; + + memcpy(ptr, memdesc->hostptr, SZ_64K); + + return SZ_64K + sizeof(*header); +} + + +static void _a5xx_do_crashdump(struct kgsl_device *device) +{ + unsigned long wait_time; + unsigned int reg = 0; + unsigned int val; + + crash_dump_valid = false; + + if (!device->snapshot_crashdumper) + return; + if (capturescript.gpuaddr == 0 || registers.gpuaddr == 0) + return; + + /* IF the SMMU is stalled we cannot do a crash dump */ + kgsl_regread(device, A5XX_RBBM_STATUS3, &val); + if (val & BIT(24)) + return; + + /* Turn on APRIV so we can access the buffers */ + kgsl_regwrite(device, A5XX_CP_CNTL, 1); + + kgsl_regwrite(device, A5XX_CP_CRASH_SCRIPT_BASE_LO, + lower_32_bits(capturescript.gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CRASH_SCRIPT_BASE_HI, + upper_32_bits(capturescript.gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CRASH_DUMP_CNTL, 1); + + wait_time = jiffies + msecs_to_jiffies(CP_CRASH_DUMPER_TIMEOUT); + while (!time_after(jiffies, wait_time)) { + kgsl_regread(device, A5XX_CP_CRASH_DUMP_CNTL, ®); + if (reg & 0x4) + break; + cpu_relax(); + } + + kgsl_regwrite(device, A5XX_CP_CNTL, 0); + + if (!(reg & 0x4)) { + dev_err(device->dev, "Crash dump timed out: 0x%X\n", reg); + return; + } + + crash_dump_valid = true; +} + +static int get_hlsq_registers(struct kgsl_device *device, + const struct a5xx_hlsq_sp_tp_regs *regs, unsigned int *data) +{ + unsigned int i; + unsigned int *src = registers.hostptr + regs->offset; + + for (i = 0; i < regs->size; i++) { + *data++ = regs->ahbaddr + i; + *data++ = *(src + i); + } + + return (2 * regs->size); +} + +static size_t a5xx_snapshot_dump_hlsq_sp_tp_regs(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + int count = 0, i; + + /* Figure out how many registers we are going to dump */ + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + count += a5xx_hlsq_sp_tp_registers[i].size; + + if (remain < (count * 8) + sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + return 0; + } + + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + data += get_hlsq_registers(device, + &a5xx_hlsq_sp_tp_registers[i], data); + + header->count = count; + + /* Return the size of the section */ + return (count * 8) + sizeof(*header); +} + +/* + * a5xx_snapshot() - A5XX GPU snapshot function + * @adreno_dev: Device being snapshotted + * @snapshot: Pointer to the snapshot instance + * + * This is where all of the A5XX specific bits and pieces are grabbed + * into the snapshot memory + */ +void a5xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_snapshot_data *snap_data = gpudev->snapshot_data; + unsigned int reg, i; + struct adreno_ringbuffer *rb; + struct registers regs; + + /* Disable Clock gating temporarily for the debug bus to work */ + a5xx_hwcg_set(adreno_dev, false); + + /* Dump the registers which get affected by crash dumper trigger */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, + snapshot, a5xx_snapshot_pre_crashdump_regs, NULL); + + /* Dump vbif registers as well which get affected by crash dumper */ + adreno_snapshot_vbif_registers(device, snapshot, + a5xx_vbif_snapshot_registers, + ARRAY_SIZE(a5xx_vbif_snapshot_registers)); + + /* Try to run the crash dumper */ + _a5xx_do_crashdump(device); + + regs.regs = a5xx_registers; + regs.size = ARRAY_SIZE(a5xx_registers); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot, + a5xx_snapshot_registers, ®s); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) { + regs.regs = a5xx_gpmu_registers; + regs.size = ARRAY_SIZE(a5xx_gpmu_registers); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, + snapshot, a5xx_snapshot_registers, ®s); + } + + + /* Dump SP TP HLSQ registers */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot, + a5xx_snapshot_dump_hlsq_sp_tp_regs, NULL); + + /* CP_PFP indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_PFP_STAT_ADDR, A5XX_CP_PFP_STAT_DATA, + 0, snap_data->sect_sizes->cp_pfp); + + /* CP_ME indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_ME_STAT_ADDR, A5XX_CP_ME_STAT_DATA, + 0, snap_data->sect_sizes->cp_me); + + /* CP_DRAW_STATE */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_DRAW_STATE_ADDR, A5XX_CP_DRAW_STATE_DATA, + 0, 1 << A5XX_CP_DRAW_STATE_ADDR_WIDTH); + + /* + * CP needs to be halted on a530v1 before reading CP_PFP_UCODE_DBG_DATA + * and CP_PM4_UCODE_DBG_DATA registers + */ + if (adreno_is_a530v1(adreno_dev)) { + adreno_readreg(adreno_dev, ADRENO_REG_CP_ME_CNTL, ®); + reg |= (1 << 27) | (1 << 28); + adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, reg); + } + + /* ME_UCODE Cache */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_ME_UCODE_DBG_ADDR, A5XX_CP_ME_UCODE_DBG_DATA, + 0, 0x53F); + + /* PFP_UCODE Cache */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_PFP_UCODE_DBG_ADDR, A5XX_CP_PFP_UCODE_DBG_DATA, + 0, 0x53F); + + /* CP MEQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_meq, + &snap_data->sect_sizes->cp_meq); + + /* CP ROQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_roq, + &snap_data->sect_sizes->roq); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, adreno_snapshot_cp_merciu, + &snap_data->sect_sizes->cp_merciu); + + /* CP PFP and PM4 */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_pfp, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_pm4, NULL); + + /* Shader memory */ + a5xx_snapshot_shader(device, snapshot); + + /* Debug bus */ + a5xx_snapshot_debugbus(device, snapshot); + + /* Preemption record */ + if (adreno_is_preemption_enabled(adreno_dev)) { + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, + snapshot, snapshot_preemption_record, + &rb->preemption_desc); + } + } + +} + +static int _a5xx_crashdump_init_shader(struct a5xx_shader_block *block, + uint64_t *ptr, uint64_t *offset) +{ + int qwords = 0; + unsigned int j; + + /* Capture each bank in the block */ + for (j = 0; j < A5XX_NUM_SHADER_BANKS; j++) { + /* Program the aperture */ + ptr[qwords++] = + (block->statetype << A5XX_SHADER_STATETYPE_SHIFT) | j; + ptr[qwords++] = (((uint64_t) A5XX_HLSQ_DBG_READ_SEL << 44)) | + (1 << 21) | 1; + + /* Read all the data in one chunk */ + ptr[qwords++] = registers.gpuaddr + *offset; + ptr[qwords++] = + (((uint64_t) A5XX_HLSQ_DBG_AHB_READ_APERTURE << 44)) | + block->sz; + + /* Remember the offset of the first bank for easy access */ + if (j == 0) + block->offset = *offset; + + *offset += block->sz * sizeof(unsigned int); + } + + return qwords; +} + +static int _a5xx_crashdump_init_hlsq(struct a5xx_hlsq_sp_tp_regs *regs, + uint64_t *ptr, uint64_t *offset) +{ + int qwords = 0; + + /* Program the aperture */ + ptr[qwords++] = + (regs->statetype << A5XX_SHADER_STATETYPE_SHIFT); + ptr[qwords++] = (((uint64_t) A5XX_HLSQ_DBG_READ_SEL << 44)) | + (1 << 21) | 1; + + /* Read all the data in one chunk */ + ptr[qwords++] = registers.gpuaddr + *offset; + ptr[qwords++] = + (((uint64_t) A5XX_HLSQ_DBG_AHB_READ_APERTURE << 44)) | + regs->size; + + /* Remember the offset of the first bank for easy access */ + regs->offset = *offset; + + *offset += regs->size * sizeof(unsigned int); + + return qwords; +} + +void a5xx_crashdump_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int script_size = 0; + unsigned int data_size = 0; + unsigned int i, j; + uint64_t *ptr; + uint64_t offset = 0; + + if (capturescript.gpuaddr != 0 && registers.gpuaddr != 0) + return; + + /* + * We need to allocate two buffers: + * 1 - the buffer to hold the draw script + * 2 - the buffer to hold the data + */ + + /* + * To save the registers, we need 16 bytes per register pair for the + * script and a dword for each register int the data + */ + + /* Each pair needs 16 bytes (2 qwords) */ + script_size += (ARRAY_SIZE(a5xx_registers) / 2) * 16; + + /* Each register needs a dword in the data */ + for (j = 0; j < ARRAY_SIZE(a5xx_registers) / 2; j++) + data_size += REG_PAIR_COUNT(a5xx_registers, j) * + sizeof(unsigned int); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) { + /* Each pair needs 16 bytes (2 qwords) */ + script_size += (ARRAY_SIZE(a5xx_gpmu_registers) / 2) * 16; + + /* Each register needs a dword in the data */ + for (j = 0; j < ARRAY_SIZE(a5xx_gpmu_registers) / 2; j++) + data_size += REG_PAIR_COUNT(a5xx_gpmu_registers, j) * + sizeof(unsigned int); + } + + /* + * To save the shader blocks for each block in each type we need 32 + * bytes for the script (16 bytes to program the aperture and 16 to + * read the data) and then a block specific number of bytes to hold + * the data + */ + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + script_size += 32 * A5XX_NUM_SHADER_BANKS; + data_size += a5xx_shader_blocks[i].sz * sizeof(unsigned int) * + A5XX_NUM_SHADER_BANKS; + } + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) { + script_size += 32; + data_size += + a5xx_hlsq_sp_tp_registers[i].size * sizeof(unsigned int); + } + + /* Now allocate the script and data buffers */ + + /* The script buffers needs 2 extra qwords on the end */ + if (kgsl_allocate_global(device, &capturescript, + script_size + 16, KGSL_MEMFLAGS_GPUREADONLY, + KGSL_MEMDESC_PRIVILEGED, "capturescript")) + return; + + if (kgsl_allocate_global(device, ®isters, data_size, 0, + KGSL_MEMDESC_PRIVILEGED, "capturescript_regs")) { + kgsl_free_global(KGSL_DEVICE(adreno_dev), &capturescript); + return; + } + /* Build the crash script */ + + ptr = (uint64_t *) capturescript.hostptr; + + /* For the registers, program a read command for each pair */ + + for (j = 0; j < ARRAY_SIZE(a5xx_registers) / 2; j++) { + unsigned int r = REG_PAIR_COUNT(a5xx_registers, j); + *ptr++ = registers.gpuaddr + offset; + *ptr++ = (((uint64_t) a5xx_registers[2 * j]) << 44) + | r; + offset += r * sizeof(unsigned int); + } + + if (ADRENO_FEATURE(adreno_dev, ADRENO_GPMU)) { + for (j = 0; j < ARRAY_SIZE(a5xx_gpmu_registers) / 2; j++) { + unsigned int r = REG_PAIR_COUNT(a5xx_gpmu_registers, j); + *ptr++ = registers.gpuaddr + offset; + *ptr++ = (((uint64_t) a5xx_gpmu_registers[2 * j]) << 44) + | r; + offset += r * sizeof(unsigned int); + } + } + + /* Program each shader block */ + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + ptr += _a5xx_crashdump_init_shader(&a5xx_shader_blocks[i], ptr, + &offset); + } + /* Program the hlsq sp tp register sets */ + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + ptr += _a5xx_crashdump_init_hlsq(&a5xx_hlsq_sp_tp_registers[i], + ptr, &offset); + + *ptr++ = 0; + *ptr++ = 0; +} diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c new file mode 100644 index 0000000000000000000000000000000000000000..14f4f2f9e22b5a55273bd1932c61c3907f2ff84b --- /dev/null +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -0,0 +1,3015 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include + +#include "adreno.h" +#include "a6xx_reg.h" +#include "adreno_a6xx.h" +#include "adreno_cp_parser.h" +#include "adreno_trace.h" +#include "adreno_pm4types.h" +#include "adreno_perfcounter.h" +#include "adreno_ringbuffer.h" +#include "adreno_llc.h" +#include "kgsl_sharedmem.h" +#include "kgsl.h" +#include "kgsl_hfi.h" +#include "kgsl_trace.h" + +#define MIN_HBB 13 + +#define A6XX_LLC_NUM_GPU_SCIDS 5 +#define A6XX_GPU_LLC_SCID_NUM_BITS 5 +#define A6XX_GPU_LLC_SCID_MASK \ + ((1 << (A6XX_LLC_NUM_GPU_SCIDS * A6XX_GPU_LLC_SCID_NUM_BITS)) - 1) +#define A6XX_GPUHTW_LLC_SCID_SHIFT 25 +#define A6XX_GPUHTW_LLC_SCID_MASK \ + (((1 << A6XX_GPU_LLC_SCID_NUM_BITS) - 1) << A6XX_GPUHTW_LLC_SCID_SHIFT) + +#define A6XX_GPU_CX_REG_BASE 0x509E000 +#define A6XX_GPU_CX_REG_SIZE 0x1000 + + +static const struct adreno_vbif_data a630_vbif[] = { + {A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009}, + {A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3}, + {0, 0}, +}; + +static const struct adreno_vbif_data a615_gbif[] = { + {A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3}, + {0, 0}, +}; + +static const struct adreno_vbif_data a640_gbif[] = { + {A6XX_GBIF_QSB_SIDE0, 0x00071620}, + {A6XX_GBIF_QSB_SIDE1, 0x00071620}, + {A6XX_GBIF_QSB_SIDE2, 0x00071620}, + {A6XX_GBIF_QSB_SIDE3, 0x00071620}, + {A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3}, + {0, 0}, +}; + +static const struct adreno_vbif_platform a6xx_vbif_platforms[] = { + { adreno_is_a630, a630_vbif }, + { adreno_is_a615, a615_gbif }, + { adreno_is_a640, a640_gbif }, + { adreno_is_a680, a640_gbif }, + { adreno_is_a608, a615_gbif }, + { adreno_is_a616, a615_gbif }, +}; + +struct kgsl_hwcg_reg { + unsigned int off; + unsigned int val; +}; +static const struct kgsl_hwcg_reg a630_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555} +}; + +static const struct kgsl_hwcg_reg a615_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555} +}; + +static const struct kgsl_hwcg_reg a640_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct kgsl_hwcg_reg a608_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x00000081}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct { + int (*devfunc)(struct adreno_device *adreno_dev); + const struct kgsl_hwcg_reg *regs; + unsigned int count; +} a6xx_hwcg_registers[] = { + {adreno_is_a630, a630_hwcg_regs, ARRAY_SIZE(a630_hwcg_regs)}, + {adreno_is_a615, a615_hwcg_regs, ARRAY_SIZE(a615_hwcg_regs)}, + {adreno_is_a640, a640_hwcg_regs, ARRAY_SIZE(a640_hwcg_regs)}, + {adreno_is_a680, a640_hwcg_regs, ARRAY_SIZE(a640_hwcg_regs)}, + {adreno_is_a608, a608_hwcg_regs, ARRAY_SIZE(a608_hwcg_regs)}, + {adreno_is_a616, a615_hwcg_regs, ARRAY_SIZE(a615_hwcg_regs)}, +}; + +static struct a6xx_protected_regs { + unsigned int base; + unsigned int count; + int read_protect; +} a6xx_protected_regs_group[] = { + { 0x600, 0x51, 0 }, + { 0xAE50, 0x2, 1 }, + { 0x9624, 0x13, 1 }, + { 0x8630, 0x8, 1 }, + { 0x9E70, 0x1, 1 }, + { 0x9E78, 0x187, 1 }, + { 0xF000, 0x810, 1 }, + { 0xFC00, 0x3, 0 }, + { 0x50E, 0x0, 1 }, + { 0x50F, 0x0, 0 }, + { 0x510, 0x0, 1 }, + { 0x0, 0x4F9, 0 }, + { 0x501, 0xA, 0 }, + { 0x511, 0x44, 0 }, + { 0xE00, 0x1, 1 }, + { 0xE03, 0xB, 1 }, + { 0x8E00, 0x0, 1 }, + { 0x8E50, 0xF, 1 }, + { 0xBE02, 0x0, 1 }, + { 0xBE20, 0x11F3, 1 }, + { 0x800, 0x82, 1 }, + { 0x8A0, 0x8, 1 }, + { 0x8AB, 0x19, 1 }, + { 0x900, 0x4D, 1 }, + { 0x98D, 0x76, 1 }, + { 0x8D0, 0x23, 0 }, + { 0x980, 0x4, 0 }, + { 0xA630, 0x0, 1 }, +}; + +/* IFPC & Preemption static powerup restore list */ +static struct reg_list_pair { + uint32_t offset; + uint32_t val; +} a6xx_pwrup_reglist[] = { + { A6XX_VSC_ADDR_MODE_CNTL, 0x0 }, + { A6XX_GRAS_ADDR_MODE_CNTL, 0x0 }, + { A6XX_RB_ADDR_MODE_CNTL, 0x0 }, + { A6XX_PC_ADDR_MODE_CNTL, 0x0 }, + { A6XX_HLSQ_ADDR_MODE_CNTL, 0x0 }, + { A6XX_VFD_ADDR_MODE_CNTL, 0x0 }, + { A6XX_VPC_ADDR_MODE_CNTL, 0x0 }, + { A6XX_UCHE_ADDR_MODE_CNTL, 0x0 }, + { A6XX_SP_ADDR_MODE_CNTL, 0x0 }, + { A6XX_TPL1_ADDR_MODE_CNTL, 0x0 }, + { A6XX_UCHE_WRITE_RANGE_MAX_LO, 0x0 }, + { A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0 }, + { A6XX_UCHE_TRAP_BASE_LO, 0x0 }, + { A6XX_UCHE_TRAP_BASE_HI, 0x0 }, + { A6XX_UCHE_WRITE_THRU_BASE_LO, 0x0 }, + { A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0 }, + { A6XX_UCHE_GMEM_RANGE_MIN_LO, 0x0 }, + { A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x0 }, + { A6XX_UCHE_GMEM_RANGE_MAX_LO, 0x0 }, + { A6XX_UCHE_GMEM_RANGE_MAX_HI, 0x0 }, + { A6XX_UCHE_FILTER_CNTL, 0x0 }, + { A6XX_UCHE_CACHE_WAYS, 0x0 }, + { A6XX_UCHE_MODE_CNTL, 0x0 }, + { A6XX_RB_NC_MODE_CNTL, 0x0 }, + { A6XX_TPL1_NC_MODE_CNTL, 0x0 }, + { A6XX_SP_NC_MODE_CNTL, 0x0 }, + { A6XX_PC_DBG_ECO_CNTL, 0x0 }, + { A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, 0x0 }, +}; + +/* IFPC only static powerup restore list */ +static struct reg_list_pair a6xx_ifpc_pwrup_reglist[] = { + { A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x0 }, + { A6XX_CP_CHICKEN_DBG, 0x0 }, + { A6XX_CP_DBG_ECO_CNTL, 0x0 }, + { A6XX_CP_PROTECT_CNTL, 0x0 }, + { A6XX_CP_PROTECT_REG, 0x0 }, + { A6XX_CP_PROTECT_REG+1, 0x0 }, + { A6XX_CP_PROTECT_REG+2, 0x0 }, + { A6XX_CP_PROTECT_REG+3, 0x0 }, + { A6XX_CP_PROTECT_REG+4, 0x0 }, + { A6XX_CP_PROTECT_REG+5, 0x0 }, + { A6XX_CP_PROTECT_REG+6, 0x0 }, + { A6XX_CP_PROTECT_REG+7, 0x0 }, + { A6XX_CP_PROTECT_REG+8, 0x0 }, + { A6XX_CP_PROTECT_REG+9, 0x0 }, + { A6XX_CP_PROTECT_REG+10, 0x0 }, + { A6XX_CP_PROTECT_REG+11, 0x0 }, + { A6XX_CP_PROTECT_REG+12, 0x0 }, + { A6XX_CP_PROTECT_REG+13, 0x0 }, + { A6XX_CP_PROTECT_REG+14, 0x0 }, + { A6XX_CP_PROTECT_REG+15, 0x0 }, + { A6XX_CP_PROTECT_REG+16, 0x0 }, + { A6XX_CP_PROTECT_REG+17, 0x0 }, + { A6XX_CP_PROTECT_REG+18, 0x0 }, + { A6XX_CP_PROTECT_REG+19, 0x0 }, + { A6XX_CP_PROTECT_REG+20, 0x0 }, + { A6XX_CP_PROTECT_REG+21, 0x0 }, + { A6XX_CP_PROTECT_REG+22, 0x0 }, + { A6XX_CP_PROTECT_REG+23, 0x0 }, + { A6XX_CP_PROTECT_REG+24, 0x0 }, + { A6XX_CP_PROTECT_REG+25, 0x0 }, + { A6XX_CP_PROTECT_REG+26, 0x0 }, + { A6XX_CP_PROTECT_REG+27, 0x0 }, + { A6XX_CP_PROTECT_REG+28, 0x0 }, + { A6XX_CP_PROTECT_REG+29, 0x0 }, + { A6XX_CP_PROTECT_REG+30, 0x0 }, + { A6XX_CP_PROTECT_REG+31, 0x0 }, + { A6XX_CP_AHB_CNTL, 0x0 }, +}; + +static struct reg_list_pair a615_pwrup_reglist[] = { + { A6XX_UCHE_GBIF_GX_CONFIG, 0x0 }, +}; + +static void _update_always_on_regs(struct adreno_device *adreno_dev) +{ + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int *const regs = gpudev->reg_offsets->offsets; + + regs[ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO] = + A6XX_CP_ALWAYS_ON_COUNTER_LO; + regs[ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI] = + A6XX_CP_ALWAYS_ON_COUNTER_HI; +} + +static void a6xx_pwrup_reglist_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (kgsl_allocate_global(device, &adreno_dev->pwrup_reglist, + PAGE_SIZE, 0, KGSL_MEMDESC_CONTIG | KGSL_MEMDESC_PRIVILEGED, + "powerup_register_list")) { + adreno_dev->pwrup_reglist.gpuaddr = 0; + return; + } + + kgsl_sharedmem_set(device, &adreno_dev->pwrup_reglist, 0, 0, + PAGE_SIZE); +} + +static void a6xx_init(struct adreno_device *adreno_dev) +{ + a6xx_crashdump_init(adreno_dev); + + /* + * If the GMU is not enabled, rewrite the offset for the always on + * counters to point to the CP always on instead of GMU always on + */ + if (!gmu_core_isenabled(KGSL_DEVICE(adreno_dev))) + _update_always_on_regs(adreno_dev); + + a6xx_pwrup_reglist_init(adreno_dev); +} + +/** + * a6xx_protect_init() - Initializes register protection on a6xx + * @device: Pointer to the device structure + * Performs register writes to enable protected access to sensitive + * registers + */ +static void a6xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_protected_registers *mmu_prot = + kgsl_mmu_get_prot_regs(&device->mmu); + int i, num_sets; + int req_sets = ARRAY_SIZE(a6xx_protected_regs_group); + int max_sets = adreno_dev->gpucore->num_protected_regs; + unsigned int mmu_base = 0, mmu_range = 0, cur_range; + + /* enable access protection to privileged registers */ + kgsl_regwrite(device, A6XX_CP_PROTECT_CNTL, 0x00000003); + + if (mmu_prot) { + mmu_base = mmu_prot->base; + mmu_range = mmu_prot->range; + req_sets += DIV_ROUND_UP(mmu_range, 0x2000); + } + + WARN(req_sets > max_sets, + "Size exceeds the num of protection regs available\n"); + + /* Protect GPU registers */ + num_sets = min_t(unsigned int, + ARRAY_SIZE(a6xx_protected_regs_group), max_sets); + for (i = 0; i < num_sets; i++) { + struct a6xx_protected_regs *regs = + &a6xx_protected_regs_group[i]; + + kgsl_regwrite(device, A6XX_CP_PROTECT_REG + i, + regs->base | (regs->count << 18) | + (regs->read_protect << 31)); + } + + /* Protect MMU registers */ + if (mmu_prot) { + while ((i < max_sets) && (mmu_range > 0)) { + cur_range = min_t(unsigned int, mmu_range, + 0x2000); + kgsl_regwrite(device, A6XX_CP_PROTECT_REG + i, + mmu_base | ((cur_range - 1) << 18) | (1 << 31)); + + mmu_base += cur_range; + mmu_range -= cur_range; + i++; + } + } +} + +static void a6xx_enable_64bit(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_regwrite(device, A6XX_CP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VSC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_GRAS_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_RB_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_PC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_HLSQ_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VFD_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VPC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_UCHE_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_SP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_TPL1_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); +} + +static inline unsigned int +__get_rbbm_clock_cntl_on(struct adreno_device *adreno_dev) +{ + if (adreno_is_a630(adreno_dev)) + return 0x8AA8AA02; + else if (adreno_is_a608(adreno_dev)) + return 0xAAA8AA82; + else + return 0x8AA8AA82; +} + +static inline unsigned int +__get_gmu_ao_cgc_mode_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a608(adreno_dev)) + return 0x00000022; + if (adreno_is_a615(adreno_dev) || adreno_is_a616(adreno_dev)) + return 0x00000222; + else + return 0x00020202; +} + +static inline unsigned int +__get_gmu_ao_cgc_delay_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a608(adreno_dev)) + return 0x00000011; + if (adreno_is_a615(adreno_dev) || adreno_is_a616(adreno_dev)) + return 0x00000111; + else + return 0x00010111; +} + +static inline unsigned int +__get_gmu_ao_cgc_hyst_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a608(adreno_dev)) + return 0x00000055; + if (adreno_is_a615(adreno_dev) || adreno_is_a616(adreno_dev)) + return 0x00000555; + else + return 0x00005555; +} + +static void a6xx_hwcg_set(struct adreno_device *adreno_dev, bool on) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct kgsl_hwcg_reg *regs; + unsigned int value; + int i, j; + + if (!test_bit(ADRENO_HWCG_CTRL, &adreno_dev->pwrctrl_flag)) + on = false; + + if (gmu_core_isenabled(device)) { + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, + on ? __get_gmu_ao_cgc_mode_cntl(adreno_dev) : 0); + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, + on ? __get_gmu_ao_cgc_delay_cntl(adreno_dev) : 0); + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, + on ? __get_gmu_ao_cgc_hyst_cntl(adreno_dev) : 0); + } + + kgsl_regread(device, A6XX_RBBM_CLOCK_CNTL, &value); + + if (value == __get_rbbm_clock_cntl_on(adreno_dev) && on) + return; + + if (value == 0 && !on) + return; + + for (i = 0; i < ARRAY_SIZE(a6xx_hwcg_registers); i++) { + if (a6xx_hwcg_registers[i].devfunc(adreno_dev)) + break; + } + + if (i == ARRAY_SIZE(a6xx_hwcg_registers)) + return; + + regs = a6xx_hwcg_registers[i].regs; + + /* + * Disable SP clock before programming HWCG registers. + * A608 GPU is not having the GX power domain. Hence + * skip GMU_GX registers for A608. + */ + + if (!adreno_is_a608(adreno_dev)) + gmu_core_regrmw(device, + A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); + + for (j = 0; j < a6xx_hwcg_registers[i].count; j++) + kgsl_regwrite(device, regs[j].off, on ? regs[j].val : 0); + + /* + * Enable SP clock after programming HWCG registers. + * A608 GPU is not having the GX power domain. Hence + * skip GMU_GX registers for A608. + */ + if (!adreno_is_a608(adreno_dev)) + gmu_core_regrmw(device, + A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); + + /* enable top level HWCG */ + kgsl_regwrite(device, A6XX_RBBM_CLOCK_CNTL, + on ? __get_rbbm_clock_cntl_on(adreno_dev) : 0); +} + +static void a6xx_patch_pwrup_reglist(struct adreno_device *adreno_dev) +{ + uint32_t i; + struct cpu_gpu_lock *lock; + struct reg_list_pair *r; + + /* Set up the register values */ + for (i = 0; i < ARRAY_SIZE(a6xx_ifpc_pwrup_reglist); i++) { + r = &a6xx_ifpc_pwrup_reglist[i]; + kgsl_regread(KGSL_DEVICE(adreno_dev), r->offset, &r->val); + } + + for (i = 0; i < ARRAY_SIZE(a6xx_pwrup_reglist); i++) { + r = &a6xx_pwrup_reglist[i]; + kgsl_regread(KGSL_DEVICE(adreno_dev), r->offset, &r->val); + } + + lock = (struct cpu_gpu_lock *) adreno_dev->pwrup_reglist.hostptr; + lock->flag_ucode = 0; + lock->flag_kmd = 0; + lock->turn = 0; + + /* + * The overall register list is composed of + * 1. Static IFPC-only registers + * 2. Static IFPC + preemption registers + * 2. Dynamic IFPC + preemption registers (ex: perfcounter selects) + * + * The CP views the second and third entries as one dynamic list + * starting from list_offset. Thus, list_length should be the sum + * of all three lists above (of which the third list will start off + * empty). And list_offset should be specified as the size in dwords + * of the static IFPC-only register list. + */ + lock->list_length = (sizeof(a6xx_ifpc_pwrup_reglist) + + sizeof(a6xx_pwrup_reglist)) >> 2; + lock->list_offset = sizeof(a6xx_ifpc_pwrup_reglist) >> 2; + + memcpy(adreno_dev->pwrup_reglist.hostptr + sizeof(*lock), + a6xx_ifpc_pwrup_reglist, sizeof(a6xx_ifpc_pwrup_reglist)); + memcpy(adreno_dev->pwrup_reglist.hostptr + sizeof(*lock) + + sizeof(a6xx_ifpc_pwrup_reglist), a6xx_pwrup_reglist, + sizeof(a6xx_pwrup_reglist)); + + if (adreno_is_a615(adreno_dev) || adreno_is_a608(adreno_dev) || + adreno_is_a616(adreno_dev)) { + for (i = 0; i < ARRAY_SIZE(a615_pwrup_reglist); i++) { + r = &a615_pwrup_reglist[i]; + kgsl_regread(KGSL_DEVICE(adreno_dev), + r->offset, &r->val); + } + + memcpy(adreno_dev->pwrup_reglist.hostptr + sizeof(*lock) + + sizeof(a6xx_ifpc_pwrup_reglist) + + sizeof(a6xx_pwrup_reglist), a615_pwrup_reglist, + sizeof(a615_pwrup_reglist)); + + lock->list_length += sizeof(a615_pwrup_reglist); + } +} + +/* + * a6xx_start() - Device start + * @adreno_dev: Pointer to adreno device + * + * a6xx device start + */ +static void a6xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int bit, mal, mode, glbl_inv; + unsigned int amsbc = 0; + static bool patch_reglist; + + /* runtime adjust callbacks based on feature sets */ + if (!gmu_core_isenabled(device)) + /* Legacy idle management if gmu is disabled */ + ADRENO_GPU_DEVICE(adreno_dev)->hw_isidle = NULL; + /* enable hardware clockgating */ + a6xx_hwcg_set(adreno_dev, true); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_LM)) + adreno_dev->lm_threshold_count = A6XX_GMU_GENERAL_1; + + adreno_vbif_start(adreno_dev, a6xx_vbif_platforms, + ARRAY_SIZE(a6xx_vbif_platforms)); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW)) + kgsl_regwrite(device, A6XX_UCHE_GBIF_GX_CONFIG, 0x10200F9); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* + * Set UCHE_WRITE_THRU_BASE to the UCHE_TRAP_BASE effectively + * disabling L2 bypass + */ + kgsl_regwrite(device, A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0); + kgsl_regwrite(device, A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff); + kgsl_regwrite(device, A6XX_UCHE_TRAP_BASE_LO, 0xfffff000); + kgsl_regwrite(device, A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff); + kgsl_regwrite(device, A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000); + kgsl_regwrite(device, A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); + + /* Program the GMEM VA range for the UCHE path */ + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MIN_LO, + ADRENO_UCHE_GMEM_BASE); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x0); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MAX_LO, + ADRENO_UCHE_GMEM_BASE + + adreno_dev->gmem_size - 1); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MAX_HI, 0x0); + + kgsl_regwrite(device, A6XX_UCHE_FILTER_CNTL, 0x804); + kgsl_regwrite(device, A6XX_UCHE_CACHE_WAYS, 0x4); + + /* ROQ sizes are twice as big on a640/a680 than on a630 */ + if (adreno_is_a640(adreno_dev) || adreno_is_a680(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362C); + } else if (adreno_is_a608(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16); + } else { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x010000C0); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362C); + } + + if (adreno_is_a608(adreno_dev)) { + /* For a608 Mem pool size is reduced to 48 */ + kgsl_regwrite(device, A6XX_CP_MEM_POOL_SIZE, 48); + kgsl_regwrite(device, A6XX_CP_MEM_POOL_DBG_ADDR, 47); + } else { + kgsl_regwrite(device, A6XX_CP_MEM_POOL_SIZE, 128); + } + + /* Setting the primFifo thresholds values */ + if (adreno_is_a640(adreno_dev)) + kgsl_regwrite(device, A6XX_PC_DBG_ECO_CNTL, (0x400 << 11)); + else if (adreno_is_a680(adreno_dev)) + kgsl_regwrite(device, A6XX_PC_DBG_ECO_CNTL, (0x800 << 11)); + else if (adreno_is_a608(adreno_dev)) + kgsl_regwrite(device, A6XX_PC_DBG_ECO_CNTL, (0x100 << 11)); + else + kgsl_regwrite(device, A6XX_PC_DBG_ECO_CNTL, (0x300 << 11)); + + /* Set the AHB default slave response to "ERROR" */ + kgsl_regwrite(device, A6XX_CP_AHB_CNTL, 0x1); + + /* Turn on performance counters */ + kgsl_regwrite(device, A6XX_RBBM_PERFCTR_CNTL, 0x1); + + /* Turn on GX_MEM retention */ + if (gmu_core_isenabled(device) && adreno_is_a608(adreno_dev)) { + kgsl_regwrite(device, A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0x7FB); + /* For CP IPC interrupt */ + kgsl_regwrite(device, A6XX_RBBM_INT_2_MASK, 0x00000010); + } + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,highest-bank-bit", &bit)) + bit = MIN_HBB; + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,min-access-length", &mal)) + mal = 32; + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,ubwc-mode", &mode)) + mode = 0; + + switch (mode) { + case KGSL_UBWC_1_0: + mode = 1; + break; + case KGSL_UBWC_2_0: + mode = 0; + break; + case KGSL_UBWC_3_0: + mode = 0; + amsbc = 1; /* Only valid for A640 and A680 */ + break; + default: + break; + } + + if (bit >= 13 && bit <= 16) + bit = (bit - 13) & 0x03; + else + bit = 0; + + mal = (mal == 64) ? 1 : 0; + + /* (1 << 29)globalInvFlushFilterDis bit needs to be set for A630 V1 */ + glbl_inv = (adreno_is_a630v1(adreno_dev)) ? 1 : 0; + + kgsl_regwrite(device, A6XX_RB_NC_MODE_CNTL, (amsbc << 4) | (mal << 3) | + (bit << 1) | mode); + kgsl_regwrite(device, A6XX_TPL1_NC_MODE_CNTL, (mal << 3) | + (bit << 1) | mode); + kgsl_regwrite(device, A6XX_SP_NC_MODE_CNTL, (mal << 3) | (bit << 1) | + mode); + + kgsl_regwrite(device, A6XX_UCHE_MODE_CNTL, (glbl_inv << 29) | + (mal << 23) | (bit << 21)); + + /* Set hang detection threshold to 0x3FFFFF * 16 cycles */ + kgsl_regwrite(device, A6XX_RBBM_INTERFACE_HANG_INT_CNTL, + (1 << 30) | 0x3fffff); + + kgsl_regwrite(device, A6XX_UCHE_CLIENT_PF, 1); + + /* Set TWOPASSUSEWFI in A6XX_PC_DBG_ECO_CNTL if requested */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_TWO_PASS_USE_WFI)) + kgsl_regrmw(device, A6XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); + + /* Enable the GMEM save/restore feature for preemption */ + if (adreno_is_preemption_enabled(adreno_dev)) + kgsl_regwrite(device, A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, + 0x1); + + a6xx_protect_init(adreno_dev); + + if (!patch_reglist && (adreno_dev->pwrup_reglist.gpuaddr != 0)) { + a6xx_patch_pwrup_reglist(adreno_dev); + patch_reglist = true; + } + + a6xx_preemption_start(adreno_dev); + + /* + * We start LM here because we want all the following to be up + * 1. GX HS + * 2. SPTPRAC + * 3. HFI + * At this point, we are guaranteed all. + */ + gmu_core_dev_enable_lm(device); +} + +/* + * a6xx_microcode_load() - Load microcode + * @adreno_dev: Pointer to adreno device + */ +static int a6xx_microcode_load(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); + uint64_t gpuaddr; + void *zap; + int ret = 0; + + gpuaddr = fw->memdesc.gpuaddr; + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_LO, + lower_32_bits(gpuaddr)); + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_HI, + upper_32_bits(gpuaddr)); + + /* + * Do not invoke to load zap shader if MMU does + * not support secure mode. + */ + if (!device->mmu.secured) + return 0; + + /* Load the zap shader firmware through PIL if its available */ + if (adreno_dev->gpucore->zap_name && !adreno_dev->zap_loaded) { + zap = subsystem_get(adreno_dev->gpucore->zap_name); + + /* Return error if the zap shader cannot be loaded */ + if (IS_ERR_OR_NULL(zap)) { + ret = (zap == NULL) ? -ENODEV : PTR_ERR(zap); + zap = NULL; + } else + adreno_dev->zap_loaded = 1; + } + + return ret; +} + + +/* + * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can + * be used at once of if they should be serialized + */ +#define CP_INIT_MAX_CONTEXT BIT(0) + +/* Enables register protection mode */ +#define CP_INIT_ERROR_DETECTION_CONTROL BIT(1) + +/* Header dump information */ +#define CP_INIT_HEADER_DUMP BIT(2) /* Reserved */ + +/* Default Reset states enabled for PFP and ME */ +#define CP_INIT_DEFAULT_RESET_STATE BIT(3) + +/* Drawcall filter range */ +#define CP_INIT_DRAWCALL_FILTER_RANGE BIT(4) + +/* Ucode workaround masks */ +#define CP_INIT_UCODE_WORKAROUND_MASK BIT(5) + +/* + * Operation mode mask + * + * This ordinal provides the option to disable the + * save/restore of performance counters across preemption. + */ +#define CP_INIT_OPERATION_MODE_MASK BIT(6) + +/* Register initialization list */ +#define CP_INIT_REGISTER_INIT_LIST BIT(7) + +/* Register initialization list with spinlock */ +#define CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK BIT(8) + +#define CP_INIT_MASK (CP_INIT_MAX_CONTEXT | \ + CP_INIT_ERROR_DETECTION_CONTROL | \ + CP_INIT_HEADER_DUMP | \ + CP_INIT_DEFAULT_RESET_STATE | \ + CP_INIT_UCODE_WORKAROUND_MASK | \ + CP_INIT_OPERATION_MODE_MASK | \ + CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK) + +static void _set_ordinals(struct adreno_device *adreno_dev, + unsigned int *cmds, unsigned int count) +{ + unsigned int *start = cmds; + + /* Enabled ordinal mask */ + *cmds++ = CP_INIT_MASK; + + if (CP_INIT_MASK & CP_INIT_MAX_CONTEXT) + *cmds++ = 0x00000003; + + if (CP_INIT_MASK & CP_INIT_ERROR_DETECTION_CONTROL) + *cmds++ = 0x20000000; + + if (CP_INIT_MASK & CP_INIT_HEADER_DUMP) { + /* Header dump address */ + *cmds++ = 0x00000000; + /* Header dump enable and dump size */ + *cmds++ = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_DRAWCALL_FILTER_RANGE) { + /* Start range */ + *cmds++ = 0x00000000; + /* End range (inclusive) */ + *cmds++ = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_UCODE_WORKAROUND_MASK) + *cmds++ = 0x00000000; + + if (CP_INIT_MASK & CP_INIT_OPERATION_MODE_MASK) + *cmds++ = 0x00000002; + + if (CP_INIT_MASK & CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK) { + uint64_t gpuaddr = adreno_dev->pwrup_reglist.gpuaddr; + + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = upper_32_bits(gpuaddr); + *cmds++ = 0; + + } else if (CP_INIT_MASK & CP_INIT_REGISTER_INIT_LIST) { + uint64_t gpuaddr = adreno_dev->pwrup_reglist.gpuaddr; + + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = upper_32_bits(gpuaddr); + /* Size is in dwords */ + *cmds++ = (sizeof(a6xx_ifpc_pwrup_reglist) + + sizeof(a6xx_pwrup_reglist)) >> 2; + } + + /* Pad rest of the cmds with 0's */ + while ((unsigned int)(cmds - start) < count) + *cmds++ = 0x0; +} + +/* + * a6xx_send_cp_init() - Initialize ringbuffer + * @adreno_dev: Pointer to adreno device + * @rb: Pointer to the ringbuffer of device + * + * Submit commands for ME initialization, + */ +static int a6xx_send_cp_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 12); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_type7_packet(CP_ME_INIT, 11); + + _set_ordinals(adreno_dev, cmds, 11); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "CP initialization failed to idle\n"); + + return ret; +} + +/* + * Follow the ME_INIT sequence with a preemption yield to allow the GPU to move + * to a different ringbuffer, if desired + */ +static int _preemption_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int *cmds, + struct kgsl_context *context) +{ + unsigned int *cmds_orig = cmds; + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + *cmds++ = cp_type7_packet(CP_SET_PSEUDO_REGISTER, 6); + *cmds++ = 1; + cmds += cp_gpuaddr(adreno_dev, cmds, + rb->preemption_desc.gpuaddr); + + *cmds++ = 2; + cmds += cp_gpuaddr(adreno_dev, cmds, + rb->secure_preemption_desc.gpuaddr); + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + *cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + cmds += cp_gpuaddr(adreno_dev, cmds, 0x0); + *cmds++ = 0; + /* generate interrupt on preemption completion */ + *cmds++ = 0; + + return cmds - cmds_orig; +} + +static int a6xx_post_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int *cmds, *start; + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_preemption_enabled(adreno_dev)) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 42); + if (IS_ERR(cmds)) { + dev_err(device->dev, + "error allocating preemption init cmds\n"); + return PTR_ERR(cmds); + } + start = cmds; + + cmds += _preemption_init(adreno_dev, rb, cmds, NULL); + + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + if (ret) + adreno_spin_idle_debug(adreno_dev, + "hw preemption initialization failed to idle\n"); + + return ret; +} + +/* + * a6xx_rb_start() - Start the ringbuffer + * @adreno_dev: Pointer to adreno device + * @start_type: Warm or cold start + */ +static int a6xx_rb_start(struct adreno_device *adreno_dev, + unsigned int start_type) +{ + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + struct kgsl_device *device = &adreno_dev->dev; + uint64_t addr; + int ret; + + addr = SCRATCH_RPTR_GPU_ADDR(device, rb->id); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, addr); + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + */ + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, + A6XX_CP_RB_CNTL_DEFAULT); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_BASE, + ADRENO_REG_CP_RB_BASE_HI, rb->buffer_desc.gpuaddr); + + ret = a6xx_microcode_load(adreno_dev); + if (ret) + return ret; + + /* Clear the SQE_HALT to start the CP engine */ + kgsl_regwrite(device, A6XX_CP_SQE_CNTL, 1); + + ret = a6xx_send_cp_init(adreno_dev, rb); + if (ret) + return ret; + + /* GPU comes up in secured mode, make it unsecured by default */ + ret = adreno_set_unsecured_mode(adreno_dev, rb); + if (ret) + return ret; + + return a6xx_post_start(adreno_dev); +} + +/* + * a6xx_sptprac_enable() - Power on SPTPRAC + * @adreno_dev: Pointer to Adreno device + */ +static int a6xx_sptprac_enable(struct adreno_device *adreno_dev) +{ + if (adreno_is_a608(adreno_dev)) + return 0; + + return a6xx_gmu_sptprac_enable(adreno_dev); +} + +/* + * a6xx_sptprac_disable() - Power off SPTPRAC + * @adreno_dev: Pointer to Adreno device + */ +static void a6xx_sptprac_disable(struct adreno_device *adreno_dev) +{ + if (adreno_is_a608(adreno_dev)) + return; + + a6xx_gmu_sptprac_disable(adreno_dev); +} + +/* + * a6xx_sptprac_is_on() - Check if SPTP is on using pwr status register + * @adreno_dev - Pointer to adreno_device + * This check should only be performed if the keepalive bit is set or it + * can be guaranteed that the power state of the GPU will remain unchanged + */ +bool a6xx_sptprac_is_on(struct adreno_device *adreno_dev) +{ + if (!adreno_has_sptprac_gdsc(adreno_dev)) + return true; + + return a6xx_gmu_sptprac_is_on(adreno_dev); +} + +unsigned int a6xx_set_marker( + unsigned int *cmds, enum adreno_cp_marker_type type) +{ + unsigned int cmd = 0; + + *cmds++ = cp_type7_packet(CP_SET_MARKER, 1); + + /* + * Indicate the beginning and end of the IB1 list with a SET_MARKER. + * Among other things, this will implicitly enable and disable + * preemption respectively. IFPC can also be disabled and enabled + * with a SET_MARKER. Bit 8 tells the CP the marker is for IFPC. + */ + switch (type) { + case IFPC_DISABLE: + cmd = 0x101; + break; + case IFPC_ENABLE: + cmd = 0x100; + break; + case IB1LIST_START: + cmd = 0xD; + break; + case IB1LIST_END: + cmd = 0xE; + break; + } + + *cmds++ = cmd; + return 2; +} + +static int _load_firmware(struct kgsl_device *device, const char *fwfile, + struct adreno_firmware *firmware) +{ + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, fwfile, device->dev); + + if (ret) { + dev_err(device->dev, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + ret = kgsl_allocate_global(device, &firmware->memdesc, fw->size - 4, + KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_UCODE, + "ucode"); + + if (!ret) { + memcpy(firmware->memdesc.hostptr, &fw->data[4], fw->size - 4); + firmware->size = (fw->size - 4) / sizeof(uint32_t); + firmware->version = *(unsigned int *)&fw->data[4]; + } + + release_firmware(fw); + return ret; +} + +/* + * a6xx_gpu_keepalive() - GMU reg write to request GPU stays on + * @adreno_dev: Pointer to the adreno device that has the GMU + * @state: State to set: true is ON, false is OFF + */ +static inline void a6xx_gpu_keepalive(struct adreno_device *adreno_dev, + bool state) +{ + if (!gmu_core_isenabled(KGSL_DEVICE(adreno_dev))) + return; + + adreno_write_gmureg(adreno_dev, + ADRENO_REG_GMU_PWR_COL_KEEPALIVE, state); +} + +/* Bitmask for GPU idle status check */ +#define GPUBUSYIGNAHB BIT(23) +static bool a6xx_hw_isidle(struct adreno_device *adreno_dev) +{ + unsigned int reg; + + gmu_core_regread(KGSL_DEVICE(adreno_dev), + A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, ®); + if (reg & GPUBUSYIGNAHB) + return false; + return true; +} + +/* + * a6xx_microcode_read() - Read microcode + * @adreno_dev: Pointer to adreno device + */ +static int a6xx_microcode_read(struct adreno_device *adreno_dev) +{ + int ret; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *sqe_fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); + + if (sqe_fw->memdesc.hostptr == NULL) { + ret = _load_firmware(device, adreno_dev->gpucore->sqefw_name, + sqe_fw); + if (ret) + return ret; + } + + return gmu_core_dev_load_firmware(device); +} + +static int a6xx_soft_reset(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + unsigned long time; + bool vbif_acked = false; + + /* + * For the soft reset case with GMU enabled this part is done + * by the GMU firmware + */ + if (gmu_core_isenabled(device) && + !test_bit(ADRENO_DEVICE_HARD_RESET, &adreno_dev->priv)) + return 0; + + + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 1); + /* + * Do a dummy read to get a brief read cycle delay for the + * reset to take effect + */ + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, ®); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 0); + + /* Wait for the VBIF reset ack to complete */ + time = jiffies + msecs_to_jiffies(VBIF_RESET_ACK_TIMEOUT); + + do { + kgsl_regread(device, A6XX_RBBM_VBIF_GX_RESET_STATUS, ®); + if ((reg & VBIF_RESET_ACK_MASK) == VBIF_RESET_ACK_MASK) { + vbif_acked = true; + break; + } + cpu_relax(); + } while (!time_after(jiffies, time)); + + if (!vbif_acked) + return -ETIMEDOUT; + + /* Clear GBIF client halt and CX arbiter halt */ + adreno_deassert_gbif_halt(adreno_dev); + + a6xx_sptprac_enable(adreno_dev); + + return 0; +} + +static int64_t a6xx_read_throttling_counters(struct adreno_device *adreno_dev) +{ + int i; + int64_t adj = 0; + uint32_t counts[ADRENO_GPMU_THROTTLE_COUNTERS]; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + + for (i = 0; i < ARRAY_SIZE(counts); i++) { + if (!adreno_dev->gpmu_throttle_counters[i]) + counts[i] = 0; + else + counts[i] = counter_delta(KGSL_DEVICE(adreno_dev), + adreno_dev->gpmu_throttle_counters[i], + &busy->throttle_cycles[i]); + } + + /* + * The adjustment is the number of cycles lost to throttling, which + * is calculated as a weighted average of the cycles throttled + * at 10%, 50%, and 90%. The adjustment is negative because in A6XX, + * the busy count includes the throttled cycles. Therefore, we want + * to remove them to prevent appearing to be busier than + * we actually are. + */ + adj = -((counts[0] * 1) + (counts[1] * 5) + (counts[2] * 9)) / 10; + + trace_kgsl_clock_throttling(0, counts[1], counts[2], + counts[0], adj); + return adj; +} + +static void a6xx_count_throttles(struct adreno_device *adreno_dev, + uint64_t adj) +{ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM) || + !test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag)) + return; + + gmu_core_regread(KGSL_DEVICE(adreno_dev), + adreno_dev->lm_threshold_count, + &adreno_dev->lm_threshold_cross); +} + +/** + * a6xx_reset() - Helper function to reset the GPU + * @device: Pointer to the KGSL device structure for the GPU + * @fault: Type of fault. Needed to skip soft reset for MMU fault + * + * Try to reset the GPU to recover from a fault. First, try to do a low latency + * soft reset. If the soft reset fails for some reason, then bring out the big + * guns and toggle the footswitch. + */ +static int a6xx_reset(struct kgsl_device *device, int fault) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int ret = -EINVAL; + int i = 0; + + /* Use the regular reset sequence for No GMU */ + if (!gmu_core_isenabled(device)) + return adreno_reset(device, fault); + + /* Transition from ACTIVE to RESET state */ + kgsl_pwrctrl_change_state(device, KGSL_STATE_RESET); + + if (ret) { + /* If soft reset failed/skipped, then pull the power */ + set_bit(ADRENO_DEVICE_HARD_RESET, &adreno_dev->priv); + /* since device is officially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* Keep trying to start the device until it works */ + for (i = 0; i < NUM_TIMES_RESET_RETRY; i++) { + ret = adreno_start(device, 0); + if (!ret) + break; + + msleep(20); + } + } + + clear_bit(ADRENO_DEVICE_HARD_RESET, &adreno_dev->priv); + + if (ret) + return ret; + + if (i != 0) + dev_warn(device->dev, + "Device hard reset tried %d tries\n", i); + + /* + * If active_cnt is non-zero then the system was active before + * going into a reset - put it back in that state + */ + + if (atomic_read(&device->active_cnt)) + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + else + kgsl_pwrctrl_change_state(device, KGSL_STATE_NAP); + + return ret; +} + +static void a6xx_cp_hw_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status1, status2; + + kgsl_regread(device, A6XX_CP_INTERRUPT_STATUS, &status1); + + if (status1 & BIT(A6XX_CP_OPCODE_ERROR)) { + unsigned int opcode; + + kgsl_regwrite(device, A6XX_CP_SQE_STAT_ADDR, 1); + kgsl_regread(device, A6XX_CP_SQE_STAT_DATA, &opcode); + dev_crit_ratelimited(device->dev, + "CP opcode error interrupt | opcode=0x%8.8x\n", opcode); + } + if (status1 & BIT(A6XX_CP_UCODE_ERROR)) + dev_crit_ratelimited(device->dev, "CP ucode error interrupt\n"); + if (status1 & BIT(A6XX_CP_HW_FAULT_ERROR)) { + kgsl_regread(device, A6XX_CP_HW_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", status2); + } + if (status1 & BIT(A6XX_CP_REGISTER_PROTECTION_ERROR)) { + kgsl_regread(device, A6XX_CP_PROTECT_STATUS, &status2); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error | %s | addr=%x | status=%x\n", + status2 & (1 << 20) ? "READ" : "WRITE", + status2 & 0x3FFFF, status2); + } + if (status1 & BIT(A6XX_CP_AHB_ERROR)) + dev_crit_ratelimited(device->dev, + "CP AHB error interrupt\n"); + if (status1 & BIT(A6XX_CP_VSD_PARITY_ERROR)) + dev_crit_ratelimited(device->dev, + "CP VSD decoder parity error\n"); + if (status1 & BIT(A6XX_CP_ILLEGAL_INSTR_ERROR)) + dev_crit_ratelimited(device->dev, + "CP Illegal instruction error\n"); + +} + +static void a6xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + switch (bit) { + case A6XX_INT_CP_AHB_ERROR: + dev_crit_ratelimited(device->dev, "CP: AHB bus error\n"); + break; + case A6XX_INT_ATB_ASYNCFIFO_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB ASYNC overflow\n"); + break; + case A6XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus overflow\n"); + break; + case A6XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + case A6XX_INT_UCHE_TRAP_INTR: + dev_crit_ratelimited(device->dev, "UCHE: Trap interrupt\n"); + break; + case A6XX_INT_TSB_WRITE_ERROR: + dev_crit_ratelimited(device->dev, "TSB: Write error interrupt\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt %d\n", + bit); + } +} + +/* GPU System Cache control registers */ +#define A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_0 0x4 +#define A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1 0x8 + +static inline void _reg_rmw(void __iomem *regaddr, + unsigned int mask, unsigned int bits) +{ + unsigned int val = 0; + + val = __raw_readl(regaddr); + /* Make sure the above read completes before we proceed */ + rmb(); + val &= ~mask; + __raw_writel(val | bits, regaddr); + /* Make sure the above write posts before we proceed*/ + wmb(); +} + +/* + * a6xx_llc_configure_gpu_scid() - Program the sub-cache ID for all GPU blocks + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_configure_gpu_scid(struct adreno_device *adreno_dev) +{ + uint32_t gpu_scid; + uint32_t gpu_cntl1_val = 0; + int i; + + gpu_scid = adreno_llc_get_scid(adreno_dev->gpu_llc_slice); + for (i = 0; i < A6XX_LLC_NUM_GPU_SCIDS; i++) + gpu_cntl1_val = (gpu_cntl1_val << A6XX_GPU_LLC_SCID_NUM_BITS) + | gpu_scid; + + if (adreno_is_a640(adreno_dev)) { + kgsl_regrmw(KGSL_DEVICE(adreno_dev), A6XX_GBIF_SCACHE_CNTL1, + A6XX_GPU_LLC_SCID_MASK, gpu_cntl1_val); + } else { + void __iomem *gpu_cx_reg; + + gpu_cx_reg = ioremap(A6XX_GPU_CX_REG_BASE, + A6XX_GPU_CX_REG_SIZE); + _reg_rmw(gpu_cx_reg + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1, + A6XX_GPU_LLC_SCID_MASK, gpu_cntl1_val); + iounmap(gpu_cx_reg); + } +} + +/* + * a6xx_llc_configure_gpuhtw_scid() - Program the SCID for GPU pagetables + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_configure_gpuhtw_scid(struct adreno_device *adreno_dev) +{ + uint32_t gpuhtw_scid; + void __iomem *gpu_cx_reg; + + /* + * On A640, the GPUHTW SCID is configured via a NoC override in the + * XBL image. + */ + if (adreno_is_a640(adreno_dev)) + return; + + gpuhtw_scid = adreno_llc_get_scid(adreno_dev->gpuhtw_llc_slice); + + gpu_cx_reg = ioremap(A6XX_GPU_CX_REG_BASE, A6XX_GPU_CX_REG_SIZE); + _reg_rmw(gpu_cx_reg + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1, + A6XX_GPUHTW_LLC_SCID_MASK, + gpuhtw_scid << A6XX_GPUHTW_LLC_SCID_SHIFT); + iounmap(gpu_cx_reg); +} + +/* + * a6xx_llc_enable_overrides() - Override the page attributes + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_enable_overrides(struct adreno_device *adreno_dev) +{ + void __iomem *gpu_cx_reg; + + /* + * Attributes override through GBIF is not supported with MMU-500. + * Attributes are used as configured through SMMU pagetable entries. + */ + if (adreno_is_a640(adreno_dev)) + return; + + /* + * 0x3: readnoallocoverrideen=0 + * read-no-alloc=0 - Allocate lines on read miss + * writenoallocoverrideen=1 + * write-no-alloc=1 - Do not allocates lines on write miss + */ + gpu_cx_reg = ioremap(A6XX_GPU_CX_REG_BASE, A6XX_GPU_CX_REG_SIZE); + __raw_writel(0x3, gpu_cx_reg + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_0); + /* Make sure the above write posts before we proceed*/ + wmb(); + iounmap(gpu_cx_reg); +} + +static const char *fault_block[8] = { + [0] = "CP", + [1] = "UCHE", + [2] = "VFD", + [3] = "UCHE", + [4] = "CCU", + [5] = "unknown", + [6] = "CDP Prefetch", + [7] = "GPMU", +}; + +static const char *uche_client[8] = { + [0] = "VFD", + [1] = "SP", + [2] = "VSC", + [3] = "VPC", + [4] = "HLSQ", + [5] = "PC", + [6] = "LRZ", + [7] = "unknown", +}; + +static const char *a6xx_iommu_fault_block(struct adreno_device *adreno_dev, + unsigned int fsynr1) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int client_id; + unsigned int uche_client_id; + + client_id = fsynr1 & 0xff; + + if (client_id >= ARRAY_SIZE(fault_block)) + return "unknown"; + else if (client_id != 3) + return fault_block[client_id]; + + mutex_lock(&device->mutex); + kgsl_regread(device, A6XX_UCHE_CLIENT_PF, &uche_client_id); + mutex_unlock(&device->mutex); + + return uche_client[uche_client_id & A6XX_UCHE_CLIENT_PF_CLIENT_ID_MASK]; +} + +static void a6xx_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_is_preemption_enabled(adreno_dev)) + a6xx_preemption_trigger(adreno_dev); + + adreno_dispatcher_schedule(device); +} + +#define A6XX_INT_MASK \ + ((1 << A6XX_INT_CP_AHB_ERROR) | \ + (1 << A6XX_INT_ATB_ASYNCFIFO_OVERFLOW) | \ + (1 << A6XX_INT_RBBM_GPC_ERROR) | \ + (1 << A6XX_INT_CP_SW) | \ + (1 << A6XX_INT_CP_HW_ERROR) | \ + (1 << A6XX_INT_CP_IB2) | \ + (1 << A6XX_INT_CP_IB1) | \ + (1 << A6XX_INT_CP_RB) | \ + (1 << A6XX_INT_CP_CACHE_FLUSH_TS) | \ + (1 << A6XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A6XX_INT_RBBM_HANG_DETECT) | \ + (1 << A6XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A6XX_INT_UCHE_TRAP_INTR) | \ + (1 << A6XX_INT_TSB_WRITE_ERROR)) + +static struct adreno_irq_funcs a6xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 2 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 3 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 4 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 5 - UNUSED */ + /* 6 - RBBM_ATB_ASYNC_OVERFLOW */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 7 - GPC_ERR */ + ADRENO_IRQ_CALLBACK(a6xx_preemption_callback),/* 8 - CP_SW */ + ADRENO_IRQ_CALLBACK(a6xx_cp_hw_err_callback), /* 9 - CP_HW_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 10 - CP_CCU_FLUSH_DEPTH_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 11 - CP_CCU_FLUSH_COLOR_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 12 - CP_CCU_RESOLVE_TS */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 15 - CP_RB_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 16 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_WT_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - UNUSED */ + ADRENO_IRQ_CALLBACK(a6xx_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 21 - UNUSED */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */ + /* 23 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 24 - UCHE_OOB_ACCESS */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 25 - UCHE_TRAP_INTR */ + ADRENO_IRQ_CALLBACK(NULL), /* 26 - DEBBUS_INTR_0 */ + ADRENO_IRQ_CALLBACK(NULL), /* 27 - DEBBUS_INTR_1 */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 28 - TSBWRITEERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 29 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 30 - ISDB_CPU_IRQ */ + ADRENO_IRQ_CALLBACK(NULL), /* 31 - ISDB_UNDER_DEBUG */ +}; + +static struct adreno_irq a6xx_irq = { + .funcs = a6xx_irq_funcs, + .mask = A6XX_INT_MASK, +}; + +static struct adreno_coresight_register a6xx_coresight_regs[] = { + { A6XX_DBGC_CFG_DBGBUS_SEL_A }, + { A6XX_DBGC_CFG_DBGBUS_SEL_B }, + { A6XX_DBGC_CFG_DBGBUS_SEL_C }, + { A6XX_DBGC_CFG_DBGBUS_SEL_D }, + { A6XX_DBGC_CFG_DBGBUS_CNTLT }, + { A6XX_DBGC_CFG_DBGBUS_CNTLM }, + { A6XX_DBGC_CFG_DBGBUS_OPL }, + { A6XX_DBGC_CFG_DBGBUS_OPE }, + { A6XX_DBGC_CFG_DBGBUS_IVTL_0 }, + { A6XX_DBGC_CFG_DBGBUS_IVTL_1 }, + { A6XX_DBGC_CFG_DBGBUS_IVTL_2 }, + { A6XX_DBGC_CFG_DBGBUS_IVTL_3 }, + { A6XX_DBGC_CFG_DBGBUS_MASKL_0 }, + { A6XX_DBGC_CFG_DBGBUS_MASKL_1 }, + { A6XX_DBGC_CFG_DBGBUS_MASKL_2 }, + { A6XX_DBGC_CFG_DBGBUS_MASKL_3 }, + { A6XX_DBGC_CFG_DBGBUS_BYTEL_0 }, + { A6XX_DBGC_CFG_DBGBUS_BYTEL_1 }, + { A6XX_DBGC_CFG_DBGBUS_IVTE_0 }, + { A6XX_DBGC_CFG_DBGBUS_IVTE_1 }, + { A6XX_DBGC_CFG_DBGBUS_IVTE_2 }, + { A6XX_DBGC_CFG_DBGBUS_IVTE_3 }, + { A6XX_DBGC_CFG_DBGBUS_MASKE_0 }, + { A6XX_DBGC_CFG_DBGBUS_MASKE_1 }, + { A6XX_DBGC_CFG_DBGBUS_MASKE_2 }, + { A6XX_DBGC_CFG_DBGBUS_MASKE_3 }, + { A6XX_DBGC_CFG_DBGBUS_NIBBLEE }, + { A6XX_DBGC_CFG_DBGBUS_PTRC0 }, + { A6XX_DBGC_CFG_DBGBUS_PTRC1 }, + { A6XX_DBGC_CFG_DBGBUS_LOADREG }, + { A6XX_DBGC_CFG_DBGBUS_IDX }, + { A6XX_DBGC_CFG_DBGBUS_CLRC }, + { A6XX_DBGC_CFG_DBGBUS_LOADIVT }, + { A6XX_DBGC_VBIF_DBG_CNTL }, + { A6XX_DBGC_DBG_LO_HI_GPIO }, + { A6XX_DBGC_EXT_TRACE_BUS_CNTL }, + { A6XX_DBGC_READ_AHB_THROUGH_DBG }, + { A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1 }, + { A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2 }, + { A6XX_DBGC_EVT_CFG }, + { A6XX_DBGC_EVT_INTF_SEL_0 }, + { A6XX_DBGC_EVT_INTF_SEL_1 }, + { A6XX_DBGC_PERF_ATB_CFG }, + { A6XX_DBGC_PERF_ATB_COUNTER_SEL_0 }, + { A6XX_DBGC_PERF_ATB_COUNTER_SEL_1 }, + { A6XX_DBGC_PERF_ATB_COUNTER_SEL_2 }, + { A6XX_DBGC_PERF_ATB_COUNTER_SEL_3 }, + { A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 }, + { A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 }, + { A6XX_DBGC_PERF_ATB_DRAIN_CMD }, + { A6XX_DBGC_ECO_CNTL }, + { A6XX_DBGC_AHB_DBG_CNTL }, +}; + +static struct adreno_coresight_register a6xx_coresight_regs_cx[] = { + { A6XX_CX_DBGC_CFG_DBGBUS_SEL_A }, + { A6XX_CX_DBGC_CFG_DBGBUS_SEL_B }, + { A6XX_CX_DBGC_CFG_DBGBUS_SEL_C }, + { A6XX_CX_DBGC_CFG_DBGBUS_SEL_D }, + { A6XX_CX_DBGC_CFG_DBGBUS_CNTLT }, + { A6XX_CX_DBGC_CFG_DBGBUS_CNTLM }, + { A6XX_CX_DBGC_CFG_DBGBUS_OPL }, + { A6XX_CX_DBGC_CFG_DBGBUS_OPE }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3 }, + { A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTE_0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTE_1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTE_2 }, + { A6XX_CX_DBGC_CFG_DBGBUS_IVTE_3 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKE_0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKE_1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKE_2 }, + { A6XX_CX_DBGC_CFG_DBGBUS_MASKE_3 }, + { A6XX_CX_DBGC_CFG_DBGBUS_NIBBLEE }, + { A6XX_CX_DBGC_CFG_DBGBUS_PTRC0 }, + { A6XX_CX_DBGC_CFG_DBGBUS_PTRC1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_LOADREG }, + { A6XX_CX_DBGC_CFG_DBGBUS_IDX }, + { A6XX_CX_DBGC_CFG_DBGBUS_CLRC }, + { A6XX_CX_DBGC_CFG_DBGBUS_LOADIVT }, + { A6XX_CX_DBGC_VBIF_DBG_CNTL }, + { A6XX_CX_DBGC_DBG_LO_HI_GPIO }, + { A6XX_CX_DBGC_EXT_TRACE_BUS_CNTL }, + { A6XX_CX_DBGC_READ_AHB_THROUGH_DBG }, + { A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1 }, + { A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2 }, + { A6XX_CX_DBGC_EVT_CFG }, + { A6XX_CX_DBGC_EVT_INTF_SEL_0 }, + { A6XX_CX_DBGC_EVT_INTF_SEL_1 }, + { A6XX_CX_DBGC_PERF_ATB_CFG }, + { A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_0 }, + { A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_1 }, + { A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_2 }, + { A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_3 }, + { A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 }, + { A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 }, + { A6XX_CX_DBGC_PERF_ATB_DRAIN_CMD }, + { A6XX_CX_DBGC_ECO_CNTL }, + { A6XX_CX_DBGC_AHB_DBG_CNTL }, +}; + +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_a, &a6xx_coresight_regs[0]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_b, &a6xx_coresight_regs[1]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_c, &a6xx_coresight_regs[2]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_d, &a6xx_coresight_regs[3]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlt, &a6xx_coresight_regs[4]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlm, &a6xx_coresight_regs[5]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_opl, &a6xx_coresight_regs[6]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ope, &a6xx_coresight_regs[7]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_0, &a6xx_coresight_regs[8]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_1, &a6xx_coresight_regs[9]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_2, &a6xx_coresight_regs[10]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_3, &a6xx_coresight_regs[11]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_0, &a6xx_coresight_regs[12]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_1, &a6xx_coresight_regs[13]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_2, &a6xx_coresight_regs[14]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_3, &a6xx_coresight_regs[15]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_0, &a6xx_coresight_regs[16]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_1, &a6xx_coresight_regs[17]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_0, &a6xx_coresight_regs[18]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_1, &a6xx_coresight_regs[19]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_2, &a6xx_coresight_regs[20]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_3, &a6xx_coresight_regs[21]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_0, &a6xx_coresight_regs[22]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_1, &a6xx_coresight_regs[23]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_2, &a6xx_coresight_regs[24]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_3, &a6xx_coresight_regs[25]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_nibblee, &a6xx_coresight_regs[26]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc0, &a6xx_coresight_regs[27]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc1, &a6xx_coresight_regs[28]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadreg, &a6xx_coresight_regs[29]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_idx, &a6xx_coresight_regs[30]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_clrc, &a6xx_coresight_regs[31]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadivt, &a6xx_coresight_regs[32]); +static ADRENO_CORESIGHT_ATTR(vbif_dbg_cntl, &a6xx_coresight_regs[33]); +static ADRENO_CORESIGHT_ATTR(dbg_lo_hi_gpio, &a6xx_coresight_regs[34]); +static ADRENO_CORESIGHT_ATTR(ext_trace_bus_cntl, &a6xx_coresight_regs[35]); +static ADRENO_CORESIGHT_ATTR(read_ahb_through_dbg, &a6xx_coresight_regs[36]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf1, &a6xx_coresight_regs[37]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf2, &a6xx_coresight_regs[38]); +static ADRENO_CORESIGHT_ATTR(evt_cfg, &a6xx_coresight_regs[39]); +static ADRENO_CORESIGHT_ATTR(evt_intf_sel_0, &a6xx_coresight_regs[40]); +static ADRENO_CORESIGHT_ATTR(evt_intf_sel_1, &a6xx_coresight_regs[41]); +static ADRENO_CORESIGHT_ATTR(perf_atb_cfg, &a6xx_coresight_regs[42]); +static ADRENO_CORESIGHT_ATTR(perf_atb_counter_sel_0, &a6xx_coresight_regs[43]); +static ADRENO_CORESIGHT_ATTR(perf_atb_counter_sel_1, &a6xx_coresight_regs[44]); +static ADRENO_CORESIGHT_ATTR(perf_atb_counter_sel_2, &a6xx_coresight_regs[45]); +static ADRENO_CORESIGHT_ATTR(perf_atb_counter_sel_3, &a6xx_coresight_regs[46]); +static ADRENO_CORESIGHT_ATTR(perf_atb_trig_intf_sel_0, + &a6xx_coresight_regs[47]); +static ADRENO_CORESIGHT_ATTR(perf_atb_trig_intf_sel_1, + &a6xx_coresight_regs[48]); +static ADRENO_CORESIGHT_ATTR(perf_atb_drain_cmd, &a6xx_coresight_regs[49]); +static ADRENO_CORESIGHT_ATTR(eco_cntl, &a6xx_coresight_regs[50]); +static ADRENO_CORESIGHT_ATTR(ahb_dbg_cntl, &a6xx_coresight_regs[51]); + +/*CX debug registers*/ +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_sel_a, + &a6xx_coresight_regs_cx[0]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_sel_b, + &a6xx_coresight_regs_cx[1]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_sel_c, + &a6xx_coresight_regs_cx[2]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_sel_d, + &a6xx_coresight_regs_cx[3]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_cntlt, + &a6xx_coresight_regs_cx[4]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_cntlm, + &a6xx_coresight_regs_cx[5]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_opl, + &a6xx_coresight_regs_cx[6]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ope, + &a6xx_coresight_regs_cx[7]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivtl_0, + &a6xx_coresight_regs_cx[8]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivtl_1, + &a6xx_coresight_regs_cx[9]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivtl_2, + &a6xx_coresight_regs_cx[10]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivtl_3, + &a6xx_coresight_regs_cx[11]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maskl_0, + &a6xx_coresight_regs_cx[12]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maskl_1, + &a6xx_coresight_regs_cx[13]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maskl_2, + &a6xx_coresight_regs_cx[14]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maskl_3, + &a6xx_coresight_regs_cx[15]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_bytel_0, + &a6xx_coresight_regs_cx[16]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_bytel_1, + &a6xx_coresight_regs_cx[17]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivte_0, + &a6xx_coresight_regs_cx[18]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivte_1, + &a6xx_coresight_regs_cx[19]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivte_2, + &a6xx_coresight_regs_cx[20]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ivte_3, + &a6xx_coresight_regs_cx[21]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maske_0, + &a6xx_coresight_regs_cx[22]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maske_1, + &a6xx_coresight_regs_cx[23]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maske_2, + &a6xx_coresight_regs_cx[24]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_maske_3, + &a6xx_coresight_regs_cx[25]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_nibblee, + &a6xx_coresight_regs_cx[26]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ptrc0, + &a6xx_coresight_regs_cx[27]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_ptrc1, + &a6xx_coresight_regs_cx[28]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_loadreg, + &a6xx_coresight_regs_cx[29]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_idx, + &a6xx_coresight_regs_cx[30]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_clrc, + &a6xx_coresight_regs_cx[31]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_loadivt, + &a6xx_coresight_regs_cx[32]); +static ADRENO_CORESIGHT_ATTR(cx_vbif_dbg_cntl, + &a6xx_coresight_regs_cx[33]); +static ADRENO_CORESIGHT_ATTR(cx_dbg_lo_hi_gpio, + &a6xx_coresight_regs_cx[34]); +static ADRENO_CORESIGHT_ATTR(cx_ext_trace_bus_cntl, + &a6xx_coresight_regs_cx[35]); +static ADRENO_CORESIGHT_ATTR(cx_read_ahb_through_dbg, + &a6xx_coresight_regs_cx[36]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_trace_buf1, + &a6xx_coresight_regs_cx[37]); +static ADRENO_CORESIGHT_ATTR(cx_cfg_dbgbus_trace_buf2, + &a6xx_coresight_regs_cx[38]); +static ADRENO_CORESIGHT_ATTR(cx_evt_cfg, + &a6xx_coresight_regs_cx[39]); +static ADRENO_CORESIGHT_ATTR(cx_evt_intf_sel_0, + &a6xx_coresight_regs_cx[40]); +static ADRENO_CORESIGHT_ATTR(cx_evt_intf_sel_1, + &a6xx_coresight_regs_cx[41]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_cfg, + &a6xx_coresight_regs_cx[42]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_counter_sel_0, + &a6xx_coresight_regs_cx[43]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_counter_sel_1, + &a6xx_coresight_regs_cx[44]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_counter_sel_2, + &a6xx_coresight_regs_cx[45]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_counter_sel_3, + &a6xx_coresight_regs_cx[46]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_trig_intf_sel_0, + &a6xx_coresight_regs_cx[47]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_trig_intf_sel_1, + &a6xx_coresight_regs_cx[48]); +static ADRENO_CORESIGHT_ATTR(cx_perf_atb_drain_cmd, + &a6xx_coresight_regs_cx[49]); +static ADRENO_CORESIGHT_ATTR(cx_eco_cntl, + &a6xx_coresight_regs_cx[50]); +static ADRENO_CORESIGHT_ATTR(cx_ahb_dbg_cntl, + &a6xx_coresight_regs_cx[51]); + +static struct attribute *a6xx_coresight_attrs[] = { + &coresight_attr_cfg_dbgbus_sel_a.attr.attr, + &coresight_attr_cfg_dbgbus_sel_b.attr.attr, + &coresight_attr_cfg_dbgbus_sel_c.attr.attr, + &coresight_attr_cfg_dbgbus_sel_d.attr.attr, + &coresight_attr_cfg_dbgbus_cntlt.attr.attr, + &coresight_attr_cfg_dbgbus_cntlm.attr.attr, + &coresight_attr_cfg_dbgbus_opl.attr.attr, + &coresight_attr_cfg_dbgbus_ope.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_3.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_0.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_1.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_2.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_3.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_0.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_3.attr.attr, + &coresight_attr_cfg_dbgbus_maske_0.attr.attr, + &coresight_attr_cfg_dbgbus_maske_1.attr.attr, + &coresight_attr_cfg_dbgbus_maske_2.attr.attr, + &coresight_attr_cfg_dbgbus_maske_3.attr.attr, + &coresight_attr_cfg_dbgbus_nibblee.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc0.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc1.attr.attr, + &coresight_attr_cfg_dbgbus_loadreg.attr.attr, + &coresight_attr_cfg_dbgbus_idx.attr.attr, + &coresight_attr_cfg_dbgbus_clrc.attr.attr, + &coresight_attr_cfg_dbgbus_loadivt.attr.attr, + &coresight_attr_vbif_dbg_cntl.attr.attr, + &coresight_attr_dbg_lo_hi_gpio.attr.attr, + &coresight_attr_ext_trace_bus_cntl.attr.attr, + &coresight_attr_read_ahb_through_dbg.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf1.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf2.attr.attr, + &coresight_attr_evt_cfg.attr.attr, + &coresight_attr_evt_intf_sel_0.attr.attr, + &coresight_attr_evt_intf_sel_1.attr.attr, + &coresight_attr_perf_atb_cfg.attr.attr, + &coresight_attr_perf_atb_counter_sel_0.attr.attr, + &coresight_attr_perf_atb_counter_sel_1.attr.attr, + &coresight_attr_perf_atb_counter_sel_2.attr.attr, + &coresight_attr_perf_atb_counter_sel_3.attr.attr, + &coresight_attr_perf_atb_trig_intf_sel_0.attr.attr, + &coresight_attr_perf_atb_trig_intf_sel_1.attr.attr, + &coresight_attr_perf_atb_drain_cmd.attr.attr, + &coresight_attr_eco_cntl.attr.attr, + &coresight_attr_ahb_dbg_cntl.attr.attr, + NULL, +}; + +/*cx*/ +static struct attribute *a6xx_coresight_attrs_cx[] = { + &coresight_attr_cx_cfg_dbgbus_sel_a.attr.attr, + &coresight_attr_cx_cfg_dbgbus_sel_b.attr.attr, + &coresight_attr_cx_cfg_dbgbus_sel_c.attr.attr, + &coresight_attr_cx_cfg_dbgbus_sel_d.attr.attr, + &coresight_attr_cx_cfg_dbgbus_cntlt.attr.attr, + &coresight_attr_cx_cfg_dbgbus_cntlm.attr.attr, + &coresight_attr_cx_cfg_dbgbus_opl.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ope.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivtl_0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivtl_1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivtl_2.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivtl_3.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maskl_0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maskl_1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maskl_2.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maskl_3.attr.attr, + &coresight_attr_cx_cfg_dbgbus_bytel_0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_bytel_1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivte_0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivte_1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivte_2.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ivte_3.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maske_0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maske_1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maske_2.attr.attr, + &coresight_attr_cx_cfg_dbgbus_maske_3.attr.attr, + &coresight_attr_cx_cfg_dbgbus_nibblee.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ptrc0.attr.attr, + &coresight_attr_cx_cfg_dbgbus_ptrc1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_loadreg.attr.attr, + &coresight_attr_cx_cfg_dbgbus_idx.attr.attr, + &coresight_attr_cx_cfg_dbgbus_clrc.attr.attr, + &coresight_attr_cx_cfg_dbgbus_loadivt.attr.attr, + &coresight_attr_cx_vbif_dbg_cntl.attr.attr, + &coresight_attr_cx_dbg_lo_hi_gpio.attr.attr, + &coresight_attr_cx_ext_trace_bus_cntl.attr.attr, + &coresight_attr_cx_read_ahb_through_dbg.attr.attr, + &coresight_attr_cx_cfg_dbgbus_trace_buf1.attr.attr, + &coresight_attr_cx_cfg_dbgbus_trace_buf2.attr.attr, + &coresight_attr_cx_evt_cfg.attr.attr, + &coresight_attr_cx_evt_intf_sel_0.attr.attr, + &coresight_attr_cx_evt_intf_sel_1.attr.attr, + &coresight_attr_cx_perf_atb_cfg.attr.attr, + &coresight_attr_cx_perf_atb_counter_sel_0.attr.attr, + &coresight_attr_cx_perf_atb_counter_sel_1.attr.attr, + &coresight_attr_cx_perf_atb_counter_sel_2.attr.attr, + &coresight_attr_cx_perf_atb_counter_sel_3.attr.attr, + &coresight_attr_cx_perf_atb_trig_intf_sel_0.attr.attr, + &coresight_attr_cx_perf_atb_trig_intf_sel_1.attr.attr, + &coresight_attr_cx_perf_atb_drain_cmd.attr.attr, + &coresight_attr_cx_eco_cntl.attr.attr, + &coresight_attr_cx_ahb_dbg_cntl.attr.attr, + NULL, +}; + +static const struct attribute_group a6xx_coresight_group = { + .attrs = a6xx_coresight_attrs, +}; + +static const struct attribute_group *a6xx_coresight_groups[] = { + &a6xx_coresight_group, + NULL, +}; + +static const struct attribute_group a6xx_coresight_group_cx = { + .attrs = a6xx_coresight_attrs_cx, +}; + +static const struct attribute_group *a6xx_coresight_groups_cx[] = { + &a6xx_coresight_group_cx, + NULL, +}; + +static struct adreno_coresight a6xx_coresight = { + .registers = a6xx_coresight_regs, + .count = ARRAY_SIZE(a6xx_coresight_regs), + .groups = a6xx_coresight_groups, +}; + +static struct adreno_coresight a6xx_coresight_cx = { + .registers = a6xx_coresight_regs_cx, + .count = ARRAY_SIZE(a6xx_coresight_regs_cx), + .groups = a6xx_coresight_groups_cx, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_0_LO, + A6XX_RBBM_PERFCTR_CP_0_HI, 0, A6XX_CP_PERFCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_1_LO, + A6XX_RBBM_PERFCTR_CP_1_HI, 1, A6XX_CP_PERFCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_2_LO, + A6XX_RBBM_PERFCTR_CP_2_HI, 2, A6XX_CP_PERFCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_3_LO, + A6XX_RBBM_PERFCTR_CP_3_HI, 3, A6XX_CP_PERFCTR_CP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_4_LO, + A6XX_RBBM_PERFCTR_CP_4_HI, 4, A6XX_CP_PERFCTR_CP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_5_LO, + A6XX_RBBM_PERFCTR_CP_5_HI, 5, A6XX_CP_PERFCTR_CP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_6_LO, + A6XX_RBBM_PERFCTR_CP_6_HI, 6, A6XX_CP_PERFCTR_CP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_7_LO, + A6XX_RBBM_PERFCTR_CP_7_HI, 7, A6XX_CP_PERFCTR_CP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_8_LO, + A6XX_RBBM_PERFCTR_CP_8_HI, 8, A6XX_CP_PERFCTR_CP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_9_LO, + A6XX_RBBM_PERFCTR_CP_9_HI, 9, A6XX_CP_PERFCTR_CP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_10_LO, + A6XX_RBBM_PERFCTR_CP_10_HI, 10, A6XX_CP_PERFCTR_CP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_11_LO, + A6XX_RBBM_PERFCTR_CP_11_HI, 11, A6XX_CP_PERFCTR_CP_SEL_11 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_12_LO, + A6XX_RBBM_PERFCTR_CP_12_HI, 12, A6XX_CP_PERFCTR_CP_SEL_12 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CP_13_LO, + A6XX_RBBM_PERFCTR_CP_13_HI, 13, A6XX_CP_PERFCTR_CP_SEL_13 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_rbbm[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RBBM_0_LO, + A6XX_RBBM_PERFCTR_RBBM_0_HI, 15, A6XX_RBBM_PERFCTR_RBBM_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RBBM_1_LO, + A6XX_RBBM_PERFCTR_RBBM_1_HI, 15, A6XX_RBBM_PERFCTR_RBBM_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RBBM_2_LO, + A6XX_RBBM_PERFCTR_RBBM_2_HI, 16, A6XX_RBBM_PERFCTR_RBBM_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RBBM_3_LO, + A6XX_RBBM_PERFCTR_RBBM_3_HI, 17, A6XX_RBBM_PERFCTR_RBBM_SEL_3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_0_LO, + A6XX_RBBM_PERFCTR_PC_0_HI, 18, A6XX_PC_PERFCTR_PC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_1_LO, + A6XX_RBBM_PERFCTR_PC_1_HI, 19, A6XX_PC_PERFCTR_PC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_2_LO, + A6XX_RBBM_PERFCTR_PC_2_HI, 20, A6XX_PC_PERFCTR_PC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_3_LO, + A6XX_RBBM_PERFCTR_PC_3_HI, 21, A6XX_PC_PERFCTR_PC_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_4_LO, + A6XX_RBBM_PERFCTR_PC_4_HI, 22, A6XX_PC_PERFCTR_PC_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_5_LO, + A6XX_RBBM_PERFCTR_PC_5_HI, 23, A6XX_PC_PERFCTR_PC_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_6_LO, + A6XX_RBBM_PERFCTR_PC_6_HI, 24, A6XX_PC_PERFCTR_PC_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_PC_7_LO, + A6XX_RBBM_PERFCTR_PC_7_HI, 25, A6XX_PC_PERFCTR_PC_SEL_7 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_0_LO, + A6XX_RBBM_PERFCTR_VFD_0_HI, 26, A6XX_VFD_PERFCTR_VFD_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_1_LO, + A6XX_RBBM_PERFCTR_VFD_1_HI, 27, A6XX_VFD_PERFCTR_VFD_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_2_LO, + A6XX_RBBM_PERFCTR_VFD_2_HI, 28, A6XX_VFD_PERFCTR_VFD_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_3_LO, + A6XX_RBBM_PERFCTR_VFD_3_HI, 29, A6XX_VFD_PERFCTR_VFD_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_4_LO, + A6XX_RBBM_PERFCTR_VFD_4_HI, 30, A6XX_VFD_PERFCTR_VFD_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_5_LO, + A6XX_RBBM_PERFCTR_VFD_5_HI, 31, A6XX_VFD_PERFCTR_VFD_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_6_LO, + A6XX_RBBM_PERFCTR_VFD_6_HI, 32, A6XX_VFD_PERFCTR_VFD_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VFD_7_LO, + A6XX_RBBM_PERFCTR_VFD_7_HI, 33, A6XX_VFD_PERFCTR_VFD_SEL_7 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_0_LO, + A6XX_RBBM_PERFCTR_HLSQ_0_HI, 34, A6XX_HLSQ_PERFCTR_HLSQ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_1_LO, + A6XX_RBBM_PERFCTR_HLSQ_1_HI, 35, A6XX_HLSQ_PERFCTR_HLSQ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_2_LO, + A6XX_RBBM_PERFCTR_HLSQ_2_HI, 36, A6XX_HLSQ_PERFCTR_HLSQ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_3_LO, + A6XX_RBBM_PERFCTR_HLSQ_3_HI, 37, A6XX_HLSQ_PERFCTR_HLSQ_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_4_LO, + A6XX_RBBM_PERFCTR_HLSQ_4_HI, 38, A6XX_HLSQ_PERFCTR_HLSQ_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_HLSQ_5_LO, + A6XX_RBBM_PERFCTR_HLSQ_5_HI, 39, A6XX_HLSQ_PERFCTR_HLSQ_SEL_5 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_0_LO, + A6XX_RBBM_PERFCTR_VPC_0_HI, 40, A6XX_VPC_PERFCTR_VPC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_1_LO, + A6XX_RBBM_PERFCTR_VPC_1_HI, 41, A6XX_VPC_PERFCTR_VPC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_2_LO, + A6XX_RBBM_PERFCTR_VPC_2_HI, 42, A6XX_VPC_PERFCTR_VPC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_3_LO, + A6XX_RBBM_PERFCTR_VPC_3_HI, 43, A6XX_VPC_PERFCTR_VPC_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_4_LO, + A6XX_RBBM_PERFCTR_VPC_4_HI, 44, A6XX_VPC_PERFCTR_VPC_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VPC_5_LO, + A6XX_RBBM_PERFCTR_VPC_5_HI, 45, A6XX_VPC_PERFCTR_VPC_SEL_5 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CCU_0_LO, + A6XX_RBBM_PERFCTR_CCU_0_HI, 46, A6XX_RB_PERFCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CCU_1_LO, + A6XX_RBBM_PERFCTR_CCU_1_HI, 47, A6XX_RB_PERFCTR_CCU_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CCU_2_LO, + A6XX_RBBM_PERFCTR_CCU_2_HI, 48, A6XX_RB_PERFCTR_CCU_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CCU_3_LO, + A6XX_RBBM_PERFCTR_CCU_3_HI, 49, A6XX_RB_PERFCTR_CCU_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CCU_4_LO, + A6XX_RBBM_PERFCTR_CCU_4_HI, 50, A6XX_RB_PERFCTR_CCU_SEL_4 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TSE_0_LO, + A6XX_RBBM_PERFCTR_TSE_0_HI, 51, A6XX_GRAS_PERFCTR_TSE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TSE_1_LO, + A6XX_RBBM_PERFCTR_TSE_1_HI, 52, A6XX_GRAS_PERFCTR_TSE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TSE_2_LO, + A6XX_RBBM_PERFCTR_TSE_2_HI, 53, A6XX_GRAS_PERFCTR_TSE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TSE_3_LO, + A6XX_RBBM_PERFCTR_TSE_3_HI, 54, A6XX_GRAS_PERFCTR_TSE_SEL_3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RAS_0_LO, + A6XX_RBBM_PERFCTR_RAS_0_HI, 55, A6XX_GRAS_PERFCTR_RAS_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RAS_1_LO, + A6XX_RBBM_PERFCTR_RAS_1_HI, 56, A6XX_GRAS_PERFCTR_RAS_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RAS_2_LO, + A6XX_RBBM_PERFCTR_RAS_2_HI, 57, A6XX_GRAS_PERFCTR_RAS_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RAS_3_LO, + A6XX_RBBM_PERFCTR_RAS_3_HI, 58, A6XX_GRAS_PERFCTR_RAS_SEL_3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_0_LO, + A6XX_RBBM_PERFCTR_UCHE_0_HI, 59, A6XX_UCHE_PERFCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_1_LO, + A6XX_RBBM_PERFCTR_UCHE_1_HI, 60, A6XX_UCHE_PERFCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_2_LO, + A6XX_RBBM_PERFCTR_UCHE_2_HI, 61, A6XX_UCHE_PERFCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_3_LO, + A6XX_RBBM_PERFCTR_UCHE_3_HI, 62, A6XX_UCHE_PERFCTR_UCHE_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_4_LO, + A6XX_RBBM_PERFCTR_UCHE_4_HI, 63, A6XX_UCHE_PERFCTR_UCHE_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_5_LO, + A6XX_RBBM_PERFCTR_UCHE_5_HI, 64, A6XX_UCHE_PERFCTR_UCHE_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_6_LO, + A6XX_RBBM_PERFCTR_UCHE_6_HI, 65, A6XX_UCHE_PERFCTR_UCHE_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_7_LO, + A6XX_RBBM_PERFCTR_UCHE_7_HI, 66, A6XX_UCHE_PERFCTR_UCHE_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_8_LO, + A6XX_RBBM_PERFCTR_UCHE_8_HI, 67, A6XX_UCHE_PERFCTR_UCHE_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_9_LO, + A6XX_RBBM_PERFCTR_UCHE_9_HI, 68, A6XX_UCHE_PERFCTR_UCHE_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_10_LO, + A6XX_RBBM_PERFCTR_UCHE_10_HI, 69, + A6XX_UCHE_PERFCTR_UCHE_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_UCHE_11_LO, + A6XX_RBBM_PERFCTR_UCHE_11_HI, 70, + A6XX_UCHE_PERFCTR_UCHE_SEL_11 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_0_LO, + A6XX_RBBM_PERFCTR_TP_0_HI, 71, A6XX_TPL1_PERFCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_1_LO, + A6XX_RBBM_PERFCTR_TP_1_HI, 72, A6XX_TPL1_PERFCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_2_LO, + A6XX_RBBM_PERFCTR_TP_2_HI, 73, A6XX_TPL1_PERFCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_3_LO, + A6XX_RBBM_PERFCTR_TP_3_HI, 74, A6XX_TPL1_PERFCTR_TP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_4_LO, + A6XX_RBBM_PERFCTR_TP_4_HI, 75, A6XX_TPL1_PERFCTR_TP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_5_LO, + A6XX_RBBM_PERFCTR_TP_5_HI, 76, A6XX_TPL1_PERFCTR_TP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_6_LO, + A6XX_RBBM_PERFCTR_TP_6_HI, 77, A6XX_TPL1_PERFCTR_TP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_7_LO, + A6XX_RBBM_PERFCTR_TP_7_HI, 78, A6XX_TPL1_PERFCTR_TP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_8_LO, + A6XX_RBBM_PERFCTR_TP_8_HI, 79, A6XX_TPL1_PERFCTR_TP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_9_LO, + A6XX_RBBM_PERFCTR_TP_9_HI, 80, A6XX_TPL1_PERFCTR_TP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_10_LO, + A6XX_RBBM_PERFCTR_TP_10_HI, 81, A6XX_TPL1_PERFCTR_TP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_TP_11_LO, + A6XX_RBBM_PERFCTR_TP_11_HI, 82, A6XX_TPL1_PERFCTR_TP_SEL_11 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_0_LO, + A6XX_RBBM_PERFCTR_SP_0_HI, 83, A6XX_SP_PERFCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_1_LO, + A6XX_RBBM_PERFCTR_SP_1_HI, 84, A6XX_SP_PERFCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_2_LO, + A6XX_RBBM_PERFCTR_SP_2_HI, 85, A6XX_SP_PERFCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_3_LO, + A6XX_RBBM_PERFCTR_SP_3_HI, 86, A6XX_SP_PERFCTR_SP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_4_LO, + A6XX_RBBM_PERFCTR_SP_4_HI, 87, A6XX_SP_PERFCTR_SP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_5_LO, + A6XX_RBBM_PERFCTR_SP_5_HI, 88, A6XX_SP_PERFCTR_SP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_6_LO, + A6XX_RBBM_PERFCTR_SP_6_HI, 89, A6XX_SP_PERFCTR_SP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_7_LO, + A6XX_RBBM_PERFCTR_SP_7_HI, 90, A6XX_SP_PERFCTR_SP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_8_LO, + A6XX_RBBM_PERFCTR_SP_8_HI, 91, A6XX_SP_PERFCTR_SP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_9_LO, + A6XX_RBBM_PERFCTR_SP_9_HI, 92, A6XX_SP_PERFCTR_SP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_10_LO, + A6XX_RBBM_PERFCTR_SP_10_HI, 93, A6XX_SP_PERFCTR_SP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_11_LO, + A6XX_RBBM_PERFCTR_SP_11_HI, 94, A6XX_SP_PERFCTR_SP_SEL_11 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_12_LO, + A6XX_RBBM_PERFCTR_SP_12_HI, 95, A6XX_SP_PERFCTR_SP_SEL_12 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_13_LO, + A6XX_RBBM_PERFCTR_SP_13_HI, 96, A6XX_SP_PERFCTR_SP_SEL_13 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_14_LO, + A6XX_RBBM_PERFCTR_SP_14_HI, 97, A6XX_SP_PERFCTR_SP_SEL_14 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_15_LO, + A6XX_RBBM_PERFCTR_SP_15_HI, 98, A6XX_SP_PERFCTR_SP_SEL_15 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_16_LO, + A6XX_RBBM_PERFCTR_SP_16_HI, 99, A6XX_SP_PERFCTR_SP_SEL_16 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_17_LO, + A6XX_RBBM_PERFCTR_SP_17_HI, 100, A6XX_SP_PERFCTR_SP_SEL_17 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_18_LO, + A6XX_RBBM_PERFCTR_SP_18_HI, 101, A6XX_SP_PERFCTR_SP_SEL_18 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_19_LO, + A6XX_RBBM_PERFCTR_SP_19_HI, 102, A6XX_SP_PERFCTR_SP_SEL_19 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_20_LO, + A6XX_RBBM_PERFCTR_SP_20_HI, 103, A6XX_SP_PERFCTR_SP_SEL_20 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_21_LO, + A6XX_RBBM_PERFCTR_SP_21_HI, 104, A6XX_SP_PERFCTR_SP_SEL_21 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_22_LO, + A6XX_RBBM_PERFCTR_SP_22_HI, 105, A6XX_SP_PERFCTR_SP_SEL_22 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_SP_23_LO, + A6XX_RBBM_PERFCTR_SP_23_HI, 106, A6XX_SP_PERFCTR_SP_SEL_23 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_0_LO, + A6XX_RBBM_PERFCTR_RB_0_HI, 107, A6XX_RB_PERFCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_1_LO, + A6XX_RBBM_PERFCTR_RB_1_HI, 108, A6XX_RB_PERFCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_2_LO, + A6XX_RBBM_PERFCTR_RB_2_HI, 109, A6XX_RB_PERFCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_3_LO, + A6XX_RBBM_PERFCTR_RB_3_HI, 110, A6XX_RB_PERFCTR_RB_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_4_LO, + A6XX_RBBM_PERFCTR_RB_4_HI, 111, A6XX_RB_PERFCTR_RB_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_5_LO, + A6XX_RBBM_PERFCTR_RB_5_HI, 112, A6XX_RB_PERFCTR_RB_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_6_LO, + A6XX_RBBM_PERFCTR_RB_6_HI, 113, A6XX_RB_PERFCTR_RB_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_RB_7_LO, + A6XX_RBBM_PERFCTR_RB_7_HI, 114, A6XX_RB_PERFCTR_RB_SEL_7 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_vsc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VSC_0_LO, + A6XX_RBBM_PERFCTR_VSC_0_HI, 115, A6XX_VSC_PERFCTR_VSC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_VSC_1_LO, + A6XX_RBBM_PERFCTR_VSC_1_HI, 116, A6XX_VSC_PERFCTR_VSC_SEL_1 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_lrz[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_LRZ_0_LO, + A6XX_RBBM_PERFCTR_LRZ_0_HI, 117, A6XX_GRAS_PERFCTR_LRZ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_LRZ_1_LO, + A6XX_RBBM_PERFCTR_LRZ_1_HI, 118, A6XX_GRAS_PERFCTR_LRZ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_LRZ_2_LO, + A6XX_RBBM_PERFCTR_LRZ_2_HI, 119, A6XX_GRAS_PERFCTR_LRZ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_LRZ_3_LO, + A6XX_RBBM_PERFCTR_LRZ_3_HI, 120, A6XX_GRAS_PERFCTR_LRZ_SEL_3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_cmp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CMP_0_LO, + A6XX_RBBM_PERFCTR_CMP_0_HI, 121, A6XX_RB_PERFCTR_CMP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CMP_1_LO, + A6XX_RBBM_PERFCTR_CMP_1_HI, 122, A6XX_RB_PERFCTR_CMP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CMP_2_LO, + A6XX_RBBM_PERFCTR_CMP_2_HI, 123, A6XX_RB_PERFCTR_CMP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_RBBM_PERFCTR_CMP_3_LO, + A6XX_RBBM_PERFCTR_CMP_3_HI, 124, A6XX_RB_PERFCTR_CMP_SEL_3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_vbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_CNT_LOW0, + A6XX_VBIF_PERF_CNT_HIGH0, -1, A6XX_VBIF_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_CNT_LOW1, + A6XX_VBIF_PERF_CNT_HIGH1, -1, A6XX_VBIF_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_CNT_LOW2, + A6XX_VBIF_PERF_CNT_HIGH2, -1, A6XX_VBIF_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_CNT_LOW3, + A6XX_VBIF_PERF_CNT_HIGH3, -1, A6XX_VBIF_PERF_CNT_SEL3 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_vbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_PWR_CNT_LOW0, + A6XX_VBIF_PERF_PWR_CNT_HIGH0, -1, A6XX_VBIF_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_PWR_CNT_LOW1, + A6XX_VBIF_PERF_PWR_CNT_HIGH1, -1, A6XX_VBIF_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_VBIF_PERF_PWR_CNT_LOW2, + A6XX_VBIF_PERF_PWR_CNT_HIGH2, -1, A6XX_VBIF_PERF_PWR_CNT_EN2 }, +}; + + +static struct adreno_perfcount_register a6xx_perfcounters_gbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PERF_CNT_LOW0, + A6XX_GBIF_PERF_CNT_HIGH0, -1, A6XX_GBIF_PERF_CNT_SEL }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PERF_CNT_LOW1, + A6XX_GBIF_PERF_CNT_HIGH1, -1, A6XX_GBIF_PERF_CNT_SEL }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PERF_CNT_LOW2, + A6XX_GBIF_PERF_CNT_HIGH2, -1, A6XX_GBIF_PERF_CNT_SEL }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PERF_CNT_LOW3, + A6XX_GBIF_PERF_CNT_HIGH3, -1, A6XX_GBIF_PERF_CNT_SEL }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_gbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PWR_CNT_LOW0, + A6XX_GBIF_PWR_CNT_HIGH0, -1, A6XX_GBIF_PERF_PWR_CNT_EN }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PWR_CNT_LOW1, + A6XX_GBIF_PWR_CNT_HIGH1, -1, A6XX_GBIF_PERF_PWR_CNT_EN }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_GBIF_PWR_CNT_LOW2, + A6XX_GBIF_PWR_CNT_HIGH2, -1, A6XX_GBIF_PERF_PWR_CNT_EN }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_pwr[] = { + { KGSL_PERFCOUNTER_BROKEN, 0, 0, 0, 0, -1, 0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H, -1, 0 }, +}; + +static struct adreno_perfcount_register a6xx_perfcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A6XX_CP_ALWAYS_ON_COUNTER_LO, + A6XX_CP_ALWAYS_ON_COUNTER_HI, -1 }, +}; + +static struct adreno_perfcount_register a6xx_pwrcounters_gpmu[] = { + /* + * A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0 is used for the GPU + * busy count (see the PWR group above). Mark it as broken + * so it's not re-used. + */ + { KGSL_PERFCOUNTER_BROKEN, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1, }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L, + A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H, -1, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1, }, +}; + +/* + * ADRENO_PERFCOUNTER_GROUP_RESTORE flag is enabled by default + * because most of the perfcounter groups need to be restored + * as part of preemption and IFPC. Perfcounter groups that are + * not restored as part of preemption and IFPC should be defined + * using A6XX_PERFCOUNTER_GROUP_FLAGS macro + */ +#define A6XX_PERFCOUNTER_GROUP(offset, name) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a6xx, offset, name, \ + ADRENO_PERFCOUNTER_GROUP_RESTORE) + +#define A6XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a6xx, offset, name, flags) + +#define A6XX_POWER_COUNTER_GROUP(offset, name) \ + ADRENO_POWER_COUNTER_GROUP(a6xx, offset, name) + +static struct adreno_perfcount_group a6xx_perfcounter_groups + [KGSL_PERFCOUNTER_GROUP_MAX] = { + A6XX_PERFCOUNTER_GROUP(CP, cp), + A6XX_PERFCOUNTER_GROUP_FLAGS(RBBM, rbbm, 0), + A6XX_PERFCOUNTER_GROUP(PC, pc), + A6XX_PERFCOUNTER_GROUP(VFD, vfd), + A6XX_PERFCOUNTER_GROUP(HLSQ, hlsq), + A6XX_PERFCOUNTER_GROUP(VPC, vpc), + A6XX_PERFCOUNTER_GROUP(CCU, ccu), + A6XX_PERFCOUNTER_GROUP(CMP, cmp), + A6XX_PERFCOUNTER_GROUP(TSE, tse), + A6XX_PERFCOUNTER_GROUP(RAS, ras), + A6XX_PERFCOUNTER_GROUP(LRZ, lrz), + A6XX_PERFCOUNTER_GROUP(UCHE, uche), + A6XX_PERFCOUNTER_GROUP(TP, tp), + A6XX_PERFCOUNTER_GROUP(SP, sp), + A6XX_PERFCOUNTER_GROUP(RB, rb), + A6XX_PERFCOUNTER_GROUP(VSC, vsc), + A6XX_PERFCOUNTER_GROUP_FLAGS(VBIF, vbif, 0), + A6XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A6XX_PERFCOUNTER_GROUP_FLAGS(PWR, pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A6XX_PERFCOUNTER_GROUP_FLAGS(ALWAYSON, alwayson, + ADRENO_PERFCOUNTER_GROUP_FIXED), + A6XX_POWER_COUNTER_GROUP(GPMU, gpmu), +}; + +static struct adreno_perfcounters a6xx_perfcounters = { + a6xx_perfcounter_groups, + ARRAY_SIZE(a6xx_perfcounter_groups), +}; + +/* Program the GMU power counter to count GPU busy cycles */ +static int a6xx_enable_pwr_counters(struct adreno_device *adreno_dev, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * We have a limited number of power counters. Since we're not using + * total GPU cycle count, return error if requested. + */ + if (counter == 0) + return -EINVAL; + + /* We can use GPU without GMU and allow it to count GPU busy cycles */ + if (!gmu_core_isenabled(device) && + !kgsl_is_register_offset(device, + A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK)) + return -ENODEV; + + kgsl_regwrite(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xFF000000); + kgsl_regrmw(device, + A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xFF, 0x20); + kgsl_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0x1); + + return 0; +} + +static void a6xx_efuse_speed_bin(struct adreno_device *adreno_dev) +{ + unsigned int val; + unsigned int speed_bin[3]; + struct kgsl_device *device = &adreno_dev->dev; + + if (of_property_read_u32_array(device->pdev->dev.of_node, + "qcom,gpu-speed-bin", speed_bin, 3)) + return; + + adreno_efuse_read_u32(adreno_dev, speed_bin[0], &val); + + adreno_dev->speed_bin = (val & speed_bin[1]) >> speed_bin[2]; +} + +static const struct { + int (*check)(struct adreno_device *adreno_dev); + void (*func)(struct adreno_device *adreno_dev); +} a6xx_efuse_funcs[] = { + { adreno_is_a615, a6xx_efuse_speed_bin }, + { adreno_is_a608, a6xx_efuse_speed_bin }, + { adreno_is_a616, a6xx_efuse_speed_bin }, +}; + +static void a6xx_check_features(struct adreno_device *adreno_dev) +{ + unsigned int i; + + if (adreno_efuse_map(adreno_dev)) + return; + for (i = 0; i < ARRAY_SIZE(a6xx_efuse_funcs); i++) { + if (a6xx_efuse_funcs[i].check(adreno_dev)) + a6xx_efuse_funcs[i].func(adreno_dev); + } + + adreno_efuse_unmap(adreno_dev); +} +static void a6xx_platform_setup(struct adreno_device *adreno_dev) +{ + uint64_t addr; + struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + /* Calculate SP local and private mem addresses */ + addr = ALIGN(ADRENO_UCHE_GMEM_BASE + adreno_dev->gmem_size, SZ_64K); + adreno_dev->sp_local_gpuaddr = addr; + adreno_dev->sp_pvt_gpuaddr = addr + SZ_64K; + + if (adreno_has_gbif(adreno_dev)) { + a6xx_perfcounter_groups[KGSL_PERFCOUNTER_GROUP_VBIF].regs = + a6xx_perfcounters_gbif; + a6xx_perfcounter_groups[KGSL_PERFCOUNTER_GROUP_VBIF].reg_count + = ARRAY_SIZE(a6xx_perfcounters_gbif); + + a6xx_perfcounter_groups[KGSL_PERFCOUNTER_GROUP_VBIF_PWR].regs = + a6xx_perfcounters_gbif_pwr; + a6xx_perfcounter_groups[ + KGSL_PERFCOUNTER_GROUP_VBIF_PWR].reg_count + = ARRAY_SIZE(a6xx_perfcounters_gbif_pwr); + + gpudev->gbif_client_halt_mask = A6XX_GBIF_CLIENT_HALT_MASK; + gpudev->gbif_arb_halt_mask = A6XX_GBIF_ARB_HALT_MASK; + } else + gpudev->vbif_xin_halt_ctrl0_mask = + A6XX_VBIF_XIN_HALT_CTRL0_MASK; + + /* Check efuse bits for various capabilties */ + a6xx_check_features(adreno_dev); +} + + +static unsigned int a6xx_ccu_invalidate(struct adreno_device *adreno_dev, + unsigned int *cmds) +{ + /* CCU_INVALIDATE_DEPTH */ + *cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1); + *cmds++ = 24; + + /* CCU_INVALIDATE_COLOR */ + *cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1); + *cmds++ = 25; + + return 4; +} + +/* Register offset defines for A6XX, in order of enum adreno_regs */ +static unsigned int a6xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A6XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A6XX_CP_RB_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, + A6XX_CP_RB_RPTR_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI, + A6XX_CP_RB_RPTR_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A6XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A6XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A6XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A6XX_CP_SQE_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A6XX_CP_MISC_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_HW_FAULT, A6XX_CP_HW_FAULT), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A6XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, A6XX_CP_IB1_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A6XX_CP_IB1_REM_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A6XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, A6XX_CP_IB2_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A6XX_CP_IB2_REM_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_ADDR, A6XX_CP_ROQ_DBG_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ROQ_DATA, A6XX_CP_ROQ_DBG_DATA), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A6XX_CP_CONTEXT_SWITCH_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, + A6XX_CP_CONTEXT_SWITCH_LEVEL_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A6XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS3, A6XX_RBBM_STATUS3), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_CTL, A6XX_RBBM_PERFCTR_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + A6XX_RBBM_PERFCTR_LOAD_CMD0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + A6XX_RBBM_PERFCTR_LOAD_CMD1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD2, + A6XX_RBBM_PERFCTR_LOAD_CMD2), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_CMD3, + A6XX_RBBM_PERFCTR_LOAD_CMD3), + + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_STATUS, A6XX_RBBM_INT_0_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A6XX_RBBM_CLOCK_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_CLEAR_CMD, + A6XX_RBBM_INT_CLEAR_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A6XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD, + A6XX_RBBM_BLOCK_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_BLOCK_SW_RESET_CMD2, + A6XX_RBBM_BLOCK_SW_RESET_CMD2), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + A6XX_RBBM_PERFCTR_LOAD_VALUE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + A6XX_RBBM_PERFCTR_LOAD_VALUE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_VERSION, A6XX_VBIF_VERSION), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL0, + A6XX_VBIF_XIN_HALT_CTRL0), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A6XX_VBIF_XIN_HALT_CTRL1), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_GPR0_CNTL, A6XX_RBBM_GPR0_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_VBIF_GX_RESET_STATUS, + A6XX_RBBM_VBIF_GX_RESET_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GBIF_HALT, A6XX_GBIF_HALT), + ADRENO_REG_DEFINE(ADRENO_REG_GBIF_HALT_ACK, A6XX_GBIF_HALT_ACK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO, + A6XX_GMU_ALWAYS_ON_COUNTER_L), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI, + A6XX_GMU_ALWAYS_ON_COUNTER_H), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_AHB_FENCE_CTRL, + A6XX_GMU_AO_AHB_FENCE_CTRL), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_INTERRUPT_EN, + A6XX_GMU_AO_INTERRUPT_EN), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, + A6XX_GMU_AO_HOST_INTERRUPT_CLR), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_STATUS, + A6XX_GMU_AO_HOST_INTERRUPT_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + A6XX_GMU_AO_HOST_INTERRUPT_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_PWR_COL_KEEPALIVE, + A6XX_GMU_GMU_PWR_COL_KEEPALIVE), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AHB_FENCE_STATUS, + A6XX_GMU_AHB_FENCE_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_CTRL_STATUS, + A6XX_GMU_HFI_CTRL_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_VERSION_INFO, + A6XX_GMU_HFI_VERSION_INFO), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_SFR_ADDR, + A6XX_GMU_HFI_SFR_ADDR), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_RPMH_POWER_STATE, + A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_CLR, + A6XX_GMU_GMU2HOST_INTR_CLR), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_INFO, + A6XX_GMU_GMU2HOST_INTR_INFO), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + A6XX_GMU_GMU2HOST_INTR_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_SET, + A6XX_GMU_HOST2GMU_INTR_SET), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_CLR, + A6XX_GMU_HOST2GMU_INTR_CLR), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_RAW_INFO, + A6XX_GMU_HOST2GMU_INTR_RAW_INFO), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_NMI_CONTROL_STATUS, + A6XX_GMU_NMI_CONTROL_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_CM3_CFG, + A6XX_GMU_CM3_CFG), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_RBBM_INT_UNMASKED_STATUS, + A6XX_GMU_RBBM_INT_UNMASKED_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TRUST_CONTROL, + A6XX_RBBM_SECVID_TRUST_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE, + A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_TRUSTED_SIZE, + A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SECVID_TSB_CONTROL, + A6XX_RBBM_SECVID_TSB_CNTL), +}; + +static const struct adreno_reg_offsets a6xx_reg_offsets = { + .offsets = a6xx_register_offsets, + .offset_0 = ADRENO_REG_REGISTER_MAX, +}; + +static void a6xx_perfcounter_init(struct adreno_device *adreno_dev) +{ + /* + * A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4/5 is not present on A608. + * Mark them as broken so that they can't be used. + */ + if (adreno_is_a608(adreno_dev)) { + a6xx_pwrcounters_gpmu[4].countable = KGSL_PERFCOUNTER_BROKEN; + a6xx_pwrcounters_gpmu[5].countable = KGSL_PERFCOUNTER_BROKEN; + } +} + +static int a6xx_perfcounter_update(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg, bool update_reg) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct cpu_gpu_lock *lock = adreno_dev->pwrup_reglist.hostptr; + struct reg_list_pair *reg_pair = (struct reg_list_pair *)(lock + 1); + unsigned int i; + unsigned long timeout = jiffies + msecs_to_jiffies(1000); + int ret = 0; + + lock->flag_kmd = 1; + /* Write flag_kmd before turn */ + wmb(); + lock->turn = 0; + /* Write these fields before looping */ + mb(); + + /* + * Spin here while GPU ucode holds the lock, lock->flag_ucode will + * be set to 0 after GPU ucode releases the lock. Minimum wait time + * is 1 second and this should be enough for GPU to release the lock + */ + while (lock->flag_ucode == 1 && lock->turn == 0) { + cpu_relax(); + /* Get the latest updates from GPU */ + rmb(); + /* + * Make sure we wait at least 1sec for the lock, + * if we did not get it after 1sec return an error. + */ + if (time_after(jiffies, timeout) && + (lock->flag_ucode == 1 && lock->turn == 0)) { + ret = -EBUSY; + goto unlock; + } + } + + /* Read flag_ucode and turn before list_length */ + rmb(); + /* + * If the perfcounter select register is already present in reglist + * update it, otherwise append the