Merge branch 'drm-next-4.17' of git://people.freedesktop.org/~agd5f/linux into drm-next

amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o

	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o

# add GMC block

amdgpu-y += \

#include "amdgpu_vce.h"

#include "amdgpu_vcn.h"

#include "amdgpu_mn.h"

#include "amdgpu_gmc.h"

#include "amdgpu_dm.h"

#include "amdgpu_virt.h"

#include "amdgpu_gart.h"

extern int amdgpu_lbpw;

extern int amdgpu_compute_multipipe;

extern int amdgpu_gpu_recovery;

extern int amdgpu_emu_mode;

#ifdef CONFIG_DRM_AMDGPU_SI

extern int amdgpu_si_support;

	void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,

			  uint64_t value, unsigned count,

			  uint32_t incr);

	/* maximum nums of PTEs/PDEs in a single operation */

	uint32_t	set_max_nums_pte_pde;

	/* number of dw to reserve per operation */

	unsigned	set_pte_pde_num_dw;

	/* for linear pte/pde updates without addr mapping */

	void (*set_pte_pde)(struct amdgpu_ib *ib,

			    uint64_t pe,

			    uint32_t incr, uint64_t flags);

};

/* provided by the gmc block */

struct amdgpu_gart_funcs {

	/* flush the vm tlb via mmio */

	void (*flush_gpu_tlb)(struct amdgpu_device *adev,

			      uint32_t vmid);

	/* write pte/pde updates using the cpu */

	int (*set_pte_pde)(struct amdgpu_device *adev,

			   void *cpu_pt_addr, /* cpu addr of page table */

			   uint32_t gpu_page_idx, /* pte/pde to update */

			   uint64_t addr, /* addr to write into pte/pde */

			   uint64_t flags); /* access flags */

	/* enable/disable PRT support */

	void (*set_prt)(struct amdgpu_device *adev, bool enable);

	/* set pte flags based per asic */

	uint64_t (*get_vm_pte_flags)(struct amdgpu_device *adev,

				     uint32_t flags);

	/* get the pde for a given mc addr */

	void (*get_vm_pde)(struct amdgpu_device *adev, int level,

			   u64 *dst, u64 *flags);

	uint32_t (*get_invalidate_req)(unsigned int vmid);

};

/* provided by the ih block */

struct amdgpu_ih_funcs {

	/* ring read/write ptr handling, called from interrupt context */

struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,

					struct drm_gem_object *gobj,

					int flags);

int amdgpu_gem_prime_pin(struct drm_gem_object *obj);

void amdgpu_gem_prime_unpin(struct drm_gem_object *obj);

struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,

					    struct dma_buf *dma_buf);

struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);

void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);

void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);

int amdgpu_fence_slab_init(void);

void amdgpu_fence_slab_fini(void);

/*

 * VMHUB structures, functions & helpers

 */

struct amdgpu_vmhub {

	uint32_t	ctx0_ptb_addr_lo32;

	uint32_t	ctx0_ptb_addr_hi32;

	uint32_t	vm_inv_eng0_req;

	uint32_t	vm_inv_eng0_ack;

	uint32_t	vm_context0_cntl;

	uint32_t	vm_l2_pro_fault_status;

	uint32_t	vm_l2_pro_fault_cntl;

};

/*

 * GPU MC structures, functions & helpers

 */

struct amdgpu_mc {

	resource_size_t		aper_size;

	resource_size_t		aper_base;

	resource_size_t		agp_base;

	/* for some chips with <= 32MB we need to lie

	 * about vram size near mc fb location */

	u64			mc_vram_size;

	u64			visible_vram_size;

	u64			gart_size;

	u64			gart_start;

	u64			gart_end;

	u64			vram_start;

	u64			vram_end;

	unsigned		vram_width;

	u64			real_vram_size;

	int			vram_mtrr;

	u64                     mc_mask;

	const struct firmware   *fw;	/* MC firmware */

	uint32_t                fw_version;

	struct amdgpu_irq_src	vm_fault;

	uint32_t		vram_type;

	uint32_t                srbm_soft_reset;

	bool			prt_warning;

	uint64_t		stolen_size;

	/* apertures */

	u64					shared_aperture_start;

	u64					shared_aperture_end;

	u64					private_aperture_start;

	u64					private_aperture_end;

	/* protects concurrent invalidation */

	spinlock_t		invalidate_lock;

	bool			translate_further;

};

/*

 * GPU doorbell structures, functions & helpers

 */

	void			*owner;

	uint64_t		fence_ctx; /* the fence_context this job uses */

	bool                    vm_needs_flush;

	unsigned		vmid;

	uint64_t		vm_pd_addr;

	unsigned		vmid;

	unsigned		pasid;

	uint32_t		gds_base, gds_size;

	uint32_t		gws_base, gws_size;

	uint32_t		oa_base, oa_size;

	void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);

	/* get config memsize register */

	u32 (*get_config_memsize)(struct amdgpu_device *adev);

	/* flush hdp write queue */

	void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring);

	/* invalidate hdp read cache */

	void (*invalidate_hdp)(struct amdgpu_device *adev,

			       struct amdgpu_ring *ring);

};

/*

	u32 (*get_pcie_data_offset)(struct amdgpu_device *adev);

	u32 (*get_rev_id)(struct amdgpu_device *adev);

	void (*mc_access_enable)(struct amdgpu_device *adev, bool enable);

	void (*hdp_flush)(struct amdgpu_device *adev);

	void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring);

	u32 (*get_memsize)(struct amdgpu_device *adev);

	void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance,

				    bool use_doorbell, int doorbell_index);

	struct amdgpu_clock            clock;

	/* MC */

	struct amdgpu_mc		mc;

	struct amdgpu_gmc		gmc;

	struct amdgpu_gart		gart;

	struct amdgpu_dummy_page	dummy_page;

	struct amdgpu_vm_manager	vm_manager;

bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);

bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);

int emu_soc_asic_init(struct amdgpu_device *adev);

/*

 * Registers read & write functions.

 */

#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))

#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))

#define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))

#define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))

#define amdgpu_gart_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gart.gart_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))

#define amdgpu_gart_get_vm_pde(adev, level, dst, flags) (adev)->gart.gart_funcs->get_vm_pde((adev), (level), (dst), (flags))

#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r))

#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r))

#define amdgpu_gmc_flush_gpu_tlb(adev, vmid) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid))

#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))

#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))

#define amdgpu_gmc_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gmc.gmc_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))

#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))

#define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags))

#define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))

#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))

#define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))

#define amdgpu_vm_get_pte_flags(adev, flags) (adev)->gart.gart_funcs->get_vm_pte_flags((adev),(flags))

#define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))

#define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))

#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))

#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))

#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))

#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))

#define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))

#define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))

#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))

#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))

#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))

#define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))

#define amdgpu_ring_emit_tmz(r, b) (r)->funcs->emit_tmz((r), (b))

#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))

#define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))

#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))

#define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))

#define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))

#define amdgpu_display_vblank_wait(adev, crtc) (adev)->mode_info.funcs->vblank_wait((adev), (crtc))

#define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))

#define amdgpu_display_backlight_get_level(adev, e) (adev)->mode_info.funcs->backlight_get_level((e))

#define amdgpu_display_hpd_sense(adev, h) (adev)->mode_info.funcs->hpd_sense((adev), (h))

			      struct amdgpu_job* job, bool force);

void amdgpu_device_pci_config_reset(struct amdgpu_device *adev);

bool amdgpu_device_need_post(struct amdgpu_device *adev);

void amdgpu_update_display_priority(struct amdgpu_device *adev);

void amdgpu_display_update_priority(struct amdgpu_device *adev);

void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,

				  u64 num_vis_bytes);

void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);

bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);

void amdgpu_device_vram_location(struct amdgpu_device *adev,

				 struct amdgpu_mc *mc, u64 base);

				 struct amdgpu_gmc *mc, u64 base);

void amdgpu_device_gart_location(struct amdgpu_device *adev,

				 struct amdgpu_mc *mc);

				 struct amdgpu_gmc *mc);

int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);

void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);

int amdgpu_ttm_init(struct amdgpu_device *adev);

		return -ENOMEM;

	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,

			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, 0,

			     &(*mem)->bo);

			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo);

	if (r) {

		dev_err(adev->dev,

			"failed to allocate BO for amdkfd (%d)\n", r);

	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;

	uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask :

					     ~((1ULL << 32) - 1);

	resource_size_t aper_limit = adev->mc.aper_base + adev->mc.aper_size;

	resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size;

	memset(mem_info, 0, sizeof(*mem_info));

	if (!(adev->mc.aper_base & address_mask || aper_limit & address_mask)) {

		mem_info->local_mem_size_public = adev->mc.visible_vram_size;

		mem_info->local_mem_size_private = adev->mc.real_vram_size -

				adev->mc.visible_vram_size;

	if (!(adev->gmc.aper_base & address_mask || aper_limit & address_mask)) {

		mem_info->local_mem_size_public = adev->gmc.visible_vram_size;

		mem_info->local_mem_size_private = adev->gmc.real_vram_size -

				adev->gmc.visible_vram_size;

	} else {

		mem_info->local_mem_size_public = 0;

		mem_info->local_mem_size_private = adev->mc.real_vram_size;

		mem_info->local_mem_size_private = adev->gmc.real_vram_size;

	}

	mem_info->vram_width = adev->mc.vram_width;

	mem_info->vram_width = adev->gmc.vram_width;

	pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n",

			&adev->mc.aper_base, &aper_limit,

			&adev->gmc.aper_base, &aper_limit,

			mem_info->local_mem_size_public,

			mem_info->local_mem_size_private);

	if (amdgpu_emu_mode == 1) {

		mem_info->mem_clk_max = 100;

		return;

	}

	if (amdgpu_sriov_vf(adev))

		mem_info->mem_clk_max = adev->clock.default_mclk / 100;

	else

	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;

	/* the sclk is in quantas of 10kHz */

	if (amdgpu_emu_mode == 1)

		return 100;

	if (amdgpu_sriov_vf(adev))

		return adev->clock.default_sclk / 100;

	/* HG _PR3 doesn't seem to work on this A+A weston board */

	{ 0x1002, 0x6900, 0x1002, 0x0124, AMDGPU_PX_QUIRK_FORCE_ATPX },

	{ 0x1002, 0x6900, 0x1028, 0x0812, AMDGPU_PX_QUIRK_FORCE_ATPX },

	{ 0x1002, 0x6900, 0x1028, 0x0813, AMDGPU_PX_QUIRK_FORCE_ATPX },

	{ 0, 0, 0, 0, 0 },

};

	n = AMDGPU_BENCHMARK_ITERATIONS;

	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, sdomain, 0, NULL,

			     NULL, 0, &sobj);

			     NULL, &sobj);

	if (r) {

		goto out_cleanup;

	}

		goto out_cleanup;

	}

	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, ddomain, 0, NULL,

			     NULL, 0, &dobj);

			     NULL, &dobj);

	if (r) {

		goto out_cleanup;

	}