Merge branch 'drm-next-4.14' of git://people.freedesktop.org/~agd5f/linux into drm-next

	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \

	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \

	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \

	amdgpu_queue_mgr.o

	amdgpu_queue_mgr.o amdgpu_vf_error.o

# add asic specific block

amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \

#include "gpu_scheduler.h"

#include "amdgpu_virt.h"

#include "amdgpu_gart.h"

/*

 * Modules parameters.

 */

extern int amdgpu_modeset;

extern int amdgpu_vram_limit;

extern int amdgpu_gart_size;

extern int amdgpu_vis_vram_limit;

extern unsigned amdgpu_gart_size;

extern int amdgpu_gtt_size;

extern int amdgpu_moverate;

extern int amdgpu_benchmarking;

extern int amdgpu_testing;

extern unsigned amdgpu_pcie_lane_cap;

extern unsigned amdgpu_cg_mask;

extern unsigned amdgpu_pg_mask;

extern unsigned amdgpu_sdma_phase_quantum;

extern char *amdgpu_disable_cu;

extern char *amdgpu_virtual_display;

extern unsigned amdgpu_pp_feature_mask;

int amdgpu_fence_slab_init(void);

void amdgpu_fence_slab_fini(void);

/*

 * GART structures, functions & helpers

 */

struct amdgpu_mc;

#define AMDGPU_GPU_PAGE_SIZE 4096

#define AMDGPU_GPU_PAGE_MASK (AMDGPU_GPU_PAGE_SIZE - 1)

#define AMDGPU_GPU_PAGE_SHIFT 12

#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)

struct amdgpu_gart {

	dma_addr_t			table_addr;

	struct amdgpu_bo		*robj;

	void				*ptr;

	unsigned			num_gpu_pages;

	unsigned			num_cpu_pages;

	unsigned			table_size;

#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS

	struct page			**pages;

#endif

	bool				ready;

	/* Asic default pte flags */

	uint64_t			gart_pte_flags;

	const struct amdgpu_gart_funcs *gart_funcs;

};

int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);

void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);

int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);

void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);

int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);

void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev);

int amdgpu_gart_init(struct amdgpu_device *adev);

void amdgpu_gart_fini(struct amdgpu_device *adev);

int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,

			int pages);

int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,

		     int pages, struct page **pagelist,

		     dma_addr_t *dma_addr, uint64_t flags);

int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);

/*

 * VMHUB structures, functions & helpers

 */

	 * about vram size near mc fb location */

	u64			mc_vram_size;

	u64			visible_vram_size;

	u64			gtt_size;

	u64			gtt_start;

	u64			gtt_end;

	u64			gart_size;

	u64			gart_start;

	u64			gart_end;

	u64			vram_start;

	u64			vram_end;

	unsigned		vram_width;

	u64			real_vram_size;

	int			vram_mtrr;

	u64                     gtt_base_align;

	u64                     mc_mask;

	const struct firmware   *fw;	/* MC firmware */

	uint32_t                fw_version;

	struct amdgpu_irq_src	vm_fault;

	uint32_t		vram_type;

	uint32_t                srbm_soft_reset;

	struct amdgpu_mode_mc_save save;

	bool			prt_warning;

	uint64_t		stolen_size;

	/* apertures */

	struct list_head		validated;

	struct dma_fence		*fence;

	uint64_t			bytes_moved_threshold;

	uint64_t			bytes_moved_vis_threshold;

	uint64_t			bytes_moved;

	uint64_t			bytes_moved_vis;

	struct amdgpu_bo_list_entry	*evictable;

	/* user fence */

int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);

void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);

int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb);

int amdgpu_wb_get_256Bit(struct amdgpu_device *adev, u32 *wb);

void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb);

void amdgpu_wb_free_256bit(struct amdgpu_device *adev, u32 wb);

void amdgpu_get_pcie_info(struct amdgpu_device *adev);

	spinlock_t gc_cac_idx_lock;

	amdgpu_rreg_t			gc_cac_rreg;

	amdgpu_wreg_t			gc_cac_wreg;

	/* protects concurrent se_cac register access */

	spinlock_t se_cac_idx_lock;

	amdgpu_rreg_t			se_cac_rreg;

	amdgpu_wreg_t			se_cac_wreg;

	/* protects concurrent ENDPOINT (audio) register access */

	spinlock_t audio_endpt_idx_lock;

	amdgpu_block_rreg_t		audio_endpt_rreg;

		spinlock_t		lock;

		s64			last_update_us;

		s64			accum_us; /* accumulated microseconds */

		s64			accum_us_vis; /* for visible VRAM */

		u32			log2_max_MBps;

	} mm_stats;

	bool has_hw_reset;

	u8				reset_magic[AMDGPU_RESET_MAGIC_NUM];

	/* record last mm index being written through WREG32*/

	unsigned long last_mm_index;

};

static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)

#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))

#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))

#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))

#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg))

#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v))

#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))

#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))

#define WREG32_P(reg, val, mask)				\

#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))

#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))

/*

 * RING helpers.

 */

static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)

{

	if (ring->count_dw <= 0)

		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");

	ring->ring[ring->wptr++ & ring->buf_mask] = v;

	ring->wptr &= ring->ptr_mask;

	ring->count_dw--;

}

static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, void *src, int count_dw)

{

	unsigned occupied, chunk1, chunk2;

	void *dst;

	if (unlikely(ring->count_dw < count_dw)) {

		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");

		return;

	}

	occupied = ring->wptr & ring->buf_mask;

	dst = (void *)&ring->ring[occupied];

	chunk1 = ring->buf_mask + 1 - occupied;

	chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1;

	chunk2 = count_dw - chunk1;

	chunk1 <<= 2;

	chunk2 <<= 2;

	if (chunk1)

		memcpy(dst, src, chunk1);

	if (chunk2) {

		src += chunk1;

		dst = (void *)ring->ring;

		memcpy(dst, src, chunk2);

	}

	ring->wptr += count_dw;

	ring->wptr &= ring->ptr_mask;

	ring->count_dw -= count_dw;

}

static inline struct amdgpu_sdma_instance *

amdgpu_get_sdma_instance(struct amdgpu_ring *ring)

{

#define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))

#define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))

#define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))

#define amdgpu_display_set_vga_render_state(adev, r) (adev)->mode_info.funcs->set_vga_render_state((adev), (r))

#define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))

#define amdgpu_display_vblank_wait(adev, crtc) (adev)->mode_info.funcs->vblank_wait((adev), (crtc))

#define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))

#define amdgpu_display_page_flip_get_scanoutpos(adev, crtc, vbl, pos) (adev)->mode_info.funcs->page_flip_get_scanoutpos((adev), (crtc), (vbl), (pos))

#define amdgpu_display_add_encoder(adev, e, s, c) (adev)->mode_info.funcs->add_encoder((adev), (e), (s), (c))

#define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r))

#define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s))

#define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))

#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))

#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))

#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))

bool amdgpu_need_post(struct amdgpu_device *adev);

void amdgpu_update_display_priority(struct amdgpu_device *adev);

void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes);

void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,

				  u64 num_vis_bytes);

void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);

bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);

int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);

uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,

				 struct ttm_mem_reg *mem);

void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);

void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);

void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);

void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);

int amdgpu_ttm_init(struct amdgpu_device *adev);

void amdgpu_ttm_fini(struct amdgpu_device *adev);

{

	uint32_t bios_6_scratch;

	bios_6_scratch = RREG32(mmBIOS_SCRATCH_6);

	bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6);

	if (lock) {

		bios_6_scratch |= ATOM_S6_CRITICAL_STATE;

		bios_6_scratch |= ATOM_S6_ACC_MODE;

	}

	WREG32(mmBIOS_SCRATCH_6, bios_6_scratch);

	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);

}

void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)

{

	uint32_t bios_2_scratch, bios_6_scratch;

	bios_2_scratch = RREG32(mmBIOS_SCRATCH_2);

	bios_6_scratch = RREG32(mmBIOS_SCRATCH_6);

	adev->bios_scratch_reg_offset = mmBIOS_SCRATCH_0;

	bios_2_scratch = RREG32(adev->bios_scratch_reg_offset + 2);

	bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6);

	/* let the bios control the backlight */

	bios_2_scratch &= ~ATOM_S2_VRI_BRIGHT_ENABLE;

	/* clear the vbios dpms state */

	bios_2_scratch &= ~ATOM_S2_DEVICE_DPMS_STATE;

	WREG32(mmBIOS_SCRATCH_2, bios_2_scratch);

	WREG32(mmBIOS_SCRATCH_6, bios_6_scratch);

	WREG32(adev->bios_scratch_reg_offset + 2, bios_2_scratch);

	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);

}

void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev)

	int i;

	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)

		adev->bios_scratch[i] = RREG32(mmBIOS_SCRATCH_0 + i);

		adev->bios_scratch[i] = RREG32(adev->bios_scratch_reg_offset + i);

}

void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)

	adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;

	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)

		WREG32(mmBIOS_SCRATCH_0 + i, adev->bios_scratch[i]);

		WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);

}

void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,

					      bool hung)

{

	u32 tmp = RREG32(mmBIOS_SCRATCH_3);

	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3);

	if (hung)

		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;

	else

		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;

	WREG32(mmBIOS_SCRATCH_3, tmp);

	WREG32(adev->bios_scratch_reg_offset + 3, tmp);

}

bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev)

{

	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 7);

	if (tmp & ATOM_S7_ASIC_INIT_COMPLETE_MASK)

		return false;

	else

		return true;

}

/* Atom needs data in little endian format

void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev);

void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,

					      bool hung);

bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev);

void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);

int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type,

	}

}

void amdgpu_atomfirmware_scratch_regs_save(struct amdgpu_device *adev)

{

	int i;

	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)

		adev->bios_scratch[i] = RREG32(adev->bios_scratch_reg_offset + i);

}

void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev)

{

	int i;

	/*

	 * VBIOS will check ASIC_INIT_COMPLETE bit to decide if

	 * execute ASIC_Init posting via driver

	 */

	adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;

	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)

		WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);

}

void amdgpu_atomfirmware_scratch_regs_engine_hung(struct amdgpu_device *adev,

						  bool hung)

{

	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3);

	if (hung)

		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;

	else

		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;

	WREG32(adev->bios_scratch_reg_offset + 3, tmp);

}

int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)

{

	struct atom_context *ctx = adev->mode_info.atom_context;

	ctx->scratch_size_bytes = usage_bytes;

	return 0;

}

union igp_info {

	struct atom_integrated_system_info_v1_11 v11;

};

/*

 * Return vram width from integrated system info table, if available,

 * or 0 if not.

 */

int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev)

{

	struct amdgpu_mode_info *mode_info = &adev->mode_info;

	int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,

						integratedsysteminfo);

	u16 data_offset, size;

	union igp_info *igp_info;

	u8 frev, crev;

	/* get any igp specific overrides */

	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size,

				   &frev, &crev, &data_offset)) {

		igp_info = (union igp_info *)

			(mode_info->atom_context->bios + data_offset);

		switch (crev) {

		case 11:

			return igp_info->v11.umachannelnumber * 64;

		default:

			return 0;

		}

	}

	return 0;

}

union firmware_info {

	struct atom_firmware_info_v3_1 v31;

};

union smu_info {

	struct atom_smu_info_v3_1 v31;

};

union umc_info {

	struct atom_umc_info_v3_1 v31;

};

int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev)

{

	struct amdgpu_mode_info *mode_info = &adev->mode_info;

	struct amdgpu_pll *spll = &adev->clock.spll;

	struct amdgpu_pll *mpll = &adev->clock.mpll;

	uint8_t frev, crev;

	uint16_t data_offset;

	int ret = -EINVAL, index;

	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,

					    firmwareinfo);

	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,

				   &frev, &crev, &data_offset)) {

		union firmware_info *firmware_info =

			(union firmware_info *)(mode_info->atom_context->bios +

						data_offset);

		adev->clock.default_sclk =

			le32_to_cpu(firmware_info->v31.bootup_sclk_in10khz);

		adev->clock.default_mclk =

			le32_to_cpu(firmware_info->v31.bootup_mclk_in10khz);

		adev->pm.current_sclk = adev->clock.default_sclk;

		adev->pm.current_mclk = adev->clock.default_mclk;

		/* not technically a clock, but... */

		adev->mode_info.firmware_flags =

			le32_to_cpu(firmware_info->v31.firmware_capability);

		ret = 0;

	}

	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,

					    smu_info);

	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,

				   &frev, &crev, &data_offset)) {

		union smu_info *smu_info =

			(union smu_info *)(mode_info->atom_context->bios +

					   data_offset);

		/* system clock */

		spll->reference_freq = le32_to_cpu(smu_info->v31.core_refclk_10khz);

		spll->reference_div = 0;

		spll->min_post_div = 1;

		spll->max_post_div = 1;

		spll->min_ref_div = 2;

		spll->max_ref_div = 0xff;

		spll->min_feedback_div = 4;

		spll->max_feedback_div = 0xff;

		spll->best_vco = 0;

		ret = 0;

	}

	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,

					    umc_info);

	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,

				   &frev, &crev, &data_offset)) {

		union umc_info *umc_info =

			(union umc_info *)(mode_info->atom_context->bios +

					   data_offset);

		/* memory clock */

		mpll->reference_freq = le32_to_cpu(umc_info->v31.mem_refclk_10khz);

		mpll->reference_div = 0;

		mpll->min_post_div = 1;

		mpll->max_post_div = 1;

		mpll->min_ref_div = 2;

		mpll->max_ref_div = 0xff;

		mpll->min_feedback_div = 4;

		mpll->max_feedback_div = 0xff;

		mpll->best_vco = 0;

		ret = 0;

	}

	return ret;

}