drm/i915: Split out i915_vma.c

	  i915_gem_dmabuf.o \

	  i915_gem_evict.o \

	  i915_gem_execbuffer.o \

	  i915_gem_fence.o \

	  i915_gem_fence_reg.o \

	  i915_gem_gtt.o \

	  i915_gem_internal.o \

	  i915_gem.o \

	  i915_gem_timeline.o \

	  i915_gem_userptr.o \

	  i915_trace_points.o \

	  i915_vma.o \

	  intel_breadcrumbs.o \

	  intel_engine_cs.o \

	  intel_hangcheck.o \

#include "intel_ringbuffer.h"

#include "i915_gem.h"

#include "i915_gem_fence_reg.h"

#include "i915_gem_object.h"

#include "i915_gem_gtt.h"

#include "i915_gem_render_state.h"

#include "i915_gem_request.h"

#include "i915_gem_timeline.h"

#include "i915_vma.h"

#include "intel_gvt.h"

/* General customization:

struct intel_overlay;

struct intel_overlay_error_state;

struct drm_i915_fence_reg {

	struct list_head link;

	struct drm_i915_private *i915;

	struct i915_vma *vma;

	int pin_count;

	int id;

	/**

	 * Whether the tiling parameters for the currently

	 * associated fence register have changed. Note that

	 * for the purposes of tracking tiling changes we also

	 * treat the unfenced register, the register slot that

	 * the object occupies whilst it executes a fenced

	 * command (such as BLT on gen2/3), as a "fence".

	 */

	bool dirty;

};

struct sdvo_device_mapping {

	u8 initialized;

	u8 dvo_port;

#define I915_GTT_OFFSET_NONE ((u32)-1)

struct drm_i915_gem_object_ops {

	unsigned int flags;

#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1

#define I915_GEM_OBJECT_IS_SHRINKABLE   0x2

	/* Interface between the GEM object and its backing storage.

	 * get_pages() is called once prior to the use of the associated set

	 * of pages before to binding them into the GTT, and put_pages() is

	 * called after we no longer need them. As we expect there to be

	 * associated cost with migrating pages between the backing storage

	 * and making them available for the GPU (e.g. clflush), we may hold

	 * onto the pages after they are no longer referenced by the GPU

	 * in case they may be used again shortly (for example migrating the

	 * pages to a different memory domain within the GTT). put_pages()

	 * will therefore most likely be called when the object itself is

	 * being released or under memory pressure (where we attempt to

	 * reap pages for the shrinker).

	 */

	struct sg_table *(*get_pages)(struct drm_i915_gem_object *);

	void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);

	int (*dmabuf_export)(struct drm_i915_gem_object *);

	void (*release)(struct drm_i915_gem_object *);

};

/*

 * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is

 * considered to be the frontbuffer for the given plane interface-wise. This

#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \

	(0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))

struct drm_i915_gem_object {

	struct drm_gem_object base;

	const struct drm_i915_gem_object_ops *ops;

	/** List of VMAs backed by this object */

	struct list_head vma_list;

	struct rb_root vma_tree;

	/** Stolen memory for this object, instead of being backed by shmem. */

	struct drm_mm_node *stolen;

	struct list_head global_link;

	union {

		struct rcu_head rcu;

		struct llist_node freed;

	};

	/**

	 * Whether the object is currently in the GGTT mmap.

	 */

	struct list_head userfault_link;

	/** Used in execbuf to temporarily hold a ref */

	struct list_head obj_exec_link;

	struct list_head batch_pool_link;

	unsigned long flags;

	/**

	 * Have we taken a reference for the object for incomplete GPU

	 * activity?

	 */

#define I915_BO_ACTIVE_REF 0

	/*

	 * Is the object to be mapped as read-only to the GPU

	 * Only honoured if hardware has relevant pte bit

	 */

	unsigned long gt_ro:1;

	unsigned int cache_level:3;

	unsigned int cache_dirty:1;

	atomic_t frontbuffer_bits;

	unsigned int frontbuffer_ggtt_origin; /* write once */

	/** Current tiling stride for the object, if it's tiled. */

	unsigned int tiling_and_stride;

#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */

#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)

#define STRIDE_MASK (~TILING_MASK)

	/** Count of VMA actually bound by this object */

	unsigned int bind_count;

	unsigned int active_count;

	unsigned int pin_display;

	struct {

		struct mutex lock; /* protects the pages and their use */

		atomic_t pages_pin_count;

		struct sg_table *pages;

		void *mapping;

		struct i915_gem_object_page_iter {

			struct scatterlist *sg_pos;

			unsigned int sg_idx; /* in pages, but 32bit eek! */

			struct radix_tree_root radix;

			struct mutex lock; /* protects this cache */

		} get_page;

		/**

		 * Advice: are the backing pages purgeable?

		 */

		unsigned int madv:2;

		/**

		 * This is set if the object has been written to since the

		 * pages were last acquired.

		 */

		bool dirty:1;

		/**

		 * This is set if the object has been pinned due to unknown

		 * swizzling.

		 */

		bool quirked:1;

	} mm;

	/** Breadcrumb of last rendering to the buffer.

	 * There can only be one writer, but we allow for multiple readers.

	 * If there is a writer that necessarily implies that all other

	 * read requests are complete - but we may only be lazily clearing

	 * the read requests. A read request is naturally the most recent

	 * request on a ring, so we may have two different write and read

	 * requests on one ring where the write request is older than the

	 * read request. This allows for the CPU to read from an active

	 * buffer by only waiting for the write to complete.

	 */

	struct reservation_object *resv;

	/** References from framebuffers, locks out tiling changes. */

	unsigned long framebuffer_references;

	/** Record of address bit 17 of each page at last unbind. */

	unsigned long *bit_17;

	struct i915_gem_userptr {

		uintptr_t ptr;

		unsigned read_only :1;

		struct i915_mm_struct *mm;

		struct i915_mmu_object *mmu_object;

		struct work_struct *work;

	} userptr;

	/** for phys allocated objects */

	struct drm_dma_handle *phys_handle;

	struct reservation_object __builtin_resv;

};

static inline struct drm_i915_gem_object *

to_intel_bo(struct drm_gem_object *gem)

{

	/* Assert that to_intel_bo(NULL) == NULL */

	BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));

	return container_of(gem, struct drm_i915_gem_object, base);

}

/**

 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle

 * @filp: DRM file private date

 * @handle: userspace handle

 *

 * Returns:

 *

 * A pointer to the object named by the handle if such exists on @filp, NULL

 * otherwise. This object is only valid whilst under the RCU read lock, and

 * note carefully the object may be in the process of being destroyed.

 */

static inline struct drm_i915_gem_object *

i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)

{

#ifdef CONFIG_LOCKDEP

	WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map));

#endif

	return idr_find(&file->object_idr, handle);

}

static inline struct drm_i915_gem_object *

i915_gem_object_lookup(struct drm_file *file, u32 handle)

{

	struct drm_i915_gem_object *obj;

	rcu_read_lock();

	obj = i915_gem_object_lookup_rcu(file, handle);

	if (obj && !kref_get_unless_zero(&obj->base.refcount))

		obj = NULL;

	rcu_read_unlock();

	return obj;

}

__deprecated

extern struct drm_gem_object *

drm_gem_object_lookup(struct drm_file *file, u32 handle);

__attribute__((nonnull))

static inline struct drm_i915_gem_object *

i915_gem_object_get(struct drm_i915_gem_object *obj)

{

	drm_gem_object_reference(&obj->base);

	return obj;

}

__deprecated

extern void drm_gem_object_reference(struct drm_gem_object *);

__attribute__((nonnull))

static inline void

i915_gem_object_put(struct drm_i915_gem_object *obj)

{

	__drm_gem_object_unreference(&obj->base);

}

__deprecated

extern void drm_gem_object_unreference(struct drm_gem_object *);

__deprecated

extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);

static inline bool

i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)

{

	return atomic_read(&obj->base.refcount.refcount) == 0;

}

static inline bool

i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)

{

	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;

}

static inline bool

i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)

{

	return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;

}

static inline bool

i915_gem_object_is_active(const struct drm_i915_gem_object *obj)

{

	return obj->active_count;

}

static inline bool

i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)

{

	return test_bit(I915_BO_ACTIVE_REF, &obj->flags);

}

static inline void

i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)

{

	lockdep_assert_held(&obj->base.dev->struct_mutex);

	__set_bit(I915_BO_ACTIVE_REF, &obj->flags);

}

static inline void

i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)

{

	lockdep_assert_held(&obj->base.dev->struct_mutex);

	__clear_bit(I915_BO_ACTIVE_REF, &obj->flags);

}

void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);

static inline unsigned int

i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)

{

	return obj->tiling_and_stride & TILING_MASK;

}

static inline bool

i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)

{

	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;

}

static inline unsigned int

i915_gem_object_get_stride(struct drm_i915_gem_object *obj)

{

	return obj->tiling_and_stride & STRIDE_MASK;

}

static inline struct intel_engine_cs *

i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)

{

	struct intel_engine_cs *engine = NULL;

	struct dma_fence *fence;

	rcu_read_lock();

	fence = reservation_object_get_excl_rcu(obj->resv);

	rcu_read_unlock();

	if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))

		engine = to_request(fence)->engine;

	dma_fence_put(fence);

	return engine;

}

static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)

{

	i915_gem_object_get(vma->obj);

	return vma;

}

static inline void i915_vma_put(struct i915_vma *vma)

{

	i915_gem_object_put(vma->obj);

}

/*

 * Optimised SGL iterator for GEM objects

 */

			 u64 alignment,

			 u64 flags);

int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,

		  u32 flags);

void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);

int __must_check i915_vma_unbind(struct i915_vma *vma);

void i915_vma_close(struct i915_vma *vma);

void i915_vma_destroy(struct i915_vma *vma);

int i915_gem_object_unbind(struct drm_i915_gem_object *obj);

void i915_gem_release_mmap(struct drm_i915_gem_object *obj);

	return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view));

}

/* i915_gem_fence.c */

/* i915_gem_fence_reg.c */

int __must_check i915_vma_get_fence(struct i915_vma *vma);

int __must_check i915_vma_put_fence(struct i915_vma *vma);

/**

 * i915_vma_pin_fence - pin fencing state

 * @vma: vma to pin fencing for

 *

 * This pins the fencing state (whether tiled or untiled) to make sure the

 * vma (and its object) is ready to be used as a scanout target. Fencing

 * status must be synchronize first by calling i915_vma_get_fence():

 *

 * The resulting fence pin reference must be released again with

 * i915_vma_unpin_fence().

 *

 * Returns:

 *

 * True if the vma has a fence, false otherwise.

 */

static inline bool

i915_vma_pin_fence(struct i915_vma *vma)

{

	lockdep_assert_held(&vma->vm->dev->struct_mutex);

	if (vma->fence) {

		vma->fence->pin_count++;

		return true;

	} else

		return false;

}

/**

 * i915_vma_unpin_fence - unpin fencing state

 * @vma: vma to unpin fencing for

 *

 * This releases the fence pin reference acquired through

 * i915_vma_pin_fence. It will handle both objects with and without an

 * attached fence correctly, callers do not need to distinguish this.

 */

static inline void

i915_vma_unpin_fence(struct i915_vma *vma)

{

	lockdep_assert_held(&vma->vm->dev->struct_mutex);

	if (vma->fence) {

		GEM_BUG_ON(vma->fence->pin_count <= 0);

		vma->fence->pin_count--;

	}

}

void i915_gem_restore_fences(struct drm_device *dev);

void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);

	return ret;

}

static void __i915_vma_iounmap(struct i915_vma *vma)

{

	GEM_BUG_ON(i915_vma_is_pinned(vma));

	if (vma->iomap == NULL)

		return;

	io_mapping_unmap(vma->iomap);

	vma->iomap = NULL;

}

int i915_vma_unbind(struct i915_vma *vma)

{

	struct drm_i915_gem_object *obj = vma->obj;

	unsigned long active;

	int ret;

	lockdep_assert_held(&obj->base.dev->struct_mutex);

	/* First wait upon any activity as retiring the request may

	 * have side-effects such as unpinning or even unbinding this vma.

	 */

	active = i915_vma_get_active(vma);

	if (active) {

		int idx;

		/* When a closed VMA is retired, it is unbound - eek.

		 * In order to prevent it from being recursively closed,

		 * take a pin on the vma so that the second unbind is

		 * aborted.

		 *

		 * Even more scary is that the retire callback may free

		 * the object (last active vma). To prevent the explosion

		 * we defer the actual object free to a worker that can

		 * only proceed once it acquires the struct_mutex (which

		 * we currently hold, therefore it cannot free this object

		 * before we are finished).

		 */

		__i915_vma_pin(vma);

		for_each_active(active, idx) {

			ret = i915_gem_active_retire(&vma->last_read[idx],

						   &vma->vm->dev->struct_mutex);

			if (ret)

				break;

		}

		__i915_vma_unpin(vma);

		if (ret)

			return ret;

		GEM_BUG_ON(i915_vma_is_active(vma));

	}

	if (i915_vma_is_pinned(vma))

		return -EBUSY;

	if (!drm_mm_node_allocated(&vma->node))

		goto destroy;

	GEM_BUG_ON(obj->bind_count == 0);

	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));

	if (i915_vma_is_map_and_fenceable(vma)) {

		/* release the fence reg _after_ flushing */

		ret = i915_vma_put_fence(vma);

		if (ret)

			return ret;

		/* Force a pagefault for domain tracking on next user access */

		i915_gem_release_mmap(obj);

		__i915_vma_iounmap(vma);

		vma->flags &= ~I915_VMA_CAN_FENCE;

	}

	if (likely(!vma->vm->closed)) {

		trace_i915_vma_unbind(vma);

		vma->vm->unbind_vma(vma);

	}

	vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);

	drm_mm_remove_node(&vma->node);

	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);

	if (vma->pages != obj->mm.pages) {

		GEM_BUG_ON(!vma->pages);

		sg_free_table(vma->pages);

		kfree(vma->pages);

	}

	vma->pages = NULL;

	/* Since the unbound list is global, only move to that list if

	 * no more VMAs exist. */

	if (--obj->bind_count == 0)

		list_move_tail(&obj->global_link,

			       &to_i915(obj->base.dev)->mm.unbound_list);

	/* And finally now the object is completely decoupled from this vma,

	 * we can drop its hold on the backing storage and allow it to be

	 * reaped by the shrinker.

	 */

	i915_gem_object_unpin_pages(obj);

destroy:

	if (unlikely(i915_vma_is_closed(vma)))

		i915_vma_destroy(vma);

	return 0;

}

static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)

{

	int ret, i;

	return 0;

}

static bool i915_gem_valid_gtt_space(struct i915_vma *vma,

				     unsigned long cache_level)

{

	struct drm_mm_node *gtt_space = &vma->node;

	struct drm_mm_node *other;

	/*

	 * On some machines we have to be careful when putting differing types

	 * of snoopable memory together to avoid the prefetcher crossing memory

	 * domains and dying. During vm initialisation, we decide whether or not

	 * these constraints apply and set the drm_mm.color_adjust

	 * appropriately.

	 */

	if (vma->vm->mm.color_adjust == NULL)

		return true;

	if (!drm_mm_node_allocated(gtt_space))

		return true;

	if (list_empty(&gtt_space->node_list))

		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);

	if (other->allocated && !other->hole_follows && other->color != cache_level)

		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);

	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)

		return false;

	return true;

}

/**

 * i915_vma_insert - finds a slot for the vma in its address space

 * @vma: the vma

 * @size: requested size in bytes (can be larger than the VMA)

 * @alignment: required alignment

 * @flags: mask of PIN_* flags to use

 *

 * First we try to allocate some free space that meets the requirements for

 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,

 * preferrably the oldest idle entry to make room for the new VMA.

 *

 * Returns:

 * 0 on success, negative error code otherwise.

 */

static int

i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)

{

	struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);

	struct drm_i915_gem_object *obj = vma->obj;

	u64 start, end;

	int ret;

	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));

	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));

	size = max(size, vma->size);

	if (flags & PIN_MAPPABLE)

		size = i915_gem_get_ggtt_size(dev_priv, size,

					      i915_gem_object_get_tiling(obj));

	alignment = max(max(alignment, vma->display_alignment),

			i915_gem_get_ggtt_alignment(dev_priv, size,

						    i915_gem_object_get_tiling(obj),

						    flags & PIN_MAPPABLE));

	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;

	end = vma->vm->total;

	if (flags & PIN_MAPPABLE)

		end = min_t(u64, end, dev_priv->ggtt.mappable_end);

	if (flags & PIN_ZONE_4G)

		end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);

	/* If binding the object/GGTT view requires more space than the entire

	 * aperture has, reject it early before evicting everything in a vain

	 * attempt to find space.

	 */

	if (size > end) {

		DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",

			  size, obj->base.size,

			  flags & PIN_MAPPABLE ? "mappable" : "total",

			  end);

		return -E2BIG;

	}

	ret = i915_gem_object_pin_pages(obj);

	if (ret)

		return ret;

	if (flags & PIN_OFFSET_FIXED) {

		u64 offset = flags & PIN_OFFSET_MASK;

		if (offset & (alignment - 1) || offset > end - size) {

			ret = -EINVAL;

			goto err_unpin;

		}

		vma->node.start = offset;

		vma->node.size = size;

		vma->node.color = obj->cache_level;

		ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);

		if (ret) {

			ret = i915_gem_evict_for_vma(vma);

			if (ret == 0)

				ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);

			if (ret)

				goto err_unpin;

		}

	} else {

		u32 search_flag, alloc_flag;

		if (flags & PIN_HIGH) {

			search_flag = DRM_MM_SEARCH_BELOW;

			alloc_flag = DRM_MM_CREATE_TOP;

		} else {

			search_flag = DRM_MM_SEARCH_DEFAULT;

			alloc_flag = DRM_MM_CREATE_DEFAULT;

		}

		/* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,

		 * so we know that we always have a minimum alignment of 4096.

		 * The drm_mm range manager is optimised to return results

		 * with zero alignment, so where possible use the optimal

		 * path.

		 */

		if (alignment <= 4096)

			alignment = 0;

search_free:

		ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,

							  &vma->node,

							  size, alignment,

							  obj->cache_level,

							  start, end,

							  search_flag,

							  alloc_flag);

		if (ret) {

			ret = i915_gem_evict_something(vma->vm, size, alignment,

						       obj->cache_level,

						       start, end,

						       flags);

			if (ret == 0)

				goto search_free;

			goto err_unpin;

		}

		GEM_BUG_ON(vma->node.start < start);

		GEM_BUG_ON(vma->node.start + vma->node.size > end);

	}

	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));

	list_move_tail(&obj->global_link, &dev_priv->mm.bound_list);

	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);

	obj->bind_count++;

	GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);

	return 0;

err_unpin:

	i915_gem_object_unpin_pages(obj);

	return ret;

}

void i915_gem_clflush_object(struct drm_i915_gem_object *obj,

			     bool force)

{