drm/vc4: Add support for drawing 3D frames.

	vc4_crtc.o \

	vc4_drv.o \

	vc4_kms.o \

	vc4_gem.o \

	vc4_hdmi.o \

	vc4_hvs.o \

	vc4_irq.o \

	vc4_plane.o \

	vc4_render_cl.o \

	vc4_trace_points.o \

	vc4_v3d.o \

	vc4_validate.o \

	vc4_validate_shaders.o

vc4-$(CONFIG_DEBUG_FS) += vc4_debugfs.o

obj-$(CONFIG_DRM_VC4)  += vc4.o

CFLAGS_vc4_trace_points.o := -I$(src)

};

static const struct drm_ioctl_desc vc4_drm_ioctls[] = {

	DRM_IOCTL_DEF_DRV(VC4_SUBMIT_CL, vc4_submit_cl_ioctl, 0),

	DRM_IOCTL_DEF_DRV(VC4_WAIT_SEQNO, vc4_wait_seqno_ioctl, 0),

	DRM_IOCTL_DEF_DRV(VC4_WAIT_BO, vc4_wait_bo_ioctl, 0),

	DRM_IOCTL_DEF_DRV(VC4_CREATE_BO, vc4_create_bo_ioctl, 0),

	DRM_IOCTL_DEF_DRV(VC4_MMAP_BO, vc4_mmap_bo_ioctl, 0),

	DRM_IOCTL_DEF_DRV(VC4_CREATE_SHADER_BO, vc4_create_shader_bo_ioctl, 0),

	.driver_features = (DRIVER_MODESET |

			    DRIVER_ATOMIC |

			    DRIVER_GEM |

			    DRIVER_HAVE_IRQ |

			    DRIVER_PRIME),

	.lastclose = vc4_lastclose,

	.preclose = vc4_drm_preclose,

	.irq_handler = vc4_irq,

	.irq_preinstall = vc4_irq_preinstall,

	.irq_postinstall = vc4_irq_postinstall,

	.irq_uninstall = vc4_irq_uninstall,

	.enable_vblank = vc4_enable_vblank,

	.disable_vblank = vc4_disable_vblank,

	.get_vblank_counter = drm_vblank_count,

	if (ret)

		goto unref;

	vc4_gem_init(drm);

	ret = component_bind_all(dev, drm);

	if (ret)

		goto unref;

		goto gem_destroy;

	ret = drm_dev_register(drm, 0);

	if (ret < 0)

	drm_dev_unregister(drm);

unbind_all:

	component_unbind_all(dev, drm);

gem_destroy:

	vc4_gem_destroy(drm);

unref:

	drm_dev_unref(drm);

	vc4_bo_cache_destroy(drm);

	/* Protects bo_cache and the BO stats. */

	struct mutex bo_lock;

	/* Sequence number for the last job queued in job_list.

	 * Starts at 0 (no jobs emitted).

	 */

	uint64_t emit_seqno;

	/* Sequence number for the last completed job on the GPU.

	 * Starts at 0 (no jobs completed).

	 */

	uint64_t finished_seqno;

	/* List of all struct vc4_exec_info for jobs to be executed.

	 * The first job in the list is the one currently programmed

	 * into ct0ca/ct1ca for execution.

	 */

	struct list_head job_list;

	/* List of the finished vc4_exec_infos waiting to be freed by

	 * job_done_work.

	 */

	struct list_head job_done_list;

	/* Spinlock used to synchronize the job_list and seqno

	 * accesses between the IRQ handler and GEM ioctls.

	 */

	spinlock_t job_lock;

	wait_queue_head_t job_wait_queue;

	struct work_struct job_done_work;

	/* The binner overflow memory that's currently set up in

	 * BPOA/BPOS registers.  When overflow occurs and a new one is

	 * allocated, the previous one will be moved to

	 * vc4->current_exec's free list.

	 */

	struct vc4_bo *overflow_mem;

	struct work_struct overflow_mem_work;

	struct {

		uint32_t last_ct0ca, last_ct1ca;

		struct timer_list timer;

		struct work_struct reset_work;

	} hangcheck;

	struct semaphore async_modeset;

};

static inline struct vc4_dev *

struct vc4_bo {

	struct drm_gem_cma_object base;

	/* seqno of the last job to render to this BO. */

	uint64_t seqno;

	/* List entry for the BO's position in either

	 * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list

	 */

#define HVS_READ(offset) readl(vc4->hvs->regs + offset)

#define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset)

struct vc4_exec_info {

	/* Sequence number for this bin/render job. */

	uint64_t seqno;

	/* Kernel-space copy of the ioctl arguments */

	struct drm_vc4_submit_cl *args;

	/* This is the array of BOs that were looked up at the start of exec.

	 * Command validation will use indices into this array.

	 */

	struct drm_gem_cma_object **bo;

	uint32_t bo_count;

	/* Pointers for our position in vc4->job_list */

	struct list_head head;

	/* List of other BOs used in the job that need to be released

	 * once the job is complete.

	 */

	struct list_head unref_list;

	/* Current unvalidated indices into @bo loaded by the non-hardware

	 * VC4_PACKET_GEM_HANDLES.

	 */

	uint32_t bo_index[2];

	/* This is the BO where we store the validated command lists, shader

	 * records, and uniforms.

	 */

	struct drm_gem_cma_object *exec_bo;

	/**

	 * This tracks the per-shader-record state (packet 64) that

	 * determines the length of the shader record and the offset

	 * it's expected to be found at.  It gets read in from the

	 * command lists.

	 */

	struct vc4_shader_state {

		uint32_t addr;

		/* Maximum vertex index referenced by any primitive using this

		 * shader state.

		 */

		uint32_t max_index;

	} *shader_state;

	/** How many shader states the user declared they were using. */

	uint32_t shader_state_size;

	/** How many shader state records the validator has seen. */

	uint32_t shader_state_count;

	bool found_tile_binning_mode_config_packet;

	bool found_start_tile_binning_packet;

	bool found_increment_semaphore_packet;

	bool found_flush;

	uint8_t bin_tiles_x, bin_tiles_y;

	struct drm_gem_cma_object *tile_bo;

	uint32_t tile_alloc_offset;

	/**

	 * Computed addresses pointing into exec_bo where we start the

	 * bin thread (ct0) and render thread (ct1).

	 */

	uint32_t ct0ca, ct0ea;

	uint32_t ct1ca, ct1ea;

	/* Pointer to the unvalidated bin CL (if present). */

	void *bin_u;

	/* Pointers to the shader recs.  These paddr gets incremented as CL

	 * packets are relocated in validate_gl_shader_state, and the vaddrs

	 * (u and v) get incremented and size decremented as the shader recs

	 * themselves are validated.

	 */

	void *shader_rec_u;

	void *shader_rec_v;

	uint32_t shader_rec_p;

	uint32_t shader_rec_size;

	/* Pointers to the uniform data.  These pointers are incremented, and

	 * size decremented, as each batch of uniforms is uploaded.

	 */

	void *uniforms_u;

	void *uniforms_v;

	uint32_t uniforms_p;

	uint32_t uniforms_size;

};

static inline struct vc4_exec_info *

vc4_first_job(struct vc4_dev *vc4)

{

	if (list_empty(&vc4->job_list))

		return NULL;

	return list_first_entry(&vc4->job_list, struct vc4_exec_info, head);

}

/**

 * struct vc4_texture_sample_info - saves the offsets into the UBO for texture

 * setup parameters.

/* vc4_drv.c */

void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index);

/* vc4_gem.c */

void vc4_gem_init(struct drm_device *dev);

void vc4_gem_destroy(struct drm_device *dev);

int vc4_submit_cl_ioctl(struct drm_device *dev, void *data,

			struct drm_file *file_priv);

int vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,

			 struct drm_file *file_priv);

int vc4_wait_bo_ioctl(struct drm_device *dev, void *data,

		      struct drm_file *file_priv);

void vc4_submit_next_job(struct drm_device *dev);

int vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno,

		       uint64_t timeout_ns, bool interruptible);

void vc4_job_handle_completed(struct vc4_dev *vc4);

/* vc4_hdmi.c */

extern struct platform_driver vc4_hdmi_driver;

int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused);

/* vc4_irq.c */

irqreturn_t vc4_irq(int irq, void *arg);

void vc4_irq_preinstall(struct drm_device *dev);

int vc4_irq_postinstall(struct drm_device *dev);

void vc4_irq_uninstall(struct drm_device *dev);

void vc4_irq_reset(struct drm_device *dev);

/* vc4_hvs.c */

extern struct platform_driver vc4_hvs_driver;

void vc4_hvs_dump_state(struct drm_device *dev);

extern struct platform_driver vc4_v3d_driver;

int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused);

int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused);

int vc4_v3d_set_power(struct vc4_dev *vc4, bool on);

/* vc4_validate.c */

int

vc4_validate_bin_cl(struct drm_device *dev,

		    void *validated,

		    void *unvalidated,

		    struct vc4_exec_info *exec);

int

vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec);

struct drm_gem_cma_object *vc4_use_bo(struct vc4_exec_info *exec,

				      uint32_t hindex);

int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec);

bool vc4_check_tex_size(struct vc4_exec_info *exec,

			struct drm_gem_cma_object *fbo,

			uint32_t offset, uint8_t tiling_format,

			uint32_t width, uint32_t height, uint8_t cpp);

/* vc4_validate_shader.c */

struct vc4_validated_shader_info *

/*

 * Copyright © 2014 Broadcom

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the "Software"),

 * to deal in the Software without restriction, including without limitation

 * the rights to use, copy, modify, merge, publish, distribute, sublicense,

 * and/or sell copies of the Software, and to permit persons to whom the

 * Software is furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice (including the next

 * paragraph) shall be included in all copies or substantial portions of the

 * Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS

 * IN THE SOFTWARE.

 */

#include <linux/module.h>

#include <linux/platform_device.h>

#include <linux/device.h>

#include <linux/io.h>

#include "uapi/drm/vc4_drm.h"

#include "vc4_drv.h"

#include "vc4_regs.h"

#include "vc4_trace.h"

static void

vc4_queue_hangcheck(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	mod_timer(&vc4->hangcheck.timer,

		  round_jiffies_up(jiffies + msecs_to_jiffies(100)));

}

static void

vc4_reset(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	DRM_INFO("Resetting GPU.\n");

	vc4_v3d_set_power(vc4, false);

	vc4_v3d_set_power(vc4, true);

	vc4_irq_reset(dev);

	/* Rearm the hangcheck -- another job might have been waiting

	 * for our hung one to get kicked off, and vc4_irq_reset()

	 * would have started it.

	 */

	vc4_queue_hangcheck(dev);

}

static void

vc4_reset_work(struct work_struct *work)

{

	struct vc4_dev *vc4 =

		container_of(work, struct vc4_dev, hangcheck.reset_work);

	vc4_reset(vc4->dev);

}

static void

vc4_hangcheck_elapsed(unsigned long data)

{

	struct drm_device *dev = (struct drm_device *)data;

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	uint32_t ct0ca, ct1ca;

	/* If idle, we can stop watching for hangs. */

	if (list_empty(&vc4->job_list))

		return;

	ct0ca = V3D_READ(V3D_CTNCA(0));

	ct1ca = V3D_READ(V3D_CTNCA(1));

	/* If we've made any progress in execution, rearm the timer

	 * and wait.

	 */

	if (ct0ca != vc4->hangcheck.last_ct0ca ||

	    ct1ca != vc4->hangcheck.last_ct1ca) {

		vc4->hangcheck.last_ct0ca = ct0ca;

		vc4->hangcheck.last_ct1ca = ct1ca;

		vc4_queue_hangcheck(dev);

		return;

	}

	/* We've gone too long with no progress, reset.  This has to

	 * be done from a work struct, since resetting can sleep and

	 * this timer hook isn't allowed to.

	 */

	schedule_work(&vc4->hangcheck.reset_work);

}

static void

submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	/* Set the current and end address of the control list.

	 * Writing the end register is what starts the job.

	 */

	V3D_WRITE(V3D_CTNCA(thread), start);

	V3D_WRITE(V3D_CTNEA(thread), end);

}

int

vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns,

		   bool interruptible)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	int ret = 0;

	unsigned long timeout_expire;

	DEFINE_WAIT(wait);

	if (vc4->finished_seqno >= seqno)

		return 0;

	if (timeout_ns == 0)

		return -ETIME;

	timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns);

	trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns);

	for (;;) {

		prepare_to_wait(&vc4->job_wait_queue, &wait,

				interruptible ? TASK_INTERRUPTIBLE :

				TASK_UNINTERRUPTIBLE);

		if (interruptible && signal_pending(current)) {

			ret = -ERESTARTSYS;

			break;

		}

		if (vc4->finished_seqno >= seqno)

			break;

		if (timeout_ns != ~0ull) {

			if (time_after_eq(jiffies, timeout_expire)) {

				ret = -ETIME;

				break;

			}

			schedule_timeout(timeout_expire - jiffies);

		} else {

			schedule();

		}

	}

	finish_wait(&vc4->job_wait_queue, &wait);

	trace_vc4_wait_for_seqno_end(dev, seqno);

	if (ret && ret != -ERESTARTSYS) {

		DRM_ERROR("timeout waiting for render thread idle\n");

		return ret;

	}

	return 0;

}

static void

vc4_flush_caches(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	/* Flush the GPU L2 caches.  These caches sit on top of system

	 * L3 (the 128kb or so shared with the CPU), and are

	 * non-allocating in the L3.

	 */

	V3D_WRITE(V3D_L2CACTL,

		  V3D_L2CACTL_L2CCLR);

	V3D_WRITE(V3D_SLCACTL,

		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |

		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) |

		  VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) |

		  VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC));

}

/* Sets the registers for the next job to be actually be executed in

 * the hardware.

 *

 * The job_lock should be held during this.

 */

void

vc4_submit_next_job(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	struct vc4_exec_info *exec = vc4_first_job(vc4);

	if (!exec)

		return;

	vc4_flush_caches(dev);

	/* Disable the binner's pre-loaded overflow memory address */

	V3D_WRITE(V3D_BPOA, 0);

	V3D_WRITE(V3D_BPOS, 0);

	if (exec->ct0ca != exec->ct0ea)

		submit_cl(dev, 0, exec->ct0ca, exec->ct0ea);

	submit_cl(dev, 1, exec->ct1ca, exec->ct1ea);

}

static void

vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)

{

	struct vc4_bo *bo;

	unsigned i;

	for (i = 0; i < exec->bo_count; i++) {

		bo = to_vc4_bo(&exec->bo[i]->base);

		bo->seqno = seqno;

	}

	list_for_each_entry(bo, &exec->unref_list, unref_head) {

		bo->seqno = seqno;

	}

}

/* Queues a struct vc4_exec_info for execution.  If no job is

 * currently executing, then submits it.

 *

 * Unlike most GPUs, our hardware only handles one command list at a

 * time.  To queue multiple jobs at once, we'd need to edit the

 * previous command list to have a jump to the new one at the end, and

 * then bump the end address.  That's a change for a later date,

 * though.

 */

static void

vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	uint64_t seqno;

	unsigned long irqflags;

	spin_lock_irqsave(&vc4->job_lock, irqflags);

	seqno = ++vc4->emit_seqno;

	exec->seqno = seqno;

	vc4_update_bo_seqnos(exec, seqno);

	list_add_tail(&exec->head, &vc4->job_list);

	/* If no job was executing, kick ours off.  Otherwise, it'll

	 * get started when the previous job's frame done interrupt

	 * occurs.

	 */

	if (vc4_first_job(vc4) == exec) {

		vc4_submit_next_job(dev);

		vc4_queue_hangcheck(dev);

	}

	spin_unlock_irqrestore(&vc4->job_lock, irqflags);

}

/**

 * Looks up a bunch of GEM handles for BOs and stores the array for

 * use in the command validator that actually writes relocated

 * addresses pointing to them.

 */

static int

vc4_cl_lookup_bos(struct drm_device *dev,

		  struct drm_file *file_priv,

		  struct vc4_exec_info *exec)

{

	struct drm_vc4_submit_cl *args = exec->args;

	uint32_t *handles;

	int ret = 0;

	int i;

	exec->bo_count = args->bo_handle_count;

	if (!exec->bo_count) {

		/* See comment on bo_index for why we have to check

		 * this.

		 */

		DRM_ERROR("Rendering requires BOs to validate\n");

		return -EINVAL;

	}

	exec->bo = kcalloc(exec->bo_count, sizeof(struct drm_gem_cma_object *),

			   GFP_KERNEL);

	if (!exec->bo) {

		DRM_ERROR("Failed to allocate validated BO pointers\n");

		return -ENOMEM;

	}

	handles = drm_malloc_ab(exec->bo_count, sizeof(uint32_t));

	if (!handles) {

		DRM_ERROR("Failed to allocate incoming GEM handles\n");

		goto fail;

	}

	ret = copy_from_user(handles,

			     (void __user *)(uintptr_t)args->bo_handles,

			     exec->bo_count * sizeof(uint32_t));

	if (ret) {

		DRM_ERROR("Failed to copy in GEM handles\n");

		goto fail;

	}

	spin_lock(&file_priv->table_lock);

	for (i = 0; i < exec->bo_count; i++) {

		struct drm_gem_object *bo = idr_find(&file_priv->object_idr,

						     handles[i]);

		if (!bo) {

			DRM_ERROR("Failed to look up GEM BO %d: %d\n",

				  i, handles[i]);

			ret = -EINVAL;

			spin_unlock(&file_priv->table_lock);

			goto fail;

		}

		drm_gem_object_reference(bo);

		exec->bo[i] = (struct drm_gem_cma_object *)bo;

	}

	spin_unlock(&file_priv->table_lock);

fail:

	kfree(handles);

	return 0;

}

static int

vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)

{

	struct drm_vc4_submit_cl *args = exec->args;

	void *temp = NULL;

	void *bin;

	int ret = 0;

	uint32_t bin_offset = 0;

	uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,

					     16);

	uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;

	uint32_t exec_size = uniforms_offset + args->uniforms_size;

	uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *

					  args->shader_rec_count);

	struct vc4_bo *bo;

	if (uniforms_offset < shader_rec_offset ||

	    exec_size < uniforms_offset ||

	    args->shader_rec_count >= (UINT_MAX /

					  sizeof(struct vc4_shader_state)) ||

	    temp_size < exec_size) {

		DRM_ERROR("overflow in exec arguments\n");

		goto fail;

	}

	/* Allocate space where we'll store the copied in user command lists

	 * and shader records.

	 *

	 * We don't just copy directly into the BOs because we need to

	 * read the contents back for validation, and I think the

	 * bo->vaddr is uncached access.

	 */

	temp = kmalloc(temp_size, GFP_KERNEL);

	if (!temp) {

		DRM_ERROR("Failed to allocate storage for copying "

			  "in bin/render CLs.\n");

		ret = -ENOMEM;

		goto fail;

	}

	bin = temp + bin_offset;

	exec->shader_rec_u = temp + shader_rec_offset;

	exec->uniforms_u = temp + uniforms_offset;

	exec->shader_state = temp + exec_size;

	exec->shader_state_size = args->shader_rec_count;

	ret = copy_from_user(bin,

			     (void __user *)(uintptr_t)args->bin_cl,

			     args->bin_cl_size);

	if (ret) {

		DRM_ERROR("Failed to copy in bin cl\n");

		goto fail;

	}

	ret = copy_from_user(exec->shader_rec_u,

			     (void __user *)(uintptr_t)args->shader_rec,

			     args->shader_rec_size);

	if (ret) {

		DRM_ERROR("Failed to copy in shader recs\n");

		goto fail;

	}

	ret = copy_from_user(exec->uniforms_u,

			     (void __user *)(uintptr_t)args->uniforms,

			     args->uniforms_size);

	if (ret) {

		DRM_ERROR("Failed to copy in uniforms cl\n");

		goto fail;

	}

	bo = vc4_bo_create(dev, exec_size, true);

	if (!bo) {

		DRM_ERROR("Couldn't allocate BO for binning\n");

		ret = PTR_ERR(exec->exec_bo);

		goto fail;

	}

	exec->exec_bo = &bo->base;

	list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,

		      &exec->unref_list);

	exec->ct0ca = exec->exec_bo->paddr + bin_offset;

	exec->bin_u = bin;

	exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;

	exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset;

	exec->shader_rec_size = args->shader_rec_size;

	exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;

	exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset;

	exec->uniforms_size = args->uniforms_size;

	ret = vc4_validate_bin_cl(dev,

				  exec->exec_bo->vaddr + bin_offset,

				  bin,

				  exec);

	if (ret)

		goto fail;

	ret = vc4_validate_shader_recs(dev, exec);

fail:

	kfree(temp);

	return ret;

}