Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b486ad97 authored by Linux Build Service Account's avatar Linux Build Service Account Committed by Gerrit - the friendly Code Review server
Browse files

Merge "msm: adsprpc: allocate all remote memory in kernel"

parents ef9aa79b 092de9c8

drivers/char/adsprpc.c

+293 −92
Original line number Diff line number Diff line
@@ -55,6 +55,8 @@ 
#define TZ_PIL_AUTH_QDSP6_PROC 1

#define ADSP_MMAP_HEAP_ADDR 4

#define ADSP_MMAP_REMOTE_HEAP_ADDR 8

#define ADSP_MMAP_ADD_PAGES 0x1000



#define FASTRPC_ENOSUCH 39

#define VMID_SSC_Q6     38

#define VMID_ADSP_Q6    6
@@ -126,6 +128,7 @@ static inline uint64_t buf_num_pages(uint64_t buf, size_t len) 
static inline uint64_t buf_page_size(uint32_t size)

{

	uint64_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;



	return sz > PAGE_SIZE ? sz : PAGE_SIZE;

}
@@ -145,10 +148,15 @@ struct fastrpc_file; 


struct fastrpc_buf {

	struct hlist_node hn;

	struct hlist_node hn_rem;

	struct fastrpc_file *fl;

	void *virt;

	uint64_t phys;

	size_t size;

	struct dma_attrs attrs;

	uintptr_t raddr;

	uint32_t flags;

	int remote;

};



struct fastrpc_ctx_lst;
@@ -292,9 +300,11 @@ struct fastrpc_file { 
	struct hlist_node hn;

	spinlock_t hlock;

	struct hlist_head maps;

	struct hlist_head bufs;

	struct hlist_head cached_bufs;

	struct hlist_head remote_bufs;

	struct fastrpc_ctx_lst clst;

	struct fastrpc_session_ctx *sctx;

	struct fastrpc_buf *init_mem;

	struct fastrpc_session_ctx *secsctx;

	uint32_t mode;

	uint32_t profile;
@@ -363,10 +373,17 @@ static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache) 
		return;

	if (cache) {

		spin_lock(&fl->hlock);

		hlist_add_head(&buf->hn, &fl->bufs);

		hlist_add_head(&buf->hn, &fl->cached_bufs);

		spin_unlock(&fl->hlock);

		return;

	}

	if (buf->remote) {

		spin_lock(&fl->hlock);

		hlist_del_init(&buf->hn_rem);

		spin_unlock(&fl->hlock);

		buf->remote = 0;

		buf->raddr = 0;

	}

	if (!IS_ERR_OR_NULL(buf->virt)) {

		int destVM[1] = {VMID_HLOS};

		int destVMperm[1] = {PERM_READ | PERM_WRITE | PERM_EXEC};
@@ -380,21 +397,22 @@ static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache) 
			hyp_assign_phys(buf->phys, buf_page_size(buf->size),

				srcVM, 2, destVM, destVMperm, 1);

		}

		dma_free_coherent(fl->sctx->smmu.dev, buf->size, buf->virt,

					buf->phys);

		dma_free_attrs(fl->sctx->smmu.dev, buf->size, buf->virt,

			buf->phys, (struct dma_attrs *)&buf->attrs);

	}

	kfree(buf);

}



static void fastrpc_buf_list_free(struct fastrpc_file *fl)

static void fastrpc_cached_buf_list_free(struct fastrpc_file *fl)

{

	struct fastrpc_buf *buf, *free;



	do {

		struct hlist_node *n;



		free = NULL;

		spin_lock(&fl->hlock);

		hlist_for_each_entry_safe(buf, n, &fl->bufs, hn) {

		hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {

			hlist_del_init(&buf->hn);

			free = buf;

			break;
@@ -405,6 +423,25 @@ static void fastrpc_buf_list_free(struct fastrpc_file *fl) 
	} while (free);

}



static void fastrpc_remote_buf_list_free(struct fastrpc_file *fl)

{

	struct fastrpc_buf *buf, *free;



	do {

		struct hlist_node *n;



		free = NULL;

		spin_lock(&fl->hlock);

		hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {

			free = buf;

			break;

		}

		spin_unlock(&fl->hlock);

		if (free)

			fastrpc_buf_free(free, 0);

	} while (free);

}



static void fastrpc_mmap_add(struct fastrpc_mmap *map)

{

	if (map->flags == ADSP_MMAP_HEAP_ADDR ||
@@ -465,23 +502,21 @@ static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd, uintptr_t va, 
	return -ENOTTY;

}



static int dma_alloc_memory(phys_addr_t *region_start, size_t size)

static int dma_alloc_memory(dma_addr_t *region_start, void **vaddr, size_t size,

			struct dma_attrs *attrs)

{

	struct fastrpc_apps *me = &gfa;

	void *vaddr = NULL;

	DEFINE_DMA_ATTRS(attrs);



	if (me->dev == NULL) {

		pr_err("device adsprpc-mem is not initialized\n");

		return -ENODEV;

	}

	dma_set_attr(DMA_ATTR_SKIP_ZEROING, &attrs);

	dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);

	vaddr = dma_alloc_attrs(me->dev, size, region_start, GFP_KERNEL,

						&attrs);

	if (!vaddr) {

		pr_err("ADSPRPC: Failed to allocate %x remote heap memory\n",

						(unsigned int)size);



	*vaddr = dma_alloc_attrs(me->dev, size, region_start,

				 GFP_KERNEL, attrs);

	if (IS_ERR_OR_NULL(*vaddr)) {

		pr_err("adsprpc: %s: %s: dma_alloc_attrs failed for size 0x%zx, returned %pK\n",

				current->comm, __func__, size, (*vaddr));

		return -ENOMEM;

	}

	return 0;
@@ -617,7 +652,8 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, unsigned attr, 
	struct fastrpc_channel_ctx *chan = &apps->channel[cid];

	struct fastrpc_mmap *map = NULL;

	struct dma_attrs attrs;

	phys_addr_t region_start = 0;

	dma_addr_t region_start = 0;

	void *region_vaddr = NULL;

	unsigned long flags;

	int err = 0, vmid;
@@ -635,14 +671,20 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, unsigned attr, 
	map->attr = attr;

	if (mflags == ADSP_MMAP_HEAP_ADDR ||

				mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {

		DEFINE_DMA_ATTRS(rh_attrs);



		dma_set_attr(DMA_ATTR_SKIP_ZEROING, &rh_attrs);

		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &rh_attrs);



		map->apps = me;

		map->fl = NULL;

		VERIFY(err, !dma_alloc_memory(&region_start, len));

		VERIFY(err, !dma_alloc_memory(&region_start,

			 &region_vaddr, len, &rh_attrs));

		if (err)

			goto bail;

		map->phys = (uintptr_t)region_start;

		map->size = len;

		map->va = (uintptr_t)map->phys;

		map->va = (uintptr_t)region_vaddr;

	} else {

		VERIFY(err, !IS_ERR_OR_NULL(map->handle =

				ion_import_dma_buf(fl->apps->client, fd)));
@@ -741,7 +783,8 @@ bail: 
}



static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,

			     struct fastrpc_buf **obuf)

				struct dma_attrs attr, uint32_t rflags,

				int remote, struct fastrpc_buf **obuf)

{

	int err = 0, vmid;

	struct fastrpc_buf *buf = NULL, *fr = NULL;
@@ -751,9 +794,10 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
	if (err)

		goto bail;



	if (!remote) {

		/* find the smallest buffer that fits in the cache */

		spin_lock(&fl->hlock);

	hlist_for_each_entry_safe(buf, n, &fl->bufs, hn) {

		hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {

			if (buf->size >= size && (!fr || fr->size > buf->size))

				fr = buf;

		}
@@ -764,6 +808,7 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
			*obuf = fr;

			return 0;

		}

	}

	buf = NULL;

	VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));

	if (err)
@@ -773,17 +818,29 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
	buf->virt = NULL;

	buf->phys = 0;

	buf->size = size;

	buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,

				       (void *)&buf->phys, GFP_KERNEL);

	memcpy(&buf->attrs, &attr, sizeof(struct dma_attrs));

	buf->flags = rflags;

	buf->raddr = 0;

	buf->remote = 0;

	buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,

				       (dma_addr_t *)&buf->phys,

					 GFP_KERNEL,

					 (struct dma_attrs *)&buf->attrs);

	if (IS_ERR_OR_NULL(buf->virt)) {

		/* free cache and retry */

		fastrpc_buf_list_free(fl);

		buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,

					       (void *)&buf->phys, GFP_KERNEL);

		fastrpc_cached_buf_list_free(fl);

		buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,

					 (dma_addr_t *)&buf->phys,

					 GFP_KERNEL,

					 (struct dma_attrs *)&buf->attrs);

		VERIFY(err, !IS_ERR_OR_NULL(buf->virt));

	}

	if (err)

	if (err) {

		err = -ENOMEM;

		pr_err("adsprpc: %s: %s: dma_alloc_attrs failed for size 0x%zx\n",

			current->comm, __func__, size);

		goto bail;

	}

	if (fl->sctx->smmu.cb)

		buf->phys += ((uint64_t)fl->sctx->smmu.cb << 32);

	vmid = fl->apps->channel[fl->cid].vmid;
@@ -799,6 +856,13 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
			goto bail;

	}



	if (remote) {

		INIT_HLIST_NODE(&buf->hn_rem);

		spin_lock(&fl->hlock);

		hlist_add_head(&buf->hn_rem, &fl->remote_bufs);

		spin_unlock(&fl->hlock);

		buf->remote = remote;

	}

	*obuf = buf;

 bail:

	if (err && buf)
@@ -806,7 +870,6 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
	return err;

}





static int context_restore_interrupted(struct fastrpc_file *fl,

				       struct fastrpc_ioctl_invoke_attrs *inv,

				       struct smq_invoke_ctx **po)
@@ -815,6 +878,7 @@ static int context_restore_interrupted(struct fastrpc_file *fl, 
	struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;

	struct hlist_node *n;

	struct fastrpc_ioctl_invoke *invoke = &inv->inv;



	spin_lock(&fl->hlock);

	hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {

		if (ictx->pid == current->pid) {
@@ -854,6 +918,7 @@ static int context_build_overlap(struct smq_invoke_ctx *ctx) 
	int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);

	int nbufs = inbufs + outbufs;

	struct overlap max;



	for (i = 0; i < nbufs; ++i) {

		ctx->overs[i].start = (uintptr_t)lpra[i].buf.pv;

		ctx->overs[i].end = ctx->overs[i].start + lpra[i].buf.len;
@@ -1003,12 +1068,13 @@ bail: 
static void context_save_interrupted(struct smq_invoke_ctx *ctx)

{

	struct fastrpc_ctx_lst *clst = &ctx->fl->clst;



	spin_lock(&ctx->fl->hlock);

	hlist_del_init(&ctx->hn);

	hlist_add_head(&ctx->hn, &clst->interrupted);

	spin_unlock(&ctx->fl->hlock);

	/* free the cache on power collapse */

	fastrpc_buf_list_free(ctx->fl);

	fastrpc_cached_buf_list_free(ctx->fl);

}



static void context_free(struct smq_invoke_ctx *ctx)
@@ -1044,11 +1110,11 @@ static void context_notify_user(struct smq_invoke_ctx *ctx, int retval) 
	complete(&ctx->work);

}





static void fastrpc_notify_users(struct fastrpc_file *me)

{

	struct smq_invoke_ctx *ictx;

	struct hlist_node *n;



	spin_lock(&me->hlock);

	hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {

		complete(&ictx->work);
@@ -1064,6 +1130,7 @@ static void fastrpc_notify_drivers(struct fastrpc_apps *me, int cid) 
{

	struct fastrpc_file *fl;

	struct hlist_node *n;



	spin_lock(&me->hlock);

	hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {

		if (fl->cid == cid)
@@ -1072,6 +1139,7 @@ static void fastrpc_notify_drivers(struct fastrpc_apps *me, int cid) 
	spin_unlock(&me->hlock);



}



static void context_list_ctor(struct fastrpc_ctx_lst *me)

{

	INIT_HLIST_HEAD(&me->interrupted);
@@ -1083,6 +1151,7 @@ static void fastrpc_context_list_dtor(struct fastrpc_file *fl) 
	struct fastrpc_ctx_lst *clst = &fl->clst;

	struct smq_invoke_ctx *ictx = NULL, *ctxfree;

	struct hlist_node *n;



	do {

		ctxfree = NULL;

		spin_lock(&fl->hlock);
@@ -1110,10 +1179,12 @@ static void fastrpc_context_list_dtor(struct fastrpc_file *fl) 
}



static int fastrpc_file_free(struct fastrpc_file *fl);



static void fastrpc_file_list_dtor(struct fastrpc_apps *me)

{

	struct fastrpc_file *fl, *free;

	struct hlist_node *n;



	do {

		free = NULL;

		spin_lock(&me->hlock);
@@ -1187,7 +1258,10 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 


	/* allocate new buffer */

	if (copylen) {

		VERIFY(err, !fastrpc_buf_alloc(ctx->fl, copylen, &ctx->buf));

		DEFINE_DMA_ATTRS(ctx_attrs);



		err = fastrpc_buf_alloc(ctx->fl, copylen, ctx_attrs,

					0, 0, &ctx->buf);

		if (err)

			goto bail;

	}
@@ -1206,6 +1280,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
	args = (uintptr_t)ctx->buf->virt + metalen;

	for (i = 0; i < bufs; ++i) {

		size_t len = lpra[i].buf.len;



		list[i].num = 0;

		list[i].pgidx = 0;

		if (!len)
@@ -1220,6 +1295,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
		struct fastrpc_mmap *map = ctx->maps[i];

		uint64_t buf = ptr_to_uint64(lpra[i].buf.pv);

		size_t len = lpra[i].buf.len;



		rpra[i].buf.pv = 0;

		rpra[i].buf.len = len;

		if (!len)
@@ -1561,6 +1637,7 @@ static void smd_event_handler(void *priv, unsigned event) 
static void fastrpc_init(struct fastrpc_apps *me)

{

	int i;



	INIT_HLIST_HEAD(&me->drivers);

	INIT_HLIST_HEAD(&me->maps);

	spin_lock_init(&me->hlock);
@@ -1684,6 +1761,7 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
	struct fastrpc_ioctl_init *init = &uproc->init;

	struct smq_phy_page pages[1];

	struct fastrpc_mmap *file = NULL, *mem = NULL;

	struct fastrpc_buf *imem = NULL;

	char *proc_name = NULL;

	int srcVM[1] = {VMID_HLOS};

	int destVM[1] = {gcinfo[0].heap_vmid};
@@ -1696,6 +1774,7 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
	if (init->flags == FASTRPC_INIT_ATTACH) {

		remote_arg_t ra[1];

		int tgid = current->tgid;



		ra[0].buf.pv = (void *)&tgid;

		ra[0].buf.len = sizeof(tgid);

		ioctl.inv.handle = 1;
@@ -1712,6 +1791,8 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
		remote_arg_t ra[6];

		int fds[6];

		int mflags = 0;

		int memlen;

		DEFINE_DMA_ATTRS(imem_dma_attr);

		struct {

			int pgid;

			unsigned int namelen;
@@ -1734,14 +1815,27 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
			if (err)

				goto bail;

		}

		if (!access_ok(1, (void const __user *)init->mem,

				init->memlen))

			goto bail;

		inbuf.pageslen = 1;

		VERIFY(err, !fastrpc_mmap_create(fl, init->memfd, 0,

				init->mem, init->memlen, mflags, &mem));



		VERIFY(err, !init->mem);

		if (err) {

			err = -EINVAL;

			pr_err("adsprpc: %s: %s: ERROR: donated memory allocated in userspace\n",

				current->comm, __func__);

			goto bail;

		}

		memlen = ALIGN(max(1024*1024*3, (int)init->filelen * 4),

						1024*1024);



		dma_set_attr(DMA_ATTR_EXEC_MAPPING, &imem_dma_attr);

		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &imem_dma_attr);

		dma_set_attr(DMA_ATTR_FORCE_NON_COHERENT, &imem_dma_attr);



		err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0, 0, &imem);

		if (err)

			goto bail;

		fl->init_mem = imem;



		inbuf.pageslen = 1;

		ra[0].buf.pv = (void *)&inbuf;

		ra[0].buf.len = sizeof(inbuf);
@@ -1755,8 +1849,8 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
		ra[2].buf.len = inbuf.filelen;

		fds[2] = init->filefd;



		pages[0].addr = mem->phys;

		pages[0].size = mem->size;

		pages[0].addr = imem->phys;

		pages[0].size = imem->size;

		ra[3].buf.pv = (void *)pages;

		ra[3].buf.len = 1 * sizeof(*pages);

		fds[3] = 0;
@@ -1896,7 +1990,8 @@ bail: 
}



static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags,

			       struct fastrpc_mmap *map)

					uintptr_t va, uint64_t phys,

					size_t size, uintptr_t *raddr)

{

	struct fastrpc_ioctl_invoke_attrs ioctl;

	struct smq_phy_page page;
@@ -1913,14 +2008,15 @@ static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags, 
	struct {

		uintptr_t vaddrout;

	} routargs;



	inargs.pid = current->tgid;

	inargs.vaddrin = (uintptr_t)map->va;

	inargs.vaddrin = (uintptr_t)va;

	inargs.flags = flags;

	inargs.num = fl->apps->compat ? num * sizeof(page) : num;

	ra[0].buf.pv = (void *)&inargs;

	ra[0].buf.len = sizeof(inargs);

	page.addr = map->phys;

	page.size = map->size;

	page.addr = phys;

	page.size = size;

	ra[1].buf.pv = (void *)&page;

	ra[1].buf.len = num * sizeof(page);
@@ -1937,15 +2033,15 @@ static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags, 
	ioctl.attrs = NULL;

	VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,

		FASTRPC_MODE_PARALLEL, 1, &ioctl)));

	map->raddr = (uintptr_t)routargs.vaddrout;

	*raddr = (uintptr_t)routargs.vaddrout;

	if (err)

		goto bail;

	if (flags == ADSP_MMAP_HEAP_ADDR) {

		struct scm_desc desc = {0};



		desc.args[0] = TZ_PIL_AUTH_QDSP6_PROC;

		desc.args[1] = map->phys;

		desc.args[2] = map->size;

		desc.args[1] = phys;

		desc.args[2] = size;

		desc.arginfo = SCM_ARGS(3);

		err = scm_call2(SCM_SIP_FNID(SCM_SVC_PIL,

			TZ_PIL_PROTECT_MEM_SUBSYS_ID), &desc);
@@ -1955,7 +2051,7 @@ static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags, 
		int destVM[1] = {gcinfo[0].heap_vmid};

		int destVMperm[1] = {PERM_READ | PERM_WRITE | PERM_EXEC};



		VERIFY(err, !hyp_assign_phys(map->phys, (uint64_t)map->size,

		VERIFY(err, !hyp_assign_phys(phys, (uint64_t)size,

				srcVM, 1, destVM, destVMperm, 1));

		if (err)

			goto bail;
@@ -1964,16 +2060,17 @@ bail: 
	return err;

}



static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl,

				 struct fastrpc_mmap *map)

static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl, uint64_t phys,

						size_t size, uint32_t flags)

{

	int err = 0;

	int srcVM[1] = {gcinfo[0].heap_vmid};

	int destVM[1] = {VMID_HLOS};

	int destVMperm[1] = {PERM_READ | PERM_WRITE | PERM_EXEC};



	if (map->flags == ADSP_MMAP_HEAP_ADDR) {

	if (flags == ADSP_MMAP_HEAP_ADDR) {

		struct fastrpc_ioctl_invoke_attrs ioctl;



		struct scm_desc desc = {0};

		remote_arg_t ra[1];

		int err = 0;
@@ -1997,14 +2094,14 @@ static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl, 
		if (err)

			goto bail;

		desc.args[0] = TZ_PIL_AUTH_QDSP6_PROC;

		desc.args[1] = map->phys;

		desc.args[2] = map->size;

		desc.args[1] = phys;

		desc.args[2] = size;

		desc.args[3] = routargs.skey;

		desc.arginfo = SCM_ARGS(4);

		err = scm_call2(SCM_SIP_FNID(SCM_SVC_PIL,

			TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID), &desc);

	} else if (map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {

		VERIFY(err, !hyp_assign_phys(map->phys, (uint64_t)map->size,

	} else if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {

		VERIFY(err, !hyp_assign_phys(phys, (uint64_t)size,

					srcVM, 1, destVM, destVMperm, 1));

		if (err)

			goto bail;
@@ -2014,8 +2111,8 @@ bail: 
	return err;

}



static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl,

				 struct fastrpc_mmap *map)

static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, uintptr_t raddr,

				uint64_t phys, size_t size, uint32_t flags)

{

	struct fastrpc_ioctl_invoke_attrs ioctl;

	remote_arg_t ra[1];
@@ -2027,8 +2124,8 @@ static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, 
	} inargs;



	inargs.pid = current->tgid;

	inargs.size = map->size;

	inargs.vaddrout = map->raddr;

	inargs.size = size;

	inargs.vaddrout = raddr;

	ra[0].buf.pv = (void *)&inargs;

	ra[0].buf.len = sizeof(inargs);
@@ -2044,9 +2141,9 @@ static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, 
		FASTRPC_MODE_PARALLEL, 1, &ioctl)));

	if (err)

		goto bail;

	if (map->flags == ADSP_MMAP_HEAP_ADDR ||

				map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {

		VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, map));

	if (flags == ADSP_MMAP_HEAP_ADDR ||

				flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {

		VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, phys, size, flags));

		if (err)

			goto bail;

	}
@@ -2073,7 +2170,8 @@ static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl) 
		spin_unlock(&me->hlock);



		if (match) {

			VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, match));

			VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, match->phys,

						match->size, match->flags));

			if (err)

				goto bail;

			if (me->channel[0].ramdumpenabled) {
@@ -2110,12 +2208,37 @@ static int fastrpc_internal_munmap(struct fastrpc_file *fl, 
{

	int err = 0;

	struct fastrpc_mmap *map = NULL;

	struct fastrpc_buf *rbuf = NULL, *free = NULL;

	struct hlist_node *n;



	mutex_lock(&fl->map_mutex);

	spin_lock(&fl->hlock);

	hlist_for_each_entry_safe(rbuf, n, &fl->remote_bufs, hn_rem) {

		if (rbuf->raddr && (rbuf->flags == ADSP_MMAP_ADD_PAGES)) {

			if ((rbuf->raddr == ud->vaddrout) &&

				(rbuf->size == ud->size)) {

				free = rbuf;

				break;

			}

		}

	}

	spin_unlock(&fl->hlock);



	if (free) {

		VERIFY(err, !fastrpc_munmap_on_dsp(fl, free->raddr,

			free->phys, free->size, free->flags));

		if (err)

			goto bail;

		fastrpc_buf_free(rbuf, 0);

		mutex_unlock(&fl->map_mutex);

		return err;

	}



	VERIFY(err, !fastrpc_mmap_remove(fl, ud->vaddrout, ud->size, &map));

	if (err)

		goto bail;

	VERIFY(err, !fastrpc_munmap_on_dsp(fl, map));

	VERIFY(err, !fastrpc_munmap_on_dsp(fl, map->raddr,

				map->phys, map->size, map->flags));

	if (err)

		goto bail;

	fastrpc_mmap_free(map);
@@ -2131,22 +2254,55 @@ static int fastrpc_internal_mmap(struct fastrpc_file *fl, 
{



	struct fastrpc_mmap *map = NULL;

	struct fastrpc_buf *rbuf = NULL;

	uintptr_t raddr = 0;

	int err = 0;



	mutex_lock(&fl->map_mutex);

	if (!fastrpc_mmap_find(fl, ud->fd, (uintptr_t)ud->vaddrin, ud->size,

			       ud->flags, &map)){

		mutex_unlock(&fl->map_mutex);

		return 0;



	if (ud->flags == ADSP_MMAP_ADD_PAGES) {

		DEFINE_DMA_ATTRS(dma_attr);



		if (ud->vaddrin) {

			err = -EINVAL;

			pr_err("adsprpc: %s: %s: ERROR: adding user allocated pages is not supported\n",

					current->comm, __func__);

			goto bail;

		}

		dma_set_attr(DMA_ATTR_EXEC_MAPPING, &dma_attr);

		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &dma_attr);

		dma_set_attr(DMA_ATTR_FORCE_NON_COHERENT, &dma_attr);



		err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,

								1, &rbuf);

		if (err)

			goto bail;

		err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,

				rbuf->phys, rbuf->size, &raddr);

		if (err)

			goto bail;

		rbuf->raddr = raddr;

	} else {

		uintptr_t va_to_dsp;



		VERIFY(err, !fastrpc_mmap_create(fl, ud->fd, 0,

			(uintptr_t)ud->vaddrin, ud->size, ud->flags, &map));

				(uintptr_t)ud->vaddrin, ud->size,

				 ud->flags, &map));

		if (err)

			goto bail;

	VERIFY(err, 0 == fastrpc_mmap_on_dsp(fl, ud->flags, map));



		if (ud->flags == ADSP_MMAP_HEAP_ADDR ||

				ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)

			va_to_dsp = 0;

		else

			va_to_dsp = (uintptr_t)map->va;

		VERIFY(err, 0 == fastrpc_mmap_on_dsp(fl, ud->flags, va_to_dsp,

				map->phys, map->size, &raddr));

		if (err)

			goto bail;

	ud->vaddrout = map->raddr;

		map->raddr = raddr;

	}

	ud->vaddrout = raddr;

 bail:

	if (err && map)

		fastrpc_mmap_free(map);
@@ -2321,15 +2477,16 @@ static int fastrpc_file_free(struct fastrpc_file *fl) 
	spin_unlock(&fl->apps->hlock);

	kfree(fl->debug_buf);



	if (!fl->sctx) {

	if (!fl->sctx)

		goto bail;

	}



	spin_lock(&fl->hlock);

	fl->file_close = 1;

	spin_unlock(&fl->hlock);

	if (!IS_ERR_OR_NULL(fl->init_mem))

		fastrpc_buf_free(fl->init_mem, 0);

	fastrpc_context_list_dtor(fl);

	fastrpc_buf_list_free(fl);

	fastrpc_cached_buf_list_free(fl);

	hlist_for_each_entry_safe(map, n, &fl->maps, hn) {

		fastrpc_mmap_free(map);

	}
@@ -2341,6 +2498,7 @@ static int fastrpc_file_free(struct fastrpc_file *fl) 
	if (fl->secsctx)

		fastrpc_session_free(&fl->apps->channel[cid], fl->secsctx);

bail:

	fastrpc_remote_buf_list_free(fl);

	mutex_destroy(&fl->map_mutex);

	kfree(fl);

	return 0;
@@ -2642,7 +2800,7 @@ static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer, 
		}

		len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,

			"\n======%s %s %s======\n", title,

			" LIST OF BUFS ", title);

			" LIST OF CACHED BUFS ", title);

		spin_lock(&fl->hlock);

		len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,

			"%-19s|%-19s|%-19s\n",
@@ -2650,7 +2808,7 @@ static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer, 
		len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,

			"%s%s%s%s%s\n", single_line, single_line,

			single_line, single_line, single_line);

		hlist_for_each_entry_safe(buf, n, &fl->bufs, hn) {

		hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {

			len += scnprintf(fileinfo + len,

			DEBUGFS_SIZE - len,

			"0x%-17p|0x%-17llX|%-19zu\n",
@@ -2800,12 +2958,15 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) 
	context_list_ctor(&fl->clst);

	spin_lock_init(&fl->hlock);

	INIT_HLIST_HEAD(&fl->maps);

	INIT_HLIST_HEAD(&fl->bufs);

	INIT_HLIST_HEAD(&fl->cached_bufs);

	INIT_HLIST_HEAD(&fl->remote_bufs);

	INIT_HLIST_NODE(&fl->hn);

	fl->tgid = current->tgid;

	fl->apps = me;

	fl->mode = FASTRPC_MODE_SERIAL;

	fl->cid = -1;

	fl->init_mem = NULL;



	if (debugfs_file != NULL)

		fl->debugfs_file = debugfs_file;

	memset(&fl->perf, 0, sizeof(fl->perf));
@@ -2843,6 +3004,30 @@ bail: 
	return err;

}



static int fastrpc_internal_control(struct fastrpc_file *fl,

					struct fastrpc_ioctl_control *cp)

{

	int err = 0;



	VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps));

	if (err)

		goto bail;

	VERIFY(err, !IS_ERR_OR_NULL(cp));

	if (err)

		goto bail;



	switch (cp->req) {

	case FASTRPC_CONTROL_KALLOC:

		cp->kalloc.kalloc_support = 1;

		break;

	default:

		err = -ENOTTY;

		break;

	}

bail:

	return err;

}



static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num,

				 unsigned long ioctl_param)

{
@@ -2852,6 +3037,7 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num, 
		struct fastrpc_ioctl_munmap munmap;

		struct fastrpc_ioctl_init_attrs init;

		struct fastrpc_ioctl_perf perf;

		struct fastrpc_ioctl_control cp;

	} p;

	void *param = (char *)ioctl_param;

	struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
@@ -2967,6 +3153,20 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num, 
		if (err)

			goto bail;

		break;

	case FASTRPC_IOCTL_CONTROL:

		K_COPY_FROM_USER(err, 0, &p.cp, param,

				sizeof(p.cp));

		if (err)

			goto bail;

		VERIFY(err, 0 == (err = fastrpc_internal_control(fl, &p.cp)));

		if (err)

			goto bail;

		if (p.cp.req == FASTRPC_CONTROL_KALLOC) {

			K_COPY_TO_USER(err, 0, param, &p.cp, sizeof(p.cp));

			if (err)

				goto bail;

		}

		break;

	case FASTRPC_IOCTL_GETINFO:

		K_COPY_FROM_USER(err, 0, &info, param, sizeof(info));

		if (err)
@@ -3283,6 +3483,7 @@ static void fastrpc_deinit(void) 
		}

		for (j = 0; j < NUM_SESSIONS; j++) {

			struct fastrpc_session_ctx *sess = &chan->session[j];



			if (sess->smmu.dev) {

				arm_iommu_detach_device(sess->smmu.dev);

				sess->smmu.dev = NULL;

drivers/char/adsprpc_compat.c

+68 −0
Original line number Diff line number Diff line
@@ -36,6 +36,8 @@ 
		_IOWR('R', 9, struct compat_fastrpc_ioctl_perf)

#define COMPAT_FASTRPC_IOCTL_INIT_ATTRS \

		_IOWR('R', 10, struct compat_fastrpc_ioctl_init_attrs)

#define COMPAT_FASTRPC_IOCTL_CONTROL \

		_IOWR('R', 12, struct compat_fastrpc_ioctl_control)

#define COMPAT_FASTRPC_IOCTL_MMAP_64 \

		_IOWR('R', 14, struct compat_fastrpc_ioctl_mmap_64)

#define COMPAT_FASTRPC_IOCTL_MUNMAP_64 \
@@ -117,6 +119,31 @@ struct compat_fastrpc_ioctl_perf { /* kernel performance data */ 
	compat_uptr_t keys;

};



#define FASTRPC_CONTROL_LATENCY		(1)

struct compat_fastrpc_ctrl_latency {

	compat_uint_t enable;	/* latency control enable */

	compat_uint_t level;	/* level of control */

};



#define FASTRPC_CONTROL_SMMU		(2)

struct compat_fastrpc_ctrl_smmu {

	compat_uint_t sharedcb;

};



#define FASTRPC_CONTROL_KALLOC		(3)

struct compat_fastrpc_ctrl_kalloc {

	compat_uint_t kalloc_support; /* Remote memory allocation from kernel */

};



struct compat_fastrpc_ioctl_control {

	compat_uint_t req;

	union {

		struct compat_fastrpc_ctrl_latency lp;

		struct compat_fastrpc_ctrl_smmu smmu;

		struct compat_fastrpc_ctrl_kalloc kalloc;

	};

};



static int compat_get_fastrpc_ioctl_invoke(

			struct compat_fastrpc_ioctl_invoke_attrs __user *inv32,

			struct fastrpc_ioctl_invoke_attrs __user **inva,
@@ -322,6 +349,19 @@ static int compat_get_fastrpc_ioctl_perf( 
	return err;

}



static int compat_get_fastrpc_ioctl_control(

			struct compat_fastrpc_ioctl_control __user *ctrl32,

			struct fastrpc_ioctl_control __user *ctrl)

{

	compat_uptr_t p;

	int err;



	err = get_user(p, &ctrl32->req);

	err |= put_user(p, &ctrl->req);



	return err;

}



static int compat_get_fastrpc_ioctl_init(

			struct compat_fastrpc_ioctl_init_attrs __user *init32,

			struct fastrpc_ioctl_init_attrs __user *init,
@@ -513,6 +553,34 @@ long compat_fastrpc_device_ioctl(struct file *filp, unsigned int cmd, 
	case FASTRPC_IOCTL_SETMODE:

		return filp->f_op->unlocked_ioctl(filp, cmd,

						(unsigned long)compat_ptr(arg));

	case COMPAT_FASTRPC_IOCTL_CONTROL:

	{

		struct compat_fastrpc_ioctl_control __user *ctrl32;

		struct fastrpc_ioctl_control __user *ctrl;

		compat_uptr_t p;



		ctrl32 = compat_ptr(arg);

		VERIFY(err, NULL != (ctrl = compat_alloc_user_space(

							sizeof(*ctrl))));

		if (err)

			return -EFAULT;

		VERIFY(err, 0 == compat_get_fastrpc_ioctl_control(ctrl32,

							ctrl));

		if (err)

			return err;

		err = filp->f_op->unlocked_ioctl(filp, FASTRPC_IOCTL_CONTROL,

							(unsigned long)ctrl);

		if (err)

			return err;

		err = get_user(p, &ctrl32->req);

		if (err)

			return err;

		if (p == FASTRPC_CONTROL_KALLOC) {

			err = get_user(p, &ctrl->kalloc.kalloc_support);

			err |= put_user(p, &ctrl32->kalloc.kalloc_support);

		}

		return err;

	}

	case COMPAT_FASTRPC_IOCTL_GETPERF:

	{

		struct compat_fastrpc_ioctl_perf __user *perf32;

drivers/char/adsprpc_shared.h

+28 −1
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#define FASTRPC_IOCTL_GETINFO	_IOWR('R', 8, uint32_t)

#define FASTRPC_IOCTL_GETPERF	_IOWR('R', 9, struct fastrpc_ioctl_perf)

#define FASTRPC_IOCTL_INIT_ATTRS _IOWR('R', 10, struct fastrpc_ioctl_init_attrs)

#define FASTRPC_IOCTL_CONTROL	_IOWR('R', 12, struct fastrpc_ioctl_control)



#define FASTRPC_GLINK_GUID "fastrpcglink-apps-dsp"

#define FASTRPC_SMD_GUID "fastrpcsmd-apps-dsp"
@@ -201,6 +202,31 @@ struct fastrpc_ioctl_perf { /* kernel performance data */ 
	uintptr_t keys;

};



#define FASTRPC_CONTROL_LATENCY (1)

struct fastrpc_ctrl_latency {

	uint32_t enable;		/* latency control enable */

	uint32_t level;			/* level of control */

};



#define FASTRPC_CONTROL_SMMU	(2)

struct fastrpc_ctrl_smmu {

	uint32_t sharedcb;

};



#define FASTRPC_CONTROL_KALLOC (3)

struct fastrpc_ctrl_kalloc {

	uint32_t kalloc_support; /* Remote memory allocation from kernel */

};



struct fastrpc_ioctl_control {

	uint32_t req;

	union {

		struct fastrpc_ctrl_latency lp;

		struct fastrpc_ctrl_smmu smmu;

		struct fastrpc_ctrl_kalloc kalloc;

	};

};



struct smq_null_invoke {

	uint64_t ctx;			/* invoke caller context */

	uint32_t handle;	    /* handle to invoke */
@@ -245,6 +271,7 @@ static inline struct smq_phy_page *smq_phy_page_start(uint32_t sc, 
						struct smq_invoke_buf *buf)

{

	uint64_t nTotal = REMOTE_SCALARS_INBUFS(sc)+REMOTE_SCALARS_OUTBUFS(sc);



	return (struct smq_phy_page *)(&buf[nTotal]);

}