Merge "msm: ADSPRPC: Add support for 32-bit fastrpc driver" (263712e5) · Commits · e / devices / android_kernel_xiaomi_markw

drivers/char/adsprpc.c

+75 −65

Original line number	Diff line number	Diff line
		@@ -59,39 +59,51 @@

		#define IS_CACHE_ALIGNED(x) (((x) & ((L1_CACHE_BYTES)-1)) == 0)

		static inline uintptr_t buf_page_start(void *buf)
		static inline uint64_t buf_page_start(uint64_t buf)
		{
		uintptr_t start = (uintptr_t) buf & PAGE_MASK;
		uint64_t start = (uint64_t) buf & PAGE_MASK;
		return start;
		}

		static inline uintptr_t buf_page_offset(void *buf)
		static inline uint64_t buf_page_offset(uint64_t buf)
		{
		uintptr_t offset = (uintptr_t) buf & (PAGE_SIZE - 1);
		uint64_t offset = (uint64_t) buf & (PAGE_SIZE - 1);
		return offset;
		}

		static inline int buf_num_pages(void *buf, ssize_t len)
		static inline int buf_num_pages(uint64_t buf, ssize_t len)
		{
		uintptr_t start = buf_page_start(buf) >> PAGE_SHIFT;
		uintptr_t end = (((uintptr_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
		uint64_t start = buf_page_start(buf) >> PAGE_SHIFT;
		uint64_t end = (((uint64_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
		int nPages = end - start + 1;
		return nPages;
		}

		static inline uint32_t buf_page_size(uint32_t size)
		static inline uint64_t buf_page_size(uint32_t size)
		{
		uint32_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
		uint64_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
		return sz > PAGE_SIZE ? sz : PAGE_SIZE;
		}

		static inline void *uint64_to_ptr(uint64_t addr)
		{
		void ptr = (void )((uintptr_t)addr);
		return ptr;
		}

		static inline uint64_t ptr_to_uint64(void *ptr)
		{
		uint64_t addr = (uint64_t)((uintptr_t)ptr);
		return addr;
		}

		struct fastrpc_file;

		struct fastrpc_buf {
		struct hlist_node hn;
		struct fastrpc_file *fl;
		void *virt;
		dma_addr_t phys;
		uint64_t phys;
		ssize_t size;
		};

		@@ -113,7 +125,7 @@ struct smq_invoke_ctx {
		int pid;
		int tgid;
		remote_arg_t *lpra;
		remote_arg_t *rpra;
		remote_arg64_t *rpra;
		int *fds;
		struct fastrpc_mmap **maps;
		struct fastrpc_buf *buf;
		@@ -184,7 +196,7 @@ struct fastrpc_mmap {
		struct sg_table *table;
		struct dma_buf_attachment *attach;
		struct ion_handle *handle;
		uintptr_t phys;
		uint64_t phys;
		ssize_t size;
		uintptr_t va;
		ssize_t len;
		@@ -229,10 +241,8 @@ static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)
		return;
		}
		if (!IS_ERR_OR_NULL(buf->virt)) {
		#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
		if (fl->sctx->smmu.cb)
		buf->phys &= ~((dma_addr_t)fl->sctx->smmu.cb << 32);
		#endif
		buf->phys &= ~((uint64_t)fl->sctx->smmu.cb << 32);
		dma_free_coherent(fl->sctx->smmu.dev, buf->size,
		buf->virt, buf->phys);
		}
		@@ -487,10 +497,8 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, uintptr_t va,
		goto bail;
		map->uncached = !ION_IS_CACHED(flags);
		map->phys = sg_dma_address(map->table->sgl);
		#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
		if (sess->smmu.cb)
		map->phys += ((dma_addr_t)sess->smmu.cb << 32);
		#endif
		map->phys += ((uint64_t)sess->smmu.cb << 32);
		map->size = sg_dma_len(map->table->sgl);
		}
		map->va = va;
		@@ -539,20 +547,18 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, ssize_t size,
		buf->phys = 0;
		buf->size = size;
		buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,
		&buf->phys, GFP_KERNEL);
		(void *)&buf->phys, GFP_KERNEL);
		if (IS_ERR_OR_NULL(buf->virt)) {
		/* free cache and retry */
		fastrpc_buf_list_free(fl);
		buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,
		&buf->phys, GFP_KERNEL);
		(void *)&buf->phys, GFP_KERNEL);
		VERIFY(err, !IS_ERR_OR_NULL(buf->virt));
		}
		if (err)
		goto bail;
		#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
		if (fl->sctx->smmu.cb)
		buf->phys += ((dma_addr_t)fl->sctx->smmu.cb << 32);
		#endif
		buf->phys += ((uint64_t)fl->sctx->smmu.cb << 32);
		*obuf = buf;
		bail:
		if (err && buf)
		@@ -833,7 +839,7 @@ static void fastrpc_file_list_dtor(struct fastrpc_apps *me)

		static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		{
		remote_arg_t *rpra;
		remote_arg64_t *rpra;
		remote_arg_t *lpra = ctx->lpra;
		struct smq_invoke_buf *list;
		struct smq_phy_page pages, ipage;
		@@ -842,7 +848,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
		int bufs = inbufs + outbufs;
		uintptr_t args;
		ssize_t rlen = 0, copylen = 0, metalen = 0, size;
		ssize_t rlen = 0, copylen = 0, metalen = 0;
		int i, inh, oix;
		int err = 0;
		int mflags = 0;
		@@ -902,7 +908,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		/* map ion buffers */
		for (i = 0; i < inbufs + outbufs; ++i) {
		struct fastrpc_mmap *map = ctx->maps[i];
		void *buf = lpra[i].buf.pv;
		uint64_t buf = ptr_to_uint64(lpra[i].buf.pv);
		ssize_t len = lpra[i].buf.len;
		rpra[i].buf.pv = 0;
		rpra[i].buf.len = len;
		@@ -930,7 +936,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		int i = ctx->overps[oix]->raix;
		struct fastrpc_mmap *map = ctx->maps[i];
		int mlen = ctx->overps[oix]->mend - ctx->overps[oix]->mstart;
		void *buf;
		uint64_t buf;
		ssize_t len = lpra[i].buf.len;
		if (!len)
		continue;
		@@ -943,16 +949,17 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		VERIFY(err, rlen >= mlen);
		if (err)
		goto bail;
		rpra[i].buf.pv = (void *)(args - ctx->overps[oix]->offset);
		rpra[i].buf.pv = (args - ctx->overps[oix]->offset);
		pages[list[i].pgidx].addr = ctx->buf->phys -
		ctx->overps[oix]->offset +
		(copylen - rlen);
		pages[list[i].pgidx].addr =
		buf_page_start((void *)pages[list[i].pgidx].addr);
		buf_page_start(pages[list[i].pgidx].addr);
		buf = rpra[i].buf.pv;
		pages[list[i].pgidx].size = buf_num_pages(buf, len) * PAGE_SIZE;
		if (i < inbufs) {
		K_COPY_FROM_USER(err, kernel, buf, lpra[i].buf.pv, len);
		K_COPY_FROM_USER(err, kernel, uint64_to_ptr(buf),
		lpra[i].buf.pv, len);
		if (err)
		goto bail;
		}
		@@ -966,15 +973,14 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
		if (map && map->uncached)
		continue;
		if (rpra[i].buf.len && ctx->overps[oix]->mstart)
		dmac_flush_range(rpra[i].buf.pv,
		(char *)rpra[i].buf.pv + rpra[i].buf.len);
		dmac_flush_range(uint64_to_ptr(rpra[i].buf.pv),
		uint64_to_ptr(rpra[i].buf.pv + rpra[i].buf.len));
		}
		size = sizeof(rpra) REMOTE_SCALARS_INHANDLES(sc);
		if (size) {
		inh = inbufs + outbufs;
		K_COPY_FROM_USER(err, kernel, &rpra[inh], &lpra[inh], size);
		if (err)
		goto bail;
		for (i = 0; i < REMOTE_SCALARS_INHANDLES(sc); i++) {
		rpra[inh + i].buf.pv = ptr_to_uint64(ctx->lpra[inh + i].buf.pv);
		rpra[inh + i].buf.len = ctx->lpra[inh + i].buf.len;
		rpra[inh + i].h = ctx->lpra[inh + i].h;
		}
		dmac_flush_range((char )rpra, (char )rpra + ctx->used);
		bail:
		@@ -985,7 +991,7 @@ static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
		remote_arg_t *upra)
		{
		uint32_t sc = ctx->sc;
		remote_arg_t *rpra = ctx->rpra;
		remote_arg64_t *rpra = ctx->rpra;
		int i, inbufs, outbufs, outh, size;
		int err = 0;

		@@ -995,7 +1001,7 @@ static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
		if (!ctx->maps[i]) {
		K_COPY_TO_USER(err, kernel,
		ctx->lpra[i].buf.pv,
		rpra[i].buf.pv,
		uint64_to_ptr(rpra[i].buf.pv),
		rpra[i].buf.len);
		if (err)
		goto bail;
		@@ -1015,7 +1021,7 @@ static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
		return err;
		}

		static void inv_args_pre(uint32_t sc, remote_arg_t *rpra)
		static void inv_args_pre(uint32_t sc, remote_arg64_t *rpra)
		{
		int i, inbufs, outbufs;
		uintptr_t end;
		@@ -1025,12 +1031,14 @@ static void inv_args_pre(uint32_t sc, remote_arg_t *rpra)
		for (i = inbufs; i < inbufs + outbufs; ++i) {
		if (!rpra[i].buf.len)
		continue;
		if (buf_page_start(rpra) == buf_page_start(rpra[i].buf.pv))
		if (buf_page_start(ptr_to_uint64((void *)rpra)) ==
		buf_page_start(rpra[i].buf.pv))
		continue;
		if (!IS_CACHE_ALIGNED((uintptr_t)rpra[i].buf.pv))
		dmac_flush_range(rpra[i].buf.pv,
		(char *)rpra[i].buf.pv + 1);
		end = (uintptr_t)rpra[i].buf.pv + rpra[i].buf.len;
		if (!IS_CACHE_ALIGNED((uintptr_t)uint64_to_ptr(rpra[i].buf.pv)))
		dmac_flush_range(uint64_to_ptr(rpra[i].buf.pv),
		(char *)(uint64_to_ptr(rpra[i].buf.pv + 1)));
		end = (uintptr_t)uint64_to_ptr(rpra[i].buf.pv +
		rpra[i].buf.len);
		if (!IS_CACHE_ALIGNED(end))
		dmac_flush_range((char *)end,
		(char *)end + 1);
		@@ -1041,7 +1049,7 @@ static void inv_args(struct smq_invoke_ctx *ctx)
		{
		int i, inbufs, outbufs;
		uint32_t sc = ctx->sc;
		remote_arg_t *rpra = ctx->rpra;
		remote_arg64_t *rpra = ctx->rpra;
		int used = ctx->used;
		int inv = 0;

		@@ -1053,17 +1061,19 @@ static void inv_args(struct smq_invoke_ctx *ctx)
		continue;
		if (!rpra[i].buf.len)
		continue;
		if (buf_page_start(rpra) == buf_page_start(rpra[i].buf.pv)) {
		if (buf_page_start(ptr_to_uint64((void *)rpra)) ==
		buf_page_start(rpra[i].buf.pv)) {
		inv = 1;
		continue;
		}
		if (map && map->handle)
		msm_ion_do_cache_op(ctx->fl->apps->client, map->handle,
		rpra[i].buf.pv, rpra[i].buf.len,
		ION_IOC_INV_CACHES);
		(char *)uint64_to_ptr(rpra[i].buf.pv),
		rpra[i].buf.len, ION_IOC_INV_CACHES);
		else
		dmac_inv_range(rpra[i].buf.pv,
		(char *)rpra[i].buf.pv + rpra[i].buf.len);
		dmac_inv_range((char *)uint64_to_ptr(rpra[i].buf.pv),
		(char *)uint64_to_ptr(rpra[i].buf.pv
		+ rpra[i].buf.len));
		}

		if (inv \|\| REMOTE_SCALARS_OUTHANDLES(sc))
		@@ -1073,7 +1083,7 @@ static void inv_args(struct smq_invoke_ctx *ctx)
		static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
		uint32_t kernel, uint32_t handle)
		{
		struct smq_msg msg;
		struct smq_msg msg = {0};
		struct fastrpc_file *fl = ctx->fl;
		int err = 0, len;

		@@ -1084,7 +1094,7 @@ static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
		msg.tid = current->pid;
		if (kernel)
		msg.pid = 0;
		msg.invoke.header.ctx = ctx;
		msg.invoke.header.ctx = ptr_to_uint64(ctx);
		msg.invoke.header.handle = handle;
		msg.invoke.header.sc = ctx->sc;
		msg.invoke.page.addr = ctx->buf ? ctx->buf->phys : 0;
		@@ -1100,7 +1110,7 @@ static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
		static void fastrpc_read_handler(int cid)
		{
		struct fastrpc_apps *me = &gfa;
		struct smq_invoke_rsp rsp;
		struct smq_invoke_rsp rsp = {0};
		int ret = 0;

		do {
		@@ -1108,7 +1118,7 @@ static void fastrpc_read_handler(int cid)
		sizeof(rsp));
		if (ret != sizeof(rsp))
		break;
		context_notify_user(rsp.ctx, rsp.retval);
		context_notify_user(uint64_to_ptr(rsp.ctx), rsp.retval);
		} while (ret == sizeof(rsp));
		}

		@@ -1218,7 +1228,7 @@ static int fastrpc_init_process(struct fastrpc_file *fl,
		if (init->flags == FASTRPC_INIT_ATTACH) {
		remote_arg_t ra[1];
		int tgid = current->tgid;
		ra[0].buf.pv = &tgid;
		ra[0].buf.pv = (void *)&tgid;
		ra[0].buf.len = sizeof(tgid);
		ioctl.inv.handle = 1;
		ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 0);
		@@ -1265,11 +1275,11 @@ static int fastrpc_init_process(struct fastrpc_file *fl,
		len += sg->length;
		}
		inbuf.pageslen = 1;
		ra[0].buf.pv = &inbuf;
		ra[0].buf.pv = (void *)&inbuf;
		ra[0].buf.len = sizeof(inbuf);
		fds[0] = 0;

		ra[1].buf.pv = current->comm;
		ra[1].buf.pv = (void *)current->comm;
		ra[1].buf.len = inbuf.namelen;
		fds[1] = 0;

		@@ -1279,7 +1289,7 @@ static int fastrpc_init_process(struct fastrpc_file *fl,

		pages[0].addr = mem->phys;
		pages[0].size = mem->size;
		ra[3].buf.pv = pages;
		ra[3].buf.pv = (void *)pages;
		ra[3].buf.len = 1 * sizeof(*pages);
		fds[3] = 0;

		@@ -1313,7 +1323,7 @@ static int fastrpc_release_current_dsp_process(struct fastrpc_file *fl)
		if (err)
		goto bail;
		tgid = fl->tgid;
		ra[0].buf.pv = &tgid;
		ra[0].buf.pv = (void *)&tgid;
		ra[0].buf.len = sizeof(tgid);
		ioctl.inv.handle = 1;
		ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
		@@ -1347,14 +1357,14 @@ static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags,
		inargs.vaddrin = (uintptr_t)map->va;
		inargs.flags = flags;
		inargs.num = fl->apps->compat ? num * sizeof(page) : num;
		ra[0].buf.pv = &inargs;
		ra[0].buf.pv = (void *)&inargs;
		ra[0].buf.len = sizeof(inargs);
		page.addr = map->phys;
		page.size = map->size;
		ra[1].buf.pv = &page;
		ra[1].buf.pv = (void *)&page;
		ra[1].buf.len = num * sizeof(page);

		ra[2].buf.pv = &routargs;
		ra[2].buf.pv = (void *)&routargs;
		ra[2].buf.len = sizeof(routargs);

		ioctl.inv.handle = 1;
		@@ -1439,7 +1449,7 @@ static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl,
		inargs.pid = current->tgid;
		inargs.size = map->size;
		inargs.vaddrout = map->raddr;
		ra[0].buf.pv = &inargs;
		ra[0].buf.pv = (void *)&inargs;
		ra[0].buf.len = sizeof(inargs);

		ioctl.inv.handle = 1;

drivers/char/adsprpc_shared.h

+18 −6

Original line number	Diff line number	Diff line
		/*
		* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved.
		* Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 and
		@@ -89,6 +89,18 @@ do {\
		} while (0)
		#endif

		#define remote_arg64_t union remote_arg64

		struct remote_buf64 {
		uint64_t pv;
		int64_t len;
		};

		union remote_arg64 {
		struct remote_buf64 buf;
		uint32_t h;
		};

		#define remote_arg_t union remote_arg

		struct remote_buf {
		@@ -137,14 +149,14 @@ struct fastrpc_ioctl_mmap {
		};

		struct smq_null_invoke {
		struct smq_invoke_ctx ctx; / invoke caller context */
		uint64_t ctx; /* invoke caller context */
		uint32_t handle; /* handle to invoke */
		uint32_t sc; /* scalars structure describing the data */
		};

		struct smq_phy_page {
		unsigned long addr; /* physical address */
		ssize_t size; /* size of contiguous region */
		uint64_t addr; /* physical address */
		uint64_t size; /* size of contiguous region */
		};

		struct smq_invoke_buf {
		@@ -164,11 +176,11 @@ struct smq_msg {
		};

		struct smq_invoke_rsp {
		struct smq_invoke_ctx ctx; / invoke caller context */
		uint64_t ctx; /* invoke caller context */
		int retval; /* invoke return value */
		};

		static inline struct smq_invoke_buf smq_invoke_buf_start(remote_arg_t pra,
		static inline struct smq_invoke_buf smq_invoke_buf_start(remote_arg64_t pra,
		uint32_t sc)
		{
		int len = REMOTE_SCALARS_LENGTH(sc);