msm: adsprpc: use persistent header buffers (674eb6f8) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/char/adsprpc.c

+195 −34

Original line number	Original line	Diff line number	Diff line
	@@ -60,10 +60,7 @@
	#define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C		#define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C
	#define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D		#define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D
	#define TZ_PIL_AUTH_QDSP6_PROC 1		#define TZ_PIL_AUTH_QDSP6_PROC 1
	#define ADSP_MMAP_HEAP_ADDR 4
	#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
	#define ADSP_MMAP_ADD_PAGES 0x1000
	#define ADSP_MMAP_ADD_PAGES_LLC 0x3000
	#define FASTRPC_DMAHANDLE_NOMAP (16)		#define FASTRPC_DMAHANDLE_NOMAP (16)

	#define DEBUGFS_SIZE 3072		#define DEBUGFS_SIZE 3072
	@@ -184,12 +181,23 @@
	#define INIT_MEMLEN_MAX (810241024)		#define INIT_MEMLEN_MAX (810241024)
	#define MAX_CACHE_BUF_SIZE (810241024)		#define MAX_CACHE_BUF_SIZE (810241024)

			/* Max no. of persistent headers pre-allocated per process */
			#define MAX_PERSISTENT_HEADERS (25)

	/* Fastrpc remote process attributes */		/* Fastrpc remote process attributes */
	enum fastrpc_proc_attr {		enum fastrpc_proc_attr {
	FASTRPC_MODE_UNSIGNED_MODULE = (1 << 3),		FASTRPC_MODE_UNSIGNED_MODULE = (1 << 3),
	FASTRPC_MODE_PRIVILEGED = (1 << 6),		FASTRPC_MODE_PRIVILEGED = (1 << 6),
	};		};

			/* Type of fastrpc DMA bufs sent to DSP */
			enum fastrpc_buf_type {
			METADATA_BUF,
			COPYDATA_BUF,
			INITMEM_BUF,
			USERHEAP_BUF,
			};

	#define PERF_END (void)0		#define PERF_END (void)0

	#define PERF(enb, cnt, ff) \		#define PERF(enb, cnt, ff) \
	@@ -289,7 +297,8 @@ struct fastrpc_buf {
	unsigned long dma_attr;		unsigned long dma_attr;
	uintptr_t raddr;		uintptr_t raddr;
	uint32_t flags;		uint32_t flags;
	int remote;		int type; /* One of "fastrpc_buf_type" */
			bool in_use; /* Used only for persistent header buffers */
	};		};

	struct fastrpc_ctx_lst;		struct fastrpc_ctx_lst;
	@@ -496,6 +505,14 @@ struct fastrpc_file {
	struct fastrpc_ctx_lst clst;		struct fastrpc_ctx_lst clst;
	struct fastrpc_session_ctx *sctx;		struct fastrpc_session_ctx *sctx;
	struct fastrpc_buf *init_mem;		struct fastrpc_buf *init_mem;

			/* No. of persistent headers */
			unsigned int num_pers_hdrs;
			/* Pre-allocated header buffer */
			struct fastrpc_buf *pers_hdr_buf;
			/* Pre-allocated buffer divided into N chunks */
			struct fastrpc_buf *hdr_bufs;

	struct fastrpc_session_ctx *secsctx;		struct fastrpc_session_ctx *secsctx;
	uint32_t mode;		uint32_t mode;
	uint32_t profile;		uint32_t profile;
	@@ -608,6 +625,9 @@ static uint32_t kernel_capabilities[FASTRPC_MAX_ATTRIBUTES -
	};		};

	static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);		static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);
			static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
			uint32_t flags, uintptr_t va, uint64_t phys,
			size_t size, uintptr_t *raddr);

	static inline int64_t getnstimediff(struct timespec64 *start)		static inline int64_t getnstimediff(struct timespec64 *start)
	{		{
	@@ -710,17 +730,24 @@ static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)

	if (!fl)		if (!fl)
	return;		return;
			if (buf->in_use) {
			/* Don't free persistent header buf. Just mark as available */
			spin_lock(&fl->hlock);
			buf->in_use = false;
			spin_unlock(&fl->hlock);
			return;
			}
	if (cache && buf->size < MAX_CACHE_BUF_SIZE) {		if (cache && buf->size < MAX_CACHE_BUF_SIZE) {
	spin_lock(&fl->hlock);		spin_lock(&fl->hlock);
	hlist_add_head(&buf->hn, &fl->cached_bufs);		hlist_add_head(&buf->hn, &fl->cached_bufs);
	spin_unlock(&fl->hlock);		spin_unlock(&fl->hlock);
			buf->type = -1;
	return;		return;
	}		}
	if (buf->remote) {		if (buf->type == USERHEAP_BUF) {
	spin_lock(&fl->hlock);		spin_lock(&fl->hlock);
	hlist_del_init(&buf->hn_rem);		hlist_del_init(&buf->hn_rem);
	spin_unlock(&fl->hlock);		spin_unlock(&fl->hlock);
	buf->remote = 0;
	buf->raddr = 0;		buf->raddr = 0;
	}		}
	if (!IS_ERR_OR_NULL(buf->virt)) {		if (!IS_ERR_OR_NULL(buf->virt)) {
	@@ -1252,21 +1279,16 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd,
	return err;		return err;
	}		}

	static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,		static inline bool fastrpc_get_cached_buf(struct fastrpc_file *fl,
	unsigned long dma_attr, uint32_t rflags,		size_t size, int buf_type, struct fastrpc_buf **obuf)
	int remote, struct fastrpc_buf **obuf)
	{		{
	int err = 0, vmid;		bool found = false;
	struct fastrpc_buf buf = NULL, fr = NULL;		struct fastrpc_buf buf = NULL, fr = NULL;
	struct hlist_node *n;		struct hlist_node *n = NULL;

	VERIFY(err, size > 0 && fl->sctx->smmu.dev);		if (buf_type == USERHEAP_BUF)
	if (err) {
	err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;
	goto bail;		goto bail;
	}

	if (!remote) {
	/* find the smallest buffer that fits in the cache */		/* find the smallest buffer that fits in the cache */
	spin_lock(&fl->hlock);		spin_lock(&fl->hlock);
	hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {		hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
	@@ -1277,12 +1299,65 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
	hlist_del_init(&fr->hn);		hlist_del_init(&fr->hn);
	spin_unlock(&fl->hlock);		spin_unlock(&fl->hlock);
	if (fr) {		if (fr) {
			fr->type = buf_type;
	*obuf = fr;		*obuf = fr;
	return 0;		found = true;
	}		}
			bail:
			return found;
			}

			static inline bool fastrpc_get_persistent_buf(struct fastrpc_file *fl,
			size_t size, int buf_type, struct fastrpc_buf **obuf)
			{
			unsigned int i = 0;
			bool found = false;
			struct fastrpc_buf *buf = NULL;

			spin_lock(&fl->hlock);
			if (!fl->num_pers_hdrs)
			goto bail;

			/*
			* Persistent header buffer can be used only if
			* metadata length is less than 1 page size.
			*/
			if (buf_type != METADATA_BUF \|\| size > PAGE_SIZE)
			goto bail;

			for (i = 0; i < fl->num_pers_hdrs; i++) {
			buf = &fl->hdr_bufs[i];
			/* If buffer not in use, then assign it for requested alloc */
			if (!buf->in_use) {
			buf->in_use = true;
			*obuf = buf;
			found = true;
			break;
			}
			}
			bail:
			spin_unlock(&fl->hlock);
			return found;
			}

			static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
			unsigned long dma_attr, uint32_t rflags,
			int buf_type, struct fastrpc_buf **obuf)
			{
			int err = 0, vmid;
			struct fastrpc_buf *buf = NULL;

			VERIFY(err, size > 0 && fl->sctx->smmu.dev);
			if (err) {
			err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;
			goto bail;
	}		}
			if (fastrpc_get_persistent_buf(fl, size, buf_type, obuf))
			return err;
			if (fastrpc_get_cached_buf(fl, size, buf_type, obuf))
			return err;

	buf = NULL;		/* If unable to get persistent or cached buf, allocate new buffer */
	VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));		VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
	if (err) {		if (err) {
	err = -ENOMEM;		err = -ENOMEM;
	@@ -1296,7 +1371,7 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
	buf->dma_attr = dma_attr;		buf->dma_attr = dma_attr;
	buf->flags = rflags;		buf->flags = rflags;
	buf->raddr = 0;		buf->raddr = 0;
	buf->remote = 0;		buf->type = buf_type;
	buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,		buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
	(dma_addr_t *)&buf->phys,		(dma_addr_t *)&buf->phys,
	GFP_KERNEL, buf->dma_attr);		GFP_KERNEL, buf->dma_attr);
	@@ -1338,12 +1413,11 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
	}		}
	}		}

	if (remote) {		if (buf_type == USERHEAP_BUF) {
	INIT_HLIST_NODE(&buf->hn_rem);		INIT_HLIST_NODE(&buf->hn_rem);
	spin_lock(&fl->hlock);		spin_lock(&fl->hlock);
	hlist_add_head(&buf->hn_rem, &fl->remote_bufs);		hlist_add_head(&buf->hn_rem, &fl->remote_bufs);
	spin_unlock(&fl->hlock);		spin_unlock(&fl->hlock);
	buf->remote = remote;
	}		}
	*obuf = buf;		*obuf = buf;
	bail:		bail:
	@@ -1957,7 +2031,8 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
	(sizeof(uint32_t) * M_CRCLIST) + sizeof(early_hint);		(sizeof(uint32_t) * M_CRCLIST) + sizeof(early_hint);

	if (metalen) {		if (metalen) {
	err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0, 0, &ctx->buf);		err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0,
			METADATA_BUF, &ctx->buf);
	if (err)		if (err)
	goto bail;		goto bail;
	VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));		VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));
	@@ -2008,7 +2083,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
	ctx->copybuf = ctx->buf;		ctx->copybuf = ctx->buf;
	rlen = totallen - metalen;		rlen = totallen - metalen;
	} else if (copylen) {		} else if (copylen) {
	err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, 0,		err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, COPYDATA_BUF,
	&ctx->copybuf);		&ctx->copybuf);
	if (err)		if (err)
	goto bail;		goto bail;
	@@ -2804,12 +2879,81 @@ static int fastrpc_get_async_response(
	return err;		return err;
	}		}

			static int fastrpc_create_persistent_headers(struct fastrpc_file *fl,
			uint32_t user_concurrency)
			{
			int err = 0, i = 0;
			uint64_t virtb = 0;
			struct fastrpc_buf pers_hdr_buf = NULL, hdr_bufs = NULL, *buf = NULL;
			unsigned int num_pers_hdrs = 0;
			size_t hdr_buf_alloc_len = 0;

			if (fl->pers_hdr_buf \|\| !user_concurrency)
			goto bail;

			/*
			* Pre-allocate memory for persistent header buffers based
			* on concurrency info passed by user. Upper limit enforced.
			*/
			num_pers_hdrs = (user_concurrency > MAX_PERSISTENT_HEADERS) ?
			MAX_PERSISTENT_HEADERS : user_concurrency;
			hdr_buf_alloc_len = num_pers_hdrs*PAGE_SIZE;
			err = fastrpc_buf_alloc(fl, hdr_buf_alloc_len, 0, 0,
			METADATA_BUF, &pers_hdr_buf);
			if (err)
			goto bail;
			virtb = ptr_to_uint64(pers_hdr_buf->virt);

			/* Map entire buffer on remote subsystem in single RPC call */
			err = fastrpc_mem_map_to_dsp(fl, -1, 0, ADSP_MMAP_PERSIST_HDR, 0,
			pers_hdr_buf->phys, pers_hdr_buf->size,
			&pers_hdr_buf->raddr);
			if (err)
			goto bail;

			/* Divide and store as N chunks, each of 1 page size */
			hdr_bufs = kcalloc(num_pers_hdrs, sizeof(struct fastrpc_buf),
			GFP_KERNEL);
			if (!hdr_bufs) {
			err = -ENOMEM;
			goto bail;
			}
			spin_lock(&fl->hlock);
			fl->pers_hdr_buf = pers_hdr_buf;
			fl->num_pers_hdrs = num_pers_hdrs;
			fl->hdr_bufs = hdr_bufs;
			for (i = 0; i < num_pers_hdrs; i++) {
			buf = &fl->hdr_bufs[i];
			buf->fl = fl;
			buf->virt = uint64_to_ptr(virtb + (i*PAGE_SIZE));
			buf->phys = pers_hdr_buf->phys + (i*PAGE_SIZE);
			buf->size = PAGE_SIZE;
			buf->dma_attr = pers_hdr_buf->dma_attr;
			buf->flags = pers_hdr_buf->flags;
			buf->type = pers_hdr_buf->type;
			buf->in_use = false;
			}
			spin_unlock(&fl->hlock);
			bail:
			if (err) {
			fl->pers_hdr_buf = NULL;
			fl->hdr_bufs = NULL;
			fl->num_pers_hdrs = 0;
			if (!IS_ERR_OR_NULL(pers_hdr_buf))
			fastrpc_buf_free(pers_hdr_buf, 0);
			if (!IS_ERR_OR_NULL(hdr_bufs))
			kfree(hdr_bufs);
			}
			return err;
			}

	static int fastrpc_internal_invoke2(struct fastrpc_file *fl,		static int fastrpc_internal_invoke2(struct fastrpc_file *fl,
	struct fastrpc_ioctl_invoke2 *inv2)		struct fastrpc_ioctl_invoke2 *inv2)
	{		{
	union {		union {
	struct fastrpc_ioctl_invoke_async inv;		struct fastrpc_ioctl_invoke_async inv;
	struct fastrpc_ioctl_async_response async_res;		struct fastrpc_ioctl_async_response async_res;
			uint32_t user_concurrency;
	} p;		} p;
	struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;		struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
	uint32_t size = 0;		uint32_t size = 0;
	@@ -2854,6 +2998,18 @@ static int fastrpc_internal_invoke2(struct fastrpc_file *fl,
	err = fastrpc_get_async_response(&p.async_res,		err = fastrpc_get_async_response(&p.async_res,
	(void *)inv2->invparam, fl);		(void *)inv2->invparam, fl);
	break;		break;
			case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:
			if (inv2->size != sizeof(uint32_t)) {
			err = -EBADE;
			goto bail;
			}
			K_COPY_FROM_USER(err, 0, &p.user_concurrency,
			(void *)inv2->invparam, size);
			if (err)
			goto bail;
			err = fastrpc_create_persistent_headers(fl,
			p.user_concurrency);
			break;
	default:		default:
	err = -ENOTTY;		err = -ENOTTY;
	break;		break;
	@@ -3040,7 +3196,8 @@ static int fastrpc_init_create_dynamic_process(struct fastrpc_file *fl,
	imem_dma_attr = DMA_ATTR_EXEC_MAPPING \|		imem_dma_attr = DMA_ATTR_EXEC_MAPPING \|
	DMA_ATTR_DELAYED_UNMAP \|		DMA_ATTR_DELAYED_UNMAP \|
	DMA_ATTR_NO_KERNEL_MAPPING;		DMA_ATTR_NO_KERNEL_MAPPING;
	err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0, 0, &imem);		err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0,
			INITMEM_BUF, &imem);
	if (err)		if (err)
	goto bail;		goto bail;
	fl->init_mem = imem;		fl->init_mem = imem;
	@@ -4180,7 +4337,7 @@ static int fastrpc_internal_mmap(struct fastrpc_file *fl,
	if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)		if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)
	dma_attr \|= DMA_ATTR_IOMMU_USE_UPSTREAM_HINT;		dma_attr \|= DMA_ATTR_IOMMU_USE_UPSTREAM_HINT;
	err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,		err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,
	1, &rbuf);		USERHEAP_BUF, &rbuf);
	if (err)		if (err)
	goto bail;		goto bail;
	err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,		err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,
	@@ -4466,6 +4623,10 @@ static int fastrpc_file_free(struct fastrpc_file *fl)
	fastrpc_buf_free(fl->init_mem, 0);		fastrpc_buf_free(fl->init_mem, 0);
	fastrpc_context_list_dtor(fl);		fastrpc_context_list_dtor(fl);
	fastrpc_cached_buf_list_free(fl);		fastrpc_cached_buf_list_free(fl);
			if (!IS_ERR_OR_NULL(fl->hdr_bufs))
			kfree(fl->hdr_bufs);
			if (!IS_ERR_OR_NULL(fl->pers_hdr_buf))
			fastrpc_buf_free(fl->pers_hdr_buf, 0);
	mutex_lock(&fl->map_mutex);		mutex_lock(&fl->map_mutex);
	do {		do {
	lmap = NULL;		lmap = NULL;

drivers/char/adsprpc_compat.c

+7 −3

Original line number	Original line	Diff line number	Diff line
	@@ -313,7 +313,7 @@ static int compat_get_fastrpc_ioctl_invoke2(
	{		{
	int err = 0;		int err = 0;
	compat_uptr_t pparam;		compat_uptr_t pparam;
	compat_uint_t req, size;		compat_uint_t req, size, ref_size = 0;
	struct fastrpc_ioctl_invoke2 __user *inv2_user = NULL;		struct fastrpc_ioctl_invoke2 __user *inv2_user = NULL;
	struct fastrpc_ioctl_invoke_async __user *asyncinv_user;		struct fastrpc_ioctl_invoke_async __user *asyncinv_user;

	@@ -362,9 +362,13 @@ static int compat_get_fastrpc_ioctl_invoke2(
	break;		break;
	}		}
	case FASTRPC_INVOKE2_ASYNC_RESPONSE:		case FASTRPC_INVOKE2_ASYNC_RESPONSE:
			ref_size = sizeof(struct fastrpc_ioctl_async_response);
			/* intentional fall through */
			case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:
	{		{
	VERIFY(err,		if (!ref_size)
	size == sizeof(struct fastrpc_ioctl_async_response));		ref_size = sizeof(uint32_t);
			VERIFY(err, size == ref_size);
	if (err) {		if (err) {
	err = -EBADE;		err = -EBADE;
	goto bail;		goto bail;

drivers/char/adsprpc_shared.h

+22 −0

Original line number	Original line	Diff line number	Diff line
	@@ -229,6 +229,7 @@ struct fastrpc_ioctl_async_response {
	enum fastrpc_invoke2_type {		enum fastrpc_invoke2_type {
	FASTRPC_INVOKE2_ASYNC = 1,		FASTRPC_INVOKE2_ASYNC = 1,
	FASTRPC_INVOKE2_ASYNC_RESPONSE = 2,		FASTRPC_INVOKE2_ASYNC_RESPONSE = 2,
			FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS,
	};		};

	struct fastrpc_ioctl_invoke2 {		struct fastrpc_ioctl_invoke2 {
	@@ -358,6 +359,27 @@ struct fastrpc_ioctl_mem_unmap {
	};		};
	};		};

			/*
			* This enum is shared with DSP. So, existing values should NOT
			* be modified. Only new members can be added.
			*/
			enum dsp_map_flags {
			/* Add memory to static PD pool, protection thru XPU */
			ADSP_MMAP_HEAP_ADDR = 4,

			/* Add memory to static PD pool, protection thru hypervisor */
			ADSP_MMAP_REMOTE_HEAP_ADDR = 8,

			/* Add memory to userPD pool, for user heap */
			ADSP_MMAP_ADD_PAGES = 0x1000,

			/* Add memory to userPD pool, for LLC heap */
			ADSP_MMAP_ADD_PAGES_LLC = 0x3000,

			/* Map persistent header buffer on DSP */
			ADSP_MMAP_PERSIST_HDR = 0x4000,
			};

	struct fastrpc_ioctl_perf { /* kernel performance data */		struct fastrpc_ioctl_perf { /* kernel performance data */
	uintptr_t data;		uintptr_t data;
	uint32_t numkeys;		uint32_t numkeys;