Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2f215da2 authored by qctecmdr's avatar qctecmdr Committed by Gerrit - the friendly Code Review server
Browse files

Merge "msm: adsprpc: use persistent header buffers"

parents d62ef4bc 674eb6f8

drivers/char/adsprpc.c

+195 −34
Original line number Diff line number Diff line
@@ -60,10 +60,7 @@ 
#define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C

#define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D

#define TZ_PIL_AUTH_QDSP6_PROC 1

#define ADSP_MMAP_HEAP_ADDR 4

#define ADSP_MMAP_REMOTE_HEAP_ADDR 8

#define ADSP_MMAP_ADD_PAGES 0x1000

#define ADSP_MMAP_ADD_PAGES_LLC  0x3000



#define FASTRPC_DMAHANDLE_NOMAP (16)



#define DEBUGFS_SIZE 3072
@@ -184,12 +181,23 @@ 
#define INIT_MEMLEN_MAX  (8*1024*1024)

#define MAX_CACHE_BUF_SIZE (8*1024*1024)



/* Max no. of persistent headers pre-allocated per process */

#define MAX_PERSISTENT_HEADERS    (25)



/* Fastrpc remote process attributes */

enum fastrpc_proc_attr {

	FASTRPC_MODE_UNSIGNED_MODULE = (1 << 3),

	FASTRPC_MODE_PRIVILEGED      = (1 << 6),

};



/* Type of fastrpc DMA bufs sent to DSP */

enum fastrpc_buf_type {

	METADATA_BUF,

	COPYDATA_BUF,

	INITMEM_BUF,

	USERHEAP_BUF,

};



#define PERF_END (void)0



#define PERF(enb, cnt, ff) \
@@ -289,7 +297,8 @@ struct fastrpc_buf { 
	unsigned long dma_attr;

	uintptr_t raddr;

	uint32_t flags;

	int remote;

	int type;		/* One of "fastrpc_buf_type" */

	bool in_use;	/* Used only for persistent header buffers */

};



struct fastrpc_ctx_lst;
@@ -496,6 +505,14 @@ struct fastrpc_file { 
	struct fastrpc_ctx_lst clst;

	struct fastrpc_session_ctx *sctx;

	struct fastrpc_buf *init_mem;



	/* No. of persistent headers */

	unsigned int num_pers_hdrs;

	/* Pre-allocated header buffer */

	struct fastrpc_buf *pers_hdr_buf;

	/* Pre-allocated buffer divided into N chunks */

	struct fastrpc_buf *hdr_bufs;



	struct fastrpc_session_ctx *secsctx;

	uint32_t mode;

	uint32_t profile;
@@ -608,6 +625,9 @@ static uint32_t kernel_capabilities[FASTRPC_MAX_ATTRIBUTES - 
};



static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);

static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,

				uint32_t flags, uintptr_t va, uint64_t phys,

				size_t size, uintptr_t *raddr);



static inline int64_t getnstimediff(struct timespec64 *start)

{
@@ -710,17 +730,24 @@ static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache) 


	if (!fl)

		return;

	if (buf->in_use) {

		/* Don't free persistent header buf. Just mark as available */

		spin_lock(&fl->hlock);

		buf->in_use = false;

		spin_unlock(&fl->hlock);

		return;

	}

	if (cache && buf->size < MAX_CACHE_BUF_SIZE) {

		spin_lock(&fl->hlock);

		hlist_add_head(&buf->hn, &fl->cached_bufs);

		spin_unlock(&fl->hlock);

		buf->type = -1;

		return;

	}

	if (buf->remote) {

	if (buf->type == USERHEAP_BUF) {

		spin_lock(&fl->hlock);

		hlist_del_init(&buf->hn_rem);

		spin_unlock(&fl->hlock);

		buf->remote = 0;

		buf->raddr = 0;

	}

	if (!IS_ERR_OR_NULL(buf->virt)) {
@@ -1252,21 +1279,16 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, 
	return err;

}



static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,

			unsigned long dma_attr, uint32_t rflags,

			int remote, struct fastrpc_buf **obuf)

static inline bool fastrpc_get_cached_buf(struct fastrpc_file *fl,

		size_t size, int buf_type, struct fastrpc_buf **obuf)

{

	int err = 0, vmid;

	bool found = false;

	struct fastrpc_buf *buf = NULL, *fr = NULL;

	struct hlist_node *n;

	struct hlist_node *n = NULL;



	VERIFY(err, size > 0 && fl->sctx->smmu.dev);

	if (err) {

		err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;

	if (buf_type == USERHEAP_BUF)

		goto bail;

	}



	if (!remote) {

	/* find the smallest buffer that fits in the cache */

	spin_lock(&fl->hlock);

	hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
@@ -1277,12 +1299,65 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
		hlist_del_init(&fr->hn);

	spin_unlock(&fl->hlock);

	if (fr) {

		fr->type = buf_type;

		*obuf = fr;

			return 0;

		found = true;

	}

bail:

	return found;

}



static inline bool fastrpc_get_persistent_buf(struct fastrpc_file *fl,

		size_t size, int buf_type, struct fastrpc_buf **obuf)

{

	unsigned int i = 0;

	bool found = false;

	struct fastrpc_buf *buf = NULL;



	spin_lock(&fl->hlock);

	if (!fl->num_pers_hdrs)

		goto bail;



	/*

	 * Persistent header buffer can be used only if

	 * metadata length is less than 1 page size.

	 */

	if (buf_type != METADATA_BUF || size > PAGE_SIZE)

		goto bail;



	for (i = 0; i < fl->num_pers_hdrs; i++) {

		buf = &fl->hdr_bufs[i];

		/* If buffer not in use, then assign it for requested alloc */

		if (!buf->in_use) {

			buf->in_use = true;

			*obuf = buf;

			found = true;

			break;

		}

	}

bail:

	spin_unlock(&fl->hlock);

	return found;

}



static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,

			unsigned long dma_attr, uint32_t rflags,

			int buf_type, struct fastrpc_buf **obuf)

{

	int err = 0, vmid;

	struct fastrpc_buf *buf = NULL;



	VERIFY(err, size > 0 && fl->sctx->smmu.dev);

	if (err) {

		err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;

		goto bail;

	}

	if (fastrpc_get_persistent_buf(fl, size, buf_type, obuf))

		return err;

	if (fastrpc_get_cached_buf(fl, size, buf_type, obuf))

		return err;



	buf = NULL;

	/* If unable to get persistent or cached buf, allocate new buffer */

	VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));

	if (err) {

		err = -ENOMEM;
@@ -1296,7 +1371,7 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
	buf->dma_attr = dma_attr;

	buf->flags = rflags;

	buf->raddr = 0;

	buf->remote = 0;

	buf->type = buf_type;

	buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,

						(dma_addr_t *)&buf->phys,

						GFP_KERNEL, buf->dma_attr);
@@ -1338,12 +1413,11 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, 
		}

	}



	if (remote) {

	if (buf_type == USERHEAP_BUF) {

		INIT_HLIST_NODE(&buf->hn_rem);

		spin_lock(&fl->hlock);

		hlist_add_head(&buf->hn_rem, &fl->remote_bufs);

		spin_unlock(&fl->hlock);

		buf->remote = remote;

	}

	*obuf = buf;

 bail:
@@ -1957,7 +2031,8 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
			(sizeof(uint32_t) * M_CRCLIST) + sizeof(early_hint);



	if (metalen) {

		err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0, 0, &ctx->buf);

		err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0,

				METADATA_BUF, &ctx->buf);

		if (err)

			goto bail;

		VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));
@@ -2008,7 +2083,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
		ctx->copybuf = ctx->buf;

		rlen = totallen - metalen;

	} else if (copylen) {

		err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, 0,

		err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, COPYDATA_BUF,

				&ctx->copybuf);

		if (err)

			goto bail;
@@ -2804,12 +2879,81 @@ static int fastrpc_get_async_response( 
	return err;

}



static int fastrpc_create_persistent_headers(struct fastrpc_file *fl,

			uint32_t user_concurrency)

{

	int err = 0, i = 0;

	uint64_t virtb = 0;

	struct fastrpc_buf *pers_hdr_buf = NULL, *hdr_bufs = NULL, *buf = NULL;

	unsigned int num_pers_hdrs = 0;

	size_t hdr_buf_alloc_len = 0;



	if (fl->pers_hdr_buf || !user_concurrency)

		goto bail;



	/*

	 * Pre-allocate memory for persistent header buffers based

	 * on concurrency info passed by user. Upper limit enforced.

	 */

	num_pers_hdrs = (user_concurrency > MAX_PERSISTENT_HEADERS) ?

		MAX_PERSISTENT_HEADERS : user_concurrency;

	hdr_buf_alloc_len = num_pers_hdrs*PAGE_SIZE;

	err = fastrpc_buf_alloc(fl, hdr_buf_alloc_len, 0, 0,

			METADATA_BUF, &pers_hdr_buf);

	if (err)

		goto bail;

	virtb = ptr_to_uint64(pers_hdr_buf->virt);



	/* Map entire buffer on remote subsystem in single RPC call */

	err = fastrpc_mem_map_to_dsp(fl, -1, 0, ADSP_MMAP_PERSIST_HDR, 0,

			pers_hdr_buf->phys, pers_hdr_buf->size,

			&pers_hdr_buf->raddr);

	if (err)

		goto bail;



	/* Divide and store as N chunks, each of 1 page size */

	hdr_bufs = kcalloc(num_pers_hdrs, sizeof(struct fastrpc_buf),

				GFP_KERNEL);

	if (!hdr_bufs) {

		err = -ENOMEM;

		goto bail;

	}

	spin_lock(&fl->hlock);

	fl->pers_hdr_buf = pers_hdr_buf;

	fl->num_pers_hdrs = num_pers_hdrs;

	fl->hdr_bufs = hdr_bufs;

	for (i = 0; i < num_pers_hdrs; i++) {

		buf = &fl->hdr_bufs[i];

		buf->fl = fl;

		buf->virt = uint64_to_ptr(virtb + (i*PAGE_SIZE));

		buf->phys = pers_hdr_buf->phys + (i*PAGE_SIZE);

		buf->size = PAGE_SIZE;

		buf->dma_attr = pers_hdr_buf->dma_attr;

		buf->flags = pers_hdr_buf->flags;

		buf->type = pers_hdr_buf->type;

		buf->in_use = false;

	}

	spin_unlock(&fl->hlock);

bail:

	if (err) {

		fl->pers_hdr_buf = NULL;

		fl->hdr_bufs = NULL;

		fl->num_pers_hdrs = 0;

		if (!IS_ERR_OR_NULL(pers_hdr_buf))

			fastrpc_buf_free(pers_hdr_buf, 0);

		if (!IS_ERR_OR_NULL(hdr_bufs))

			kfree(hdr_bufs);

	}

	return err;

}



static int fastrpc_internal_invoke2(struct fastrpc_file *fl,

				struct fastrpc_ioctl_invoke2 *inv2)

{

	union {

		struct fastrpc_ioctl_invoke_async inv;

		struct fastrpc_ioctl_async_response async_res;

		uint32_t user_concurrency;

	} p;

	struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;

	uint32_t size = 0;
@@ -2854,6 +2998,18 @@ static int fastrpc_internal_invoke2(struct fastrpc_file *fl, 
		err = fastrpc_get_async_response(&p.async_res,

						(void *)inv2->invparam, fl);

		break;

	case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:

		if (inv2->size != sizeof(uint32_t)) {

			err = -EBADE;

			goto bail;

		}

		K_COPY_FROM_USER(err, 0, &p.user_concurrency,

				(void *)inv2->invparam, size);

		if (err)

			goto bail;

		err = fastrpc_create_persistent_headers(fl,

				p.user_concurrency);

		break;

	default:

		err = -ENOTTY;

		break;
@@ -3040,7 +3196,8 @@ static int fastrpc_init_create_dynamic_process(struct fastrpc_file *fl, 
	imem_dma_attr = DMA_ATTR_EXEC_MAPPING |

					DMA_ATTR_DELAYED_UNMAP |

					DMA_ATTR_NO_KERNEL_MAPPING;

	err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0, 0, &imem);

	err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0,

				INITMEM_BUF, &imem);

	if (err)

		goto bail;

	fl->init_mem = imem;
@@ -4180,7 +4337,7 @@ static int fastrpc_internal_mmap(struct fastrpc_file *fl, 
		if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)

			dma_attr |= DMA_ATTR_IOMMU_USE_UPSTREAM_HINT;

		err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,

								1, &rbuf);

						USERHEAP_BUF, &rbuf);

		if (err)

			goto bail;

		err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,
@@ -4466,6 +4623,10 @@ static int fastrpc_file_free(struct fastrpc_file *fl) 
		fastrpc_buf_free(fl->init_mem, 0);

	fastrpc_context_list_dtor(fl);

	fastrpc_cached_buf_list_free(fl);

	if (!IS_ERR_OR_NULL(fl->hdr_bufs))

		kfree(fl->hdr_bufs);

	if (!IS_ERR_OR_NULL(fl->pers_hdr_buf))

		fastrpc_buf_free(fl->pers_hdr_buf, 0);

	mutex_lock(&fl->map_mutex);

	do {

		lmap = NULL;

drivers/char/adsprpc_compat.c

+7 −3
Original line number Diff line number Diff line
@@ -313,7 +313,7 @@ static int compat_get_fastrpc_ioctl_invoke2( 
{

	int err = 0;

	compat_uptr_t pparam;

	compat_uint_t req, size;

	compat_uint_t req, size, ref_size = 0;

	struct fastrpc_ioctl_invoke2 __user *inv2_user = NULL;

	struct fastrpc_ioctl_invoke_async __user *asyncinv_user;
@@ -362,9 +362,13 @@ static int compat_get_fastrpc_ioctl_invoke2( 
		break;

	}

	case FASTRPC_INVOKE2_ASYNC_RESPONSE:

		ref_size = sizeof(struct fastrpc_ioctl_async_response);

		/* intentional fall through */

	case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:

	{

		VERIFY(err,

			size == sizeof(struct fastrpc_ioctl_async_response));

		if (!ref_size)

			ref_size = sizeof(uint32_t);

		VERIFY(err, size == ref_size);

		if (err) {

			err = -EBADE;

			goto bail;

drivers/char/adsprpc_shared.h

+22 −0
Original line number Diff line number Diff line
@@ -229,6 +229,7 @@ struct fastrpc_ioctl_async_response { 
enum fastrpc_invoke2_type {

	FASTRPC_INVOKE2_ASYNC		   = 1,

	FASTRPC_INVOKE2_ASYNC_RESPONSE = 2,

	FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS,

};



struct fastrpc_ioctl_invoke2 {
@@ -358,6 +359,27 @@ struct fastrpc_ioctl_mem_unmap { 
	};

};



/*

 * This enum is shared with DSP. So, existing values should NOT

 * be modified. Only new members can be added.

 */

enum dsp_map_flags {

	/* Add memory to static PD pool, protection thru XPU */

	ADSP_MMAP_HEAP_ADDR = 4,



	/* Add memory to static PD pool, protection thru hypervisor */

	ADSP_MMAP_REMOTE_HEAP_ADDR = 8,



	/* Add memory to userPD pool, for user heap */

	ADSP_MMAP_ADD_PAGES = 0x1000,



	/* Add memory to userPD pool, for LLC heap */

	ADSP_MMAP_ADD_PAGES_LLC = 0x3000,



	/* Map persistent header buffer on DSP */

	ADSP_MMAP_PERSIST_HDR = 0x4000,

};



struct fastrpc_ioctl_perf {			/* kernel performance data */

	uintptr_t data;

	uint32_t numkeys;