msm: ADSPRPC: Add early wake up signal support (770ae38a) · Commits · e / devices / android_kernel_fairphone_FP4

drivers/char/adsprpc.c

+300 −34

Original line number	Diff line number	Diff line
		@@ -38,6 +38,7 @@
		#include "adsprpc_compat.h"
		#include "adsprpc_shared.h"
		#include <soc/qcom/ramdump.h>
		#include <linux/delay.h>
		#include <linux/debugfs.h>
		#include <linux/pm_qos.h>
		#include <linux/stat.h>
		@@ -106,6 +107,28 @@
		#define FASTRPC_STATIC_HANDLE_MAX (20)
		#define FASTRPC_LATENCY_CTRL_ENB (1)

		/* timeout in us for busy polling after early response from remote processor */
		#define FASTRPC_POLL_TIME (4000)

		/* timeout in us for polling without preempt */
		#define FASTRPC_POLL_TIME_WITHOUT_PREEMPT (500)

		/* timeout in us for polling completion signal after user early hint */
		#define FASTRPC_USER_EARLY_HINT_TIMEOUT (500)

		/* Early wake up poll completion number received from remote processor */
		#define FASTRPC_EARLY_WAKEUP_POLL (0xabbccdde)

		/* latency in us, early wake up signal used below this value */
		#define FASTRPC_EARLY_WAKEUP_LATENCY (200)

		/* response version number */
		#define FASTRPC_RSP_VERSION2 (2)

		/* CPU feature information to DSP */
		#define FASTRPC_CPUINFO_DEFAULT (0)
		#define FASTRPC_CPUINFO_EARLY_WAKEUP (1)

		#define INIT_FILELEN_MAX (210241024)
		#define INIT_MEMLEN_MAX (810241024)
		#define MAX_CACHE_BUF_SIZE (810241024)
		@@ -146,6 +169,11 @@ static int fastrpc_pdr_notifier_cb(struct notifier_block *nb,
		static struct dentry *debugfs_root;
		static struct dentry *debugfs_global_file;

		static inline void mem_barrier(void)
		{
		__asm__ __volatile__("dmb sy":::"memory");
		}

		static inline uint64_t buf_page_start(uint64_t buf)
		{
		uint64_t start = (uint64_t) buf & PAGE_MASK;
		@@ -242,6 +270,12 @@ struct smq_invoke_ctx {
		uint32_t *crc;
		unsigned int magic;
		uint64_t ctxid;
		/* response flags from remote processor */
		enum fastrpc_response_flags rspFlags;
		/* user hint of completion time in us */
		uint32_t earlyWakeTime;
		/* work done status flag */
		bool isWorkDone;
		};

		struct fastrpc_ctx_lst {
		@@ -307,6 +341,9 @@ struct fastrpc_channel_ctx {
		int secure;
		struct fastrpc_dsp_capabilities dsp_cap_kernel;
		void *ipc_log_ctx;
		/* cpu capabilities shared to DSP */
		uint64_t cpuinfo_todsp;
		bool cpuinfo_status;
		};

		struct fastrpc_apps {
		@@ -435,6 +472,8 @@ static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
		.cid = ADSP_DOMAIN_ID,
		}
		},
		.cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
		.cpuinfo_status = false,
		},
		{
		.name = "mdsprpc-smd",
		@@ -444,6 +483,8 @@ static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
		.cid = MDSP_DOMAIN_ID,
		}
		},
		.cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
		.cpuinfo_status = false,
		},
		{
		.name = "sdsprpc-smd",
		@@ -458,6 +499,8 @@ static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
		.cid = SDSP_DOMAIN_ID,
		}
		},
		.cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
		.cpuinfo_status = false,
		},
		{
		.name = "cdsprpc-smd",
		@@ -467,6 +510,8 @@ static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
		.cid = CDSP_DOMAIN_ID,
		}
		},
		.cpuinfo_todsp = FASTRPC_CPUINFO_EARLY_WAKEUP,
		.cpuinfo_status = false,
		},
		};

		@@ -523,6 +568,48 @@ static inline int64_t getperfcounter(struct fastrpc_file fl, int key)
		return val;
		}

		static inline int poll_on_early_response(struct smq_invoke_ctx *ctx)
		{
		int ii, jj, err = -EIO;
		uint32_t sc = ctx->sc;
		struct smq_invoke_buf *list;
		struct smq_phy_page *pages;
		uint64_t *fdlist;
		uint32_t crclist, poll;
		unsigned int inbufs, outbufs, handles;

		/* calculate poll memory location */
		inbufs = REMOTE_SCALARS_INBUFS(sc);
		outbufs = REMOTE_SCALARS_OUTBUFS(sc);
		handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
		list = smq_invoke_buf_start(ctx->rpra, sc);
		pages = smq_phy_page_start(sc, list);
		fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
		crclist = (uint32_t *)(fdlist + M_FDLIST);
		poll = (uint32_t *)(crclist + M_CRCLIST);

		/*
		* poll on memory for actual completion after receiving
		* early response from DSP. Return failure on timeout.
		*/
		preempt_disable();
		for (ii = 0, jj = 0; ii < FASTRPC_POLL_TIME; ii++, jj++) {
		if (*poll == FASTRPC_EARLY_WAKEUP_POLL) {
		err = 0;
		break;
		}
		if (jj == FASTRPC_POLL_TIME_WITHOUT_PREEMPT) {
		/* limit preempt disable time with no rescheduling */
		preempt_enable();
		mem_barrier();
		preempt_disable();
		jj = 0;
		}
		udelay(1);
		}
		preempt_enable_no_resched();
		return err;
		}

		static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)
		{
		@@ -1243,6 +1330,8 @@ static int context_alloc(struct fastrpc_file *fl, uint32_t kernel,
		ctx->tgid = fl->tgid;
		init_completion(&ctx->work);
		ctx->magic = FASTRPC_CTX_MAGIC;
		ctx->rspFlags = NORMAL_RESPONSE;
		ctx->isWorkDone = false;

		spin_lock(&fl->hlock);
		hlist_add_head(&ctx->hn, &clst->pending);
		@@ -1312,13 +1401,35 @@ static void context_free(struct smq_invoke_ctx *ctx)
		kfree(ctx);
		}

		static void context_notify_user(struct smq_invoke_ctx *ctx, int retval)
		static void context_notify_user(struct smq_invoke_ctx *ctx,
		int retval, uint32_t rspFlags, uint32_t earlyWakeTime)
		{
		ctx->retval = retval;
		switch (rspFlags) {
		case NORMAL_RESPONSE:
		/* normal response with return value */
		ctx->retval = retval;
		ctx->isWorkDone = true;
		break;
		case USER_EARLY_SIGNAL:
		/* user hint of approximate time of completion */
		ctx->earlyWakeTime = earlyWakeTime;
		break;
		case EARLY_RESPONSE:
		/* rpc framework early response with return value */
		ctx->retval = retval;
		break;
		case COMPLETE_SIGNAL:
		/* rpc framework signal to clear if pending on ctx */
		ctx->isWorkDone = true;
		break;
		default:
		break;
		}
		ctx->rspFlags = (enum fastrpc_response_flags)rspFlags;
		complete(&ctx->work);
		}


		static void fastrpc_notify_users(struct fastrpc_file *me)
		{
		struct smq_invoke_ctx *ictx;
		@@ -1326,9 +1437,11 @@ static void fastrpc_notify_users(struct fastrpc_file *me)

		spin_lock(&me->hlock);
		hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
		ictx->isWorkDone = true;
		complete(&ictx->work);
		}
		hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
		ictx->isWorkDone = true;
		complete(&ictx->work);
		}
		spin_unlock(&me->hlock);
		@@ -1342,13 +1455,17 @@ static void fastrpc_notify_users_staticpd_pdr(struct fastrpc_file *me)

		spin_lock(&me->hlock);
		hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
		if (ictx->msg.pid)
		if (ictx->msg.pid) {
		ictx->isWorkDone = true;
		complete(&ictx->work);
		}
		}
		hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
		if (ictx->msg.pid)
		if (ictx->msg.pid) {
		ictx->isWorkDone = true;
		complete(&ictx->work);
		}
		}
		spin_unlock(&me->hlock);
		}

		@@ -1910,6 +2027,106 @@ static void fastrpc_init(struct fastrpc_apps *me)

		static int fastrpc_release_current_dsp_process(struct fastrpc_file *fl);

		static inline int fastrpc_wait_for_response(struct smq_invoke_ctx *ctx,
		uint32_t kernel)
		{
		int interrupted = 0;

		if (kernel)
		wait_for_completion(&ctx->work);
		else
		interrupted = wait_for_completion_interruptible(&ctx->work);

		return interrupted;
		}

		static void fastrpc_wait_for_completion(struct smq_invoke_ctx *ctx,
		int *pInterrupted, uint32_t kernel)
		{
		int interrupted = 0, err = 0;
		int jj;
		bool wait_resp;
		uint32_t wTimeout = FASTRPC_USER_EARLY_HINT_TIMEOUT;
		uint32_t wakeTime = ctx->earlyWakeTime;

		while (ctx && !ctx->isWorkDone) {
		switch (ctx->rspFlags) {
		/* try polling on completion with timeout */
		case USER_EARLY_SIGNAL:
		/* try wait if completion time is less than timeout */
		/* disable preempt to avoid context switch latency */
		preempt_disable();
		jj = 0;
		wait_resp = false;
		for (; wakeTime < wTimeout && jj < wTimeout; jj++) {
		wait_resp = try_wait_for_completion(&ctx->work);
		if (wait_resp)
		break;
		udelay(1);
		}
		preempt_enable_no_resched();
		if (!wait_resp) {
		interrupted = fastrpc_wait_for_response(ctx,
		kernel);
		*pInterrupted = interrupted;
		if (interrupted \|\| ctx->isWorkDone)
		return;
		}
		break;

		/* busy poll on memory for actual job done */
		case EARLY_RESPONSE:
		err = poll_on_early_response(ctx);

		/* Mark job done if poll on memory successful */
		/* Wait for completion if poll on memory timoeut */
		if (!err) {
		ctx->isWorkDone = true;
		} else if (!ctx->isWorkDone) {
		pr_info("poll timeout ctxid 0x%llx sc 0x%x\n",
		ctx->ctxid, ctx->sc);
		interrupted = fastrpc_wait_for_response(ctx,
		kernel);
		*pInterrupted = interrupted;
		if (interrupted \|\| ctx->isWorkDone)
		return;
		}
		break;

		case COMPLETE_SIGNAL:
		case NORMAL_RESPONSE:
		interrupted = fastrpc_wait_for_response(ctx, kernel);
		*pInterrupted = interrupted;
		if (interrupted \|\| ctx->isWorkDone)
		return;
		break;

		default:
		pr_err("adsprpc: unsupported response flags 0x%x\n",
		ctx->rspFlags);
		return;
		} /* end of switch */
		} /* end of while loop */
		}

		static void fastrpc_update_invoke_count(uint32_t handle, int64_t *perf_counter,
		struct timespec *invoket)
		{
		/* update invoke count for dynamic handles */
		if (handle != FASTRPC_STATIC_HANDLE_LISTENER) {
		int64_t *count = GET_COUNTER(perf_counter, PERF_INVOKE);

		if (count)
		*count += getnstimediff(invoket);
		}
		if (handle > FASTRPC_STATIC_HANDLE_MAX) {
		int64_t *count = GET_COUNTER(perf_counter, PERF_COUNT);

		if (count)
		*count += 1;
		}
		}

		static int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
		uint32_t kernel,
		struct fastrpc_ioctl_invoke_crc *inv)
		@@ -1920,10 +2137,12 @@ static int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
		int interrupted = 0;
		int err = 0;
		struct timespec invoket = {0};
		int64_t *perf_counter = getperfcounter(fl, PERF_COUNT);
		int64_t *perf_counter = NULL;

		if (fl->profile)
		if (fl->profile) {
		perf_counter = getperfcounter(fl, PERF_COUNT);
		getnstimeofday(&invoket);
		}

		if (!kernel) {
		VERIFY(err, invoke->handle !=
		@@ -1985,13 +2204,16 @@ static int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
		if (err)
		goto bail;
		wait:
		if (kernel)
		wait_for_completion(&ctx->work);
		else {
		interrupted = wait_for_completion_interruptible(&ctx->work);
		fastrpc_wait_for_completion(ctx, &interrupted, kernel);
		VERIFY(err, 0 == (err = interrupted));
		if (err)
		goto bail;

		VERIFY(err, ctx->isWorkDone);
		if (err) {
		pr_err("adsprpc: ctx work done failed, sc 0x%x handle 0x%x\n",
		ctx->sc, invoke->handle);
		goto bail;
		}

		PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
		@@ -2015,20 +2237,9 @@ static int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
		if (fl->ssrcount != fl->apps->channel[cid].ssrcount)
		err = ECONNRESET;

		if (fl->profile && !interrupted) {
		if (invoke->handle != FASTRPC_STATIC_HANDLE_LISTENER) {
		int64_t *count = GET_COUNTER(perf_counter, PERF_INVOKE);

		if (count)
		*count += getnstimediff(&invoket);
		}
		if (invoke->handle > FASTRPC_STATIC_HANDLE_MAX) {
		int64_t *count = GET_COUNTER(perf_counter, PERF_COUNT);

		if (count)
		count = count+1;
		}
		}
		if (fl->profile && !interrupted)
		fastrpc_update_invoke_count(invoke->handle, perf_counter,
		&invoket);
		return err;
		}

		@@ -2343,6 +2554,40 @@ static int fastrpc_kstat(const char filename, struct kstat stat)
		return result;
		}

		static int fastrpc_send_cpuinfo_to_dsp(struct fastrpc_file *fl)
		{
		int err = 0;
		uint64_t cpuinfo = 0;
		struct fastrpc_apps *me = &gfa;
		struct fastrpc_ioctl_invoke_crc ioctl;
		remote_arg_t ra[2];

		VERIFY(err, fl && fl->cid >= 0 && fl->cid < NUM_CHANNELS);
		if (err)
		goto bail;

		cpuinfo = me->channel[fl->cid].cpuinfo_todsp;
		/* return success if already updated to remote processor */
		if (me->channel[fl->cid].cpuinfo_status)
		return 0;

		ra[0].buf.pv = (void *)&cpuinfo;
		ra[0].buf.len = sizeof(cpuinfo);
		ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
		ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
		ioctl.inv.pra = ra;
		ioctl.fds = NULL;
		ioctl.attrs = NULL;
		ioctl.crc = NULL;
		fl->pd = 1;

		err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, 1, &ioctl);
		if (!err)
		me->channel[fl->cid].cpuinfo_status = true;
		bail:
		return err;
		}

		static int fastrpc_get_info_from_dsp(struct fastrpc_file *fl,
		uint32_t *dsp_attr, uint32_t dsp_attr_size,
		uint32_t domain)
		@@ -3066,22 +3311,32 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device rpdev, void data,
		int len, void *priv, u32 addr)
		{
		struct smq_invoke_rsp rsp = (struct smq_invoke_rsp )data;
		struct smq_invoke_rspv2 *rspv2 = NULL;
		struct fastrpc_apps *me = &gfa;
		uint32_t index;
		uint32_t index, flags = 0, earlyWakeTime = 0;
		int err = 0;
		#if IS_ENABLED(CONFIG_ADSPRPC_DEBUG)
		int cid = -1;
		uint32_t logFlags = 0, logEarlyWakeTime = 0;
		#endif

		VERIFY(err, (rsp && len >= sizeof(*rsp)));
		if (err)
		goto bail;

		#if IS_ENABLED(CONFIG_ADSPRPC_DEBUG)
		int cid = -1;
		if (len >= sizeof(struct smq_invoke_rspv2))
		rspv2 = (struct smq_invoke_rspv2 *)data;

		#if IS_ENABLED(CONFIG_ADSPRPC_DEBUG)
		cid = get_cid_from_rpdev(rpdev);
		if (rspv2) {
		logEarlyWakeTime = rspv2->earlyWakeTime;
		logFlags = rspv2->flags;
		}
		if (cid >= 0 && cid < NUM_CHANNELS) {
		LOG_FASTRPC_GLINK_MSG(gcinfo[cid].ipc_log_ctx,
		"recvd pkt %pK (sz %d): ctx 0x%llx, retVal %d",
		data, len, rsp->ctx, rsp->retval);
		"recvd pkt %pK (sz %d): ctx 0x%llx, retVal %d flags %u earlyWake %u",
		data, len, rsp->ctx, rsp->retval, logFlags, logEarlyWakeTime);
		}
		#endif

		@@ -3099,7 +3354,15 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device rpdev, void data,
		if (err)
		goto bail;

		context_notify_user(me->ctxtable[index], rsp->retval);
		if (rspv2) {
		VERIFY(err, rspv2->version == FASTRPC_RSP_VERSION2);
		if (err)
		goto bail;
		flags = rspv2->flags;
		earlyWakeTime = rspv2->earlyWakeTime;
		}
		context_notify_user(me->ctxtable[index], rsp->retval,
		flags, earlyWakeTime);
		bail:
		if (err)
		pr_err("adsprpc: ERROR: %s: invalid response (data %pK, len %d) from remote subsystem (err %d)\n",
		@@ -3607,7 +3870,7 @@ static int fastrpc_internal_control(struct fastrpc_file *fl,
		struct fastrpc_ioctl_control *cp)
		{
		int err = 0;
		int latency;
		unsigned int latency;

		VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps));
		if (err)
		@@ -3629,6 +3892,9 @@ static int fastrpc_internal_control(struct fastrpc_file *fl,
		fl->qos_request = 1;
		} else
		pm_qos_update_request(&fl->pm_qos_req, latency);

		/* Ensure CPU feature map updated to DSP for early WakeUp */
		fastrpc_send_cpuinfo_to_dsp(fl);
		break;
		case FASTRPC_CONTROL_KALLOC:
		cp->kalloc.kalloc_support = 1;

drivers/char/adsprpc_compat.c

+3 −3

Original line number	Diff line number	Diff line
		@@ -122,7 +122,7 @@ struct compat_fastrpc_ioctl_perf { /* kernel performance data */
		#define FASTRPC_CONTROL_LATENCY (1)
		struct compat_fastrpc_ctrl_latency {
		compat_uint_t enable; /* latency control enable */
		compat_uint_t level; /* level of control */
		compat_uint_t latency; /* target latency in us */
		};

		#define FASTRPC_CONTROL_KALLOC (3)
		@@ -334,8 +334,8 @@ static int compat_get_fastrpc_ioctl_control(
		if (p == FASTRPC_CONTROL_LATENCY) {
		err \|= get_user(p, &ctrl32->lp.enable);
		err \|= put_user(p, &ctrl->lp.enable);
		err \|= get_user(p, &ctrl32->lp.level);
		err \|= put_user(p, &ctrl->lp.level);
		err \|= get_user(p, &ctrl32->lp.latency);
		err \|= put_user(p, &ctrl->lp.latency);
		}

		return err;

drivers/char/adsprpc_shared.h

+17 −1

Original line number	Diff line number	Diff line
		@@ -237,13 +237,14 @@ struct fastrpc_ioctl_perf { /* kernel performance data */
		#define FASTRPC_CONTROL_LATENCY (1)
		struct fastrpc_ctrl_latency {
		uint32_t enable; /* latency control enable */
		uint32_t level; /* level of control */
		uint32_t latency; /* latency request in us */
		};

		#define FASTRPC_CONTROL_KALLOC (3)
		struct fastrpc_ctrl_kalloc {
		uint32_t kalloc_support; /* Remote memory allocation from kernel */
		};

		/* FASTRPC_CONTROL value 2 is reserved in user space */
		struct fastrpc_ioctl_control {
		uint32_t req;
		@@ -291,6 +292,21 @@ struct smq_invoke_rsp {
		int retval; /* invoke return value */
		};

		enum fastrpc_response_flags {
		NORMAL_RESPONSE = 0,
		EARLY_RESPONSE = 1,
		USER_EARLY_SIGNAL = 2,
		COMPLETE_SIGNAL = 3
		};

		struct smq_invoke_rspv2 {
		uint64_t ctx; /* invoke caller context */
		int retval; /* invoke return value */
		uint32_t flags; /* early response flags */
		uint32_t earlyWakeTime; /* user predicted early wakeup time in us */
		uint32_t version; /* Version number for validation */
		};

		static inline struct smq_invoke_buf smq_invoke_buf_start(remote_arg64_t pra,
		uint32_t sc)
		{