Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 009a82f6 authored by Trond Myklebust's avatar Trond Myklebust
Browse files

SUNRPC: Micro-optimise when the task is known not to be sleeping 



In cases where we know the task is not sleeping, try to optimise
away the indirect call to task->tk_action() by replacing it with
a direct call.
Only change tail calls, to allow gcc to perform tail call
elimination.

Signed-off-by: default avatarTrond Myklebust <trond.myklebust@hammerspace.com>
parent 03e51d32

include/linux/sunrpc/sched.h

+8 −0
Original line number Diff line number Diff line
@@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt) 
}

#endif /* CONFIG_SUNRPC_SWAP */



static inline bool

rpc_task_need_resched(const struct rpc_task *task)

{

	if (RPC_IS_QUEUED(task) || task->tk_callback)

		return true;

	return false;

}



#endif /* _LINUX_SUNRPC_SCHED_H_ */

net/sunrpc/clnt.c

+65 −34
Original line number Diff line number Diff line
@@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task) 
	clnt->cl_stats->rpccnt++;

	task->tk_action = call_reserve;

	rpc_task_set_transport(task, clnt);

	call_reserve(task);

}



/*
@@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task) 
	task->tk_status  = 0;

	task->tk_action  = call_reserveresult;

	xprt_reserve(task);

	if (rpc_task_need_resched(task))

		return;

	 call_reserveresult(task);

}



static void call_retry_reserve(struct rpc_task *task);
@@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task) 
	if (status >= 0) {

		if (task->tk_rqstp) {

			task->tk_action = call_refresh;

			call_refresh(task);

			return;

		}
@@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task) 
		/* fall through */

	case -EAGAIN:	/* woken up; retry */

		task->tk_action = call_retry_reserve;

		call_retry_reserve(task);

		return;

	case -EIO:	/* probably a shutdown */

		break;
@@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task) 
	task->tk_status  = 0;

	task->tk_action  = call_reserveresult;

	xprt_retry_reserve(task);

	if (rpc_task_need_resched(task))

		return;

	call_reserveresult(task);

}



/*
@@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task) 
	task->tk_status = 0;

	task->tk_client->cl_stats->rpcauthrefresh++;

	rpcauth_refreshcred(task);

	if (rpc_task_need_resched(task))

		return;

	call_refreshresult(task);

}



/*
@@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task) 
	case 0:

		if (rpcauth_uptodatecred(task)) {

			task->tk_action = call_allocate;

			call_allocate(task);

			return;

		}

		/* Use rate-limiting and a max number of retries if refresh
@@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task) 
		task->tk_cred_retry--;

		dprintk("RPC: %5u %s: retry refresh creds\n",

				task->tk_pid, __func__);

		call_refresh(task);

		return;

	}

	dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
@@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task) 
	task->tk_status = 0;

	task->tk_action = call_encode;



	if (req->rq_buffer)

	if (req->rq_buffer) {

		call_encode(task);

		return;

	}



	if (proc->p_proc != 0) {

		BUG_ON(proc->p_arglen == 0);
@@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task) 


	status = xprt->ops->buf_alloc(task);

	xprt_inject_disconnect(xprt);

	if (status == 0)

	if (status == 0) {

		if (rpc_task_need_resched(task))

			return;

		call_encode(task);

		return;

	}

	if (status != -ENOMEM) {

		rpc_exit(task, status);

		return;
@@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task) 
		xprt_request_enqueue_receive(task);

	xprt_request_enqueue_transmit(task);

out:

	task->tk_action = call_transmit;

	/* Check that the connection is OK */

	if (!xprt_bound(task->tk_xprt))

	task->tk_action = call_bind;

	else if (!xprt_connected(task->tk_xprt))

		task->tk_action = call_connect;

	call_bind(task);

}



/*
@@ -1842,15 +1858,18 @@ call_bind(struct rpc_task *task) 
		return;

	}



	if (xprt_bound(xprt)) {

		task->tk_action = call_connect;

		call_connect(task);

		return;

	}



	dprint_status(task);



	task->tk_action = call_connect;

	if (!xprt_bound(xprt)) {

	task->tk_action = call_bind_status;

	task->tk_timeout = xprt->bind_timeout;

	xprt->ops->rpcbind(task);

}

}



/*

 * 4a.	Sort out bind result
@@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task) 
		dprint_status(task);

		task->tk_status = 0;

		task->tk_action = call_connect;

		call_connect(task);

		return;

	}
@@ -1949,12 +1969,16 @@ call_connect(struct rpc_task *task) 
		return;

	}



	if (xprt_connected(xprt)) {

		task->tk_action = call_transmit;

		call_transmit(task);

		return;

	}



	dprintk("RPC: %5u call_connect xprt %p %s connected\n",

			task->tk_pid, xprt,

			(xprt_connected(xprt) ? "is" : "is not"));



	task->tk_action = call_transmit;

	if (!xprt_connected(xprt)) {

	task->tk_action = call_connect_status;

	if (task->tk_status < 0)

		return;
@@ -1964,7 +1988,6 @@ call_connect(struct rpc_task *task) 
	}

	xprt_connect(task);

}

}



/*

 * 4c.	Sort out connect result
@@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task) 
	case 0:

		clnt->cl_stats->netreconn++;

		task->tk_action = call_transmit;

		call_transmit(task);

		return;

	}

	rpc_exit(task, status);
@@ -2040,7 +2064,6 @@ call_transmit(struct rpc_task *task) 
	dprint_status(task);



	task->tk_action = call_transmit_status;

	if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {

	if (!xprt_prepare_transmit(task))

		return;

	task->tk_status = 0;
@@ -2051,8 +2074,10 @@ call_transmit(struct rpc_task *task) 
		}

		xprt_transmit(task);

	}

	}

	xprt_end_transmit(task);

	if (rpc_task_need_resched(task))

		return;

	call_transmit_status(task);

}



/*
@@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task) 
	 * Common case: success.  Force the compiler to put this

	 * test first.

	 */

	if (task->tk_status == 0) {

	if (rpc_task_transmitted(task)) {

		if (task->tk_status == 0)

			xprt_request_wait_receive(task);

		if (rpc_task_need_resched(task))

			return;

		call_status(task);

		return;

	}
@@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task) 
{

	xprt_request_enqueue_transmit(task);

	task->tk_action = call_bc_transmit;

	call_bc_transmit(task);

}



/*
@@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task) 
	status = task->tk_status;

	if (status >= 0) {

		task->tk_action = call_decode;

		call_decode(task);

		return;

	}