ANDROID: binder: add support for RT prio inheritance. (ce388e00) · Commits · e / devices / android_kernel_fairphone_FP4

drivers/android/binder.c

+212 −32

Original line number	Diff line number	Diff line
		@@ -73,6 +73,7 @@
		#include <linux/ratelimit.h>

		#include <uapi/linux/android/binder.h>
		#include <uapi/linux/sched/types.h>

		#include <asm/cacheflush.h>

		@@ -350,10 +351,14 @@ struct binder_error {
		* and by @lock)
		* @has_async_transaction: async transaction to node in progress
		* (protected by @lock)
		* @sched_policy: minimum scheduling policy for node
		* (invariant after initialized)
		* @accept_fds: file descriptor operations supported for node
		* (invariant after initialized)
		* @min_priority: minimum scheduling priority
		* (invariant after initialized)
		* @inherit_rt: inherit RT scheduling policy from caller
		* (invariant after initialized)
		* @async_todo: list of async work items
		* (protected by @proc->inner_lock)
		*
		@@ -389,6 +394,8 @@ struct binder_node {
		/*
		* invariant after initialization
		*/
		u8 sched_policy:2;
		u8 inherit_rt:1;
		u8 accept_fds:1;
		u8 min_priority;
		};
		@@ -462,6 +469,22 @@ enum binder_deferred_state {
		BINDER_DEFERRED_RELEASE = 0x04,
		};

		/**
		* struct binder_priority - scheduler policy and priority
		* @sched_policy scheduler policy
		* @prio [100..139] for SCHED_NORMAL, [0..99] for FIFO/RT
		*
		* The binder driver supports inheriting the following scheduler policies:
		* SCHED_NORMAL
		* SCHED_BATCH
		* SCHED_FIFO
		* SCHED_RR
		*/
		struct binder_priority {
		unsigned int sched_policy;
		int prio;
		};

		/**
		* struct binder_proc - binder process bookkeeping
		* @proc_node: element for binder_procs list
		@@ -540,7 +563,7 @@ struct binder_proc {
		int requested_threads;
		int requested_threads_started;
		int tmp_ref;
		long default_priority;
		struct binder_priority default_priority;
		struct dentry *debugfs_entry;
		struct binder_alloc alloc;
		struct binder_context *context;
		@@ -590,6 +613,7 @@ enum {
		* @is_dead: thread is dead and awaiting free
		* when outstanding transactions are cleaned up
		* (protected by @proc->inner_lock)
		* @task: struct task_struct for this thread
		*
		* Bookkeeping structure for binder threads.
		*/
		@@ -609,6 +633,7 @@ struct binder_thread {
		struct binder_stats stats;
		atomic_t tmp_ref;
		bool is_dead;
		struct task_struct *task;
		};

		struct binder_transaction {
		@@ -625,8 +650,9 @@ struct binder_transaction {
		struct binder_buffer *buffer;
		unsigned int code;
		unsigned int flags;
		long priority;
		long saved_priority;
		struct binder_priority priority;
		struct binder_priority saved_priority;
		bool set_priority_called;
		kuid_t sender_euid;
		/**
		* @lock: protects @from, @to_proc, and @to_thread
		@@ -1107,22 +1133,145 @@ static void binder_wakeup_proc_ilocked(struct binder_proc *proc)
		binder_wakeup_thread_ilocked(proc, thread, /* sync = */false);
		}

		static void binder_set_nice(long nice)
		static bool is_rt_policy(int policy)
		{
		return policy == SCHED_FIFO \|\| policy == SCHED_RR;
		}

		static bool is_fair_policy(int policy)
		{
		long min_nice;
		return policy == SCHED_NORMAL \|\| policy == SCHED_BATCH;
		}

		static bool binder_supported_policy(int policy)
		{
		return is_fair_policy(policy) \|\| is_rt_policy(policy);
		}

		static int to_userspace_prio(int policy, int kernel_priority)
		{
		if (is_fair_policy(policy))
		return PRIO_TO_NICE(kernel_priority);
		else
		return MAX_USER_RT_PRIO - 1 - kernel_priority;
		}

		if (can_nice(current, nice)) {
		set_user_nice(current, nice);
		static int to_kernel_prio(int policy, int user_priority)
		{
		if (is_fair_policy(policy))
		return NICE_TO_PRIO(user_priority);
		else
		return MAX_USER_RT_PRIO - 1 - user_priority;
		}

		static void binder_do_set_priority(struct task_struct *task,
		struct binder_priority desired,
		bool verify)
		{
		int priority; /* user-space prio value */
		bool has_cap_nice;
		unsigned int policy = desired.sched_policy;

		if (task->policy == policy && task->normal_prio == desired.prio)
		return;

		has_cap_nice = has_capability_noaudit(task, CAP_SYS_NICE);

		priority = to_userspace_prio(policy, desired.prio);

		if (verify && is_rt_policy(policy) && !has_cap_nice) {
		long max_rtprio = task_rlimit(task, RLIMIT_RTPRIO);

		if (max_rtprio == 0) {
		policy = SCHED_NORMAL;
		priority = MIN_NICE;
		} else if (priority > max_rtprio) {
		priority = max_rtprio;
		}
		}

		if (verify && is_fair_policy(policy) && !has_cap_nice) {
		long min_nice = rlimit_to_nice(task_rlimit(task, RLIMIT_NICE));

		if (min_nice > MAX_NICE) {
		binder_user_error("%d RLIMIT_NICE not set\n",
		task->pid);
		return;
		} else if (priority < min_nice) {
		priority = min_nice;
		}
		min_nice = rlimit_to_nice(rlimit(RLIMIT_NICE));
		}

		if (policy != desired.sched_policy \|\|
		to_kernel_prio(policy, priority) != desired.prio)
		binder_debug(BINDER_DEBUG_PRIORITY_CAP,
		"%d: nice value %ld not allowed use %ld instead\n",
		current->pid, nice, min_nice);
		set_user_nice(current, min_nice);
		if (min_nice <= MAX_NICE)
		"%d: priority %d not allowed, using %d instead\n",
		task->pid, desired.prio,
		to_kernel_prio(policy, priority));

		trace_binder_set_priority(task->tgid, task->pid, task->normal_prio,
		to_kernel_prio(policy, priority),
		desired.prio);

		/* Set the actual priority */
		if (task->policy != policy \|\| is_rt_policy(policy)) {
		struct sched_param params;

		params.sched_priority = is_rt_policy(policy) ? priority : 0;

		sched_setscheduler_nocheck(task,
		policy \| SCHED_RESET_ON_FORK,
		&params);
		}
		if (is_fair_policy(policy))
		set_user_nice(task, priority);
		}

		static void binder_set_priority(struct task_struct *task,
		struct binder_priority desired)
		{
		binder_do_set_priority(task, desired, /* verify = */ true);
		}

		static void binder_restore_priority(struct task_struct *task,
		struct binder_priority desired)
		{
		binder_do_set_priority(task, desired, /* verify = */ false);
		}

		static void binder_transaction_priority(struct task_struct *task,
		struct binder_transaction *t,
		struct binder_priority node_prio,
		bool inherit_rt)
		{
		struct binder_priority desired_prio = t->priority;

		if (t->set_priority_called)
		return;
		binder_user_error("%d RLIMIT_NICE not set\n", current->pid);

		t->set_priority_called = true;
		t->saved_priority.sched_policy = task->policy;
		t->saved_priority.prio = task->normal_prio;

		if (!inherit_rt && is_rt_policy(desired_prio.sched_policy)) {
		desired_prio.prio = NICE_TO_PRIO(0);
		desired_prio.sched_policy = SCHED_NORMAL;
		}

		if (node_prio.prio < t->priority.prio \|\|
		(node_prio.prio == t->priority.prio &&
		node_prio.sched_policy == SCHED_FIFO)) {
		/*
		* In case the minimum priority on the node is
		* higher (lower value), use that priority. If
		* the priority is the same, but the node uses
		* SCHED_FIFO, prefer SCHED_FIFO, since it can
		* run unbounded, unlike SCHED_RR.
		*/
		desired_prio = node_prio;
		}

		binder_set_priority(task, desired_prio);
		}

		static struct binder_node binder_get_node_ilocked(struct binder_proc proc,
		@@ -1175,6 +1324,7 @@ static struct binder_node *binder_init_node_ilocked(
		binder_uintptr_t ptr = fp ? fp->binder : 0;
		binder_uintptr_t cookie = fp ? fp->cookie : 0;
		__u32 flags = fp ? fp->flags : 0;
		s8 priority;

		assert_spin_locked(&proc->inner_lock);

		@@ -1207,8 +1357,12 @@ static struct binder_node *binder_init_node_ilocked(
		node->ptr = ptr;
		node->cookie = cookie;
		node->work.type = BINDER_WORK_NODE;
		node->min_priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
		priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
		node->sched_policy = (flags & FLAT_BINDER_FLAG_SCHED_POLICY_MASK) >>
		FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT;
		node->min_priority = to_kernel_prio(node->sched_policy, priority);
		node->accept_fds = !!(flags & FLAT_BINDER_FLAG_ACCEPTS_FDS);
		node->inherit_rt = !!(flags & FLAT_BINDER_FLAG_INHERIT_RT);
		spin_lock_init(&node->lock);
		INIT_LIST_HEAD(&node->work.entry);
		INIT_LIST_HEAD(&node->async_todo);
		@@ -2633,11 +2787,15 @@ static bool binder_proc_transaction(struct binder_transaction *t,
		struct binder_thread *thread)
		{
		struct binder_node *node = t->buffer->target_node;
		struct binder_priority node_prio;
		bool oneway = !!(t->flags & TF_ONE_WAY);
		bool pending_async = false;

		BUG_ON(!node);
		binder_node_lock(node);
		node_prio.prio = node->min_priority;
		node_prio.sched_policy = node->sched_policy;

		if (oneway) {
		BUG_ON(thread);
		if (node->has_async_transaction) {
		@@ -2658,12 +2816,15 @@ static bool binder_proc_transaction(struct binder_transaction *t,
		if (!thread && !pending_async)
		thread = binder_select_thread_ilocked(proc);

		if (thread)
		if (thread) {
		binder_transaction_priority(thread->task, t, node_prio,
		node->inherit_rt);
		binder_enqueue_thread_work_ilocked(thread, &t->work);
		else if (!pending_async)
		} else if (!pending_async) {
		binder_enqueue_work_ilocked(&t->work, &proc->todo);
		else
		} else {
		binder_enqueue_work_ilocked(&t->work, &node->async_todo);
		}

		if (!pending_async)
		binder_wakeup_thread_ilocked(proc, thread, !oneway /* sync */);
		@@ -2780,7 +2941,6 @@ static void binder_transaction(struct binder_proc *proc,
		}
		thread->transaction_stack = in_reply_to->to_parent;
		binder_inner_proc_unlock(proc);
		binder_set_nice(in_reply_to->saved_priority);
		target_thread = binder_get_txn_from_and_acq_inner(in_reply_to);
		if (target_thread == NULL) {
		return_error = BR_DEAD_REPLY;
		@@ -2953,7 +3113,15 @@ static void binder_transaction(struct binder_proc *proc,
		t->to_thread = target_thread;
		t->code = tr->code;
		t->flags = tr->flags;
		t->priority = task_nice(current);
		if (!(t->flags & TF_ONE_WAY) &&
		binder_supported_policy(current->policy)) {
		/* Inherit supported policies for synchronous transactions */
		t->priority.sched_policy = current->policy;
		t->priority.prio = current->normal_prio;
		} else {
		/* Otherwise, fall back to the default priority */
		t->priority = target_proc->default_priority;
		}

		trace_binder_transaction(reply, t, target_node);

		@@ -3182,6 +3350,7 @@ static void binder_transaction(struct binder_proc *proc,
		binder_enqueue_thread_work_ilocked(target_thread, &t->work);
		binder_inner_proc_unlock(target_proc);
		wake_up_interruptible_sync(&target_thread->wait);
		binder_restore_priority(current, in_reply_to->saved_priority);
		binder_free_transaction(in_reply_to);
		} else if (!(t->flags & TF_ONE_WAY)) {
		BUG_ON(t->buffer->async_transaction != 0);
		@@ -3285,6 +3454,7 @@ static void binder_transaction(struct binder_proc *proc,

		BUG_ON(thread->return_error.cmd != BR_OK);
		if (in_reply_to) {
		binder_restore_priority(current, in_reply_to->saved_priority);
		thread->return_error.cmd = BR_TRANSACTION_COMPLETE;
		binder_enqueue_thread_work(thread, &thread->return_error.work);
		binder_send_failed_reply(in_reply_to, return_error);
		@@ -3865,7 +4035,7 @@ static int binder_thread_read(struct binder_proc *proc,
		wait_event_interruptible(binder_user_error_wait,
		binder_stop_on_user_error < 2);
		}
		binder_set_nice(proc->default_priority);
		binder_restore_priority(current, proc->default_priority);
		}

		if (non_block) {
		@@ -4080,16 +4250,14 @@ static int binder_thread_read(struct binder_proc *proc,
		BUG_ON(t->buffer == NULL);
		if (t->buffer->target_node) {
		struct binder_node *target_node = t->buffer->target_node;
		struct binder_priority node_prio;

		tr.target.ptr = target_node->ptr;
		tr.cookie = target_node->cookie;
		t->saved_priority = task_nice(current);
		if (t->priority < target_node->min_priority &&
		!(t->flags & TF_ONE_WAY))
		binder_set_nice(t->priority);
		else if (!(t->flags & TF_ONE_WAY) \|\|
		t->saved_priority > target_node->min_priority)
		binder_set_nice(target_node->min_priority);
		node_prio.sched_policy = target_node->sched_policy;
		node_prio.prio = target_node->min_priority;
		binder_transaction_priority(current, t, node_prio,
		target_node->inherit_rt);
		cmd = BR_TRANSACTION;
		} else {
		tr.target.ptr = 0;
		@@ -4267,6 +4435,8 @@ static struct binder_thread *binder_get_thread_ilocked(
		binder_stats_created(BINDER_STAT_THREAD);
		thread->proc = proc;
		thread->pid = current->pid;
		get_task_struct(current);
		thread->task = current;
		atomic_set(&thread->tmp_ref, 0);
		init_waitqueue_head(&thread->wait);
		INIT_LIST_HEAD(&thread->todo);
		@@ -4317,6 +4487,7 @@ static void binder_free_thread(struct binder_thread *thread)
		BUG_ON(!list_empty(&thread->todo));
		binder_stats_deleted(BINDER_STAT_THREAD);
		binder_proc_dec_tmpref(thread->proc);
		put_task_struct(thread->task);
		kfree(thread);
		}

		@@ -4822,7 +4993,14 @@ static int binder_open(struct inode nodp, struct file filp)
		proc->tsk = current->group_leader;
		mutex_init(&proc->files_lock);
		INIT_LIST_HEAD(&proc->todo);
		proc->default_priority = task_nice(current);
		if (binder_supported_policy(current->policy)) {
		proc->default_priority.sched_policy = current->policy;
		proc->default_priority.prio = current->normal_prio;
		} else {
		proc->default_priority.sched_policy = SCHED_NORMAL;
		proc->default_priority.prio = NICE_TO_PRIO(0);
		}

		binder_dev = container_of(filp->private_data, struct binder_device,
		miscdev);
		proc->context = &binder_dev->context;
		@@ -5116,13 +5294,14 @@ static void print_binder_transaction_ilocked(struct seq_file *m,
		spin_lock(&t->lock);
		to_proc = t->to_proc;
		seq_printf(m,
		"%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %ld r%d",
		"%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %d:%d r%d",
		prefix, t->debug_id, t,
		t->from ? t->from->proc->pid : 0,
		t->from ? t->from->pid : 0,
		to_proc ? to_proc->pid : 0,
		t->to_thread ? t->to_thread->pid : 0,
		t->code, t->flags, t->priority, t->need_reply);
		t->code, t->flags, t->priority.sched_policy,
		t->priority.prio, t->need_reply);
		spin_unlock(&t->lock);

		if (proc != to_proc) {
		@@ -5240,8 +5419,9 @@ static void print_binder_node_nilocked(struct seq_file *m,
		hlist_for_each_entry(ref, &node->refs, node_entry)
		count++;

		seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d tr %d",
		seq_printf(m, " node %d: u%016llx c%016llx pri %d:%d hs %d hw %d ls %d lw %d is %d iw %d tr %d",
		node->debug_id, (u64)node->ptr, (u64)node->cookie,
		node->sched_policy, node->min_priority,
		node->has_strong_ref, node->has_weak_ref,
		node->local_strong_refs, node->local_weak_refs,
		node->internal_strong_refs, count, node->tmp_refs);

drivers/android/binder_trace.h

+24 −0

Original line number	Diff line number	Diff line
		@@ -85,6 +85,30 @@ DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done);
		DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done);
		DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done);

		TRACE_EVENT(binder_set_priority,
		TP_PROTO(int proc, int thread, unsigned int old_prio,
		unsigned int desired_prio, unsigned int new_prio),
		TP_ARGS(proc, thread, old_prio, new_prio, desired_prio),

		TP_STRUCT__entry(
		__field(int, proc)
		__field(int, thread)
		__field(unsigned int, old_prio)
		__field(unsigned int, new_prio)
		__field(unsigned int, desired_prio)
		),
		TP_fast_assign(
		__entry->proc = proc;
		__entry->thread = thread;
		__entry->old_prio = old_prio;
		__entry->new_prio = new_prio;
		__entry->desired_prio = desired_prio;
		),
		TP_printk("proc=%d thread=%d old=%d => new=%d desired=%d",
		__entry->proc, __entry->thread, __entry->old_prio,
		__entry->new_prio, __entry->desired_prio)
		);

		TRACE_EVENT(binder_wait_for_work,
		TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo),
		TP_ARGS(proc_work, transaction_stack, thread_todo),

include/uapi/linux/android/binder.h

+48 −1

Original line number	Diff line number	Diff line
		@@ -38,9 +38,56 @@ enum {
		BINDER_TYPE_PTR = B_PACK_CHARS('p', 't', '*', B_TYPE_LARGE),
		};

		enum {
		/**
		* enum flat_binder_object_shifts: shift values for flat_binder_object_flags
		* @FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT: shift for getting scheduler policy.
		*
		*/
		enum flat_binder_object_shifts {
		FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT = 9,
		};

		/**
		* enum flat_binder_object_flags - flags for use in flat_binder_object.flags
		*/
		enum flat_binder_object_flags {
		/**
		* @FLAT_BINDER_FLAG_PRIORITY_MASK: bit-mask for min scheduler priority
		*
		* These bits can be used to set the minimum scheduler priority
		* at which transactions into this node should run. Valid values
		* in these bits depend on the scheduler policy encoded in
		* @FLAT_BINDER_FLAG_SCHED_POLICY_MASK.
		*
		* For SCHED_NORMAL/SCHED_BATCH, the valid range is between [-20..19]
		* For SCHED_FIFO/SCHED_RR, the value can run between [1..99]
		*/
		FLAT_BINDER_FLAG_PRIORITY_MASK = 0xff,
		/**
		* @FLAT_BINDER_FLAG_ACCEPTS_FDS: whether the node accepts fds.
		*/
		FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100,
		/**
		* @FLAT_BINDER_FLAG_SCHED_POLICY_MASK: bit-mask for scheduling policy
		*
		* These two bits can be used to set the min scheduling policy at which
		* transactions on this node should run. These match the UAPI
		* scheduler policy values, eg:
		* 00b: SCHED_NORMAL
		* 01b: SCHED_FIFO
		* 10b: SCHED_RR
		* 11b: SCHED_BATCH
		*/
		FLAT_BINDER_FLAG_SCHED_POLICY_MASK =
		3U << FLAT_BINDER_FLAG_SCHED_POLICY_SHIFT,

		/**
		* @FLAT_BINDER_FLAG_INHERIT_RT: whether the node inherits RT policy
		*
		* Only when set, calls into this node will inherit a real-time
		* scheduling policy from the caller (for synchronous transactions).
		*/
		FLAT_BINDER_FLAG_INHERIT_RT = 0x800,
		};

		#ifdef BINDER_IPC_32BIT