Merge "psi: pressure stall information for CPU, memory, and IO"

================================

PSI - Pressure Stall Information

================================

:Date: April, 2018

:Author: Johannes Weiner <hannes@cmpxchg.org>

When CPU, memory or IO devices are contended, workloads experience

latency spikes, throughput losses, and run the risk of OOM kills.

Without an accurate measure of such contention, users are forced to

either play it safe and under-utilize their hardware resources, or

roll the dice and frequently suffer the disruptions resulting from

excessive overcommit.

The psi feature identifies and quantifies the disruptions caused by

such resource crunches and the time impact it has on complex workloads

or even entire systems.

Having an accurate measure of productivity losses caused by resource

scarcity aids users in sizing workloads to hardware--or provisioning

hardware according to workload demand.

As psi aggregates this information in realtime, systems can be managed

dynamically using techniques such as load shedding, migrating jobs to

other systems or data centers, or strategically pausing or killing low

priority or restartable batch jobs.

This allows maximizing hardware utilization without sacrificing

workload health or risking major disruptions such as OOM kills.

Pressure interface

==================

Pressure information for each resource is exported through the

respective file in /proc/pressure/ -- cpu, memory, and io.

The format for CPU is as such:

some avg10=0.00 avg60=0.00 avg300=0.00 total=0

and for memory and IO:

some avg10=0.00 avg60=0.00 avg300=0.00 total=0

full avg10=0.00 avg60=0.00 avg300=0.00 total=0

The "some" line indicates the share of time in which at least some

tasks are stalled on a given resource.

The "full" line indicates the share of time in which all non-idle

tasks are stalled on a given resource simultaneously. In this state

actual CPU cycles are going to waste, and a workload that spends

extended time in this state is considered to be thrashing. This has

severe impact on performance, and it's useful to distinguish this

situation from a state where some tasks are stalled but the CPU is

still doing productive work. As such, time spent in this subset of the

stall state is tracked separately and exported in the "full" averages.

The ratios are tracked as recent trends over ten, sixty, and three

hundred second windows, which gives insight into short term events as

well as medium and long term trends. The total absolute stall time is

tracked and exported as well, to allow detection of latency spikes

which wouldn't necessarily make a dent in the time averages, or to

average trends over custom time frames.

#ifndef _LINUX_PSI_H

#define _LINUX_PSI_H

#include <linux/psi_types.h>

#include <linux/sched.h>

#ifdef CONFIG_PSI

extern bool psi_disabled;

void psi_init(void);

void psi_task_change(struct task_struct *task, int clear, int set);

void psi_memstall_tick(struct task_struct *task, int cpu);

void psi_memstall_enter(unsigned long *flags);

void psi_memstall_leave(unsigned long *flags);

#else /* CONFIG_PSI */

static inline void psi_init(void) {}

static inline void psi_memstall_enter(unsigned long *flags) {}

static inline void psi_memstall_leave(unsigned long *flags) {}

#endif /* CONFIG_PSI */

#endif /* _LINUX_PSI_H */

#ifndef _LINUX_PSI_TYPES_H

#define _LINUX_PSI_TYPES_H

#include <linux/seqlock.h>

#include <linux/types.h>

#ifdef CONFIG_PSI

/* Tracked task states */

enum psi_task_count {

	NR_IOWAIT,

	NR_MEMSTALL,

	NR_RUNNING,

	NR_PSI_TASK_COUNTS,

};

/* Task state bitmasks */

#define TSK_IOWAIT	(1 << NR_IOWAIT)

#define TSK_MEMSTALL	(1 << NR_MEMSTALL)

#define TSK_RUNNING	(1 << NR_RUNNING)

/* Resources that workloads could be stalled on */

enum psi_res {

	PSI_IO,

	PSI_MEM,

	PSI_CPU,

	NR_PSI_RESOURCES,

};

/*

 * Pressure states for each resource:

 *

 * SOME: Stalled tasks & working tasks

 * FULL: Stalled tasks & no working tasks

 */

enum psi_states {

	PSI_IO_SOME,

	PSI_IO_FULL,

	PSI_MEM_SOME,

	PSI_MEM_FULL,

	PSI_CPU_SOME,

	/* Only per-CPU, to weigh the CPU in the global average: */

	PSI_NONIDLE,

	NR_PSI_STATES,

};

struct psi_group_cpu {

	/* 1st cacheline updated by the scheduler */

	/* Aggregator needs to know of concurrent changes */

	seqcount_t seq ____cacheline_aligned_in_smp;

	/* States of the tasks belonging to this group */

	unsigned int tasks[NR_PSI_TASK_COUNTS];

	/* Period time sampling buckets for each state of interest (ns) */

	u32 times[NR_PSI_STATES];

	/* Time of last task change in this group (rq_clock) */

	u64 state_start;

	/* 2nd cacheline updated by the aggregator */

	/* Delta detection against the sampling buckets */

	u32 times_prev[NR_PSI_STATES] ____cacheline_aligned_in_smp;

};

struct psi_group {

	/* Protects data updated during an aggregation */

	struct mutex stat_lock;

	/* Per-cpu task state & time tracking */

	struct psi_group_cpu __percpu *pcpu;

	/* Periodic aggregation state */

	u64 total_prev[NR_PSI_STATES - 1];

	u64 last_update;

	u64 next_update;

	struct delayed_work clock_work;

	/* Total stall times and sampled pressure averages */

	u64 total[NR_PSI_STATES - 1];

	unsigned long avg[NR_PSI_STATES - 1][3];

};

#else /* CONFIG_PSI */

struct psi_group { };

#endif /* CONFIG_PSI */

#endif /* _LINUX_PSI_TYPES_H */

#include <linux/latencytop.h>

#include <linux/sched/prio.h>

#include <linux/signal_types.h>

#include <linux/psi_types.h>

#include <linux/mm_types_task.h>

#include <linux/task_io_accounting.h>

#include <linux/rseq.h>

	unsigned			sched_contributes_to_load:1;

	unsigned			sched_migrated:1;

	unsigned			sched_remote_wakeup:1;

#ifdef CONFIG_PSI

	unsigned			sched_psi_wake_requeue:1;

#endif

	/* Force alignment to the next boundary: */

	unsigned			:0;

	siginfo_t			*last_siginfo;

	struct task_io_accounting	ioac;

#ifdef CONFIG_PSI

	/* Pressure stall state */

	unsigned int			psi_flags;

#endif

#ifdef CONFIG_TASK_XACCT

	/* Accumulated RSS usage: */

	u64				acct_rss_mem1;

#define PF_WAKE_UP_IDLE         0x01000000	/* TTWU on an idle CPU */

#define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_allowed */

#define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */

#define PF_MEMSTALL		0x10000000	/* Stalled due to lack of memory */

#define PF_MUTEX_TESTER		0x20000000	/* Thread belongs to the rt mutex tester */

#define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */

#define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */

	return newload / FIXED_1;

}

extern unsigned long calc_load_n(unsigned long load, unsigned long exp,

				 unsigned long active, unsigned int n);

#define LOAD_INT(x) ((x) >> FSHIFT)

#define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100)