timekeeping: Cache optimize struct timekeeper

#include <linux/jiffies.h>

#include <linux/time.h>

/* Structure holding internal timekeeping values. */

/*

 * Structure holding internal timekeeping values.

 *

 * Note: wall_to_monotonic is what we need to add to xtime (or xtime

 * corrected for sub jiffie times) to get to monotonic time.

 * Monotonic is pegged at zero at system boot time, so

 * wall_to_monotonic will be negative, however, we will ALWAYS keep

 * the tv_nsec part positive so we can use the usual normalization.

 *

 * wall_to_monotonic is moved after resume from suspend for the

 * monotonic time not to jump. We need to add total_sleep_time to

 * wall_to_monotonic to get the real boot based time offset.

 *

 * - wall_to_monotonic is no longer the boot time, getboottime must be

 * used instead.

 */

struct timekeeper {

	/* Current clocksource used for timekeeping. */

	struct clocksource	*clock;

	u32			mult;

	/* The shift value of the current clocksource. */

	u32			shift;

	/* Clock shifted nano seconds */

	u64			xtime_nsec;

	/* Current CLOCK_REALTIME time in seconds */

	u64			xtime_sec;

	/* CLOCK_REALTIME to CLOCK_MONOTONIC offset */

	struct timespec64	wall_to_monotonic;

	/* Offset clock monotonic -> clock realtime */

	ktime_t			offs_real;

	/* Offset clock monotonic -> clock boottime */

	ktime_t			offs_boot;

	/* Offset clock monotonic -> clock tai */

	ktime_t			offs_tai;

	/* time spent in suspend */

	struct timespec64	total_sleep_time;

	/* The current UTC to TAI offset in seconds */

	s32			tai_offset;

	/* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */

	struct timespec64	raw_time;

	/* Number of clock cycles in one NTP interval. */

	cycle_t			cycle_interval;

	/* Last cycle value (also stored in clock->cycle_last) */

	/* Raw nano seconds accumulated per NTP interval. */

	u32			raw_interval;

	/* Current CLOCK_REALTIME time in seconds */

	u64			xtime_sec;

	/* Clock shifted nano seconds */

	u64			xtime_nsec;

	/* Difference between accumulated time and NTP time in ntp

	 * shifted nano seconds. */

	/*

	 * Difference between accumulated time and NTP time in ntp

	 * shifted nano seconds.

	 */

	s64			ntp_error;

	/* Shift conversion between clock shifted nano seconds and

	 * ntp shifted nano seconds. */

	u32			ntp_error_shift;

	/*

	 * wall_to_monotonic is what we need to add to xtime (or xtime corrected

	 * for sub jiffie times) to get to monotonic time.  Monotonic is pegged

	 * at zero at system boot time, so wall_to_monotonic will be negative,

	 * however, we will ALWAYS keep the tv_nsec part positive so we can use

	 * the usual normalization.

	 *

	 * wall_to_monotonic is moved after resume from suspend for the

	 * monotonic time not to jump. We need to add total_sleep_time to

	 * wall_to_monotonic to get the real boot based time offset.

	 *

	 * - wall_to_monotonic is no longer the boot time, getboottime must be

	 * used instead.

	 * Shift conversion between clock shifted nano seconds and

	 * ntp shifted nano seconds.

	 */

	struct timespec64	wall_to_monotonic;

	/* Offset clock monotonic -> clock realtime */

	ktime_t			offs_real;

	/* time spent in suspend */

	struct timespec64	total_sleep_time;

	/* Offset clock monotonic -> clock boottime */

	ktime_t			offs_boot;

	/* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */

	struct timespec64	raw_time;

	/* The current UTC to TAI offset in seconds */

	s32			tai_offset;

	/* Offset clock monotonic -> clock tai */

	ktime_t			offs_tai;

	u32			ntp_error_shift;

};

#ifdef CONFIG_GENERIC_TIME_VSYSCALL

#define TK_MIRROR		(1 << 1)

#define TK_CLOCK_WAS_SET	(1 << 2)

static struct timekeeper timekeeper;

/*

 * The most important data for readout fits into a single 64 byte

 * cache line.

 */

static struct {

	seqcount_t		seq;

	struct timekeeper	timekeeper;

} tk_core ____cacheline_aligned;

static DEFINE_RAW_SPINLOCK(timekeeper_lock);

static seqcount_t timekeeper_seq;

static struct timekeeper shadow_timekeeper;

/* flag for if timekeeping is suspended */

 */

int pvclock_gtod_register_notifier(struct notifier_block *nb)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long flags;

	int ret;

	update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);

	if (action & TK_MIRROR)

		memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));

		memcpy(&shadow_timekeeper, &tk_core.timekeeper,

		       sizeof(tk_core.timekeeper));

}

/**

 */

int __getnstimeofday64(struct timespec64 *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long seq;

	s64 nsecs = 0;

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ts->tv_sec = tk->xtime_sec;

		nsecs = timekeeping_get_ns(tk);

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	ts->tv_nsec = 0;

	timespec64_add_ns(ts, nsecs);

ktime_t ktime_get(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned int seq;

	s64 secs, nsecs;

	WARN_ON(timekeeping_suspended);

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;

		nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ktime_set(secs, nsecs);

}

 */

void ktime_get_ts64(struct timespec64 *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 tomono;

	s64 nsec;

	unsigned int seq;

	WARN_ON(timekeeping_suspended);

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ts->tv_sec = tk->xtime_sec;

		nsec = timekeeping_get_ns(tk);

		tomono = tk->wall_to_monotonic;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	ts->tv_sec += tomono.tv_sec;

	ts->tv_nsec = 0;

 */

void timekeeping_clocktai(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 ts64;

	unsigned long seq;

	u64 nsecs;

	WARN_ON(timekeeping_suspended);

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ts64.tv_sec = tk->xtime_sec + tk->tai_offset;

		nsecs = timekeeping_get_ns(tk);

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	ts64.tv_nsec = 0;

	timespec64_add_ns(&ts64, nsecs);

 */

void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long seq;

	s64 nsecs_raw, nsecs_real;

	WARN_ON_ONCE(timekeeping_suspended);

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		*ts_raw = timespec64_to_timespec(tk->raw_time);

		ts_real->tv_sec = tk->xtime_sec;

		nsecs_raw = timekeeping_get_ns_raw(tk);

		nsecs_real = timekeeping_get_ns(tk);

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	timespec_add_ns(ts_raw, nsecs_raw);

	timespec_add_ns(ts_real, nsecs_real);

 */

int do_settimeofday(const struct timespec *tv)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 ts_delta, xt, tmp;

	unsigned long flags;

		return -EINVAL;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	timekeeping_forward_now(tk);

	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	/* signal hrtimers about time change */

 */

int timekeeping_inject_offset(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long flags;

	struct timespec64 ts64, tmp;

	int ret = 0;

	ts64 = timespec_to_timespec64(*ts);

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	timekeeping_forward_now(tk);

error: /* even if we error out, we forwarded the time, so call update */

	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	/* signal hrtimers about time change */

 */

s32 timekeeping_get_tai_offset(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned int seq;

	s32 ret;

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ret = tk->tai_offset;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ret;

}

 */

void timekeeping_set_tai_offset(s32 tai_offset)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long flags;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	__timekeeping_set_tai_offset(tk, tai_offset);

	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	clock_was_set();

}

 */

static int change_clocksource(void *data)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct clocksource *new, *old;

	unsigned long flags;

	new = (struct clocksource *) data;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	timekeeping_forward_now(tk);

	/*

	}

	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	return 0;

 */

int timekeeping_notify(struct clocksource *clock)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	if (tk->clock == clock)

		return 0;

 */

void getrawmonotonic(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 ts64;

	unsigned long seq;

	s64 nsecs;

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		nsecs = timekeeping_get_ns_raw(tk);

		ts64 = tk->raw_time;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	timespec64_add_ns(&ts64, nsecs);

	*ts = timespec64_to_timespec(ts64);

 */

int timekeeping_valid_for_hres(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long seq;

	int ret;

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ret;

}

 */

u64 timekeeping_max_deferment(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long seq;

	u64 ret;

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ret = tk->clock->max_idle_ns;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ret;

}

 */

void __init timekeeping_init(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct clocksource *clock;

	unsigned long flags;

	struct timespec64 now, boot, tmp;

	}

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	ntp_init();

	clock = clocksource_default_clock();

	tmp.tv_nsec = 0;

	tk_set_sleep_time(tk, tmp);

	memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper));

	memcpy(&shadow_timekeeper, &tk_core.timekeeper,

	       sizeof(tk_core.timekeeper));

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

}

 */

void timekeeping_inject_sleeptime(struct timespec *delta)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 tmp;

	unsigned long flags;

		return;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	timekeeping_forward_now(tk);

	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	/* signal hrtimers about time change */

 */

static void timekeeping_resume(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct clocksource *clock = tk->clock;

	unsigned long flags;

	struct timespec64 ts_new, ts_delta;

	clocksource_resume();

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	/*

	 * After system resumes, we need to calculate the suspended time and

	tk->ntp_error = 0;

	timekeeping_suspended = 0;

	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	touch_softlockup_watchdog();

static int timekeeping_suspend(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	unsigned long flags;

	struct timespec64		delta, delta_delta;

	static struct timespec64	old_delta;

		persistent_clock_exist = true;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	timekeeping_forward_now(tk);

	timekeeping_suspended = 1;

	}

	timekeeping_update(tk, TK_MIRROR);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);

void update_wall_time(void)

{

	struct clocksource *clock;

	struct timekeeper *real_tk = &timekeeper;

	struct timekeeper *real_tk = &tk_core.timekeeper;

	struct timekeeper *tk = &shadow_timekeeper;

	cycle_t offset;

	int shift = 0, maxshift;

	 */

	clock_set |= accumulate_nsecs_to_secs(tk);

	write_seqcount_begin(&timekeeper_seq);

	write_seqcount_begin(&tk_core.seq);

	/* Update clock->cycle_last with the new value */

	clock->cycle_last = tk->cycle_last;

	/*

	 * requires changes to all other timekeeper usage sites as

	 * well, i.e. move the timekeeper pointer getter into the

	 * spinlocked/seqcount protected sections. And we trade this

	 * memcpy under the timekeeper_seq against one before we start

	 * memcpy under the tk_core.seq against one before we start

	 * updating.

	 */

	memcpy(real_tk, tk, sizeof(*tk));

	timekeeping_update(real_tk, clock_set);

	write_seqcount_end(&timekeeper_seq);

	write_seqcount_end(&tk_core.seq);

out:

	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

	if (clock_set)

 */

void getboottime(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec boottime = {

		.tv_sec = tk->wall_to_monotonic.tv_sec +

				tk->total_sleep_time.tv_sec,

 */

void get_monotonic_boottime(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 tomono, sleep, ret;

	s64 nsec;

	unsigned int seq;

	WARN_ON(timekeeping_suspended);

	do {

		seq = read_seqcount_begin(&timekeeper_seq);

		seq = read_seqcount_begin(&tk_core.seq);

		ret.tv_sec = tk->xtime_sec;

		nsec = timekeeping_get_ns(tk);

		tomono = tk->wall_to_monotonic;

		sleep = tk->total_sleep_time;

	} while (read_seqcount_retry(&timekeeper_seq, seq));

	} while (read_seqcount_retry(&tk_core.seq, seq));

	ret.tv_sec += tomono.tv_sec + sleep.tv_sec;

	ret.tv_nsec = 0;

 */

void monotonic_to_bootbased(struct timespec *ts)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	struct timespec64 ts64;

	ts64 = timespec_to_timespec64(*ts);

unsigned long get_seconds(void)

{

	struct timekeeper *tk = &timekeeper;

	struct timekeeper *tk = &tk_core.timekeeper;

	return tk->xtime_sec;

}

struct timespec __current_kernel_time(void)