Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 91ec87d5 authored by Andy Lutomirski's avatar Andy Lutomirski Committed by John Stultz
Browse files

x86-64: Simplify and optimize vdso clock_gettime monotonic variants 



We used to store the wall-to-monotonic offset and the realtime base.
It's faster to precompute the monotonic base.

This is about a 3% speedup on Sandy Bridge for CLOCK_MONOTONIC.
It's much more impressive for CLOCK_MONOTONIC_COARSE.

Signed-off-by: default avatarAndy Lutomirski <luto@amacapital.net>
Signed-off-by: default avatarJohn Stultz <john.stultz@linaro.org>
parent 88b28adf

arch/x86/include/asm/vgtod.h

+9 −6
Original line number Diff line number Diff line
@@ -7,11 +7,6 @@ 
struct vsyscall_gtod_data {

	seqcount_t	seq;



	/* open coded 'struct timespec' */

	time_t		wall_time_sec;

	u32		wall_time_nsec;



	struct timezone sys_tz;

	struct { /* extract of a clocksource struct */

		int vclock_mode;

		cycle_t	cycle_last;
@@ -19,8 +14,16 @@ struct vsyscall_gtod_data { 
		u32	mult;

		u32	shift;

	} clock;

	struct timespec wall_to_monotonic;



	/* open coded 'struct timespec' */

	time_t		wall_time_sec;

	u32		wall_time_nsec;

	u32		monotonic_time_nsec;

	time_t		monotonic_time_sec;



	struct timezone sys_tz;

	struct timespec wall_time_coarse;

	struct timespec monotonic_time_coarse;

};

extern struct vsyscall_gtod_data vsyscall_gtod_data;

arch/x86/kernel/vsyscall_64.c

+9 −1
Original line number Diff line number Diff line
@@ -84,6 +84,7 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 
			struct clocksource *clock, u32 mult)

{

	write_seqcount_begin(&vsyscall_gtod_data.seq);

	struct timespec monotonic;



	/* copy vsyscall data */

	vsyscall_gtod_data.clock.vclock_mode	= clock->archdata.vclock_mode;
@@ -91,10 +92,17 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 
	vsyscall_gtod_data.clock.mask		= clock->mask;

	vsyscall_gtod_data.clock.mult		= mult;

	vsyscall_gtod_data.clock.shift		= clock->shift;



	vsyscall_gtod_data.wall_time_sec	= wall_time->tv_sec;

	vsyscall_gtod_data.wall_time_nsec	= wall_time->tv_nsec;

	vsyscall_gtod_data.wall_to_monotonic	= *wtm;



	monotonic = timespec_add(*wall_time, *wtm);

	vsyscall_gtod_data.monotonic_time_sec	= monotonic.tv_sec;

	vsyscall_gtod_data.monotonic_time_nsec	= monotonic.tv_nsec;



	vsyscall_gtod_data.wall_time_coarse	= __current_kernel_time();

	vsyscall_gtod_data.monotonic_time_coarse =

		timespec_add(vsyscall_gtod_data.wall_time_coarse, *wtm);



	write_seqcount_end(&vsyscall_gtod_data.seq);

}

arch/x86/vdso/vclock_gettime.c

+8 −30
Original line number Diff line number Diff line
@@ -113,27 +113,17 @@ notrace static noinline int do_realtime(struct timespec *ts) 


notrace static noinline int do_monotonic(struct timespec *ts)

{

	unsigned long seq, ns, secs;

	unsigned long seq, ns;

	int mode;



	do {

		seq = read_seqcount_begin(&gtod->seq);

		mode = gtod->clock.vclock_mode;

		secs = gtod->wall_time_sec;

		ns = gtod->wall_time_nsec + vgetns();

		secs += gtod->wall_to_monotonic.tv_sec;

		ns += gtod->wall_to_monotonic.tv_nsec;

		ts->tv_sec = gtod->monotonic_time_sec;

		ts->tv_nsec = gtod->monotonic_time_nsec;

		ns = vgetns();

	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));



	/* wall_time_nsec, vgetns(), and wall_to_monotonic.tv_nsec

	 * are all guaranteed to be nonnegative.

	 */

	while (ns >= NSEC_PER_SEC) {

		ns -= NSEC_PER_SEC;

		++secs;

	}

	ts->tv_sec = secs;

	ts->tv_nsec = ns;

	timespec_add_ns(ts, ns);



	return mode;

}
@@ -151,25 +141,13 @@ notrace static noinline int do_realtime_coarse(struct timespec *ts) 


notrace static noinline int do_monotonic_coarse(struct timespec *ts)

{

	unsigned long seq, ns, secs;

	unsigned long seq;

	do {

		seq = read_seqcount_begin(&gtod->seq);

		secs = gtod->wall_time_coarse.tv_sec;

		ns = gtod->wall_time_coarse.tv_nsec;

		secs += gtod->wall_to_monotonic.tv_sec;

		ns += gtod->wall_to_monotonic.tv_nsec;

		ts->tv_sec = gtod->monotonic_time_coarse.tv_sec;

		ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec;

	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));



	/* wall_time_nsec and wall_to_monotonic.tv_nsec are

	 * guaranteed to be between 0 and NSEC_PER_SEC.

	 */

	if (ns >= NSEC_PER_SEC) {

		ns -= NSEC_PER_SEC;

		++secs;

	}

	ts->tv_sec = secs;

	ts->tv_nsec = ns;



	return 0;

}