Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 539eb11e authored by John Stultz's avatar John Stultz Committed by Linus Torvalds
Browse files

[PATCH] Time: i386 Conversion - part 2: Rework TSC Support 



As part of the i386 conversion to the generic timekeeping infrastructure, this
introduces a new tsc.c file.  The code in this file replaces the TSC
initialization, management and access code currently in timer_tsc.c (which
will be removed) that we want to preserve.

The code also introduces the following functionality:

o tsc_khz: like cpu_khz but stores the TSC frequency on systems that do not
  change TSC frequency w/ CPU frequency

o check/mark_tsc_unstable: accessor/modifier flag for TSC timekeeping
  usability

o minor cleanups to calibration math.

This patch also includes a one line __cpuinitdata fix from Zwane Mwaikambo.

Signed-off-by: default avatarJohn Stultz <johnstul@us.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 8d016ef1

arch/i386/kernel/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ extra-y := head.o init_task.o vmlinux.lds 
obj-y	:= process.o semaphore.o signal.o entry.o traps.o irq.o \

		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_i386.o \

		pci-dma.o i386_ksyms.o i387.o bootflag.o \

		quirks.o i8237.o topology.o alternative.o i8253.o

		quirks.o i8237.o topology.o alternative.o i8253.o tsc.o



obj-y				+= cpu/

obj-y				+= timers/

arch/i386/kernel/numaq.c

+6 −4
Original line number Diff line number Diff line
@@ -79,10 +79,12 @@ int __init get_memcfg_numaq(void) 
	return 1;

}



static int __init numaq_dsc_disable(void)

static int __init numaq_tsc_disable(void)

{

	if (num_online_nodes() > 1) {

		printk(KERN_DEBUG "NUMAQ: disabling TSC\n");

		tsc_disable = 1;

	}

	return 0;

}

core_initcall(numaq_dsc_disable);
 
arch_initcall(numaq_tsc_disable);

arch/i386/kernel/setup.c

+1 −0
Original line number Diff line number Diff line
@@ -1575,6 +1575,7 @@ void __init setup_arch(char **cmdline_p) 
	conswitchp = &dummy_con;

#endif

#endif

	tsc_init();

}



static __init int add_pcspkr(void)

arch/i386/kernel/timers/timer_tsc.c

+0 −178
Original line number Diff line number Diff line
@@ -32,10 +32,6 @@ static unsigned long hpet_last; 
static struct timer_opts timer_tsc;

#endif



static inline void cpufreq_delayed_get(void);



int tsc_disable __devinitdata = 0;



static int use_tsc;

/* Number of usecs that the last interrupt was delayed */

static int delay_at_last_interrupt;
@@ -144,30 +140,6 @@ static unsigned long long monotonic_clock_tsc(void) 
	return base + cycles_2_ns(this_offset - last_offset);

}



/*

 * Scheduler clock - returns current time in nanosec units.

 */

unsigned long long sched_clock(void)

{

	unsigned long long this_offset;



	/*

	 * In the NUMA case we dont use the TSC as they are not

	 * synchronized across all CPUs.

	 */

#ifndef CONFIG_NUMA

	if (!use_tsc)

#endif

		/* no locking but a rare wrong value is not a big deal */

		return jiffies_64 * (1000000000 / HZ);



	/* Read the Time Stamp Counter */

	rdtscll(this_offset);



	/* return the value in ns */

	return cycles_2_ns(this_offset);

}



static void delay_tsc(unsigned long loops)

{

	unsigned long bclock, now;
@@ -231,136 +203,6 @@ static void mark_offset_tsc_hpet(void) 
}

#endif





#ifdef CONFIG_CPU_FREQ

#include <linux/workqueue.h>



static unsigned int cpufreq_delayed_issched = 0;

static unsigned int cpufreq_init = 0;

static struct work_struct cpufreq_delayed_get_work;



static void handle_cpufreq_delayed_get(void *v)

{

	unsigned int cpu;

	for_each_online_cpu(cpu) {

		cpufreq_get(cpu);

	}

	cpufreq_delayed_issched = 0;

}



/* if we notice lost ticks, schedule a call to cpufreq_get() as it tries

 * to verify the CPU frequency the timing core thinks the CPU is running

 * at is still correct.

 */

static inline void cpufreq_delayed_get(void) 

{

	if (cpufreq_init && !cpufreq_delayed_issched) {

		cpufreq_delayed_issched = 1;

		printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");

		schedule_work(&cpufreq_delayed_get_work);

	}

}



/* If the CPU frequency is scaled, TSC-based delays will need a different

 * loops_per_jiffy value to function properly.

 */



static unsigned int  ref_freq = 0;

static unsigned long loops_per_jiffy_ref = 0;



#ifndef CONFIG_SMP

static unsigned long fast_gettimeoffset_ref = 0;

static unsigned int cpu_khz_ref = 0;

#endif



static int

time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,

		       void *data)

{

	struct cpufreq_freqs *freq = data;



	if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)

		write_seqlock_irq(&xtime_lock);

	if (!ref_freq) {

		if (!freq->old){

			ref_freq = freq->new;

			goto end;

		}

		ref_freq = freq->old;

		loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;

#ifndef CONFIG_SMP

		fast_gettimeoffset_ref = fast_gettimeoffset_quotient;

		cpu_khz_ref = cpu_khz;

#endif

	}



	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||

	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||

	    (val == CPUFREQ_RESUMECHANGE)) {

		if (!(freq->flags & CPUFREQ_CONST_LOOPS))

			cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);

#ifndef CONFIG_SMP

		if (cpu_khz)

			cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);

		if (use_tsc) {

			if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {

				fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);

				set_cyc2ns_scale(cpu_khz);

			}

		}

#endif

	}



end:

	if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)

		write_sequnlock_irq(&xtime_lock);



	return 0;

}



static struct notifier_block time_cpufreq_notifier_block = {

	.notifier_call	= time_cpufreq_notifier

};





static int __init cpufreq_tsc(void)

{

	int ret;

	INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);

	ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,

					CPUFREQ_TRANSITION_NOTIFIER);

	if (!ret)

		cpufreq_init = 1;

	return ret;

}

core_initcall(cpufreq_tsc);



#else /* CONFIG_CPU_FREQ */

static inline void cpufreq_delayed_get(void) { return; }

#endif 



int recalibrate_cpu_khz(void)

{

#ifndef CONFIG_SMP

	unsigned int cpu_khz_old = cpu_khz;



	if (cpu_has_tsc) {

		local_irq_disable();

		init_cpu_khz();

		local_irq_enable();

		cpu_data[0].loops_per_jiffy =

		    cpufreq_scale(cpu_data[0].loops_per_jiffy,

			          cpu_khz_old,

				  cpu_khz);

		return 0;

	} else

		return -ENODEV;

#else

	return -ENODEV;

#endif

}

EXPORT_SYMBOL(recalibrate_cpu_khz);



static void mark_offset_tsc(void)

{

	unsigned long lost,delay;
@@ -451,9 +293,6 @@ static void mark_offset_tsc(void) 


			clock_fallback();

		}

		/* ... but give the TSC a fair chance */

		if (lost_count > 25)

			cpufreq_delayed_get();

	} else

		lost_count = 0;

	/* update the monotonic base value */
@@ -578,23 +417,6 @@ static int tsc_resume(void) 
	return 0;

}



#ifndef CONFIG_X86_TSC

/* disable flag for tsc.  Takes effect by clearing the TSC cpu flag

 * in cpu/common.c */

static int __init tsc_setup(char *str)

{

	tsc_disable = 1;

	return 1;

}

#else

static int __init tsc_setup(char *str)

{

	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "

				"cannot disable TSC.\n");

	return 1;

}

#endif

__setup("notsc", tsc_setup);

arch/i386/kernel/tsc.c

0 → 100644
+316 −0
Original line number Diff line number Diff line 
/*

 * This code largely moved from arch/i386/kernel/timer/timer_tsc.c

 * which was originally moved from arch/i386/kernel/time.c.

 * See comments there for proper credits.

 */



#include <linux/workqueue.h>

#include <linux/cpufreq.h>

#include <linux/jiffies.h>

#include <linux/init.h>



#include <asm/tsc.h>

#include <asm/io.h>



#include "mach_timer.h"



/*

 * On some systems the TSC frequency does not

 * change with the cpu frequency. So we need

 * an extra value to store the TSC freq

 */

unsigned int tsc_khz;



int tsc_disable __cpuinitdata = 0;



#ifdef CONFIG_X86_TSC

static int __init tsc_setup(char *str)

{

	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "

				"cannot disable TSC.\n");

	return 1;

}

#else

/*

 * disable flag for tsc. Takes effect by clearing the TSC cpu flag

 * in cpu/common.c

 */

static int __init tsc_setup(char *str)

{

	tsc_disable = 1;



	return 1;

}

#endif



__setup("notsc", tsc_setup);





/*

 * code to mark and check if the TSC is unstable

 * due to cpufreq or due to unsynced TSCs

 */

static int tsc_unstable;



static inline int check_tsc_unstable(void)

{

	return tsc_unstable;

}



void mark_tsc_unstable(void)

{

	tsc_unstable = 1;

}

EXPORT_SYMBOL_GPL(mark_tsc_unstable);



/* Accellerators for sched_clock()

 * convert from cycles(64bits) => nanoseconds (64bits)

 *  basic equation:

 *		ns = cycles / (freq / ns_per_sec)

 *		ns = cycles * (ns_per_sec / freq)

 *		ns = cycles * (10^9 / (cpu_khz * 10^3))

 *		ns = cycles * (10^6 / cpu_khz)

 *

 *	Then we use scaling math (suggested by george@mvista.com) to get:

 *		ns = cycles * (10^6 * SC / cpu_khz) / SC

 *		ns = cycles * cyc2ns_scale / SC

 *

 *	And since SC is a constant power of two, we can convert the div

 *  into a shift.

 *

 *  We can use khz divisor instead of mhz to keep a better percision, since

 *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.

 *  (mathieu.desnoyers@polymtl.ca)

 *

 *			-johnstul@us.ibm.com "math is hard, lets go shopping!"

 */

static unsigned long cyc2ns_scale __read_mostly;



#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */



static inline void set_cyc2ns_scale(unsigned long cpu_khz)

{

	cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;

}



static inline unsigned long long cycles_2_ns(unsigned long long cyc)

{

	return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;

}



/*

 * Scheduler clock - returns current time in nanosec units.

 */

unsigned long long sched_clock(void)

{

	unsigned long long this_offset;



	/*

	 * in the NUMA case we dont use the TSC as they are not

	 * synchronized across all CPUs.

	 */

#ifndef CONFIG_NUMA

	if (!cpu_khz || check_tsc_unstable())

#endif

		/* no locking but a rare wrong value is not a big deal */

		return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);



	/* read the Time Stamp Counter: */

	rdtscll(this_offset);



	/* return the value in ns */

	return cycles_2_ns(this_offset);

}



static unsigned long calculate_cpu_khz(void)

{

	unsigned long long start, end;

	unsigned long count;

	u64 delta64;

	int i;

	unsigned long flags;



	local_irq_save(flags);



	/* run 3 times to ensure the cache is warm */

	for (i = 0; i < 3; i++) {

		mach_prepare_counter();

		rdtscll(start);

		mach_countup(&count);

		rdtscll(end);

	}

	/*

	 * Error: ECTCNEVERSET

	 * The CTC wasn't reliable: we got a hit on the very first read,

	 * or the CPU was so fast/slow that the quotient wouldn't fit in

	 * 32 bits..

	 */

	if (count <= 1)

		goto err;



	delta64 = end - start;



	/* cpu freq too fast: */

	if (delta64 > (1ULL<<32))

		goto err;



	/* cpu freq too slow: */

	if (delta64 <= CALIBRATE_TIME_MSEC)

		goto err;



	delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */

	do_div(delta64,CALIBRATE_TIME_MSEC);



	local_irq_restore(flags);

	return (unsigned long)delta64;

err:

	local_irq_restore(flags);

	return 0;

}



int recalibrate_cpu_khz(void)

{

#ifndef CONFIG_SMP

	unsigned long cpu_khz_old = cpu_khz;



	if (cpu_has_tsc) {

		cpu_khz = calculate_cpu_khz();

		tsc_khz = cpu_khz;

		cpu_data[0].loops_per_jiffy =

			cpufreq_scale(cpu_data[0].loops_per_jiffy,

					cpu_khz_old, cpu_khz);

		return 0;

	} else

		return -ENODEV;

#else

	return -ENODEV;

#endif

}



EXPORT_SYMBOL(recalibrate_cpu_khz);



void tsc_init(void)

{

	if (!cpu_has_tsc || tsc_disable)

		return;



	cpu_khz = calculate_cpu_khz();

	tsc_khz = cpu_khz;



	if (!cpu_khz)

		return;



	printk("Detected %lu.%03lu MHz processor.\n",

				(unsigned long)cpu_khz / 1000,

				(unsigned long)cpu_khz % 1000);



	set_cyc2ns_scale(cpu_khz);

}



#ifdef CONFIG_CPU_FREQ



static unsigned int cpufreq_delayed_issched = 0;

static unsigned int cpufreq_init = 0;

static struct work_struct cpufreq_delayed_get_work;



static void handle_cpufreq_delayed_get(void *v)

{

	unsigned int cpu;



	for_each_online_cpu(cpu)

		cpufreq_get(cpu);



	cpufreq_delayed_issched = 0;

}



/*

 * if we notice cpufreq oddness, schedule a call to cpufreq_get() as it tries

 * to verify the CPU frequency the timing core thinks the CPU is running

 * at is still correct.

 */

static inline void cpufreq_delayed_get(void)

{

	if (cpufreq_init && !cpufreq_delayed_issched) {

		cpufreq_delayed_issched = 1;

		printk(KERN_DEBUG "Checking if CPU frequency changed.\n");

		schedule_work(&cpufreq_delayed_get_work);

	}

}



/*

 * if the CPU frequency is scaled, TSC-based delays will need a different

 * loops_per_jiffy value to function properly.

 */

static unsigned int ref_freq = 0;

static unsigned long loops_per_jiffy_ref = 0;

static unsigned long cpu_khz_ref = 0;



static int

time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)

{

	struct cpufreq_freqs *freq = data;



	if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)

		write_seqlock_irq(&xtime_lock);



	if (!ref_freq) {

		if (!freq->old){

			ref_freq = freq->new;

			goto end;

		}

		ref_freq = freq->old;

		loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;

		cpu_khz_ref = cpu_khz;

	}



	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||

	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||

	    (val == CPUFREQ_RESUMECHANGE)) {

		if (!(freq->flags & CPUFREQ_CONST_LOOPS))

			cpu_data[freq->cpu].loops_per_jiffy =

				cpufreq_scale(loops_per_jiffy_ref,

						ref_freq, freq->new);



		if (cpu_khz) {



			if (num_online_cpus() == 1)

				cpu_khz = cpufreq_scale(cpu_khz_ref,

						ref_freq, freq->new);

			if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {

				tsc_khz = cpu_khz;

				set_cyc2ns_scale(cpu_khz);

				/*

				 * TSC based sched_clock turns

				 * to junk w/ cpufreq

				 */

				mark_tsc_unstable();

			}

		}

	}

end:

	if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)

		write_sequnlock_irq(&xtime_lock);



	return 0;

}



static struct notifier_block time_cpufreq_notifier_block = {

	.notifier_call	= time_cpufreq_notifier

};



static int __init cpufreq_tsc(void)

{

	int ret;



	INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);

	ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,

					CPUFREQ_TRANSITION_NOTIFIER);

	if (!ret)

		cpufreq_init = 1;



	return ret;

}



core_initcall(cpufreq_tsc);



#endif