Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9e74e7c8 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu 

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu:
  module: add stub for is_module_percpu_address
  percpu, module: implement and use is_kernel/module_percpu_address()
  module: encapsulate percpu handling better and record percpu_size
parents 4946d54c d5e50daf

include/linux/module.h

+10 −1
Original line number Diff line number Diff line
@@ -330,8 +330,11 @@ struct module 
	struct module_notes_attrs *notes_attrs;

#endif



#ifdef CONFIG_SMP

	/* Per-cpu data. */

	void *percpu;

	void __percpu *percpu;

	unsigned int percpu_size;

#endif



	/* The command line arguments (may be mangled).  People like

	   keeping pointers to this stuff */
@@ -392,6 +395,7 @@ static inline int module_is_live(struct module *mod) 
struct module *__module_text_address(unsigned long addr);

struct module *__module_address(unsigned long addr);

bool is_module_address(unsigned long addr);

bool is_module_percpu_address(unsigned long addr);

bool is_module_text_address(unsigned long addr);



static inline int within_module_core(unsigned long addr, struct module *mod)
@@ -563,6 +567,11 @@ static inline bool is_module_address(unsigned long addr) 
	return false;

}



static inline bool is_module_percpu_address(unsigned long addr)

{

	return false;

}



static inline bool is_module_text_address(unsigned long addr)

{

	return false;

include/linux/percpu.h

+7 −0
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, 
extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);

extern void __percpu *__alloc_percpu(size_t size, size_t align);

extern void free_percpu(void __percpu *__pdata);

extern bool is_kernel_percpu_address(unsigned long addr);

extern phys_addr_t per_cpu_ptr_to_phys(void *addr);



#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
@@ -163,6 +164,12 @@ static inline void free_percpu(void __percpu *p) 
	kfree(p);

}



/* can't distinguish from other static vars, always false */

static inline bool is_kernel_percpu_address(unsigned long addr)

{

	return false;

}



static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)

{

	return __pa(addr);

kernel/lockdep.c

+5 −16
Original line number Diff line number Diff line
@@ -582,9 +582,6 @@ static int static_obj(void *obj) 
	unsigned long start = (unsigned long) &_stext,

		      end   = (unsigned long) &_end,

		      addr  = (unsigned long) obj;

#ifdef CONFIG_SMP

	int i;

#endif



	/*

	 * static variable?
@@ -595,24 +592,16 @@ static int static_obj(void *obj) 
	if (arch_is_kernel_data(addr))

		return 1;



#ifdef CONFIG_SMP

	/*

	 * percpu var?

	 * in-kernel percpu var?

	 */

	for_each_possible_cpu(i) {

		start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);

		end   = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM

					+ per_cpu_offset(i);



		if ((addr >= start) && (addr < end))

	if (is_kernel_percpu_address(addr))

		return 1;

	}

#endif



	/*

	 * module var?

	 * module static or percpu var?

	 */

	return is_module_address(addr);

	return is_module_address(addr) || is_module_percpu_address(addr);

}



/*

kernel/module.c

+73 −31
Original line number Diff line number Diff line
@@ -370,27 +370,33 @@ EXPORT_SYMBOL_GPL(find_module); 


#ifdef CONFIG_SMP



static void *percpu_modalloc(unsigned long size, unsigned long align,

			     const char *name)

static inline void __percpu *mod_percpu(struct module *mod)

{

	void *ptr;

	return mod->percpu;

}



static int percpu_modalloc(struct module *mod,

			   unsigned long size, unsigned long align)

{

	if (align > PAGE_SIZE) {

		printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",

		       name, align, PAGE_SIZE);

		       mod->name, align, PAGE_SIZE);

		align = PAGE_SIZE;

	}



	ptr = __alloc_reserved_percpu(size, align);

	if (!ptr)

	mod->percpu = __alloc_reserved_percpu(size, align);

	if (!mod->percpu) {

		printk(KERN_WARNING

		       "Could not allocate %lu bytes percpu data\n", size);

	return ptr;

		return -ENOMEM;

	}

	mod->percpu_size = size;

	return 0;

}



static void percpu_modfree(void *freeme)

static void percpu_modfree(struct module *mod)

{

	free_percpu(freeme);

	free_percpu(mod->percpu);

}



static unsigned int find_pcpusec(Elf_Ehdr *hdr,
@@ -400,24 +406,62 @@ static unsigned int find_pcpusec(Elf_Ehdr *hdr, 
	return find_sec(hdr, sechdrs, secstrings, ".data.percpu");

}



static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size)

static void percpu_modcopy(struct module *mod,

			   const void *from, unsigned long size)

{

	int cpu;



	for_each_possible_cpu(cpu)

		memcpy(pcpudest + per_cpu_offset(cpu), from, size);

		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);

}



/**

 * is_module_percpu_address - test whether address is from module static percpu

 * @addr: address to test

 *

 * Test whether @addr belongs to module static percpu area.

 *

 * RETURNS:

 * %true if @addr is from module static percpu area

 */

bool is_module_percpu_address(unsigned long addr)

{

	struct module *mod;

	unsigned int cpu;



	preempt_disable();



	list_for_each_entry_rcu(mod, &modules, list) {

		if (!mod->percpu_size)

			continue;

		for_each_possible_cpu(cpu) {

			void *start = per_cpu_ptr(mod->percpu, cpu);



			if ((void *)addr >= start &&

			    (void *)addr < start + mod->percpu_size) {

				preempt_enable();

				return true;

			}

		}

	}



	preempt_enable();

	return false;

}



#else /* ... !CONFIG_SMP */



static inline void *percpu_modalloc(unsigned long size, unsigned long align,

				    const char *name)

static inline void __percpu *mod_percpu(struct module *mod)

{

	return NULL;

}

static inline void percpu_modfree(void *pcpuptr)

static inline int percpu_modalloc(struct module *mod,

				  unsigned long size, unsigned long align)

{

	return -ENOMEM;

}

static inline void percpu_modfree(struct module *mod)

{

	BUG();

}

static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,

					Elf_Shdr *sechdrs,
@@ -425,12 +469,16 @@ static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, 
{

	return 0;

}

static inline void percpu_modcopy(void *pcpudst, const void *src,

				  unsigned long size)

static inline void percpu_modcopy(struct module *mod,

				  const void *from, unsigned long size)

{

	/* pcpusec should be 0, and size of that section should be 0. */

	BUG_ON(size != 0);

}

bool is_module_percpu_address(unsigned long addr)

{

	return false;

}



#endif /* CONFIG_SMP */
@@ -1400,8 +1448,7 @@ static void free_module(struct module *mod) 
	/* This may be NULL, but that's OK */

	module_free(mod, mod->module_init);

	kfree(mod->args);

	if (mod->percpu)

		percpu_modfree(mod->percpu);

	percpu_modfree(mod);

#if defined(CONFIG_MODULE_UNLOAD)

	if (mod->refptr)

		free_percpu(mod->refptr);
@@ -1520,7 +1567,7 @@ static int simplify_symbols(Elf_Shdr *sechdrs, 
		default:

			/* Divert to percpu allocation if a percpu var. */

			if (sym[i].st_shndx == pcpuindex)

				secbase = (unsigned long)mod->percpu;

				secbase = (unsigned long)mod_percpu(mod);

			else

				secbase = sechdrs[sym[i].st_shndx].sh_addr;

			sym[i].st_value += secbase;
@@ -1954,7 +2001,7 @@ static noinline struct module *load_module(void __user *umod, 
	unsigned int modindex, versindex, infoindex, pcpuindex;

	struct module *mod;

	long err = 0;

	void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */

	void *ptr = NULL; /* Stops spurious gcc warning */

	unsigned long symoffs, stroffs, *strmap;



	mm_segment_t old_fs;
@@ -2094,15 +2141,11 @@ static noinline struct module *load_module(void __user *umod, 


	if (pcpuindex) {

		/* We have a special allocation for this section. */

		percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,

					 sechdrs[pcpuindex].sh_addralign,

					 mod->name);

		if (!percpu) {

			err = -ENOMEM;

		err = percpu_modalloc(mod, sechdrs[pcpuindex].sh_size,

				      sechdrs[pcpuindex].sh_addralign);

		if (err)

			goto free_mod;

		}

		sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;

		mod->percpu = percpu;

	}



	/* Determine total sizes, and put offsets in sh_entsize.  For now
@@ -2317,7 +2360,7 @@ static noinline struct module *load_module(void __user *umod, 
	sort_extable(mod->extable, mod->extable + mod->num_exentries);



	/* Finally, copy percpu area over. */

	percpu_modcopy(mod->percpu, (void *)sechdrs[pcpuindex].sh_addr,

	percpu_modcopy(mod, (void *)sechdrs[pcpuindex].sh_addr,

		       sechdrs[pcpuindex].sh_size);



	add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex,
@@ -2409,8 +2452,7 @@ static noinline struct module *load_module(void __user *umod, 
	module_free(mod, mod->module_core);

	/* mod will be freed with core. Don't access it beyond this line! */

 free_percpu:

	if (percpu)

		percpu_modfree(percpu);

	percpu_modfree(mod);

 free_mod:

	kfree(args);

	kfree(strmap);

mm/percpu.c

+26 −0
Original line number Diff line number Diff line
@@ -1303,6 +1303,32 @@ void free_percpu(void __percpu *ptr) 
}

EXPORT_SYMBOL_GPL(free_percpu);



/**

 * is_kernel_percpu_address - test whether address is from static percpu area

 * @addr: address to test

 *

 * Test whether @addr belongs to in-kernel static percpu area.  Module

 * static percpu areas are not considered.  For those, use

 * is_module_percpu_address().

 *

 * RETURNS:

 * %true if @addr is from in-kernel static percpu area, %false otherwise.

 */

bool is_kernel_percpu_address(unsigned long addr)

{

	const size_t static_size = __per_cpu_end - __per_cpu_start;

	void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);

	unsigned int cpu;



	for_each_possible_cpu(cpu) {

		void *start = per_cpu_ptr(base, cpu);



		if ((void *)addr >= start && (void *)addr < start + static_size)

			return true;

        }

	return false;

}



/**

 * per_cpu_ptr_to_phys - convert translated percpu address to physical address

 * @addr: the address to be converted to physical address