Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1527bc8b authored by Peter Zijlstra's avatar Peter Zijlstra Committed by H. Peter Anvin
Browse files

bitops: Optimize hweight() by making use of compile-time evaluation 



Rename the extisting runtime hweight() implementations to
__arch_hweight(), rename the compile-time versions to __const_hweight()
and then have hweight() pick between them.

Suggested-by: default avatarH. Peter Anvin <hpa@zytor.com>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20100318111929.GB11152@aftab>
Acked-by: default avatarH. Peter Anvin <hpa@zytor.com>
LKML-Reference: <1265028224.24455.154.camel@laptop>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
parent 0fdf8675

arch/alpha/include/asm/bitops.h

+10 −8
Original line number Diff line number Diff line
@@ -405,29 +405,31 @@ static inline int fls(int x) 


#if defined(CONFIG_ALPHA_EV6) && defined(CONFIG_ALPHA_EV67)

/* Whee.  EV67 can calculate it directly.  */

static inline unsigned long hweight64(unsigned long w)

static inline unsigned long __arch_hweight64(unsigned long w)

{

	return __kernel_ctpop(w);

}



static inline unsigned int hweight32(unsigned int w)

static inline unsigned int __arch_weight32(unsigned int w)

{

	return hweight64(w);

	return __arch_hweight64(w);

}



static inline unsigned int hweight16(unsigned int w)

static inline unsigned int __arch_hweight16(unsigned int w)

{

	return hweight64(w & 0xffff);

	return __arch_hweight64(w & 0xffff);

}



static inline unsigned int hweight8(unsigned int w)

static inline unsigned int __arch_hweight8(unsigned int w)

{

	return hweight64(w & 0xff);

	return __arch_hweight64(w & 0xff);

}

#else

#include <asm-generic/bitops/hweight.h>

#include <asm-generic/bitops/arch_hweight.h>

#endif



#include <asm-generic/bitops/const_hweight.h>



#endif /* __KERNEL__ */



#include <asm-generic/bitops/find.h>

arch/ia64/include/asm/bitops.h

+6 −5
Original line number Diff line number Diff line
@@ -437,17 +437,18 @@ __fls (unsigned long x) 
 * hweightN: returns the hamming weight (i.e. the number

 * of bits set) of a N-bit word

 */

static __inline__ unsigned long

hweight64 (unsigned long x)

static __inline__ unsigned long __arch_hweight64(unsigned long x)

{

	unsigned long result;

	result = ia64_popcnt(x);

	return result;

}



#define hweight32(x)	(unsigned int) hweight64((x) & 0xfffffffful)

#define hweight16(x)	(unsigned int) hweight64((x) & 0xfffful)

#define hweight8(x)	(unsigned int) hweight64((x) & 0xfful)

#define __arch_hweight32(x) ((unsigned int) __arch_hweight64((x) & 0xfffffffful))

#define __arch_hweight16(x) ((unsigned int) __arch_hweight64((x) & 0xfffful))

#define __arch_hweight8(x)  ((unsigned int) __arch_hweight64((x) & 0xfful))



#include <asm-generic/bitops/const_hweight.h>



#endif /* __KERNEL__ */

arch/sparc/include/asm/bitops_64.h

+6 −5
Original line number Diff line number Diff line
@@ -44,7 +44,7 @@ extern void change_bit(unsigned long nr, volatile unsigned long *addr); 


#ifdef ULTRA_HAS_POPULATION_COUNT



static inline unsigned int hweight64(unsigned long w)

static inline unsigned int __arch_hweight64(unsigned long w)

{

	unsigned int res;
@@ -52,7 +52,7 @@ static inline unsigned int hweight64(unsigned long w) 
	return res;

}



static inline unsigned int hweight32(unsigned int w)

static inline unsigned int __arch_hweight32(unsigned int w)

{

	unsigned int res;
@@ -60,7 +60,7 @@ static inline unsigned int hweight32(unsigned int w) 
	return res;

}



static inline unsigned int hweight16(unsigned int w)

static inline unsigned int __arch_hweight16(unsigned int w)

{

	unsigned int res;
@@ -68,7 +68,7 @@ static inline unsigned int hweight16(unsigned int w) 
	return res;

}



static inline unsigned int hweight8(unsigned int w)

static inline unsigned int __arch_hweight8(unsigned int w)

{

	unsigned int res;
@@ -78,9 +78,10 @@ static inline unsigned int hweight8(unsigned int w) 


#else



#include <asm-generic/bitops/hweight.h>

#include <asm-generic/bitops/arch_hweight.h>



#endif

#include <asm-generic/bitops/const_hweight.h>

#include <asm-generic/bitops/lock.h>

#endif /* __KERNEL__ */

include/asm-generic/bitops/arch_hweight.h

0 → 100644
+11 −0
Original line number Diff line number Diff line 
#ifndef _ASM_GENERIC_BITOPS_ARCH_HWEIGHT_H_

#define _ASM_GENERIC_BITOPS_ARCH_HWEIGHT_H_



#include <asm/types.h>



extern unsigned int __arch_hweight32(unsigned int w);

extern unsigned int __arch_hweight16(unsigned int w);

extern unsigned int __arch_hweight8(unsigned int w);

extern unsigned long __arch_hweight64(__u64 w);



#endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */

include/asm-generic/bitops/const_hweight.h

0 → 100644
+42 −0
Original line number Diff line number Diff line 
#ifndef _ASM_GENERIC_BITOPS_CONST_HWEIGHT_H_

#define _ASM_GENERIC_BITOPS_CONST_HWEIGHT_H_



/*

 * Compile time versions of __arch_hweightN()

 */

#define __const_hweight8(w)		\

      (	(!!((w) & (1ULL << 0))) +	\

	(!!((w) & (1ULL << 1))) +	\

	(!!((w) & (1ULL << 2))) +	\

	(!!((w) & (1ULL << 3))) +	\

	(!!((w) & (1ULL << 4))) +	\

	(!!((w) & (1ULL << 5))) +	\

	(!!((w) & (1ULL << 6))) +	\

	(!!((w) & (1ULL << 7)))	)



#define __const_hweight16(w) (__const_hweight8(w)  + __const_hweight8((w)  >> 8 ))

#define __const_hweight32(w) (__const_hweight16(w) + __const_hweight16((w) >> 16))

#define __const_hweight64(w) (__const_hweight32(w) + __const_hweight32((w) >> 32))



/*

 * Generic interface.

 */

#define hweight8(w)  (__builtin_constant_p(w) ? __const_hweight8(w)  : __arch_hweight8(w))

#define hweight16(w) (__builtin_constant_p(w) ? __const_hweight16(w) : __arch_hweight16(w))

#define hweight32(w) (__builtin_constant_p(w) ? __const_hweight32(w) : __arch_hweight32(w))

#define hweight64(w) (__builtin_constant_p(w) ? __const_hweight64(w) : __arch_hweight64(w))



/*

 * Interface for known constant arguments

 */

#define HWEIGHT8(w)  (BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + __const_hweight8(w))

#define HWEIGHT16(w) (BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + __const_hweight16(w))

#define HWEIGHT32(w) (BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + __const_hweight32(w))

#define HWEIGHT64(w) (BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + __const_hweight64(w))



/*

 * Type invariant interface to the compile time constant hweight functions.

 */

#define HWEIGHT(w)   HWEIGHT64((u64)w)



#endif /* _ASM_GENERIC_BITOPS_CONST_HWEIGHT_H_ */