RISC-V: Atomic and Locking Code

/*

 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.

 * Copyright (C) 2012 Regents of the University of California

 * Copyright (C) 2017 SiFive

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public Licence

 * as published by the Free Software Foundation; either version

 * 2 of the Licence, or (at your option) any later version.

 */

#ifndef _ASM_RISCV_ATOMIC_H

#define _ASM_RISCV_ATOMIC_H

#ifdef CONFIG_GENERIC_ATOMIC64

# include <asm-generic/atomic64.h>

#else

# if (__riscv_xlen < 64)

#  error "64-bit atomics require XLEN to be at least 64"

# endif

#endif

#include <asm/cmpxchg.h>

#include <asm/barrier.h>

#define ATOMIC_INIT(i)	{ (i) }

static __always_inline int atomic_read(const atomic_t *v)

{

	return READ_ONCE(v->counter);

}

static __always_inline void atomic_set(atomic_t *v, int i)

{

	WRITE_ONCE(v->counter, i);

}

#ifndef CONFIG_GENERIC_ATOMIC64

#define ATOMIC64_INIT(i) { (i) }

static __always_inline long atomic64_read(const atomic64_t *v)

{

	return READ_ONCE(v->counter);

}

static __always_inline void atomic64_set(atomic64_t *v, long i)

{

	WRITE_ONCE(v->counter, i);

}

#endif

/*

 * First, the atomic ops that have no ordering constraints and therefor don't

 * have the AQ or RL bits set.  These don't return anything, so there's only

 * one version to worry about.

 */

#define ATOMIC_OP(op, asm_op, c_op, I, asm_type, c_type, prefix)				\

static __always_inline void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)		\

{												\

	__asm__ __volatile__ (									\

		"amo" #asm_op "." #asm_type " zero, %1, %0"					\

		: "+A" (v->counter)								\

		: "r" (I)									\

		: "memory");									\

}

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(op, asm_op, c_op, I)			\

        ATOMIC_OP (op, asm_op, c_op, I, w,  int,   )

#else

#define ATOMIC_OPS(op, asm_op, c_op, I)			\

        ATOMIC_OP (op, asm_op, c_op, I, w,  int,   )	\

        ATOMIC_OP (op, asm_op, c_op, I, d, long, 64)

#endif

ATOMIC_OPS(add, add, +,  i)

ATOMIC_OPS(sub, add, +, -i)

ATOMIC_OPS(and, and, &,  i)

ATOMIC_OPS( or,  or, |,  i)

ATOMIC_OPS(xor, xor, ^,  i)

#undef ATOMIC_OP

#undef ATOMIC_OPS

/*

 * Atomic ops that have ordered, relaxed, acquire, and relese variants.

 * There's two flavors of these: the arithmatic ops have both fetch and return

 * versions, while the logical ops only have fetch versions.

 */

#define ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix)			\

static __always_inline c_type atomic##prefix##_fetch_##op##c_or(c_type i, atomic##prefix##_t *v)	\

{													\

	register c_type ret;										\

	__asm__ __volatile__ (										\

		"amo" #asm_op "." #asm_type #asm_or " %1, %2, %0"					\

		: "+A" (v->counter), "=r" (ret)								\

		: "r" (I)										\

		: "memory");										\

	return ret;											\

}

#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, asm_type, c_type, prefix)			\

static __always_inline c_type atomic##prefix##_##op##_return##c_or(c_type i, atomic##prefix##_t *v)	\

{													\

        return atomic##prefix##_fetch_##op##c_or(i, v) c_op I;						\

}

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, w,  int,   )	\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )

#else

#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, w,  int,   )	\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )	\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I, asm_or, c_or, d, long, 64)	\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)

#endif

ATOMIC_OPS(add, add, +,  i,      , _relaxed)

ATOMIC_OPS(add, add, +,  i, .aq  , _acquire)

ATOMIC_OPS(add, add, +,  i, .rl  , _release)

ATOMIC_OPS(add, add, +,  i, .aqrl,         )

ATOMIC_OPS(sub, add, +, -i,      , _relaxed)

ATOMIC_OPS(sub, add, +, -i, .aq  , _acquire)

ATOMIC_OPS(sub, add, +, -i, .rl  , _release)

ATOMIC_OPS(sub, add, +, -i, .aqrl,         )

#undef ATOMIC_OPS

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\

        ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )

#else

#define ATOMIC_OPS(op, asm_op, c_op, I, asm_or, c_or)				\

        ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, w,  int,   )		\

        ATOMIC_FETCH_OP(op, asm_op, c_op, I, asm_or, c_or, d, long, 64)

#endif

ATOMIC_OPS(and, and, &,  i,      , _relaxed)

ATOMIC_OPS(and, and, &,  i, .aq  , _acquire)

ATOMIC_OPS(and, and, &,  i, .rl  , _release)

ATOMIC_OPS(and, and, &,  i, .aqrl,         )

ATOMIC_OPS( or,  or, |,  i,      , _relaxed)

ATOMIC_OPS( or,  or, |,  i, .aq  , _acquire)

ATOMIC_OPS( or,  or, |,  i, .rl  , _release)

ATOMIC_OPS( or,  or, |,  i, .aqrl,         )

ATOMIC_OPS(xor, xor, ^,  i,      , _relaxed)

ATOMIC_OPS(xor, xor, ^,  i, .aq  , _acquire)

ATOMIC_OPS(xor, xor, ^,  i, .rl  , _release)

ATOMIC_OPS(xor, xor, ^,  i, .aqrl,         )

#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

/*

 * The extra atomic operations that are constructed from one of the core

 * AMO-based operations above (aside from sub, which is easier to fit above).

 * These are required to perform a barrier, but they're OK this way because

 * atomic_*_return is also required to perform a barrier.

 */

#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix)			\

static __always_inline bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \

{										\

	return atomic##prefix##_##func_op##_return(i, v) comp_op I;		\

}

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(op, func_op, comp_op, I)			\

        ATOMIC_OP (op, func_op, comp_op, I,  int,   )

#else

#define ATOMIC_OPS(op, func_op, comp_op, I)			\

        ATOMIC_OP (op, func_op, comp_op, I,  int,   )		\

        ATOMIC_OP (op, func_op, comp_op, I, long, 64)

#endif

ATOMIC_OPS(add_and_test, add, ==, 0)

ATOMIC_OPS(sub_and_test, sub, ==, 0)

ATOMIC_OPS(add_negative, add,  <, 0)

#undef ATOMIC_OP

#undef ATOMIC_OPS

#define ATOMIC_OP(op, func_op, c_op, I, c_type, prefix)				\

static __always_inline void atomic##prefix##_##op(atomic##prefix##_t *v)	\

{										\

	atomic##prefix##_##func_op(I, v);					\

}

#define ATOMIC_FETCH_OP(op, func_op, c_op, I, c_type, prefix)				\

static __always_inline c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v)	\

{											\

	return atomic##prefix##_fetch_##func_op(I, v);					\

}

#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix)				\

static __always_inline c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v)	\

{											\

        return atomic##prefix##_fetch_##op(v) c_op I;					\

}

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(op, asm_op, c_op, I)						\

        ATOMIC_OP       (op, asm_op, c_op, I,  int,   )				\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I,  int,   )				\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )

#else

#define ATOMIC_OPS(op, asm_op, c_op, I)						\

        ATOMIC_OP       (op, asm_op, c_op, I,  int,   )				\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I,  int,   )				\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I,  int,   )				\

        ATOMIC_OP       (op, asm_op, c_op, I, long, 64)				\

        ATOMIC_FETCH_OP (op, asm_op, c_op, I, long, 64)				\

        ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64)

#endif

ATOMIC_OPS(inc, add, +,  1)

ATOMIC_OPS(dec, add, +, -1)

#undef ATOMIC_OPS

#undef ATOMIC_OP

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#define ATOMIC_OP(op, func_op, comp_op, I, prefix)				\

static __always_inline bool atomic##prefix##_##op(atomic##prefix##_t *v)	\

{										\

	return atomic##prefix##_##func_op##_return(v) comp_op I;		\

}

ATOMIC_OP(inc_and_test, inc, ==, 0,   )

ATOMIC_OP(dec_and_test, dec, ==, 0,   )

#ifndef CONFIG_GENERIC_ATOMIC64

ATOMIC_OP(inc_and_test, inc, ==, 0, 64)

ATOMIC_OP(dec_and_test, dec, ==, 0, 64)

#endif

#undef ATOMIC_OP

/* This is required to provide a barrier on success. */

static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)

{

       int prev, rc;

	__asm__ __volatile__ (

		"0:\n\t"

		"lr.w.aqrl  %[p],  %[c]\n\t"

		"beq        %[p],  %[u], 1f\n\t"

		"add       %[rc],  %[p], %[a]\n\t"

		"sc.w.aqrl %[rc], %[rc], %[c]\n\t"

		"bnez      %[rc], 0b\n\t"

		"1:"

		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)

		: [a]"r" (a), [u]"r" (u)

		: "memory");

	return prev;

}

#ifndef CONFIG_GENERIC_ATOMIC64

static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u)

{

       long prev, rc;

	__asm__ __volatile__ (

		"0:\n\t"

		"lr.d.aqrl  %[p],  %[c]\n\t"

		"beq        %[p],  %[u], 1f\n\t"

		"add       %[rc],  %[p], %[a]\n\t"

		"sc.d.aqrl %[rc], %[rc], %[c]\n\t"

		"bnez      %[rc], 0b\n\t"

		"1:"

		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)

		: [a]"r" (a), [u]"r" (u)

		: "memory");

	return prev;

}

static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u)

{

	return __atomic64_add_unless(v, a, u) != u;

}

#endif

/*

 * The extra atomic operations that are constructed from one of the core

 * LR/SC-based operations above.

 */

static __always_inline int atomic_inc_not_zero(atomic_t *v)

{

        return __atomic_add_unless(v, 1, 0);

}

#ifndef CONFIG_GENERIC_ATOMIC64

static __always_inline long atomic64_inc_not_zero(atomic64_t *v)

{

        return atomic64_add_unless(v, 1, 0);

}

#endif

/*

 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as

 * {cmp,}xchg and the operations that return, so they need a barrier.  We just

 * use the other implementations directly.

 */

#define ATOMIC_OP(c_t, prefix, c_or, size, asm_or)						\

static __always_inline c_t atomic##prefix##_cmpxchg##c_or(atomic##prefix##_t *v, c_t o, c_t n) 	\

{												\

	return __cmpxchg(&(v->counter), o, n, size, asm_or, asm_or);				\

}												\

static __always_inline c_t atomic##prefix##_xchg##c_or(atomic##prefix##_t *v, c_t n) 		\

{												\

	return __xchg(n, &(v->counter), size, asm_or);						\

}

#ifdef CONFIG_GENERIC_ATOMIC64

#define ATOMIC_OPS(c_or, asm_or)			\

	ATOMIC_OP( int,   , c_or, 4, asm_or)

#else

#define ATOMIC_OPS(c_or, asm_or)			\

	ATOMIC_OP( int,   , c_or, 4, asm_or)		\

	ATOMIC_OP(long, 64, c_or, 8, asm_or)

#endif

ATOMIC_OPS(        , .aqrl)

ATOMIC_OPS(_acquire,   .aq)

ATOMIC_OPS(_release,   .rl)

ATOMIC_OPS(_relaxed,      )

#undef ATOMIC_OPS

#undef ATOMIC_OP

static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)

{

       int prev, rc;

	__asm__ __volatile__ (

		"0:\n\t"

		"lr.w.aqrl  %[p],  %[c]\n\t"

		"sub       %[rc],  %[p], %[o]\n\t"

		"bltz      %[rc],    1f\n\t"

		"sc.w.aqrl %[rc], %[rc], %[c]\n\t"

		"bnez      %[rc],    0b\n\t"

		"1:"

		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)

		: [o]"r" (offset)

		: "memory");

	return prev - offset;

}

#define atomic_dec_if_positive(v)	atomic_sub_if_positive(v, 1)

#ifndef CONFIG_GENERIC_ATOMIC64

static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)

{

       long prev, rc;

	__asm__ __volatile__ (

		"0:\n\t"

		"lr.d.aqrl  %[p],  %[c]\n\t"

		"sub       %[rc],  %[p], %[o]\n\t"

		"bltz      %[rc],    1f\n\t"

		"sc.d.aqrl %[rc], %[rc], %[c]\n\t"

		"bnez      %[rc],    0b\n\t"

		"1:"

		: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)

		: [o]"r" (offset)

		: "memory");

	return prev - offset;

}

#define atomic64_dec_if_positive(v)	atomic64_sub_if_positive(v, 1)

#endif

#endif /* _ASM_RISCV_ATOMIC_H */

/*

 * Based on arch/arm/include/asm/barrier.h

 *

 * Copyright (C) 2012 ARM Ltd.

 * Copyright (C) 2013 Regents of the University of California

 * Copyright (C) 2017 SiFive

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

#ifndef _ASM_RISCV_BARRIER_H

#define _ASM_RISCV_BARRIER_H

#ifndef __ASSEMBLY__

#define nop()		__asm__ __volatile__ ("nop")

#define RISCV_FENCE(p, s) \

	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")

/* These barriers need to enforce ordering on both devices or memory. */

#define mb()		RISCV_FENCE(iorw,iorw)

#define rmb()		RISCV_FENCE(ir,ir)

#define wmb()		RISCV_FENCE(ow,ow)

/* These barriers do not need to enforce ordering on devices, just memory. */

#define smp_mb()	RISCV_FENCE(rw,rw)

#define smp_rmb()	RISCV_FENCE(r,r)

#define smp_wmb()	RISCV_FENCE(w,w)

/*

 * These fences exist to enforce ordering around the relaxed AMOs.  The

 * documentation defines that

 * "

 *     atomic_fetch_add();

 *   is equivalent to:

 *     smp_mb__before_atomic();

 *     atomic_fetch_add_relaxed();

 *     smp_mb__after_atomic();

 * "

 * So we emit full fences on both sides.

 */

#define __smb_mb__before_atomic()	smp_mb()

#define __smb_mb__after_atomic()	smp_mb()

/*

 * These barriers prevent accesses performed outside a spinlock from being moved

 * inside a spinlock.  Since RISC-V sets the aq/rl bits on our spinlock only

 * enforce release consistency, we need full fences here.

 */

#define smb_mb__before_spinlock()	smp_mb()

#define smb_mb__after_spinlock()	smp_mb()

#include <asm-generic/barrier.h>

#endif /* __ASSEMBLY__ */

#endif /* _ASM_RISCV_BARRIER_H */

/*

 * Copyright (C) 2012 Regents of the University of California

 *

 *   This program is free software; you can redistribute it and/or

 *   modify it under the terms of the GNU General Public License

 *   as published by the Free Software Foundation, version 2.

 *

 *   This program is distributed in the hope that it will be useful,

 *   but WITHOUT ANY WARRANTY; without even the implied warranty of

 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 *   GNU General Public License for more details.

 */

#ifndef _ASM_RISCV_BITOPS_H

#define _ASM_RISCV_BITOPS_H

#ifndef _LINUX_BITOPS_H

#error "Only <linux/bitops.h> can be included directly"

#endif /* _LINUX_BITOPS_H */

#include <linux/compiler.h>

#include <linux/irqflags.h>

#include <asm/barrier.h>

#include <asm/bitsperlong.h>

#ifndef smp_mb__before_clear_bit

#define smp_mb__before_clear_bit()  smp_mb()

#define smp_mb__after_clear_bit()   smp_mb()

#endif /* smp_mb__before_clear_bit */

#include <asm-generic/bitops/__ffs.h>

#include <asm-generic/bitops/ffz.h>

#include <asm-generic/bitops/fls.h>

#include <asm-generic/bitops/__fls.h>

#include <asm-generic/bitops/fls64.h>

#include <asm-generic/bitops/find.h>

#include <asm-generic/bitops/sched.h>

#include <asm-generic/bitops/ffs.h>

#include <asm-generic/bitops/hweight.h>

#if (BITS_PER_LONG == 64)

#define __AMO(op)	"amo" #op ".d"

#elif (BITS_PER_LONG == 32)

#define __AMO(op)	"amo" #op ".w"

#else

#error "Unexpected BITS_PER_LONG"

#endif

#define __test_and_op_bit_ord(op, mod, nr, addr, ord)		\

({								\

	unsigned long __res, __mask;				\

	__mask = BIT_MASK(nr);					\

	__asm__ __volatile__ (					\

		__AMO(op) #ord " %0, %2, %1"			\

		: "=r" (__res), "+A" (addr[BIT_WORD(nr)])	\

		: "r" (mod(__mask))				\

		: "memory");					\

	((__res & __mask) != 0);				\

})

#define __op_bit_ord(op, mod, nr, addr, ord)			\

	__asm__ __volatile__ (					\

		__AMO(op) #ord " zero, %1, %0"			\

		: "+A" (addr[BIT_WORD(nr)])			\

		: "r" (mod(BIT_MASK(nr)))			\

		: "memory");

#define __test_and_op_bit(op, mod, nr, addr) 			\

	__test_and_op_bit_ord(op, mod, nr, addr, )

#define __op_bit(op, mod, nr, addr)				\

	__op_bit_ord(op, mod, nr, addr, )

/* Bitmask modifiers */

#define __NOP(x)	(x)

#define __NOT(x)	(~(x))

/**

 * test_and_set_bit - Set a bit and return its old value

 * @nr: Bit to set

 * @addr: Address to count from

 *

 * This operation may be reordered on other architectures than x86.

 */

static inline int test_and_set_bit(int nr, volatile unsigned long *addr)

{

	return __test_and_op_bit(or, __NOP, nr, addr);

}

/**

 * test_and_clear_bit - Clear a bit and return its old value

 * @nr: Bit to clear

 * @addr: Address to count from

 *

 * This operation can be reordered on other architectures other than x86.

 */

static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)

{

	return __test_and_op_bit(and, __NOT, nr, addr);

}

/**

 * test_and_change_bit - Change a bit and return its old value

 * @nr: Bit to change

 * @addr: Address to count from

 *

 * This operation is atomic and cannot be reordered.

 * It also implies a memory barrier.

 */

static inline int test_and_change_bit(int nr, volatile unsigned long *addr)

{

	return __test_and_op_bit(xor, __NOP, nr, addr);

}

/**

 * set_bit - Atomically set a bit in memory

 * @nr: the bit to set

 * @addr: the address to start counting from

 *

 * Note: there are no guarantees that this function will not be reordered

 * on non x86 architectures, so if you are writing portable code,

 * make sure not to rely on its reordering guarantees.

 *

 * Note that @nr may be almost arbitrarily large; this function is not

 * restricted to acting on a single-word quantity.

 */

static inline void set_bit(int nr, volatile unsigned long *addr)

{

	__op_bit(or, __NOP, nr, addr);

}

/**

 * clear_bit - Clears a bit in memory

 * @nr: Bit to clear

 * @addr: Address to start counting from

 *

 * Note: there are no guarantees that this function will not be reordered

 * on non x86 architectures, so if you are writing portable code,

 * make sure not to rely on its reordering guarantees.

 */

static inline void clear_bit(int nr, volatile unsigned long *addr)

{

	__op_bit(and, __NOT, nr, addr);

}

/**

 * change_bit - Toggle a bit in memory

 * @nr: Bit to change

 * @addr: Address to start counting from

 *

 * change_bit()  may be reordered on other architectures than x86.

 * Note that @nr may be almost arbitrarily large; this function is not

 * restricted to acting on a single-word quantity.

 */

static inline void change_bit(int nr, volatile unsigned long *addr)

{

	__op_bit(xor, __NOP, nr, addr);

}

/**

 * test_and_set_bit_lock - Set a bit and return its old value, for lock

 * @nr: Bit to set

 * @addr: Address to count from

 *

 * This operation is atomic and provides acquire barrier semantics.

 * It can be used to implement bit locks.

 */

static inline int test_and_set_bit_lock(

	unsigned long nr, volatile unsigned long *addr)

{

	return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq);

}

/**

 * clear_bit_unlock - Clear a bit in memory, for unlock

 * @nr: the bit to set

 * @addr: the address to start counting from

 *

 * This operation is atomic and provides release barrier semantics.

 */

static inline void clear_bit_unlock(

	unsigned long nr, volatile unsigned long *addr)

{

	__op_bit_ord(and, __NOT, nr, addr, .rl);

}

/**

 * __clear_bit_unlock - Clear a bit in memory, for unlock

 * @nr: the bit to set

 * @addr: the address to start counting from

 *

 * This operation is like clear_bit_unlock, however it is not atomic.

 * It does provide release barrier semantics so it can be used to unlock

 * a bit lock, however it would only be used if no other CPU can modify

 * any bits in the memory until the lock is released (a good example is

 * if the bit lock itself protects access to the other bits in the word).

 *

 * On RISC-V systems there seems to be no benefit to taking advantage of the

 * non-atomic property here: it's a lot more instructions and we still have to

 * provide release semantics anyway.

 */

static inline void __clear_bit_unlock(

	unsigned long nr, volatile unsigned long *addr)

{

	clear_bit_unlock(nr, addr);

}

#undef __test_and_op_bit

#undef __op_bit

#undef __NOP

#undef __NOT

#undef __AMO

#include <asm-generic/bitops/non-atomic.h>

#include <asm-generic/bitops/le.h>

#include <asm-generic/bitops/ext2-atomic.h>

#endif /* _ASM_RISCV_BITOPS_H */

/*

 * Copyright (C) 2015 Regents of the University of California

 *

 *   This program is free software; you can redistribute it and/or

 *   modify it under the terms of the GNU General Public License

 *   as published by the Free Software Foundation, version 2.

 *

 *   This program is distributed in the hope that it will be useful,

 *   but WITHOUT ANY WARRANTY; without even the implied warranty of

 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 *   GNU General Public License for more details.

 */

#ifndef _ASM_RISCV_CACHEFLUSH_H

#define _ASM_RISCV_CACHEFLUSH_H

#include <asm-generic/cacheflush.h>

#undef flush_icache_range

#undef flush_icache_user_range

static inline void local_flush_icache_all(void)

{

	asm volatile ("fence.i" ::: "memory");

}

#ifndef CONFIG_SMP

#define flush_icache_range(start, end) local_flush_icache_all()

#define flush_icache_user_range(vma, pg, addr, len) local_flush_icache_all()

#else /* CONFIG_SMP */

#define flush_icache_range(start, end) sbi_remote_fence_i(0)

#define flush_icache_user_range(vma, pg, addr, len) sbi_remote_fence_i(0)

#endif /* CONFIG_SMP */

#endif /* _ASM_RISCV_CACHEFLUSH_H */

/*

 * Copyright (C) 2014 Regents of the University of California

 *

 *   This program is free software; you can redistribute it and/or

 *   modify it under the terms of the GNU General Public License

 *   as published by the Free Software Foundation, version 2.

 *

 *   This program is distributed in the hope that it will be useful,

 *   but WITHOUT ANY WARRANTY; without even the implied warranty of

 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 *   GNU General Public License for more details.

 */

#ifndef _ASM_RISCV_CMPXCHG_H

#define _ASM_RISCV_CMPXCHG_H

#include <linux/bug.h>

#include <asm/barrier.h>

#define __xchg(new, ptr, size, asm_or)				\

({								\

	__typeof__(ptr) __ptr = (ptr);				\

	__typeof__(new) __new = (new);				\

	__typeof__(*(ptr)) __ret;				\

	switch (size) {						\

	case 4:							\

		__asm__ __volatile__ (				\

			"amoswap.w" #asm_or " %0, %2, %1"	\

			: "=r" (__ret), "+A" (*__ptr)		\

			: "r" (__new)				\

			: "memory");				\

		break;						\

	case 8:							\

		__asm__ __volatile__ (				\

			"amoswap.d" #asm_or " %0, %2, %1"	\

			: "=r" (__ret), "+A" (*__ptr)		\

			: "r" (__new)				\

			: "memory");				\

		break;						\

	default:						\

		BUILD_BUG();					\

	}							\

	__ret;							\

})

#define xchg(ptr, x)    (__xchg((x), (ptr), sizeof(*(ptr)), .aqrl))