* [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro
@ 2024-04-30 9:17 Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 2/5] x86/percpu: Move some percpu macros around Uros Bizjak
` (4 more replies)
0 siblings, 5 replies; 10+ messages in thread
From: Uros Bizjak @ 2024-04-30 9:17 UTC (permalink / raw
To: x86, linux-kernel
Cc: Uros Bizjak, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, H. Peter Anvin
Introduce pcpu_binary_op() macro, a copy of percpu_to_op() macro.
Update percpu binary operators to use the new macro, since
percpu_to_op() will be re-purposed as a raw percpu write accessor
in a follow-up patch.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/percpu.h | 47 ++++++++++++++++++++++-------------
1 file changed, 30 insertions(+), 17 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3bedee1801e2..cc40d8d9c272 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -163,6 +163,19 @@ do { \
: [var] "+m" (__my_cpu_var(_var))); \
})
+#define percpu_binary_op(size, qual, op, _var, _val) \
+do { \
+ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
+ if (0) { \
+ typeof(_var) pto_tmp__; \
+ pto_tmp__ = (_val); \
+ (void)pto_tmp__; \
+ } \
+ asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \
+ : [var] "+m" (__my_cpu_var(_var)) \
+ : [val] __pcpu_reg_imm_##size(pto_val__)); \
+} while (0)
+
/*
* Generate a percpu add to memory instruction and optimize code
* if one is added or subtracted.
@@ -182,7 +195,7 @@ do { \
else if (pao_ID__ == -1) \
percpu_unary_op(size, qual, "dec", var); \
else \
- percpu_to_op(size, qual, "add", var, val); \
+ percpu_binary_op(size, qual, "add", var, val); \
} while (0)
#define percpu_from_op(size, qual, op, _var) \
@@ -492,12 +505,12 @@ do { \
#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val)
#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val)
#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val)
-#define raw_cpu_and_1(pcp, val) percpu_to_op(1, , "and", (pcp), val)
-#define raw_cpu_and_2(pcp, val) percpu_to_op(2, , "and", (pcp), val)
-#define raw_cpu_and_4(pcp, val) percpu_to_op(4, , "and", (pcp), val)
-#define raw_cpu_or_1(pcp, val) percpu_to_op(1, , "or", (pcp), val)
-#define raw_cpu_or_2(pcp, val) percpu_to_op(2, , "or", (pcp), val)
-#define raw_cpu_or_4(pcp, val) percpu_to_op(4, , "or", (pcp), val)
+#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val)
+#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val)
+#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val)
+#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val)
+#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val)
+#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val)
#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val)
@@ -505,12 +518,12 @@ do { \
#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val)
#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val)
#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val)
-#define this_cpu_and_1(pcp, val) percpu_to_op(1, volatile, "and", (pcp), val)
-#define this_cpu_and_2(pcp, val) percpu_to_op(2, volatile, "and", (pcp), val)
-#define this_cpu_and_4(pcp, val) percpu_to_op(4, volatile, "and", (pcp), val)
-#define this_cpu_or_1(pcp, val) percpu_to_op(1, volatile, "or", (pcp), val)
-#define this_cpu_or_2(pcp, val) percpu_to_op(2, volatile, "or", (pcp), val)
-#define this_cpu_or_4(pcp, val) percpu_to_op(4, volatile, "or", (pcp), val)
+#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val)
+#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val)
+#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val)
+#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val)
+#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val)
+#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)
#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval)
@@ -543,16 +556,16 @@ do { \
#define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp)
#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val)
-#define raw_cpu_and_8(pcp, val) percpu_to_op(8, , "and", (pcp), val)
-#define raw_cpu_or_8(pcp, val) percpu_to_op(8, , "or", (pcp), val)
+#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val)
+#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val)
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val)
#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval)
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val)
-#define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), val)
-#define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), val)
+#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val)
+#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH -tip 2/5] x86/percpu: Move some percpu macros around
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
@ 2024-04-30 9:17 ` Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Move some percpu macros around for readability tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 3/5] x86/percpu: Unify percpu read-write accessors Uros Bizjak
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Uros Bizjak @ 2024-04-30 9:17 UTC (permalink / raw
To: x86, linux-kernel
Cc: Uros Bizjak, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, H. Peter Anvin
Move some percpu macros around to make a follow-up
patch more readable.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/percpu.h | 63 +++++++++++++++++++----------------
1 file changed, 34 insertions(+), 29 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index cc40d8d9c272..08113a2e5377 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -144,6 +144,29 @@
#define __pcpu_reg_imm_4(x) "ri" (x)
#define __pcpu_reg_imm_8(x) "re" (x)
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+
+#define __raw_cpu_read(qual, pcp) \
+({ \
+ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
+})
+
+#define __raw_cpu_write(qual, pcp, val) \
+do { \
+ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
+} while (0)
+
+#else /* CONFIG_USE_X86_SEG_SUPPORT */
+
+#define percpu_from_op(size, qual, op, _var) \
+({ \
+ __pcpu_type_##size pfo_val__; \
+ asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
+ : [val] __pcpu_reg_##size("=", pfo_val__) \
+ : [var] "m" (__my_cpu_var(_var))); \
+ (typeof(_var))(unsigned long) pfo_val__; \
+})
+
#define percpu_to_op(size, qual, op, _var, _val) \
do { \
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
@@ -157,6 +180,17 @@ do { \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
+#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+
+#define percpu_stable_op(size, op, _var) \
+({ \
+ __pcpu_type_##size pfo_val__; \
+ asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \
+ : [val] __pcpu_reg_##size("=", pfo_val__) \
+ : [var] "i" (&(_var))); \
+ (typeof(_var))(unsigned long) pfo_val__; \
+})
+
#define percpu_unary_op(size, qual, op, _var) \
({ \
asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
@@ -198,24 +232,6 @@ do { \
percpu_binary_op(size, qual, "add", var, val); \
} while (0)
-#define percpu_from_op(size, qual, op, _var) \
-({ \
- __pcpu_type_##size pfo_val__; \
- asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
- : [val] __pcpu_reg_##size("=", pfo_val__) \
- : [var] "m" (__my_cpu_var(_var))); \
- (typeof(_var))(unsigned long) pfo_val__; \
-})
-
-#define percpu_stable_op(size, op, _var) \
-({ \
- __pcpu_type_##size pfo_val__; \
- asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \
- : [val] __pcpu_reg_##size("=", pfo_val__) \
- : [var] "i" (&(_var))); \
- (typeof(_var))(unsigned long) pfo_val__; \
-})
-
/*
* Add return operation
*/
@@ -433,17 +449,6 @@ do { \
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
#ifdef CONFIG_USE_X86_SEG_SUPPORT
-
-#define __raw_cpu_read(qual, pcp) \
-({ \
- *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
-})
-
-#define __raw_cpu_write(qual, pcp, val) \
-do { \
- *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
-} while (0)
-
#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp)
#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp)
#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp)
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH -tip 3/5] x86/percpu: Unify percpu read-write accessors
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 2/5] x86/percpu: Move some percpu macros around Uros Bizjak
@ 2024-04-30 9:17 ` Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 4/5] x86/percpu: Introduce __raw_cpu_read_const() macro Uros Bizjak
` (2 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Uros Bizjak @ 2024-04-30 9:17 UTC (permalink / raw
To: x86, linux-kernel
Cc: Uros Bizjak, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, H. Peter Anvin
Redefine percpu_from_op() and percpu_to_op() as __raw_cpu_read()
and __raw_cpu_write(). Unify __raw_cpu_{read,write}() macros
between configs w/ and w/o USE_X86_SEG_SUPPORT in order to
unify {raw,this}_cpu{read_write}_N() accessors between configs.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/percpu.h | 72 ++++++++++++-----------------------
1 file changed, 25 insertions(+), 47 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 08113a2e5377..f360ac5fccde 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -146,28 +146,28 @@
#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define __raw_cpu_read(qual, pcp) \
+#define __raw_cpu_read(size, qual, pcp) \
({ \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
})
-#define __raw_cpu_write(qual, pcp, val) \
+#define __raw_cpu_write(size, qual, pcp, val) \
do { \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
} while (0)
#else /* CONFIG_USE_X86_SEG_SUPPORT */
-#define percpu_from_op(size, qual, op, _var) \
+#define __raw_cpu_read(size, qual, _var) \
({ \
__pcpu_type_##size pfo_val__; \
- asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
+ asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \
: [val] __pcpu_reg_##size("=", pfo_val__) \
: [var] "m" (__my_cpu_var(_var))); \
(typeof(_var))(unsigned long) pfo_val__; \
})
-#define percpu_to_op(size, qual, op, _var, _val) \
+#define __raw_cpu_write(size, qual, _var, _val) \
do { \
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
if (0) { \
@@ -175,7 +175,7 @@ do { \
pto_tmp__ = (_val); \
(void)pto_tmp__; \
} \
- asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \
+ asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
: [var] "+m" (__my_cpu_var(_var)) \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
@@ -448,54 +448,32 @@ do { \
*/
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
-#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_write_1(pcp, val) __raw_cpu_write(, pcp, val)
-#define raw_cpu_write_2(pcp, val) __raw_cpu_write(, pcp, val)
-#define raw_cpu_write_4(pcp, val) __raw_cpu_write(, pcp, val)
-
-#define this_cpu_read_1(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_read_2(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_read_4(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_write_1(pcp, val) __raw_cpu_write(volatile, pcp, val)
-#define this_cpu_write_2(pcp, val) __raw_cpu_write(volatile, pcp, val)
-#define this_cpu_write_4(pcp, val) __raw_cpu_write(volatile, pcp, val)
+#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp)
+#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp)
+#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp)
+#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val)
+#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val)
+#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val)
+
+#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp)
+#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp)
+#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp)
+#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val)
+#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val)
+#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val)
#ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_write_8(pcp, val) __raw_cpu_write(, pcp, val)
+#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp)
+#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val)
-#define this_cpu_read_8(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_write_8(pcp, val) __raw_cpu_write(volatile, pcp, val)
+#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp)
+#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
#endif
-#define this_cpu_read_const(pcp) __raw_cpu_read(, pcp)
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
#else /* CONFIG_USE_X86_SEG_SUPPORT */
-#define raw_cpu_read_1(pcp) percpu_from_op(1, , "mov", pcp)
-#define raw_cpu_read_2(pcp) percpu_from_op(2, , "mov", pcp)
-#define raw_cpu_read_4(pcp) percpu_from_op(4, , "mov", pcp)
-#define raw_cpu_write_1(pcp, val) percpu_to_op(1, , "mov", (pcp), val)
-#define raw_cpu_write_2(pcp, val) percpu_to_op(2, , "mov", (pcp), val)
-#define raw_cpu_write_4(pcp, val) percpu_to_op(4, , "mov", (pcp), val)
-
-#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp)
-#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp)
-#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp)
-#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val)
-#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val)
-#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val)
-
-#ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp)
-#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val)
-
-#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp)
-#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val)
-#endif
-
/*
* The generic per-cpu infrastrucutre is not suitable for
* reading const-qualified variables.
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH -tip 4/5] x86/percpu: Introduce __raw_cpu_read_const() macro
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 2/5] x86/percpu: Move some percpu macros around Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 3/5] x86/percpu: Unify percpu read-write accessors Uros Bizjak
@ 2024-04-30 9:17 ` Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Introduce the " tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 5/5] x86/percpu: Fix operand constraint modifier in __raw_cpu_write() Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Introduce the pcpu_binary_op() macro tip-bot2 for Uros Bizjak
4 siblings, 1 reply; 10+ messages in thread
From: Uros Bizjak @ 2024-04-30 9:17 UTC (permalink / raw
To: x86, linux-kernel
Cc: Uros Bizjak, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, H. Peter Anvin
Introduce __raw_cpu_read_const() macro to further reduce ifdeffery
and differences between configs w/ and w/o USE_X86_SEG_SUPPORT.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/percpu.h | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index f360ac5fccde..d20255138426 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -156,6 +156,8 @@ do { \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
} while (0)
+#define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
+
#else /* CONFIG_USE_X86_SEG_SUPPORT */
#define __raw_cpu_read(size, qual, _var) \
@@ -180,6 +182,12 @@ do { \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
+/*
+ * The generic per-cpu infrastrucutre is not suitable for
+ * reading const-qualified variables.
+ */
+#define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+
#endif /* CONFIG_USE_X86_SEG_SUPPORT */
#define percpu_stable_op(size, op, _var) \
@@ -470,16 +478,7 @@ do { \
#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
#endif
-#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
-#else /* CONFIG_USE_X86_SEG_SUPPORT */
-
-/*
- * The generic per-cpu infrastrucutre is not suitable for
- * reading const-qualified variables.
- */
-#define this_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
-#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp)
#define this_cpu_read_stable_1(pcp) percpu_stable_op(1, "mov", pcp)
#define this_cpu_read_stable_2(pcp) percpu_stable_op(2, "mov", pcp)
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH -tip 5/5] x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
` (2 preceding siblings ...)
2024-04-30 9:17 ` [PATCH -tip 4/5] x86/percpu: Introduce __raw_cpu_read_const() macro Uros Bizjak
@ 2024-04-30 9:17 ` Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Introduce the pcpu_binary_op() macro tip-bot2 for Uros Bizjak
4 siblings, 1 reply; 10+ messages in thread
From: Uros Bizjak @ 2024-04-30 9:17 UTC (permalink / raw
To: x86, linux-kernel
Cc: Uros Bizjak, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, H. Peter Anvin
__raw_cpu_write() with !USE_X86_SEG_SUPPORT config uses read/write
operand constraint modifier "+" for its memory location. This signals
the compiler that the location is both read and written by the asm.
This is not true, because MOV insn only writes to the output.
Correct the modifier to "=" to inform the compiler that the memory
location is only written to. This also prevents the compiler from
value tracking the undefined value from the uninitialized memory.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
---
arch/x86/include/asm/percpu.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d20255138426..c77393cd0273 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -178,7 +178,7 @@ do { \
(void)pto_tmp__; \
} \
asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
- : [var] "+m" (__my_cpu_var(_var)) \
+ : [var] "=m" (__my_cpu_var(_var)) \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
--
2.44.0
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [tip: x86/percpu] x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
2024-04-30 9:17 ` [PATCH -tip 5/5] x86/percpu: Fix operand constraint modifier in __raw_cpu_write() Uros Bizjak
@ 2024-05-18 9:44 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 10+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-05-18 9:44 UTC (permalink / raw
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Josh Poimboeuf,
Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 1fe67aee8ab3fdab4357afc983a9e9ff3892d694
Gitweb: https://git.kernel.org/tip/1fe67aee8ab3fdab4357afc983a9e9ff3892d694
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Tue, 30 Apr 2024 11:17:24 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 18 May 2024 11:18:42 +02:00
x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
__raw_cpu_write() with !USE_X86_SEG_SUPPORT config uses read/write
operand constraint modifier "+" for its memory location. This signals
the compiler that the location is both read and written by the asm.
This is not true, because MOV insn only writes to the output.
Correct the modifier to "=" to inform the compiler that the memory
location is only written to. This also prevents the compiler from
value tracking the undefined value from the uninitialized memory.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240430091833.196482-5-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d202551..c77393c 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -178,7 +178,7 @@ do { \
(void)pto_tmp__; \
} \
asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
- : [var] "+m" (__my_cpu_var(_var)) \
+ : [var] "=m" (__my_cpu_var(_var)) \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [tip: x86/percpu] x86/percpu: Introduce the __raw_cpu_read_const() macro
2024-04-30 9:17 ` [PATCH -tip 4/5] x86/percpu: Introduce __raw_cpu_read_const() macro Uros Bizjak
@ 2024-05-18 9:44 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 10+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-05-18 9:44 UTC (permalink / raw
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Josh Poimboeuf,
Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 539615de7004a46778020183622856f4ca14e4ac
Gitweb: https://git.kernel.org/tip/539615de7004a46778020183622856f4ca14e4ac
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Tue, 30 Apr 2024 11:17:23 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 18 May 2024 11:18:42 +02:00
x86/percpu: Introduce the __raw_cpu_read_const() macro
Introduce the __raw_cpu_read_const() macro to further reduce ifdeffery
and differences between configs w/ and w/o USE_X86_SEG_SUPPORT.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240430091833.196482-4-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index f360ac5..d202551 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -156,6 +156,8 @@ do { \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
} while (0)
+#define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
+
#else /* CONFIG_USE_X86_SEG_SUPPORT */
#define __raw_cpu_read(size, qual, _var) \
@@ -180,6 +182,12 @@ do { \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
+/*
+ * The generic per-cpu infrastrucutre is not suitable for
+ * reading const-qualified variables.
+ */
+#define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
+
#endif /* CONFIG_USE_X86_SEG_SUPPORT */
#define percpu_stable_op(size, op, _var) \
@@ -470,16 +478,7 @@ do { \
#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
#endif
-#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
-#else /* CONFIG_USE_X86_SEG_SUPPORT */
-
-/*
- * The generic per-cpu infrastrucutre is not suitable for
- * reading const-qualified variables.
- */
-#define this_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
-#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp)
#define this_cpu_read_stable_1(pcp) percpu_stable_op(1, "mov", pcp)
#define this_cpu_read_stable_2(pcp) percpu_stable_op(2, "mov", pcp)
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [tip: x86/percpu] x86/percpu: Unify percpu read-write accessors
2024-04-30 9:17 ` [PATCH -tip 3/5] x86/percpu: Unify percpu read-write accessors Uros Bizjak
@ 2024-05-18 9:44 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 10+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-05-18 9:44 UTC (permalink / raw
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Josh Poimboeuf,
Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: a50ea641296699af1947336c2e75f6234f53548a
Gitweb: https://git.kernel.org/tip/a50ea641296699af1947336c2e75f6234f53548a
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Tue, 30 Apr 2024 11:17:22 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 18 May 2024 11:18:41 +02:00
x86/percpu: Unify percpu read-write accessors
Redefine percpu_from_op() and percpu_to_op() as __raw_cpu_read()
and __raw_cpu_write(). Unify __raw_cpu_{read,write}() macros
between configs w/ and w/o USE_X86_SEG_SUPPORT in order to
unify {raw,this}_cpu{read_write}_N() accessors between configs.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240430091833.196482-3-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 72 +++++++++++-----------------------
1 file changed, 25 insertions(+), 47 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 08113a2..f360ac5 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -146,28 +146,28 @@
#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define __raw_cpu_read(qual, pcp) \
+#define __raw_cpu_read(size, qual, pcp) \
({ \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
})
-#define __raw_cpu_write(qual, pcp, val) \
+#define __raw_cpu_write(size, qual, pcp, val) \
do { \
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
} while (0)
#else /* CONFIG_USE_X86_SEG_SUPPORT */
-#define percpu_from_op(size, qual, op, _var) \
+#define __raw_cpu_read(size, qual, _var) \
({ \
__pcpu_type_##size pfo_val__; \
- asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
+ asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \
: [val] __pcpu_reg_##size("=", pfo_val__) \
: [var] "m" (__my_cpu_var(_var))); \
(typeof(_var))(unsigned long) pfo_val__; \
})
-#define percpu_to_op(size, qual, op, _var, _val) \
+#define __raw_cpu_write(size, qual, _var, _val) \
do { \
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
if (0) { \
@@ -175,7 +175,7 @@ do { \
pto_tmp__ = (_val); \
(void)pto_tmp__; \
} \
- asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \
+ asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
: [var] "+m" (__my_cpu_var(_var)) \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
@@ -448,54 +448,32 @@ do { \
*/
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
-#ifdef CONFIG_USE_X86_SEG_SUPPORT
-#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_write_1(pcp, val) __raw_cpu_write(, pcp, val)
-#define raw_cpu_write_2(pcp, val) __raw_cpu_write(, pcp, val)
-#define raw_cpu_write_4(pcp, val) __raw_cpu_write(, pcp, val)
-
-#define this_cpu_read_1(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_read_2(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_read_4(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_write_1(pcp, val) __raw_cpu_write(volatile, pcp, val)
-#define this_cpu_write_2(pcp, val) __raw_cpu_write(volatile, pcp, val)
-#define this_cpu_write_4(pcp, val) __raw_cpu_write(volatile, pcp, val)
+#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp)
+#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp)
+#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp)
+#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val)
+#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val)
+#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val)
+
+#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp)
+#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp)
+#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp)
+#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val)
+#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val)
+#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val)
#ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp) __raw_cpu_read(, pcp)
-#define raw_cpu_write_8(pcp, val) __raw_cpu_write(, pcp, val)
+#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp)
+#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val)
-#define this_cpu_read_8(pcp) __raw_cpu_read(volatile, pcp)
-#define this_cpu_write_8(pcp, val) __raw_cpu_write(volatile, pcp, val)
+#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp)
+#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
#endif
-#define this_cpu_read_const(pcp) __raw_cpu_read(, pcp)
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
#else /* CONFIG_USE_X86_SEG_SUPPORT */
-#define raw_cpu_read_1(pcp) percpu_from_op(1, , "mov", pcp)
-#define raw_cpu_read_2(pcp) percpu_from_op(2, , "mov", pcp)
-#define raw_cpu_read_4(pcp) percpu_from_op(4, , "mov", pcp)
-#define raw_cpu_write_1(pcp, val) percpu_to_op(1, , "mov", (pcp), val)
-#define raw_cpu_write_2(pcp, val) percpu_to_op(2, , "mov", (pcp), val)
-#define raw_cpu_write_4(pcp, val) percpu_to_op(4, , "mov", (pcp), val)
-
-#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp)
-#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp)
-#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp)
-#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val)
-#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val)
-#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val)
-
-#ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp)
-#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val)
-
-#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp)
-#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val)
-#endif
-
/*
* The generic per-cpu infrastrucutre is not suitable for
* reading const-qualified variables.
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [tip: x86/percpu] x86/percpu: Move some percpu macros around for readability
2024-04-30 9:17 ` [PATCH -tip 2/5] x86/percpu: Move some percpu macros around Uros Bizjak
@ 2024-05-18 9:44 ` tip-bot2 for Uros Bizjak
0 siblings, 0 replies; 10+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-05-18 9:44 UTC (permalink / raw
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Josh Poimboeuf,
Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 08d564ad699ef32ceaf99d238b3d9c1f4ce5c998
Gitweb: https://git.kernel.org/tip/08d564ad699ef32ceaf99d238b3d9c1f4ce5c998
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Tue, 30 Apr 2024 11:17:21 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 18 May 2024 11:18:41 +02:00
x86/percpu: Move some percpu macros around for readability
Move some percpu macros around to make a follow-up
patch more readable.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240430091833.196482-2-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 63 ++++++++++++++++++----------------
1 file changed, 34 insertions(+), 29 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index cc40d8d..08113a2 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -144,6 +144,29 @@
#define __pcpu_reg_imm_4(x) "ri" (x)
#define __pcpu_reg_imm_8(x) "re" (x)
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+
+#define __raw_cpu_read(qual, pcp) \
+({ \
+ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
+})
+
+#define __raw_cpu_write(qual, pcp, val) \
+do { \
+ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
+} while (0)
+
+#else /* CONFIG_USE_X86_SEG_SUPPORT */
+
+#define percpu_from_op(size, qual, op, _var) \
+({ \
+ __pcpu_type_##size pfo_val__; \
+ asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
+ : [val] __pcpu_reg_##size("=", pfo_val__) \
+ : [var] "m" (__my_cpu_var(_var))); \
+ (typeof(_var))(unsigned long) pfo_val__; \
+})
+
#define percpu_to_op(size, qual, op, _var, _val) \
do { \
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
@@ -157,6 +180,17 @@ do { \
: [val] __pcpu_reg_imm_##size(pto_val__)); \
} while (0)
+#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+
+#define percpu_stable_op(size, op, _var) \
+({ \
+ __pcpu_type_##size pfo_val__; \
+ asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \
+ : [val] __pcpu_reg_##size("=", pfo_val__) \
+ : [var] "i" (&(_var))); \
+ (typeof(_var))(unsigned long) pfo_val__; \
+})
+
#define percpu_unary_op(size, qual, op, _var) \
({ \
asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
@@ -198,24 +232,6 @@ do { \
percpu_binary_op(size, qual, "add", var, val); \
} while (0)
-#define percpu_from_op(size, qual, op, _var) \
-({ \
- __pcpu_type_##size pfo_val__; \
- asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
- : [val] __pcpu_reg_##size("=", pfo_val__) \
- : [var] "m" (__my_cpu_var(_var))); \
- (typeof(_var))(unsigned long) pfo_val__; \
-})
-
-#define percpu_stable_op(size, op, _var) \
-({ \
- __pcpu_type_##size pfo_val__; \
- asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \
- : [val] __pcpu_reg_##size("=", pfo_val__) \
- : [var] "i" (&(_var))); \
- (typeof(_var))(unsigned long) pfo_val__; \
-})
-
/*
* Add return operation
*/
@@ -433,17 +449,6 @@ do { \
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
#ifdef CONFIG_USE_X86_SEG_SUPPORT
-
-#define __raw_cpu_read(qual, pcp) \
-({ \
- *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
-})
-
-#define __raw_cpu_write(qual, pcp, val) \
-do { \
- *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
-} while (0)
-
#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp)
#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp)
#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp)
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [tip: x86/percpu] x86/percpu: Introduce the pcpu_binary_op() macro
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
` (3 preceding siblings ...)
2024-04-30 9:17 ` [PATCH -tip 5/5] x86/percpu: Fix operand constraint modifier in __raw_cpu_write() Uros Bizjak
@ 2024-05-18 9:44 ` tip-bot2 for Uros Bizjak
4 siblings, 0 replies; 10+ messages in thread
From: tip-bot2 for Uros Bizjak @ 2024-05-18 9:44 UTC (permalink / raw
To: linux-tip-commits
Cc: Uros Bizjak, Ingo Molnar, Andy Lutomirski, Josh Poimboeuf,
Linus Torvalds, x86, linux-kernel
The following commit has been merged into the x86/percpu branch of tip:
Commit-ID: 455ca134a7b2121dd739d425d3be313fb52f0651
Gitweb: https://git.kernel.org/tip/455ca134a7b2121dd739d425d3be313fb52f0651
Author: Uros Bizjak <ubizjak@gmail.com>
AuthorDate: Tue, 30 Apr 2024 11:17:20 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Sat, 18 May 2024 11:18:40 +02:00
x86/percpu: Introduce the pcpu_binary_op() macro
Introduce the pcpu_binary_op() macro, a copy of the percpu_to_op() macro.
Update percpu binary operators to use the new macro, since
percpu_to_op() will be re-purposed as a raw percpu write accessor
in a follow-up patch.
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240430091833.196482-1-ubizjak@gmail.com
---
arch/x86/include/asm/percpu.h | 47 +++++++++++++++++++++-------------
1 file changed, 30 insertions(+), 17 deletions(-)
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3bedee1..cc40d8d 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -163,6 +163,19 @@ do { \
: [var] "+m" (__my_cpu_var(_var))); \
})
+#define percpu_binary_op(size, qual, op, _var, _val) \
+do { \
+ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
+ if (0) { \
+ typeof(_var) pto_tmp__; \
+ pto_tmp__ = (_val); \
+ (void)pto_tmp__; \
+ } \
+ asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \
+ : [var] "+m" (__my_cpu_var(_var)) \
+ : [val] __pcpu_reg_imm_##size(pto_val__)); \
+} while (0)
+
/*
* Generate a percpu add to memory instruction and optimize code
* if one is added or subtracted.
@@ -182,7 +195,7 @@ do { \
else if (pao_ID__ == -1) \
percpu_unary_op(size, qual, "dec", var); \
else \
- percpu_to_op(size, qual, "add", var, val); \
+ percpu_binary_op(size, qual, "add", var, val); \
} while (0)
#define percpu_from_op(size, qual, op, _var) \
@@ -492,12 +505,12 @@ do { \
#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val)
#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val)
#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val)
-#define raw_cpu_and_1(pcp, val) percpu_to_op(1, , "and", (pcp), val)
-#define raw_cpu_and_2(pcp, val) percpu_to_op(2, , "and", (pcp), val)
-#define raw_cpu_and_4(pcp, val) percpu_to_op(4, , "and", (pcp), val)
-#define raw_cpu_or_1(pcp, val) percpu_to_op(1, , "or", (pcp), val)
-#define raw_cpu_or_2(pcp, val) percpu_to_op(2, , "or", (pcp), val)
-#define raw_cpu_or_4(pcp, val) percpu_to_op(4, , "or", (pcp), val)
+#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val)
+#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val)
+#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val)
+#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val)
+#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val)
+#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val)
#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val)
@@ -505,12 +518,12 @@ do { \
#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val)
#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val)
#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val)
-#define this_cpu_and_1(pcp, val) percpu_to_op(1, volatile, "and", (pcp), val)
-#define this_cpu_and_2(pcp, val) percpu_to_op(2, volatile, "and", (pcp), val)
-#define this_cpu_and_4(pcp, val) percpu_to_op(4, volatile, "and", (pcp), val)
-#define this_cpu_or_1(pcp, val) percpu_to_op(1, volatile, "or", (pcp), val)
-#define this_cpu_or_2(pcp, val) percpu_to_op(2, volatile, "or", (pcp), val)
-#define this_cpu_or_4(pcp, val) percpu_to_op(4, volatile, "or", (pcp), val)
+#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val)
+#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val)
+#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val)
+#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val)
+#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val)
+#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)
#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval)
@@ -543,16 +556,16 @@ do { \
#define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp)
#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val)
-#define raw_cpu_and_8(pcp, val) percpu_to_op(8, , "and", (pcp), val)
-#define raw_cpu_or_8(pcp, val) percpu_to_op(8, , "or", (pcp), val)
+#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val)
+#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val)
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val)
#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval)
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val)
-#define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), val)
-#define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), val)
+#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val)
+#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
^ permalink raw reply related [flat|nested] 10+ messages in thread
end of thread, other threads:[~2024-05-18 9:44 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-04-30 9:17 [PATCH -tip 1/5] x86/percpu: Introduce pcpu_binary_op() macro Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 2/5] x86/percpu: Move some percpu macros around Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Move some percpu macros around for readability tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 3/5] x86/percpu: Unify percpu read-write accessors Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 4/5] x86/percpu: Introduce __raw_cpu_read_const() macro Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Introduce the " tip-bot2 for Uros Bizjak
2024-04-30 9:17 ` [PATCH -tip 5/5] x86/percpu: Fix operand constraint modifier in __raw_cpu_write() Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] " tip-bot2 for Uros Bizjak
2024-05-18 9:44 ` [tip: x86/percpu] x86/percpu: Introduce the pcpu_binary_op() macro tip-bot2 for Uros Bizjak
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).