From: Frederic Weisbecker <frederic@kernel.org>
To: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: linux-kernel@vger.kernel.org, rcu@vger.kernel.org,
x86@kernel.org, Lai Jiangshan <jiangshan.ljs@antgroup.com>,
"Paul E. McKenney" <paulmck@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Thomas Gleixner <tglx@linutronix.de>,
Ingo Molnar <mingo@redhat.com>, Borislav Petkov <bp@alien8.de>,
Dave Hansen <dave.hansen@linux.intel.com>,
"H. Peter Anvin" <hpa@zytor.com>, Uros Bizjak <ubizjak@gmail.com>,
Josh Poimboeuf <jpoimboe@kernel.org>,
Nadav Amit <namit@vmware.com>, Breno Leitao <leitao@debian.org>,
Kent Overstreet <kent.overstreet@linux.dev>,
Pawan Gupta <pawan.kumar.gupta@linux.intel.com>,
Rick Edgecombe <rick.p.edgecombe@intel.com>,
Vegard Nossum <vegard.nossum@oracle.com>,
Daniel Sneddon <daniel.sneddon@linux.intel.com>,
Nikolay Borisov <nik.borisov@suse.com>
Subject: Re: [PATCH 09/10] x86/rcu: Add rcu_preempt_count
Date: Mon, 22 Apr 2024 13:05:20 +0200 [thread overview]
Message-ID: <ZiZEcBEvK8NOQvwU@localhost.localdomain> (raw)
In-Reply-To: <20240328075318.83039-10-jiangshanlai@gmail.com>
Le Thu, Mar 28, 2024 at 03:53:17PM +0800, Lai Jiangshan a écrit :
> From: Lai Jiangshan <jiangshan.ljs@antgroup.com>
>
> Implement PCPU_RCU_PREEMPT_COUNT for x86.
> Mainly copied from asm/preempt.h
>
> Make rcu_read_[un]lock() inlined for rcu-preempt.
> Make rcu_read_lock() only one instruction.
> Make rcu_read_unlock() only two instructions in the fast path.
>
> Cc: "Paul E. McKenney" <paulmck@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Frederic Weisbecker <frederic@kernel.org>
> Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>
> ---
> arch/x86/Kconfig | 1 +
> arch/x86/include/asm/current.h | 3 +
> arch/x86/include/asm/rcu_preempt.h | 107 +++++++++++++++++++++++++++++
> arch/x86/kernel/cpu/common.c | 7 +-
> 4 files changed, 115 insertions(+), 3 deletions(-)
> create mode 100644 arch/x86/include/asm/rcu_preempt.h
>
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index 78050d5d7fac..7eb17c12f7b7 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -257,6 +257,7 @@ config X86
> select HAVE_OBJTOOL if X86_64
> select HAVE_OPTPROBES
> select HAVE_PAGE_SIZE_4KB
> + select HAVE_PCPU_RCU_PREEMPT_COUNT
> select HAVE_PCSPKR_PLATFORM
> select HAVE_PERF_EVENTS
> select HAVE_PERF_EVENTS_NMI
> diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
> index bf5953883ec3..dcc2ef784120 100644
> --- a/arch/x86/include/asm/current.h
> +++ b/arch/x86/include/asm/current.h
> @@ -24,6 +24,9 @@ struct pcpu_hot {
> unsigned long top_of_stack;
> void *hardirq_stack_ptr;
> u16 softirq_pending;
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> + int rcu_preempt_count;
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> #ifdef CONFIG_X86_64
> bool hardirq_stack_inuse;
> #else
> diff --git a/arch/x86/include/asm/rcu_preempt.h b/arch/x86/include/asm/rcu_preempt.h
> new file mode 100644
> index 000000000000..cb25ebe038a5
> --- /dev/null
> +++ b/arch/x86/include/asm/rcu_preempt.h
> @@ -0,0 +1,107 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_RCU_PREEMPT_H
> +#define __ASM_RCU_PREEMPT_H
> +
> +#include <asm/rmwcc.h>
> +#include <asm/percpu.h>
> +#include <asm/current.h>
> +
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +/* We use the MSB mostly because its available */
I think you can safely remove the "We " from all the comments :-)
> +#define RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED 0x80000000
How about RCU_PREEMPT_UNLOCK_FASTPATH ?
> +
> +/*
> + * We use the RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED bit as an inverted
> + * current->rcu_read_unlock_special.s such that a decrement hitting 0
> + * means we can and should call rcu_read_unlock_special().
> + */
> +#define RCU_PREEMPT_INIT (0 + RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED)
Or simply:
#define RCU_PREEMPT_INIT RCU_PREEMPT_UNLOCK_FASTPATH
Or you can even remove RCU_PREEMPT_INIT and use RCU_PREEMPT_UNLOCK_FASTPATH directly.
> +/*
> + * Because we keep RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED set when we do
> + * _not_ need to handle unlock-special for a fast-path decrement.
> + */
> +static __always_inline bool pcpu_rcu_preempt_count_dec_and_test(void)
> +{
> + return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.rcu_preempt_count), e,
> + __percpu_arg([var]));
> +}
> +
> +#define pcpu_rcu_read_unlock_special() \
> +do { \
> + rcu_read_unlock_special(); \
> +} while (0)
Why not just call that directly?
> +
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +#endif /* __ASM_RCU_PREEMPT_H */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index ba8cf5e9ce56..0b204a649442 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -1992,9 +1992,10 @@ static __init int setup_clearcpuid(char *arg)
> __setup("clearcpuid=", setup_clearcpuid);
>
> DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
> - .current_task = &init_task,
> - .preempt_count = INIT_PREEMPT_COUNT,
> - .top_of_stack = TOP_OF_INIT_STACK,
> + .current_task = &init_task,
> + .preempt_count = INIT_PREEMPT_COUNT,
> + .top_of_stack = TOP_OF_INIT_STACK,
> + .rcu_preempt_count = RCU_PREEMPT_INIT,
#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT ?
Thanks.
> };
> EXPORT_PER_CPU_SYMBOL(pcpu_hot);
> EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
> --
> 2.19.1.6.gb485710b
>
next prev parent reply other threads:[~2024-04-22 11:05 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-03-28 7:53 [PATCH 00/10] rcu/x86: Use per-cpu rcu preempt count Lai Jiangshan
2024-03-28 7:53 ` [PATCH 01/10] lib: Use rcu_preempt_depth() to replace current->rcu_read_lock_nesting Lai Jiangshan
2024-03-28 7:53 ` [PATCH 02/10] rcu: Move rcu_preempt_depth_set() to rcupdate.h Lai Jiangshan
2024-03-31 11:09 ` Joel Fernandes
2024-03-31 16:16 ` Lai Jiangshan
2024-04-01 2:27 ` Joel Fernandes
2024-04-01 11:40 ` Alan Huang
2024-04-01 11:57 ` Alan Huang
2024-03-28 7:53 ` [PATCH 03/10] rcu: Reorder tree_exp.h after tree_plugin.h Lai Jiangshan
2024-03-28 7:53 ` [PATCH 04/10] rcu: Add macros set_rcu_preempt_special() and clear_rcu_preempt_special() Lai Jiangshan
2024-03-28 7:53 ` [PATCH 05/10] rcu: Make rcu_read_unlock_special() global Lai Jiangshan
2024-03-28 7:53 ` [PATCH 06/10] rcu: Rename marco __LINUX_RCU_H to __KERNEL_RCU_H Lai Jiangshan
2024-03-28 7:53 ` [PATCH 07/10] sched/core: Add rcu_preempt_switch() Lai Jiangshan
2024-03-28 7:53 ` [PATCH 08/10] rcu: Implement PCPU_RCU_PREEMPT_COUNT framework Lai Jiangshan
2024-04-22 10:41 ` Frederic Weisbecker
2024-03-28 7:53 ` [PATCH 09/10] x86/rcu: Add rcu_preempt_count Lai Jiangshan
2024-03-29 7:37 ` kernel test robot
2024-03-29 7:48 ` kernel test robot
2024-04-22 11:05 ` Frederic Weisbecker [this message]
2024-04-23 9:02 ` Lai Jiangshan
2024-04-23 11:33 ` Frederic Weisbecker
2024-03-28 7:53 ` [PATCH 10/10] x86/rcu: Add THUNK rcu_read_unlock_special_thunk Lai Jiangshan
2024-03-29 14:46 ` kernel test robot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=ZiZEcBEvK8NOQvwU@localhost.localdomain \
--to=frederic@kernel.org \
--cc=bp@alien8.de \
--cc=daniel.sneddon@linux.intel.com \
--cc=dave.hansen@linux.intel.com \
--cc=hpa@zytor.com \
--cc=jiangshan.ljs@antgroup.com \
--cc=jiangshanlai@gmail.com \
--cc=jpoimboe@kernel.org \
--cc=kent.overstreet@linux.dev \
--cc=leitao@debian.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=namit@vmware.com \
--cc=nik.borisov@suse.com \
--cc=paulmck@kernel.org \
--cc=pawan.kumar.gupta@linux.intel.com \
--cc=peterz@infradead.org \
--cc=rcu@vger.kernel.org \
--cc=rick.p.edgecombe@intel.com \
--cc=tglx@linutronix.de \
--cc=ubizjak@gmail.com \
--cc=vegard.nossum@oracle.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).