From: "Paul E. McKenney" <paulmck@kernel.org>
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, kernel-team@fb.com,
mingo@kernel.org, jiangshanlai@gmail.com,
akpm@linux-foundation.org, mathieu.desnoyers@efficios.com,
josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com,
fweisbec@gmail.com, oleg@redhat.com, joel@joelfernandes.org,
"Paul E. McKenney" <paulmck@kernel.org>
Subject: [PATCH tip/core/rcu 21/26] rcutorture: Forgive RCU boost failures when CPUs don't pass through QS
Date: Tue, 11 May 2021 16:12:18 -0700 [thread overview]
Message-ID: <20210511231223.2895398-21-paulmck@kernel.org> (raw)
In-Reply-To: <20210511231149.GA2895263@paulmck-ThinkPad-P17-Gen-1>
Currently, rcu_torture_boost() runs CPU-bound at real-time priority
to force RCU priority inversions. It then checks that grace periods
progress during this CPU-bound time. If grace periods fail to progress,
it reports and RCU priority boosting failure.
However, it is possible (and sometimes does happen) that the grace period
fails to progress due to a CPU failing to pass through a quiescent state
for an extended time period (3.5 seconds by default). This can happen
due to vCPU preemption, long-running interrupts, and much else besides.
There is nothing that RCU priority boosting can do about these situations,
and so they should not be counted as RCU priority boosting failures.
This commit therefore checks for CPUs (as opposed to preempted tasks)
holding up a grace period, and flags the resulting RCU priority boosting
failures, but does not splat nor count them as errors. It does rate-limit
them to avoid flooding the console log.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
kernel/rcu/rcu.h | 2 ++
kernel/rcu/rcutorture.c | 67 +++++++++++++++++++++++++----------------
kernel/rcu/tree_stall.h | 36 ++++++++++++++++++++++
3 files changed, 79 insertions(+), 26 deletions(-)
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index bf0827d4b659..daf0cd3f2926 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -519,6 +519,7 @@ static inline unsigned long rcu_exp_batches_completed(void) { return 0; }
static inline unsigned long
srcu_batches_completed(struct srcu_struct *sp) { return 0; }
static inline void rcu_force_quiescent_state(void) { }
+static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { return true; }
static inline void show_rcu_gp_kthreads(void) { }
static inline int rcu_get_gp_kthreads_prio(void) { return 0; }
static inline void rcu_fwd_progress_check(unsigned long j) { }
@@ -527,6 +528,7 @@ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp);
unsigned long rcu_get_gp_seq(void);
unsigned long rcu_exp_batches_completed(void);
unsigned long srcu_batches_completed(struct srcu_struct *sp);
+bool rcu_check_boost_fail(unsigned long gp_state, int *cpup);
void show_rcu_gp_kthreads(void);
int rcu_get_gp_kthreads_prio(void);
void rcu_fwd_progress_check(unsigned long j);
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 02a14dfcae67..5ae4dcc6ba27 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -340,6 +340,7 @@ struct rcu_torture_ops {
void (*fqs)(void);
void (*stats)(void);
void (*gp_kthread_dbg)(void);
+ bool (*check_boost_failed)(unsigned long gp_state, int *cpup);
int (*stall_dur)(void);
int irq_capable;
int can_boost;
@@ -483,31 +484,32 @@ static void rcu_sync_torture_init(void)
}
static struct rcu_torture_ops rcu_ops = {
- .ttype = RCU_FLAVOR,
- .init = rcu_sync_torture_init,
- .readlock = rcu_torture_read_lock,
- .read_delay = rcu_read_delay,
- .readunlock = rcu_torture_read_unlock,
- .readlock_held = torture_readlock_not_held,
- .get_gp_seq = rcu_get_gp_seq,
- .gp_diff = rcu_seq_diff,
- .deferred_free = rcu_torture_deferred_free,
- .sync = synchronize_rcu,
- .exp_sync = synchronize_rcu_expedited,
- .get_gp_state = get_state_synchronize_rcu,
- .start_gp_poll = start_poll_synchronize_rcu,
- .poll_gp_state = poll_state_synchronize_rcu,
- .cond_sync = cond_synchronize_rcu,
- .call = call_rcu,
- .cb_barrier = rcu_barrier,
- .fqs = rcu_force_quiescent_state,
- .stats = NULL,
- .gp_kthread_dbg = show_rcu_gp_kthreads,
- .stall_dur = rcu_jiffies_till_stall_check,
- .irq_capable = 1,
- .can_boost = IS_ENABLED(CONFIG_RCU_BOOST),
- .extendables = RCUTORTURE_MAX_EXTEND,
- .name = "rcu"
+ .ttype = RCU_FLAVOR,
+ .init = rcu_sync_torture_init,
+ .readlock = rcu_torture_read_lock,
+ .read_delay = rcu_read_delay,
+ .readunlock = rcu_torture_read_unlock,
+ .readlock_held = torture_readlock_not_held,
+ .get_gp_seq = rcu_get_gp_seq,
+ .gp_diff = rcu_seq_diff,
+ .deferred_free = rcu_torture_deferred_free,
+ .sync = synchronize_rcu,
+ .exp_sync = synchronize_rcu_expedited,
+ .get_gp_state = get_state_synchronize_rcu,
+ .start_gp_poll = start_poll_synchronize_rcu,
+ .poll_gp_state = poll_state_synchronize_rcu,
+ .cond_sync = cond_synchronize_rcu,
+ .call = call_rcu,
+ .cb_barrier = rcu_barrier,
+ .fqs = rcu_force_quiescent_state,
+ .stats = NULL,
+ .gp_kthread_dbg = show_rcu_gp_kthreads,
+ .check_boost_failed = rcu_check_boost_fail,
+ .stall_dur = rcu_jiffies_till_stall_check,
+ .irq_capable = 1,
+ .can_boost = IS_ENABLED(CONFIG_RCU_BOOST),
+ .extendables = RCUTORTURE_MAX_EXTEND,
+ .name = "rcu"
};
/*
@@ -918,14 +920,27 @@ static void rcu_torture_enable_rt_throttle(void)
static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long start, unsigned long end)
{
+ int cpu;
static int dbg_done;
bool gp_done;
+ unsigned long j;
+ static unsigned long last_persist;
+ unsigned long lp;
+ unsigned long mininterval = test_boost_duration * HZ - HZ / 2;
- if (end - start > test_boost_duration * HZ - HZ / 2) {
+ if (end - start > mininterval) {
// Recheck after checking time to avoid false positives.
smp_mb(); // Time check before grace-period check.
if (cur_ops->poll_gp_state(gp_state))
return false; // passed, though perhaps just barely
+ if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {
+ // At most one persisted message per boost test.
+ j = jiffies;
+ lp = READ_ONCE(last_persist);
+ if (time_after(j, lp + mininterval) && cmpxchg(&last_persist, lp, j) == lp)
+ pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu);
+ return false; // passed on a technicality
+ }
VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
n_rcu_torture_boost_failure++;
if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index 59b95cc5cbdf..af92d9fee0d4 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -716,6 +716,42 @@ static void check_cpu_stall(struct rcu_data *rdp)
// RCU forward-progress mechanisms, including of callback invocation.
+/*
+ * Check to see if a failure to end RCU priority inversion was due to
+ * a CPU not passing through a quiescent state. When this happens, there
+ * is nothing that RCU priority boosting can do to help, so we shouldn't
+ * count this as an RCU priority boosting failure. A return of true says
+ * RCU priority boosting is to blame, and false says otherwise. If false
+ * is returned, the first of the CPUs to blame is stored through cpup.
+ */
+bool rcu_check_boost_fail(unsigned long gp_state, int *cpup)
+{
+ int cpu;
+ unsigned long flags;
+ struct rcu_node *rnp;
+
+ rcu_for_each_leaf_node(rnp) {
+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
+ if (!rnp->qsmask) {
+ // No CPUs without quiescent states for this rnp.
+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
+ continue;
+ }
+ // Find the first holdout CPU.
+ for_each_leaf_node_possible_cpu(rnp, cpu) {
+ if (rnp->qsmask & (1UL << (cpu - rnp->grplo))) {
+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
+ *cpup = cpu;
+ return false;
+ }
+ }
+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
+ }
+ // Can't blame CPUs, so must blame RCU priority boosting.
+ return true;
+}
+EXPORT_SYMBOL_GPL(rcu_check_boost_fail);
+
/*
* Show the state of the grace-period kthreads.
*/
--
2.31.1.189.g2e36527f23
next prev parent reply other threads:[~2021-05-11 23:13 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-11 23:11 [PATCH tip/core/rcu 0/26] Torture-test updates for v5.14 Paul E. McKenney
2021-05-11 23:11 ` [PATCH tip/core/rcu 01/26] torture: Fix remaining erroneous torture.sh instance of $* Paul E. McKenney
2021-05-11 23:11 ` [PATCH tip/core/rcu 02/26] torture: Add "scenarios" option to kvm.sh --dryrun parameter Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 03/26] torture: Make kvm-again.sh use "scenarios" rather than "batches" file Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 04/26] refscale: Allow CPU hotplug to be enabled Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 05/26] rcuscale: " Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 06/26] torture: Add kvm-remote.sh script for distributed rcutorture test runs Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 07/26] refscale: Add acqrel, lock, and lock-irq Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 08/26] rcutorture: Abstract read-lock-held checks Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 09/26] torture: Fix grace-period rate output Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 10/26] torture: Abstract end-of-run summary Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 11/26] torture: Make kvm.sh use abstracted kvm-end-run-stats.sh Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 12/26] torture: Make the build machine control N in "make -jN" Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 13/26] torture: Make kvm-find-errors.sh account for kvm-remote.sh Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 14/26] rcutorture: Judge RCU priority boosting on grace periods, not callbacks Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 15/26] torture: Correctly fetch number of CPUs for non-English languages Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 16/26] torture: Set kvm.sh language to English Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 17/26] rcutorture: Delay-based false positives for RCU priority boosting tests Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 18/26] rcutorture: Consolidate rcu_torture_boost() timing and statistics Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 19/26] rcutorture: Make rcu_torture_boost_failed() check for GP end Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 20/26] rcutorture: Add BUSTED-BOOST to test RCU priority boosting tests Paul E. McKenney
2021-05-11 23:12 ` Paul E. McKenney [this message]
2021-05-11 23:12 ` [PATCH tip/core/rcu 22/26] rcutorture: Don't count CPU-stalled time against priority boosting Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 23/26] torture: Make kvm-remote.sh account for network failure in pathname checks Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 24/26] torture: Don't cap remote runs by build-system number of CPUs Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 25/26] rcutorture: Move mem_dump_obj() tests into separate function Paul E. McKenney
2021-05-11 23:12 ` [PATCH tip/core/rcu 26/26] rcu: Don't penalize priority boosting when there is nothing to boost Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 01/10] kcsan: Add pointer to access-marking.txt to data_race() bullet Paul E. McKenney
2021-05-13 10:47 ` Akira Yokosawa
2021-05-13 10:53 ` Marco Elver
2021-05-13 17:50 ` Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 02/10] kcsan: Simplify value change detection Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 03/10] kcsan: Distinguish kcsan_report() calls Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 04/10] kcsan: Refactor passing watchpoint/other_info Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 05/10] kcsan: Fold panic() call into print_report() Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 06/10] kcsan: Refactor access_info initialization Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 07/10] kcsan: Remove reporting indirection Paul E. McKenney
2021-05-11 23:23 ` [PATCH tip/core/rcu 08/10] kcsan: Remove kcsan_report_type Paul E. McKenney
2021-05-11 23:24 ` [PATCH tip/core/rcu 09/10] kcsan: Report observed value changes Paul E. McKenney
2021-05-11 23:24 ` [PATCH tip/core/rcu 10/10] kcsan: Document "value changed" line Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210511231223.2895398-21-paulmck@kernel.org \
--to=paulmck@kernel.org \
--cc=akpm@linux-foundation.org \
--cc=dhowells@redhat.com \
--cc=edumazet@google.com \
--cc=fweisbec@gmail.com \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=kernel-team@fb.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mingo@kernel.org \
--cc=oleg@redhat.com \
--cc=peterz@infradead.org \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.