From: Waiman Long <Waiman.Long@hp.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Thomas Gleixner <tglx@linutronix.de>,
"H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org, linux-kernel@vger.kernel.org,
Scott J Norton <scott.norton@hp.com>,
Douglas Hatch <doug.hatch@hp.com>,
Waiman Long <Waiman.Long@hp.com>
Subject: [PATCH 4/7] locking/pvqspinlock: Collect slowpath lock statistics
Date: Sat, 11 Jul 2015 16:36:55 -0400 [thread overview]
Message-ID: <1436647018-49734-5-git-send-email-Waiman.Long@hp.com> (raw)
In-Reply-To: <1436647018-49734-1-git-send-email-Waiman.Long@hp.com>
This patch enables the accumulation of kicking and waiting related
PV qspinlock statistics when the new QUEUED_LOCK_STAT configuration
option is selected. It also enables the collection of kicking and
wakeup latencies which have a heavy dependency on the CPUs being used.
The measured latencies for different CPUs are:
CPU Wakeup Kicking
--- ------ -------
Haswell-EX 26.4us 9.2us
Westmere-EX 99.4US 25.5us
So Haswell is much faster than Westmere.
The accumulated lock statistics will be reported in debugfs under the
pv-qspinlock directory.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/Kconfig | 7 ++
kernel/locking/qspinlock_paravirt.h | 173 ++++++++++++++++++++++++++++++++++-
2 files changed, 177 insertions(+), 3 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 55bced1..299a1c4 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -679,6 +679,13 @@ config PARAVIRT_SPINLOCKS
If you are unsure how to answer this question, answer Y.
+config QUEUED_LOCK_STAT
+ bool "Paravirt queued lock statistics"
+ depends on PARAVIRT && DEBUG_FS && QUEUED_SPINLOCKS
+ ---help---
+ Enable the collection of statistical data on the behavior of
+ paravirtualized queued spinlocks and report them on debugfs.
+
source "arch/x86/xen/Kconfig"
config KVM_GUEST
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index b3fe5bb..efc9a72 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -60,6 +60,155 @@ struct pv_node {
};
/*
+ * PV qspinlock statistics
+ */
+enum pv_qlock_stat {
+ pvstat_wait_head,
+ pvstat_wait_node,
+ pvstat_wait_early,
+ pvstat_kick_wake,
+ pvstat_kick_cpu,
+ pvstat_kick_ahead,
+ pvstat_no_kick,
+ pvstat_spurious,
+ pvstat_hash,
+ pvstat_hops,
+ pvstat_num /* Total number of statistics counts */
+};
+
+#ifdef CONFIG_QUEUED_LOCK_STAT
+/*
+ * Collect pvqspinlock statiatics
+ */
+#include <linux/debugfs.h>
+#include <linux/sched.h>
+
+static const char * const stat_fsnames[pvstat_num] = {
+ [pvstat_wait_head] = "wait_head_count",
+ [pvstat_wait_node] = "wait_node_count",
+ [pvstat_wait_early] = "wait_early_count",
+ [pvstat_kick_wake] = "kick_wake_count",
+ [pvstat_kick_cpu] = "kick_cpu_count",
+ [pvstat_kick_ahead] = "kick_ahead_count",
+ [pvstat_no_kick] = "no_kick_count",
+ [pvstat_spurious] = "spurious_wakeup",
+ [pvstat_hash] = "hash_count",
+ [pvstat_hops] = "hash_hops_count",
+};
+
+static atomic_t pvstats[pvstat_num];
+
+/*
+ * pv_kick_latencies = sum of all pv_kick latencies in ns
+ * pv_wake_latencies = sum of all wakeup latencies in ns
+ *
+ * Avg kick latency = pv_kick_latencies/(kick_cpu_count + kick_ahead_count)
+ * Avg wake latency = pv_wake_latencies/kick_wake_count
+ */
+static atomic64_t pv_kick_latencies, pv_wake_latencies;
+static DEFINE_PER_CPU(u64, pv_kick_time);
+
+/*
+ * Reset all the statistics counts if set
+ */
+static bool reset_cnts __read_mostly;
+
+/*
+ * Initialize debugfs for the PV qspinlock statistics
+ */
+static int __init pv_qspinlock_debugfs(void)
+{
+ struct dentry *d_pvqlock = debugfs_create_dir("pv-qspinlock", NULL);
+ int i;
+
+ if (!d_pvqlock)
+ printk(KERN_WARNING
+ "Could not create 'pv-qspinlock' debugfs directory\n");
+
+ for (i = 0; i < pvstat_num; i++)
+ debugfs_create_u32(stat_fsnames[i], 0444, d_pvqlock,
+ (u32 *)&pvstats[i]);
+ debugfs_create_u64("kick_latencies", 0444, d_pvqlock,
+ (u64 *)&pv_kick_latencies);
+ debugfs_create_u64("wake_latencies", 0444, d_pvqlock,
+ (u64 *)&pv_wake_latencies);
+ debugfs_create_bool("reset_cnts", 0644, d_pvqlock, (u32 *)&reset_cnts);
+ return 0;
+}
+fs_initcall(pv_qspinlock_debugfs);
+
+/*
+ * Reset all the counts
+ */
+static noinline void pvstat_reset(void)
+{
+ int i;
+
+ for (i = 0; i < pvstat_num; i++)
+ atomic_set(&pvstats[i], 0);
+ atomic64_set(&pv_kick_latencies, 0);
+ atomic64_set(&pv_wake_latencies, 0);
+ reset_cnts = 0;
+}
+
+/*
+ * Increment the PV qspinlock statistics counts
+ */
+static inline void pvstat_inc(enum pv_qlock_stat stat)
+{
+ atomic_inc(&pvstats[stat]);
+ if (unlikely(reset_cnts))
+ pvstat_reset();
+}
+
+/*
+ * PV hash hop count
+ */
+static inline void pvstat_hop(int hopcnt)
+{
+ atomic_inc(&pvstats[pvstat_hash]);
+ atomic_add(hopcnt, &pvstats[pvstat_hops]);
+}
+
+/*
+ * Replacement function for pv_kick()
+ */
+static inline void __pv_kick(int cpu)
+{
+ u64 start = sched_clock();
+
+ *per_cpu_ptr(&pv_kick_time, cpu) = start;
+ pv_kick(cpu);
+ atomic64_add(sched_clock() - start, &pv_kick_latencies);
+}
+
+/*
+ * Replacement function for pv_wait()
+ */
+static inline void __pv_wait(u8 *ptr, u8 val)
+{
+ u64 kick_time;
+
+ pv_wait(ptr, val);
+ kick_time = *this_cpu_ptr(&pv_kick_time);
+ if (kick_time) {
+ atomic64_add(sched_clock() - kick_time, &pv_wake_latencies);
+ pvstat_inc(pvstat_kick_wake);
+ *this_cpu_ptr(&pv_kick_time) = 0;
+ }
+}
+
+#define pv_kick(c) __pv_kick(c)
+#define pv_wait(p, v) __pv_wait(p, v)
+
+#else /* CONFIG_QUEUED_LOCK_STAT */
+
+static inline void pvstat_inc(enum pv_qlock_stat stat) { }
+static inline void pvstat_hop(int hopcnt) { }
+
+#endif /* CONFIG_QUEUED_LOCK_STAT */
+
+/*
* Lock and MCS node addresses hash table for fast lookup
*
* Hashing is done on a per-cacheline basis to minimize the need to access
@@ -146,10 +295,13 @@ static struct qspinlock **pv_hash(struct qspinlock *lock, struct pv_node *node)
{
unsigned long offset, hash = hash_ptr(lock, pv_lock_hash_bits);
struct pv_hash_entry *he;
+ int hopcnt = 0;
for_each_hash_entry(he, offset, hash) {
+ hopcnt++;
if (!cmpxchg(&he->lock, NULL, lock)) {
WRITE_ONCE(he->node, node);
+ pvstat_hop(hopcnt);
return &he->lock;
}
}
@@ -221,6 +373,8 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)
prev_halted = (READ_ONCE(pp->state) != vcpu_running);
loop = prev_halted ? QNODE_SPIN_THRESHOLD_SHORT
: QNODE_SPIN_THRESHOLD;
+ if (prev_halted)
+ pvstat_inc(pvstat_wait_early);
while (loop--) {
if (READ_ONCE(node->locked))
return;
@@ -236,6 +390,7 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)
bool halted = (READ_ONCE(pp->state)
!= vcpu_running);
if (!prev_halted && halted) {
+ pvstat_inc(pvstat_wait_early);
break;
} else if (prev_halted && !halted) {
loop = QNODE_SPIN_THRESHOLD;
@@ -256,14 +411,18 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)
*/
smp_store_mb(pn->state, vcpu_halted);
- if (!READ_ONCE(node->locked))
+ if (!READ_ONCE(node->locked)) {
+ pvstat_inc(pvstat_wait_node);
pv_wait(&pn->state, vcpu_halted);
+ }
/*
* Reset the state except when vcpu_hashed is set.
*/
cmpxchg(&pn->state, vcpu_halted, vcpu_running);
+ if (READ_ONCE(node->locked))
+ break;
/*
* If the locked flag is still not set after wakeup, it is a
* spurious wakeup and the vCPU should wait again. However,
@@ -271,6 +430,7 @@ static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev)
* So it is better to spin for a while in the hope that the
* MCS lock will be released soon.
*/
+ pvstat_inc(pvstat_spurious);
}
/*
@@ -372,6 +532,7 @@ static void pv_wait_head(struct qspinlock *lock, struct mcs_spinlock *node)
return;
}
}
+ pvstat_inc(pvstat_wait_head);
pv_wait(&l->locked, _Q_SLOW_VAL);
/*
@@ -459,14 +620,20 @@ __visible void __pv_queued_spin_unlock(struct qspinlock *lock)
* vCPU is harmless other than the additional latency in completing
* the unlock.
*/
- if (READ_ONCE(node->state) == vcpu_hashed)
+ if (READ_ONCE(node->state) == vcpu_hashed) {
+ pvstat_inc(pvstat_kick_cpu);
pv_kick(node->cpu);
+ } else {
+ pvstat_inc(pvstat_no_kick);
+ }
/*
* Kick the next group of vCPUs, if available.
*/
- for (i = 0; i < nr_kick; i++)
+ for (i = 0; i < nr_kick; i++) {
+ pvstat_inc(pvstat_kick_ahead);
pv_kick(next[i]->cpu);
+ }
}
/*
* Include the architecture specific callee-save thunk of the
--
1.7.1
next prev parent reply other threads:[~2015-07-11 20:37 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-07-11 20:36 [PATCH 0/7] locking/qspinlock: Enhance pvqspinlock & introduce queued unfair lock Waiman Long
2015-07-11 20:36 ` [PATCH 1/7] locking/pvqspinlock: Only kick CPU at unlock time Waiman Long
2015-07-13 12:02 ` Peter Zijlstra
2015-07-13 12:31 ` Peter Zijlstra
2015-07-15 1:24 ` Waiman Long
2015-07-13 13:48 ` Peter Zijlstra
2015-07-14 9:31 ` Peter Zijlstra
2015-07-15 1:31 ` Waiman Long
2015-08-03 17:00 ` [tip:locking/core] " tip-bot for Waiman Long
2015-07-11 20:36 ` [PATCH 2/7] locking/pvqspinlock: Allow vCPUs kick-ahead Waiman Long
2015-07-13 13:52 ` Peter Zijlstra
2015-07-15 1:38 ` Waiman Long
2015-07-11 20:36 ` [PATCH 3/7] locking/pvqspinlock: Implement wait-early for overcommitted guest Waiman Long
2015-07-12 8:23 ` Peter Zijlstra
2015-07-13 19:50 ` Davidlohr Bueso
2015-07-15 1:39 ` Waiman Long
2015-07-11 20:36 ` Waiman Long [this message]
2015-07-12 8:22 ` [PATCH 4/7] locking/pvqspinlock: Collect slowpath lock statistics Peter Zijlstra
2015-07-14 18:48 ` Waiman Long
2015-07-11 20:36 ` [PATCH 5/7] locking/pvqspinlock: Add pending bit support Waiman Long
2015-07-12 8:21 ` Peter Zijlstra
2015-07-14 18:47 ` Waiman Long
2015-07-11 20:36 ` [PATCH 6/7] locking/qspinlock: A fairer queued unfair lock Waiman Long
2015-07-12 8:21 ` Peter Zijlstra
2015-07-14 18:47 ` Waiman Long
2015-07-14 20:45 ` Peter Zijlstra
2015-07-11 20:36 ` [PATCH 7/7] locking/qspinlock: Collect queued unfair lock slowpath statistics Waiman Long
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1436647018-49734-5-git-send-email-Waiman.Long@hp.com \
--to=waiman.long@hp.com \
--cc=doug.hatch@hp.com \
--cc=hpa@zytor.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=scott.norton@hp.com \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.