From: Waiman Long <Waiman.Long@hp.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Thomas Gleixner <tglx@linutronix.de>,
"H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org, linux-kernel@vger.kernel.org,
Scott J Norton <scott.norton@hp.com>,
Douglas Hatch <doug.hatch@hp.com>,
Waiman Long <Waiman.Long@hp.com>
Subject: [PATCH 5/7] locking/pvqspinlock: Add pending bit support
Date: Sat, 11 Jul 2015 16:36:56 -0400 [thread overview]
Message-ID: <1436647018-49734-6-git-send-email-Waiman.Long@hp.com> (raw)
In-Reply-To: <1436647018-49734-1-git-send-email-Waiman.Long@hp.com>
Like the native qspinlock, using the pending bit when it is lightly
loaded to acquire the lock is faster than going through the PV queuing
process which is even slower than the native queuing process. It also
avoids loading two additional cachelines (the MCS and PV nodes).
This patch adds the pending bit support for PV qspinlock. The
pending bit code has a smaller spin threshold. It will default back
to the queuing method if it cannot acquired the lock within a certain
time limit.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 27 +++++++++++++++-
kernel/locking/qspinlock_paravirt.h | 61 +++++++++++++++++++++++++++++++++++
2 files changed, 87 insertions(+), 1 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 782bc18..5a25e89 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -162,6 +162,17 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
WRITE_ONCE(l->locked_pending, _Q_LOCKED_VAL);
}
+/**
+ * clear_pending - clear the pending bit.
+ * @lock: Pointer to queued spinlock structure
+ */
+static __always_inline void clear_pending(struct qspinlock *lock)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ WRITE_ONCE(l->pending, 0);
+}
+
/*
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queued spinlock structure
@@ -193,6 +204,15 @@ static __always_inline void clear_pending_set_locked(struct qspinlock *lock)
}
/**
+ * clear_pending - clear the pending bit.
+ * @lock: Pointer to queued spinlock structure
+ */
+static __always_inline void clear_pending(struct qspinlock *lock)
+{
+ atomic_add(-_Q_PENDING_VAL, &lock->val);
+}
+
+/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queued spinlock structure
* @tail : The new queue tail code word
@@ -246,6 +266,7 @@ static __always_inline void __pv_wait_head(struct qspinlock *lock,
struct mcs_spinlock *node) { }
#define pv_enabled() false
+#define pv_pending_lock(l, v) false
#define pv_init_node __pv_init_node
#define pv_wait_node __pv_wait_node
@@ -287,8 +308,11 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
- if (pv_enabled())
+ if (pv_enabled()) {
+ if (pv_pending_lock(lock, val))
+ return; /* Got the lock via pending bit */
goto queue;
+ }
if (virt_queued_spin_lock(lock))
return;
@@ -464,6 +488,7 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath);
#undef pv_wait_node
#undef pv_scan_next
#undef pv_wait_head
+#undef pv_pending_lock
#undef queued_spin_lock_slowpath
#define queued_spin_lock_slowpath __pv_queued_spin_lock_slowpath
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index efc9a72..d770694 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -40,6 +40,7 @@
#define QNODE_SPIN_THRESHOLD SPIN_THRESHOLD
#define QNODE_SPIN_THRESHOLD_SHORT (QNODE_SPIN_THRESHOLD >> 4)
#define QNODE_SPIN_CHECK_MASK 0xff
+#define PENDING_SPIN_THRESHOLD QNODE_SPIN_THRESHOLD_SHORT
/*
* Queue node uses: vcpu_running & vcpu_halted.
@@ -70,6 +71,8 @@ enum pv_qlock_stat {
pvstat_kick_cpu,
pvstat_kick_ahead,
pvstat_no_kick,
+ pvstat_pend_lock,
+ pvstat_pend_fail,
pvstat_spurious,
pvstat_hash,
pvstat_hops,
@@ -91,6 +94,8 @@ static const char * const stat_fsnames[pvstat_num] = {
[pvstat_kick_cpu] = "kick_cpu_count",
[pvstat_kick_ahead] = "kick_ahead_count",
[pvstat_no_kick] = "no_kick_count",
+ [pvstat_pend_lock] = "pending_lock_count",
+ [pvstat_pend_fail] = "pending_fail_count",
[pvstat_spurious] = "spurious_wakeup",
[pvstat_hash] = "hash_count",
[pvstat_hops] = "hash_hops_count",
@@ -355,6 +360,62 @@ static void pv_init_node(struct mcs_spinlock *node)
}
/*
+ * Try to acquire the lock and wait using the pending bit
+ */
+static int pv_pending_lock(struct qspinlock *lock, u32 val)
+{
+ int loop = PENDING_SPIN_THRESHOLD;
+ u32 new, old;
+
+ /*
+ * wait for in-progress pending->locked hand-overs
+ */
+ if (val == _Q_PENDING_VAL) {
+ while (((val = atomic_read(&lock->val)) == _Q_PENDING_VAL) &&
+ loop--)
+ cpu_relax();
+ }
+
+ /*
+ * trylock || pending
+ */
+ for (;;) {
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+ if (loop-- <= 0)
+ goto queue;
+ }
+
+ if (new == _Q_LOCKED_VAL)
+ goto gotlock;
+ /*
+ * We are pending, wait for the owner to go away.
+ */
+ while (((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
+ && (loop-- > 0))
+ cpu_relax();
+ if (!(val & _Q_LOCKED_MASK)) {
+ clear_pending_set_locked(lock);
+ goto gotlock;
+ }
+ clear_pending(lock); /* Clear the pending bit only */
+ pvstat_inc(pvstat_pend_fail);
+
+queue:
+ return 0;
+
+gotlock:
+ pvstat_inc(pvstat_pend_lock);
+ return 1;
+}
+
+/*
* Wait for node->locked to become true, halt the vcpu after a short spin.
* pv_scan_next() is used to set _Q_SLOW_VAL and fill in hash table on its
* behalf.
--
1.7.1
next prev parent reply other threads:[~2015-07-11 20:37 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-07-11 20:36 [PATCH 0/7] locking/qspinlock: Enhance pvqspinlock & introduce queued unfair lock Waiman Long
2015-07-11 20:36 ` [PATCH 1/7] locking/pvqspinlock: Only kick CPU at unlock time Waiman Long
2015-07-13 12:02 ` Peter Zijlstra
2015-07-13 12:31 ` Peter Zijlstra
2015-07-15 1:24 ` Waiman Long
2015-07-13 13:48 ` Peter Zijlstra
2015-07-14 9:31 ` Peter Zijlstra
2015-07-15 1:31 ` Waiman Long
2015-08-03 17:00 ` [tip:locking/core] " tip-bot for Waiman Long
2015-07-11 20:36 ` [PATCH 2/7] locking/pvqspinlock: Allow vCPUs kick-ahead Waiman Long
2015-07-13 13:52 ` Peter Zijlstra
2015-07-15 1:38 ` Waiman Long
2015-07-11 20:36 ` [PATCH 3/7] locking/pvqspinlock: Implement wait-early for overcommitted guest Waiman Long
2015-07-12 8:23 ` Peter Zijlstra
2015-07-13 19:50 ` Davidlohr Bueso
2015-07-15 1:39 ` Waiman Long
2015-07-11 20:36 ` [PATCH 4/7] locking/pvqspinlock: Collect slowpath lock statistics Waiman Long
2015-07-12 8:22 ` Peter Zijlstra
2015-07-14 18:48 ` Waiman Long
2015-07-11 20:36 ` Waiman Long [this message]
2015-07-12 8:21 ` [PATCH 5/7] locking/pvqspinlock: Add pending bit support Peter Zijlstra
2015-07-14 18:47 ` Waiman Long
2015-07-11 20:36 ` [PATCH 6/7] locking/qspinlock: A fairer queued unfair lock Waiman Long
2015-07-12 8:21 ` Peter Zijlstra
2015-07-14 18:47 ` Waiman Long
2015-07-14 20:45 ` Peter Zijlstra
2015-07-11 20:36 ` [PATCH 7/7] locking/qspinlock: Collect queued unfair lock slowpath statistics Waiman Long
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1436647018-49734-6-git-send-email-Waiman.Long@hp.com \
--to=waiman.long@hp.com \
--cc=doug.hatch@hp.com \
--cc=hpa@zytor.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=scott.norton@hp.com \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.