2006-01-10 07:59:19 +08:00
|
|
|
/*
|
2013-11-08 15:26:39 +08:00
|
|
|
* kernel/locking/mutex.c
|
2006-01-10 07:59:19 +08:00
|
|
|
*
|
|
|
|
* Mutexes: blocking mutual exclusion locks
|
|
|
|
*
|
|
|
|
* Started by Ingo Molnar:
|
|
|
|
*
|
|
|
|
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
|
|
*
|
|
|
|
* Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
|
|
|
|
* David Howells for suggestions and improvements.
|
|
|
|
*
|
2009-01-12 21:01:47 +08:00
|
|
|
* - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
|
|
|
|
* from the -rt tree, where it was originally implemented for rtmutexes
|
|
|
|
* by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
|
|
|
|
* and Sven Dietrich.
|
|
|
|
*
|
2014-07-31 04:41:55 +08:00
|
|
|
* Also see Documentation/locking/mutex-design.txt.
|
2006-01-10 07:59:19 +08:00
|
|
|
*/
|
|
|
|
#include <linux/mutex.h>
|
2013-07-05 15:29:32 +08:00
|
|
|
#include <linux/ww_mutex.h>
|
2006-01-10 07:59:19 +08:00
|
|
|
#include <linux/sched.h>
|
2013-02-07 23:47:07 +08:00
|
|
|
#include <linux/sched/rt.h>
|
2011-05-24 02:51:41 +08:00
|
|
|
#include <linux/export.h>
|
2006-01-10 07:59:19 +08:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/interrupt.h>
|
2006-07-03 15:24:33 +08:00
|
|
|
#include <linux/debug_locks.h>
|
2015-01-30 17:14:25 +08:00
|
|
|
#include <linux/osq_lock.h>
|
2006-01-10 07:59:19 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
# include "mutex-debug.h"
|
|
|
|
#else
|
|
|
|
# include "mutex.h"
|
|
|
|
#endif
|
|
|
|
|
2006-07-03 15:24:55 +08:00
|
|
|
void
|
|
|
|
__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2016-08-23 19:36:04 +08:00
|
|
|
atomic_long_set(&lock->owner, 0);
|
2006-01-10 07:59:19 +08:00
|
|
|
spin_lock_init(&lock->wait_lock);
|
|
|
|
INIT_LIST_HEAD(&lock->wait_list);
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
2014-07-15 01:27:50 +08:00
|
|
|
osq_lock_init(&lock->osq);
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
#endif
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2006-07-03 15:24:55 +08:00
|
|
|
debug_mutex_init(lock, name, key);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__mutex_init);
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
/*
|
|
|
|
* @owner: contains: 'struct task_struct *' to the current lock owner,
|
|
|
|
* NULL means not owned. Since task_struct pointers are aligned at
|
|
|
|
* ARCH_MIN_TASKALIGN (which is at least sizeof(void *)), we have low
|
|
|
|
* bits to store extra state.
|
|
|
|
*
|
|
|
|
* Bit0 indicates a non-empty waiter list; unlock must issue a wakeup.
|
2016-08-23 20:40:16 +08:00
|
|
|
* Bit1 indicates unlock needs to hand the lock to the top-waiter
|
2016-08-23 19:36:04 +08:00
|
|
|
*/
|
|
|
|
#define MUTEX_FLAG_WAITERS 0x01
|
2016-08-23 20:40:16 +08:00
|
|
|
#define MUTEX_FLAG_HANDOFF 0x02
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
#define MUTEX_FLAGS 0x03
|
|
|
|
|
|
|
|
static inline struct task_struct *__owner_task(unsigned long owner)
|
|
|
|
{
|
|
|
|
return (struct task_struct *)(owner & ~MUTEX_FLAGS);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long __owner_flags(unsigned long owner)
|
|
|
|
{
|
|
|
|
return owner & MUTEX_FLAGS;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Actual trylock that will work on any unlocked state.
|
2016-08-23 20:40:16 +08:00
|
|
|
*
|
|
|
|
* When setting the owner field, we must preserve the low flag bits.
|
|
|
|
*
|
|
|
|
* Be careful with @handoff, only set that in a wait-loop (where you set
|
|
|
|
* HANDOFF) to avoid recursive lock attempts.
|
2016-08-23 19:36:04 +08:00
|
|
|
*/
|
2016-08-23 20:40:16 +08:00
|
|
|
static inline bool __mutex_trylock(struct mutex *lock, const bool handoff)
|
2016-08-23 19:36:04 +08:00
|
|
|
{
|
|
|
|
unsigned long owner, curr = (unsigned long)current;
|
|
|
|
|
|
|
|
owner = atomic_long_read(&lock->owner);
|
|
|
|
for (;;) { /* must loop, can race against a flag */
|
2016-08-23 20:40:16 +08:00
|
|
|
unsigned long old, flags = __owner_flags(owner);
|
|
|
|
|
|
|
|
if (__owner_task(owner)) {
|
|
|
|
if (handoff && unlikely(__owner_task(owner) == current)) {
|
|
|
|
/*
|
|
|
|
* Provide ACQUIRE semantics for the lock-handoff.
|
|
|
|
*
|
|
|
|
* We cannot easily use load-acquire here, since
|
|
|
|
* the actual load is a failed cmpxchg, which
|
|
|
|
* doesn't imply any barriers.
|
|
|
|
*
|
|
|
|
* Also, this is a fairly unlikely scenario, and
|
|
|
|
* this contains the cost.
|
|
|
|
*/
|
|
|
|
smp_mb(); /* ACQUIRE */
|
|
|
|
return true;
|
|
|
|
}
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
return false;
|
2016-08-23 20:40:16 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We set the HANDOFF bit, we must make sure it doesn't live
|
|
|
|
* past the point where we acquire it. This would be possible
|
|
|
|
* if we (accidentally) set the bit on an unlocked mutex.
|
|
|
|
*/
|
|
|
|
if (handoff)
|
|
|
|
flags &= ~MUTEX_FLAG_HANDOFF;
|
2016-08-23 19:36:04 +08:00
|
|
|
|
2016-08-23 20:40:16 +08:00
|
|
|
old = atomic_long_cmpxchg_acquire(&lock->owner, owner, curr | flags);
|
2016-08-23 19:36:04 +08:00
|
|
|
if (old == owner)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
owner = old;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
/*
|
|
|
|
* Lockdep annotations are contained to the slow paths for simplicity.
|
|
|
|
* There is nothing that would stop spreading the lockdep annotations outwards
|
|
|
|
* except more code.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimistic trylock that only works in the uncontended case. Make sure to
|
|
|
|
* follow with a __mutex_trylock() before failing.
|
|
|
|
*/
|
|
|
|
static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
|
|
|
|
{
|
|
|
|
unsigned long curr = (unsigned long)current;
|
|
|
|
|
|
|
|
if (!atomic_long_cmpxchg_acquire(&lock->owner, 0UL, curr))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
|
|
|
|
{
|
|
|
|
unsigned long curr = (unsigned long)current;
|
|
|
|
|
|
|
|
if (atomic_long_cmpxchg_release(&lock->owner, curr, 0UL) == curr)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
|
|
|
|
{
|
|
|
|
atomic_long_or(flag, &lock->owner);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
|
|
|
|
{
|
|
|
|
atomic_long_andnot(flag, &lock->owner);
|
|
|
|
}
|
|
|
|
|
2016-08-23 20:40:16 +08:00
|
|
|
static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
|
|
|
|
{
|
|
|
|
return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Give up ownership to a specific task, when @task = NULL, this is equivalent
|
|
|
|
* to a regular unlock. Clears HANDOFF, preserves WAITERS. Provides RELEASE
|
|
|
|
* semantics like a regular unlock, the __mutex_trylock() provides matching
|
|
|
|
* ACQUIRE semantics for the handoff.
|
|
|
|
*/
|
|
|
|
static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
|
|
|
|
{
|
|
|
|
unsigned long owner = atomic_long_read(&lock->owner);
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
unsigned long old, new;
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
new = (owner & MUTEX_FLAG_WAITERS);
|
|
|
|
new |= (unsigned long)task;
|
|
|
|
|
|
|
|
old = atomic_long_cmpxchg_release(&lock->owner, owner, new);
|
|
|
|
if (old == owner)
|
|
|
|
break;
|
|
|
|
|
|
|
|
owner = old;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-10-12 04:11:12 +08:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
2006-01-10 07:59:19 +08:00
|
|
|
/*
|
|
|
|
* We split the mutex lock/unlock logic into separate fastpath and
|
|
|
|
* slowpath functions, to reduce the register pressure on the fastpath.
|
|
|
|
* We also put the fastpath first in the kernel image, to make sure the
|
|
|
|
* branch is predicted by the CPU as default-untaken.
|
|
|
|
*/
|
2016-08-23 19:36:04 +08:00
|
|
|
static void __sched __mutex_lock_slowpath(struct mutex *lock);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2010-09-03 06:48:16 +08:00
|
|
|
/**
|
2006-01-10 07:59:19 +08:00
|
|
|
* mutex_lock - acquire the mutex
|
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Lock the mutex exclusively for this task. If the mutex is not
|
|
|
|
* available right now, it will sleep until it can get it.
|
|
|
|
*
|
|
|
|
* The mutex must later on be released by the same task that
|
|
|
|
* acquired it. Recursive locking is not allowed. The task
|
|
|
|
* may not exit without first unlocking the mutex. Also, kernel
|
2015-02-02 05:47:32 +08:00
|
|
|
* memory where the mutex resides must not be freed with
|
2006-01-10 07:59:19 +08:00
|
|
|
* the mutex still locked. The mutex must first be initialized
|
|
|
|
* (or statically defined) before it can be locked. memset()-ing
|
|
|
|
* the mutex to 0 is not allowed.
|
|
|
|
*
|
|
|
|
* ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
|
|
|
|
* checks that will enforce the restrictions and will also do
|
|
|
|
* deadlock debugging. )
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) down().
|
|
|
|
*/
|
2009-04-02 08:21:56 +08:00
|
|
|
void __sched mutex_lock(struct mutex *lock)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2006-01-11 05:10:36 +08:00
|
|
|
might_sleep();
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
if (!__mutex_trylock_fast(lock))
|
|
|
|
__mutex_lock_slowpath(lock);
|
|
|
|
}
|
2006-01-10 07:59:19 +08:00
|
|
|
EXPORT_SYMBOL(mutex_lock);
|
2007-10-12 04:11:12 +08:00
|
|
|
#endif
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2014-07-31 04:41:53 +08:00
|
|
|
static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
|
|
|
|
struct ww_acquire_ctx *ww_ctx)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
/*
|
|
|
|
* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
|
|
|
|
* but released with a normal mutex_unlock in this call.
|
|
|
|
*
|
|
|
|
* This should never happen, always use ww_mutex_unlock.
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww->ctx);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Not quite done after calling ww_acquire_done() ?
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
|
|
|
|
|
|
|
|
if (ww_ctx->contending_lock) {
|
|
|
|
/*
|
|
|
|
* After -EDEADLK you tried to
|
|
|
|
* acquire a different ww_mutex? Bad!
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* You called ww_mutex_lock after receiving -EDEADLK,
|
|
|
|
* but 'forgot' to unlock everything else first?
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
|
|
|
|
ww_ctx->contending_lock = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Naughty, using a different class will lead to undefined behavior!
|
|
|
|
*/
|
|
|
|
DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
|
|
|
|
#endif
|
|
|
|
ww_ctx->acquired++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2015-01-07 03:45:06 +08:00
|
|
|
* After acquiring lock with fastpath or when we lost out in contested
|
2014-07-31 04:41:53 +08:00
|
|
|
* slowpath, set ctx and wake up any waiters so they can recheck.
|
|
|
|
*/
|
|
|
|
static __always_inline void
|
|
|
|
ww_mutex_set_context_fastpath(struct ww_mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
struct mutex_waiter *cur;
|
|
|
|
|
|
|
|
ww_mutex_lock_acquired(lock, ctx);
|
|
|
|
|
|
|
|
lock->ctx = ctx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The lock->ctx update should be visible on all cores before
|
|
|
|
* the atomic read is done, otherwise contended waiters might be
|
|
|
|
* missed. The contended waiters will either see ww_ctx == NULL
|
|
|
|
* and keep spinning, or it will acquire wait_lock, add itself
|
|
|
|
* to waiter list and sleep.
|
|
|
|
*/
|
|
|
|
smp_mb(); /* ^^^ */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if lock is contended, if not there is nobody to wake up
|
|
|
|
*/
|
2016-08-23 19:36:04 +08:00
|
|
|
if (likely(!(atomic_long_read(&lock->base.owner) & MUTEX_FLAG_WAITERS)))
|
2014-07-31 04:41:53 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Uh oh, we raced in fastpath, wake up everyone in this case,
|
|
|
|
* so they can see the new lock->ctx.
|
|
|
|
*/
|
|
|
|
spin_lock_mutex(&lock->base.wait_lock, flags);
|
|
|
|
list_for_each_entry(cur, &lock->base.wait_list, list) {
|
|
|
|
debug_mutex_wake_waiter(&lock->base, cur);
|
|
|
|
wake_up_process(cur->task);
|
|
|
|
}
|
|
|
|
spin_unlock_mutex(&lock->base.wait_lock, flags);
|
|
|
|
}
|
|
|
|
|
2015-01-07 03:45:06 +08:00
|
|
|
/*
|
|
|
|
* After acquiring lock in the slowpath set ctx and wake up any
|
|
|
|
* waiters so they can recheck.
|
|
|
|
*
|
|
|
|
* Callers must hold the mutex wait_lock.
|
|
|
|
*/
|
|
|
|
static __always_inline void
|
|
|
|
ww_mutex_set_context_slowpath(struct ww_mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
struct mutex_waiter *cur;
|
|
|
|
|
|
|
|
ww_mutex_lock_acquired(lock, ctx);
|
|
|
|
lock->ctx = ctx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Give any possible sleeping processes the chance to wake up,
|
|
|
|
* so they can recheck if they have to back off.
|
|
|
|
*/
|
|
|
|
list_for_each_entry(cur, &lock->base.wait_list, list) {
|
|
|
|
debug_mutex_wake_waiter(&lock->base, cur);
|
|
|
|
wake_up_process(cur->task);
|
|
|
|
}
|
|
|
|
}
|
2014-07-31 04:41:53 +08:00
|
|
|
|
2013-04-18 03:23:11 +08:00
|
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
|
|
|
/*
|
|
|
|
* Look out! "owner" is an entirely speculative pointer
|
|
|
|
* access and not reliable.
|
|
|
|
*/
|
|
|
|
static noinline
|
2015-02-03 05:59:27 +08:00
|
|
|
bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
|
2013-04-18 03:23:11 +08:00
|
|
|
{
|
2015-04-09 03:39:19 +08:00
|
|
|
bool ret = true;
|
2015-02-03 05:59:27 +08:00
|
|
|
|
2013-04-18 03:23:11 +08:00
|
|
|
rcu_read_lock();
|
2016-08-23 19:36:04 +08:00
|
|
|
while (__mutex_owner(lock) == owner) {
|
2015-02-03 05:59:27 +08:00
|
|
|
/*
|
|
|
|
* Ensure we emit the owner->on_cpu, dereference _after_
|
2015-04-09 03:39:19 +08:00
|
|
|
* checking lock->owner still matches owner. If that fails,
|
|
|
|
* owner might point to freed memory. If it still matches,
|
2015-02-03 05:59:27 +08:00
|
|
|
* the rcu_read_lock() ensures the memory stays valid.
|
|
|
|
*/
|
|
|
|
barrier();
|
|
|
|
|
|
|
|
if (!owner->on_cpu || need_resched()) {
|
|
|
|
ret = false;
|
|
|
|
break;
|
|
|
|
}
|
2013-04-18 03:23:11 +08:00
|
|
|
|
arch, locking: Ciao arch_mutex_cpu_relax()
The arch_mutex_cpu_relax() function, introduced by 34b133f, is
hacky and ugly. It was added a few years ago to address the fact
that common cpu_relax() calls include yielding on s390, and thus
impact the optimistic spinning functionality of mutexes. Nowadays
we use this function well beyond mutexes: rwsem, qrwlock, mcs and
lockref. Since the macro that defines the call is in the mutex header,
any users must include mutex.h and the naming is misleading as well.
This patch (i) renames the call to cpu_relax_lowlatency ("relax, but
only if you can do it with very low latency") and (ii) defines it in
each arch's asm/processor.h local header, just like for regular cpu_relax
functions. On all archs, except s390, cpu_relax_lowlatency is simply cpu_relax,
and thus we can take it out of mutex.h. While this can seem redundant,
I believe it is a good choice as it allows us to move out arch specific
logic from generic locking primitives and enables future(?) archs to
transparently define it, similarly to System Z.
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Anton Blanchard <anton@samba.org>
Cc: Aurelien Jacquiot <a-jacquiot@ti.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Bharat Bhushan <r65777@freescale.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chen Liqin <liqin.linux@gmail.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: David Howells <dhowells@redhat.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Deepthi Dharwar <deepthi@linux.vnet.ibm.com>
Cc: Dominik Dingel <dingel@linux.vnet.ibm.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
Cc: Haavard Skinnemoen <hskinnemoen@gmail.com>
Cc: Hans-Christian Egtvedt <egtvedt@samfundet.no>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: James E.J. Bottomley <jejb@parisc-linux.org>
Cc: James Hogan <james.hogan@imgtec.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: Joe Perches <joe@perches.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Joseph Myers <joseph@codesourcery.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: Lennox Wu <lennox.wu@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mark Salter <msalter@redhat.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Neuling <mikey@neuling.org>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Mikael Starvik <starvik@axis.com>
Cc: Nicolas Pitre <nico@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Paul Burton <paul.burton@imgtec.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Qais Yousef <qais.yousef@imgtec.com>
Cc: Qiaowei Ren <qiaowei.ren@intel.com>
Cc: Rafael Wysocki <rafael.j.wysocki@intel.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Steven Miao <realmz6@gmail.com>
Cc: Steven Rostedt <srostedt@redhat.com>
Cc: Stratos Karafotis <stratosk@semaphore.gr>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Vasily Kulikov <segoon@openwall.com>
Cc: Vineet Gupta <vgupta@synopsys.com>
Cc: Vineet Gupta <Vineet.Gupta1@synopsys.com>
Cc: Waiman Long <Waiman.Long@hp.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Wolfram Sang <wsa@the-dreams.de>
Cc: adi-buildroot-devel@lists.sourceforge.net
Cc: linux390@de.ibm.com
Cc: linux-alpha@vger.kernel.org
Cc: linux-am33-list@redhat.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-c6x-dev@linux-c6x.org
Cc: linux-cris-kernel@axis.com
Cc: linux-hexagon@vger.kernel.org
Cc: linux-ia64@vger.kernel.org
Cc: linux@lists.openrisc.net
Cc: linux-m32r-ja@ml.linux-m32r.org
Cc: linux-m32r@ml.linux-m32r.org
Cc: linux-m68k@lists.linux-m68k.org
Cc: linux-metag@vger.kernel.org
Cc: linux-mips@linux-mips.org
Cc: linux-parisc@vger.kernel.org
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-s390@vger.kernel.org
Cc: linux-sh@vger.kernel.org
Cc: linux-xtensa@linux-xtensa.org
Cc: sparclinux@vger.kernel.org
Link: http://lkml.kernel.org/r/1404079773.2619.4.camel@buesod1.americas.hpqcorp.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-06-30 06:09:33 +08:00
|
|
|
cpu_relax_lowlatency();
|
2013-04-18 03:23:11 +08:00
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
2015-02-03 05:59:27 +08:00
|
|
|
return ret;
|
2013-04-18 03:23:11 +08:00
|
|
|
}
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Initial check for entering the mutex spinning loop
|
|
|
|
*/
|
|
|
|
static inline int mutex_can_spin_on_owner(struct mutex *lock)
|
|
|
|
{
|
2013-07-20 02:31:01 +08:00
|
|
|
struct task_struct *owner;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
int retval = 1;
|
|
|
|
|
2014-01-29 03:13:12 +08:00
|
|
|
if (need_resched())
|
|
|
|
return 0;
|
|
|
|
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
rcu_read_lock();
|
2016-08-23 19:36:04 +08:00
|
|
|
owner = __mutex_owner(lock);
|
2013-07-20 02:31:01 +08:00
|
|
|
if (owner)
|
|
|
|
retval = owner->on_cpu;
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
rcu_read_unlock();
|
2016-08-23 19:36:04 +08:00
|
|
|
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
/*
|
2016-08-23 19:36:04 +08:00
|
|
|
* If lock->owner is not set, the mutex has been released. Return true
|
|
|
|
* such that we'll trylock in the spin path, which is a faster option
|
|
|
|
* than the blocking slow path.
|
mutex: Queue mutex spinners with MCS lock to reduce cacheline contention
The current mutex spinning code (with MUTEX_SPIN_ON_OWNER option
turned on) allow multiple tasks to spin on a single mutex
concurrently. A potential problem with the current approach is
that when the mutex becomes available, all the spinning tasks
will try to acquire the mutex more or less simultaneously. As a
result, there will be a lot of cacheline bouncing especially on
systems with a large number of CPUs.
This patch tries to reduce this kind of contention by putting
the mutex spinners into a queue so that only the first one in
the queue will try to acquire the mutex. This will reduce
contention and allow all the tasks to move forward faster.
The queuing of mutex spinners is done using an MCS lock based
implementation which will further reduce contention on the mutex
cacheline than a similar ticket spinlock based implementation.
This patch will add a new field into the mutex data structure
for holding the MCS lock. This expands the mutex size by 8 bytes
for 64-bit system and 4 bytes for 32-bit system. This overhead
will be avoid if the MUTEX_SPIN_ON_OWNER option is turned off.
The following table shows the jobs per minute (JPM) scalability
data on an 8-node 80-core Westmere box with a 3.7.10 kernel. The
numactl command is used to restrict the running of the fserver
workloads to 1/2/4/8 nodes with hyperthreading off.
+-----------------+-----------+-----------+-------------+----------+
| Configuration | Mean JPM | Mean JPM | Mean JPM | % Change |
| | w/o patch | patch 1 | patches 1&2 | 1->1&2 |
+-----------------+------------------------------------------------+
| | User Range 1100 - 2000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 227972 | 227237 | 305043 | +34.2% |
| 4 nodes, HT off | 393503 | 381558 | 394650 | +3.4% |
| 2 nodes, HT off | 334957 | 325240 | 338853 | +4.2% |
| 1 node , HT off | 198141 | 197972 | 198075 | +0.1% |
+-----------------+------------------------------------------------+
| | User Range 200 - 1000 |
+-----------------+------------------------------------------------+
| 8 nodes, HT off | 282325 | 312870 | 332185 | +6.2% |
| 4 nodes, HT off | 390698 | 378279 | 393419 | +4.0% |
| 2 nodes, HT off | 336986 | 326543 | 340260 | +4.2% |
| 1 node , HT off | 197588 | 197622 | 197582 | 0.0% |
+-----------------+-----------+-----------+-------------+----------+
At low user range 10-100, the JPM differences were within +/-1%.
So they are not that interesting.
The fserver workload uses mutex spinning extensively. With just
the mutex change in the first patch, there is no noticeable
change in performance. Rather, there is a slight drop in
performance. This mutex spinning patch more than recovers the
lost performance and show a significant increase of +30% at high
user load with the full 8 nodes. Similar improvements were also
seen in a 3.8 kernel.
The table below shows the %time spent by different kernel
functions as reported by perf when running the fserver workload
at 1500 users with all 8 nodes.
+-----------------------+-----------+---------+-------------+
| Function | % time | % time | % time |
| | w/o patch | patch 1 | patches 1&2 |
+-----------------------+-----------+---------+-------------+
| __read_lock_failed | 34.96% | 34.91% | 29.14% |
| __write_lock_failed | 10.14% | 10.68% | 7.51% |
| mutex_spin_on_owner | 3.62% | 3.42% | 2.33% |
| mspin_lock | N/A | N/A | 9.90% |
| __mutex_lock_slowpath | 1.46% | 0.81% | 0.14% |
| _raw_spin_lock | 2.25% | 2.50% | 1.10% |
+-----------------------+-----------+---------+-------------+
The fserver workload for an 8-node system is dominated by the
contention in the read/write lock. Mutex contention also plays a
role. With the first patch only, mutex contention is down (as
shown by the __mutex_lock_slowpath figure) which help a little
bit. We saw only a few percents improvement with that.
By applying patch 2 as well, the single mutex_spin_on_owner
figure is now split out into an additional mspin_lock figure.
The time increases from 3.42% to 11.23%. It shows a great
reduction in contention among the spinners leading to a 30%
improvement. The time ratio 9.9/2.33=4.3 indicates that there
are on average 4+ spinners waiting in the spin_lock loop for
each spinner in the mutex_spin_on_owner loop. Contention in
other locking functions also go down by quite a lot.
The table below shows the performance change of both patches 1 &
2 over patch 1 alone in other AIM7 workloads (at 8 nodes,
hyperthreading off).
+--------------+---------------+----------------+-----------------+
| Workload | mean % change | mean % change | mean % change |
| | 10-100 users | 200-1000 users | 1100-2000 users |
+--------------+---------------+----------------+-----------------+
| alltests | 0.0% | -0.8% | +0.6% |
| five_sec | -0.3% | +0.8% | +0.8% |
| high_systime | +0.4% | +2.4% | +2.1% |
| new_fserver | +0.1% | +14.1% | +34.2% |
| shared | -0.5% | -0.3% | -0.4% |
| short | -1.7% | -9.8% | -8.3% |
+--------------+---------------+----------------+-----------------+
The short workload is the only one that shows a decline in
performance probably due to the spinner locking and queuing
overhead.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Reviewed-by: Davidlohr Bueso <davidlohr.bueso@hp.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Chandramouleeswaran Aswin <aswin@hp.com>
Cc: Norton Scott J <scott.norton@hp.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Dave Jones <davej@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1366226594-5506-4-git-send-email-Waiman.Long@hp.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-04-18 03:23:13 +08:00
|
|
|
*/
|
|
|
|
return retval;
|
|
|
|
}
|
2014-07-31 04:41:53 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Optimistic spinning.
|
|
|
|
*
|
|
|
|
* We try to spin for acquisition when we find that the lock owner
|
|
|
|
* is currently running on a (different) CPU and while we don't
|
|
|
|
* need to reschedule. The rationale is that if the lock owner is
|
|
|
|
* running, it is likely to release the lock soon.
|
|
|
|
*
|
|
|
|
* The mutex spinners are queued up using MCS lock so that only one
|
|
|
|
* spinner can compete for the mutex. However, if mutex spinning isn't
|
|
|
|
* going to happen, there is no point in going through the lock/unlock
|
|
|
|
* overhead.
|
|
|
|
*
|
|
|
|
* Returns true when the lock was taken, otherwise false, indicating
|
|
|
|
* that we need to jump to the slowpath and sleep.
|
|
|
|
*/
|
|
|
|
static bool mutex_optimistic_spin(struct mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
|
|
|
{
|
|
|
|
struct task_struct *task = current;
|
|
|
|
|
|
|
|
if (!mutex_can_spin_on_owner(lock))
|
|
|
|
goto done;
|
|
|
|
|
2015-01-07 03:45:05 +08:00
|
|
|
/*
|
|
|
|
* In order to avoid a stampede of mutex spinners trying to
|
|
|
|
* acquire the mutex all at once, the spinners need to take a
|
|
|
|
* MCS (queued) lock first before spinning on the owner field.
|
|
|
|
*/
|
2014-07-31 04:41:53 +08:00
|
|
|
if (!osq_lock(&lock->osq))
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
struct task_struct *owner;
|
|
|
|
|
|
|
|
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
|
|
|
struct ww_mutex *ww;
|
|
|
|
|
|
|
|
ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
/*
|
|
|
|
* If ww->ctx is set the contents are undefined, only
|
|
|
|
* by acquiring wait_lock there is a guarantee that
|
|
|
|
* they are not invalid when reading.
|
|
|
|
*
|
|
|
|
* As such, when deadlock detection needs to be
|
|
|
|
* performed the optimistic spinning cannot be done.
|
|
|
|
*/
|
2015-02-23 11:31:41 +08:00
|
|
|
if (READ_ONCE(ww->ctx))
|
2014-07-31 04:41:53 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there's an owner, wait for it to either
|
|
|
|
* release the lock or go to sleep.
|
|
|
|
*/
|
2016-08-23 19:36:04 +08:00
|
|
|
owner = __mutex_owner(lock);
|
2014-07-31 04:41:53 +08:00
|
|
|
if (owner && !mutex_spin_on_owner(lock, owner))
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* Try to acquire the mutex if it is unlocked. */
|
2016-08-23 20:40:16 +08:00
|
|
|
if (__mutex_trylock(lock, false)) {
|
2014-07-31 04:41:53 +08:00
|
|
|
osq_unlock(&lock->osq);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The cpu_relax() call is a compiler barrier which forces
|
|
|
|
* everything in this loop to be re-loaded. We don't need
|
|
|
|
* memory barriers as we'll eventually observe the right
|
|
|
|
* values at the cost of a few extra spins.
|
|
|
|
*/
|
|
|
|
cpu_relax_lowlatency();
|
|
|
|
}
|
|
|
|
|
|
|
|
osq_unlock(&lock->osq);
|
|
|
|
done:
|
|
|
|
/*
|
|
|
|
* If we fell out of the spin path because of need_resched(),
|
|
|
|
* reschedule now, before we try-lock the mutex. This avoids getting
|
|
|
|
* scheduled out right after we obtained the mutex.
|
|
|
|
*/
|
2014-09-24 16:18:46 +08:00
|
|
|
if (need_resched()) {
|
|
|
|
/*
|
|
|
|
* We _should_ have TASK_RUNNING here, but just in case
|
|
|
|
* we do not, make it so, otherwise we might get stuck.
|
|
|
|
*/
|
|
|
|
__set_current_state(TASK_RUNNING);
|
2014-07-31 04:41:53 +08:00
|
|
|
schedule_preempt_disabled();
|
2014-09-24 16:18:46 +08:00
|
|
|
}
|
2014-07-31 04:41:53 +08:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static bool mutex_optimistic_spin(struct mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
2013-04-18 03:23:11 +08:00
|
|
|
#endif
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2010-09-03 06:48:16 +08:00
|
|
|
/**
|
2006-01-10 07:59:19 +08:00
|
|
|
* mutex_unlock - release the mutex
|
|
|
|
* @lock: the mutex to be released
|
|
|
|
*
|
|
|
|
* Unlock a mutex that has been locked by this task previously.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. Unlocking
|
|
|
|
* of a not locked mutex is not allowed.
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) up().
|
|
|
|
*/
|
2008-02-08 20:19:53 +08:00
|
|
|
void __sched mutex_unlock(struct mutex *lock)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2016-08-23 19:36:04 +08:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
if (__mutex_unlock_fast(lock))
|
|
|
|
return;
|
2009-01-12 21:01:47 +08:00
|
|
|
#endif
|
2016-08-23 19:36:04 +08:00
|
|
|
__mutex_unlock_slowpath(lock, _RET_IP_);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mutex_unlock);
|
|
|
|
|
2013-06-24 16:30:04 +08:00
|
|
|
/**
|
|
|
|
* ww_mutex_unlock - release the w/w mutex
|
|
|
|
* @lock: the mutex to be released
|
|
|
|
*
|
|
|
|
* Unlock a mutex that has been locked by this task previously with any of the
|
|
|
|
* ww_mutex_lock* functions (with or without an acquire context). It is
|
|
|
|
* forbidden to release the locks after releasing the acquire context.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. Unlocking
|
|
|
|
* of a unlocked mutex is not allowed.
|
|
|
|
*/
|
|
|
|
void __sched ww_mutex_unlock(struct ww_mutex *lock)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The unlocking fastpath is the 0->1 transition from 'locked'
|
|
|
|
* into 'unlocked' state:
|
|
|
|
*/
|
|
|
|
if (lock->ctx) {
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
|
|
|
|
#endif
|
|
|
|
if (lock->ctx->acquired > 0)
|
|
|
|
lock->ctx->acquired--;
|
|
|
|
lock->ctx = NULL;
|
|
|
|
}
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
mutex_unlock(&lock->base);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(ww_mutex_unlock);
|
|
|
|
|
|
|
|
static inline int __sched
|
2015-01-07 03:45:04 +08:00
|
|
|
__ww_mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
|
2013-06-24 16:30:04 +08:00
|
|
|
{
|
|
|
|
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
|
2015-02-23 11:31:41 +08:00
|
|
|
struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
|
2013-06-24 16:30:04 +08:00
|
|
|
|
|
|
|
if (!hold_ctx)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
|
|
|
|
(ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
|
|
|
|
ctx->contending_lock = ww;
|
|
|
|
#endif
|
|
|
|
return -EDEADLK;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-01-10 07:59:19 +08:00
|
|
|
/*
|
|
|
|
* Lock a mutex (possibly interruptible), slowpath:
|
|
|
|
*/
|
2013-06-24 16:30:04 +08:00
|
|
|
static __always_inline int __sched
|
2007-10-12 04:11:12 +08:00
|
|
|
__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
|
2013-06-24 16:30:04 +08:00
|
|
|
struct lockdep_map *nest_lock, unsigned long ip,
|
2013-10-17 18:45:29 +08:00
|
|
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
|
|
|
struct task_struct *task = current;
|
|
|
|
struct mutex_waiter waiter;
|
2006-06-26 15:24:31 +08:00
|
|
|
unsigned long flags;
|
2016-08-23 20:40:16 +08:00
|
|
|
bool first = false;
|
2013-06-24 16:30:04 +08:00
|
|
|
int ret;
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2016-05-27 04:08:17 +08:00
|
|
|
if (use_ww_ctx) {
|
|
|
|
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
|
|
|
|
return -EALREADY;
|
|
|
|
}
|
|
|
|
|
2009-01-14 22:36:26 +08:00
|
|
|
preempt_disable();
|
2011-05-25 08:12:03 +08:00
|
|
|
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
|
2009-12-03 03:49:16 +08:00
|
|
|
|
2016-08-23 20:40:16 +08:00
|
|
|
if (__mutex_trylock(lock, false) ||
|
|
|
|
mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
|
2014-07-31 04:41:53 +08:00
|
|
|
/* got the lock, yay! */
|
2016-08-23 19:36:04 +08:00
|
|
|
lock_acquired(&lock->dep_map, ip);
|
|
|
|
if (use_ww_ctx) {
|
|
|
|
struct ww_mutex *ww;
|
|
|
|
ww = container_of(lock, struct ww_mutex, base);
|
|
|
|
|
|
|
|
ww_mutex_set_context_fastpath(ww, ww_ctx);
|
|
|
|
}
|
2014-07-31 04:41:53 +08:00
|
|
|
preempt_enable();
|
|
|
|
return 0;
|
2009-01-12 21:01:47 +08:00
|
|
|
}
|
2014-07-31 04:41:53 +08:00
|
|
|
|
2006-06-26 15:24:31 +08:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2014-06-12 02:37:21 +08:00
|
|
|
/*
|
2016-08-23 19:36:04 +08:00
|
|
|
* After waiting to acquire the wait_lock, try again.
|
2014-06-12 02:37:21 +08:00
|
|
|
*/
|
2016-08-23 20:40:16 +08:00
|
|
|
if (__mutex_trylock(lock, false))
|
2013-06-29 04:13:18 +08:00
|
|
|
goto skip_wait;
|
|
|
|
|
2006-07-03 15:24:33 +08:00
|
|
|
debug_mutex_lock_common(lock, &waiter);
|
2016-06-24 03:11:17 +08:00
|
|
|
debug_mutex_add_waiter(lock, &waiter, task);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
|
|
|
/* add waiting tasks to the end of the waitqueue (FIFO): */
|
|
|
|
list_add_tail(&waiter.list, &lock->wait_list);
|
|
|
|
waiter.task = task;
|
|
|
|
|
2016-08-23 20:40:16 +08:00
|
|
|
if (__mutex_waiter_is_first(lock, &waiter))
|
2016-08-23 19:36:04 +08:00
|
|
|
__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
|
|
|
|
|
2007-10-12 04:11:12 +08:00
|
|
|
lock_contended(&lock->dep_map, ip);
|
2007-07-19 16:48:58 +08:00
|
|
|
|
2016-09-02 19:42:12 +08:00
|
|
|
set_task_state(task, state);
|
2006-01-10 07:59:19 +08:00
|
|
|
for (;;) {
|
2016-09-02 19:42:12 +08:00
|
|
|
/*
|
|
|
|
* Once we hold wait_lock, we're serialized against
|
|
|
|
* mutex_unlock() handing the lock off to us, do a trylock
|
|
|
|
* before testing the error conditions to make sure we pick up
|
|
|
|
* the handoff.
|
|
|
|
*/
|
2016-08-23 20:40:16 +08:00
|
|
|
if (__mutex_trylock(lock, first))
|
2016-09-02 19:42:12 +08:00
|
|
|
goto acquired;
|
2006-01-10 07:59:19 +08:00
|
|
|
|
|
|
|
/*
|
2016-09-02 19:42:12 +08:00
|
|
|
* Check for signals and wound conditions while holding
|
|
|
|
* wait_lock. This ensures the lock cancellation is ordered
|
|
|
|
* against mutex_unlock() and wake-ups do not go missing.
|
2006-01-10 07:59:19 +08:00
|
|
|
*/
|
2008-06-09 01:20:42 +08:00
|
|
|
if (unlikely(signal_pending_state(state, task))) {
|
2013-06-24 16:30:04 +08:00
|
|
|
ret = -EINTR;
|
|
|
|
goto err;
|
|
|
|
}
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2013-10-17 18:45:29 +08:00
|
|
|
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
2015-01-07 03:45:04 +08:00
|
|
|
ret = __ww_mutex_lock_check_stamp(lock, ww_ctx);
|
2013-06-24 16:30:04 +08:00
|
|
|
if (ret)
|
|
|
|
goto err;
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
2013-06-24 16:30:04 +08:00
|
|
|
|
2006-06-26 15:24:31 +08:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2011-03-21 19:33:18 +08:00
|
|
|
schedule_preempt_disabled();
|
2016-08-23 20:40:16 +08:00
|
|
|
|
|
|
|
if (!first && __mutex_waiter_is_first(lock, &waiter)) {
|
|
|
|
first = true;
|
|
|
|
__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
|
|
|
|
}
|
2016-09-02 19:42:12 +08:00
|
|
|
|
|
|
|
set_task_state(task, state);
|
|
|
|
/*
|
|
|
|
* Here we order against unlock; we must either see it change
|
|
|
|
* state back to RUNNING and fall through the next schedule(),
|
|
|
|
* or we must see its unlock and acquire.
|
|
|
|
*/
|
|
|
|
if (__mutex_trylock(lock, first))
|
|
|
|
break;
|
|
|
|
|
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
2016-09-02 19:42:12 +08:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
|
|
|
acquired:
|
2015-01-20 09:39:21 +08:00
|
|
|
__set_task_state(task, TASK_RUNNING);
|
|
|
|
|
2016-06-24 03:11:17 +08:00
|
|
|
mutex_remove_waiter(lock, &waiter, task);
|
2013-06-29 04:13:18 +08:00
|
|
|
if (likely(list_empty(&lock->wait_list)))
|
2016-08-23 20:40:16 +08:00
|
|
|
__mutex_clear_flag(lock, MUTEX_FLAGS);
|
2016-08-23 19:36:04 +08:00
|
|
|
|
2013-06-29 04:13:18 +08:00
|
|
|
debug_mutex_free_waiter(&waiter);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2013-06-29 04:13:18 +08:00
|
|
|
skip_wait:
|
|
|
|
/* got the lock - cleanup and rejoice! */
|
2008-10-17 05:17:09 +08:00
|
|
|
lock_acquired(&lock->dep_map, ip);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2013-10-17 18:45:29 +08:00
|
|
|
if (use_ww_ctx) {
|
2013-06-29 04:13:18 +08:00
|
|
|
struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
|
2015-01-07 03:45:06 +08:00
|
|
|
ww_mutex_set_context_slowpath(ww, ww_ctx);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
|
2006-06-26 15:24:31 +08:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2009-01-14 22:36:26 +08:00
|
|
|
preempt_enable();
|
2006-01-10 07:59:19 +08:00
|
|
|
return 0;
|
2013-06-24 16:30:04 +08:00
|
|
|
|
|
|
|
err:
|
2016-09-02 19:42:12 +08:00
|
|
|
__set_task_state(task, TASK_RUNNING);
|
2016-06-24 03:11:17 +08:00
|
|
|
mutex_remove_waiter(lock, &waiter, task);
|
2013-06-24 16:30:04 +08:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
|
|
|
debug_mutex_free_waiter(&waiter);
|
|
|
|
mutex_release(&lock->dep_map, 1, ip);
|
|
|
|
preempt_enable();
|
|
|
|
return ret;
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
|
2006-07-03 15:24:55 +08:00
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
void __sched
|
|
|
|
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 16:30:04 +08:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2006-07-03 15:24:55 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_nested);
|
2006-12-08 18:36:17 +08:00
|
|
|
|
2011-05-25 08:12:03 +08:00
|
|
|
void __sched
|
|
|
|
_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 16:30:04 +08:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
0, nest, _RET_IP_, NULL, 0);
|
2011-05-25 08:12:03 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
|
|
|
|
|
2007-12-07 06:37:59 +08:00
|
|
|
int __sched
|
|
|
|
mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2013-06-24 16:30:04 +08:00
|
|
|
return __mutex_lock_common(lock, TASK_KILLABLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2007-12-07 06:37:59 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
|
|
|
|
|
2006-12-08 18:36:17 +08:00
|
|
|
int __sched
|
|
|
|
mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
|
|
|
|
{
|
|
|
|
might_sleep();
|
2009-01-12 21:01:47 +08:00
|
|
|
return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
subclass, NULL, _RET_IP_, NULL, 0);
|
2006-12-08 18:36:17 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
|
2013-06-24 16:30:04 +08:00
|
|
|
|
2013-06-20 19:31:17 +08:00
|
|
|
static inline int
|
|
|
|
ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
|
|
|
|
unsigned tmp;
|
|
|
|
|
|
|
|
if (ctx->deadlock_inject_countdown-- == 0) {
|
|
|
|
tmp = ctx->deadlock_inject_interval;
|
|
|
|
if (tmp > UINT_MAX/4)
|
|
|
|
tmp = UINT_MAX;
|
|
|
|
else
|
|
|
|
tmp = tmp*2 + tmp + tmp/2;
|
|
|
|
|
|
|
|
ctx->deadlock_inject_interval = tmp;
|
|
|
|
ctx->deadlock_inject_countdown = tmp;
|
|
|
|
ctx->contending_lock = lock;
|
|
|
|
|
|
|
|
ww_mutex_unlock(lock);
|
|
|
|
|
|
|
|
return -EDEADLK;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2013-06-24 16:30:04 +08:00
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
2013-06-20 19:31:17 +08:00
|
|
|
int ret;
|
|
|
|
|
2013-06-24 16:30:04 +08:00
|
|
|
might_sleep();
|
2013-06-20 19:31:17 +08:00
|
|
|
ret = __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
0, &ctx->dep_map, _RET_IP_, ctx, 1);
|
2013-07-30 16:13:41 +08:00
|
|
|
if (!ret && ctx->acquired > 1)
|
2013-06-20 19:31:17 +08:00
|
|
|
return ww_mutex_deadlock_injection(lock, ctx);
|
|
|
|
|
|
|
|
return ret;
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__ww_mutex_lock);
|
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
2013-06-20 19:31:17 +08:00
|
|
|
int ret;
|
|
|
|
|
2013-06-24 16:30:04 +08:00
|
|
|
might_sleep();
|
2013-06-20 19:31:17 +08:00
|
|
|
ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
|
2013-10-17 18:45:29 +08:00
|
|
|
0, &ctx->dep_map, _RET_IP_, ctx, 1);
|
2013-06-20 19:31:17 +08:00
|
|
|
|
2013-07-30 16:13:41 +08:00
|
|
|
if (!ret && ctx->acquired > 1)
|
2013-06-20 19:31:17 +08:00
|
|
|
return ww_mutex_deadlock_injection(lock, ctx);
|
|
|
|
|
|
|
|
return ret;
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
|
|
|
|
|
2006-07-03 15:24:55 +08:00
|
|
|
#endif
|
|
|
|
|
2006-01-10 07:59:19 +08:00
|
|
|
/*
|
|
|
|
* Release the lock, slowpath:
|
|
|
|
*/
|
2016-08-23 19:36:04 +08:00
|
|
|
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2016-08-23 20:40:16 +08:00
|
|
|
struct task_struct *next = NULL;
|
2016-08-23 19:36:04 +08:00
|
|
|
unsigned long owner, flags;
|
2016-01-25 10:23:43 +08:00
|
|
|
WAKE_Q(wake_q);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
mutex_release(&lock->dep_map, 1, ip);
|
|
|
|
|
2006-01-10 07:59:19 +08:00
|
|
|
/*
|
2016-08-23 20:40:16 +08:00
|
|
|
* Release the lock before (potentially) taking the spinlock such that
|
|
|
|
* other contenders can get on with things ASAP.
|
|
|
|
*
|
|
|
|
* Except when HANDOFF, in that case we must not clear the owner field,
|
|
|
|
* but instead set it to the top waiter.
|
2006-01-10 07:59:19 +08:00
|
|
|
*/
|
2016-08-23 20:40:16 +08:00
|
|
|
owner = atomic_long_read(&lock->owner);
|
|
|
|
for (;;) {
|
|
|
|
unsigned long old;
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (owner & MUTEX_FLAG_HANDOFF)
|
|
|
|
break;
|
|
|
|
|
|
|
|
old = atomic_long_cmpxchg_release(&lock->owner, owner,
|
|
|
|
__owner_flags(owner));
|
|
|
|
if (old == owner) {
|
|
|
|
if (owner & MUTEX_FLAG_WAITERS)
|
|
|
|
break;
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
owner = old;
|
|
|
|
}
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2014-01-29 03:13:14 +08:00
|
|
|
spin_lock_mutex(&lock->wait_lock, flags);
|
|
|
|
debug_mutex_unlock(lock);
|
2006-01-10 07:59:19 +08:00
|
|
|
if (!list_empty(&lock->wait_list)) {
|
|
|
|
/* get the first entry from the wait-list: */
|
|
|
|
struct mutex_waiter *waiter =
|
2016-08-23 20:40:16 +08:00
|
|
|
list_first_entry(&lock->wait_list,
|
|
|
|
struct mutex_waiter, list);
|
|
|
|
|
|
|
|
next = waiter->task;
|
2006-01-10 07:59:19 +08:00
|
|
|
|
|
|
|
debug_mutex_wake_waiter(lock, waiter);
|
2016-08-23 20:40:16 +08:00
|
|
|
wake_q_add(&wake_q, next);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
|
2016-08-23 20:40:16 +08:00
|
|
|
if (owner & MUTEX_FLAG_HANDOFF)
|
|
|
|
__mutex_handoff(lock, next);
|
|
|
|
|
2006-06-26 15:24:31 +08:00
|
|
|
spin_unlock_mutex(&lock->wait_lock, flags);
|
2016-08-23 20:40:16 +08:00
|
|
|
|
2016-01-25 10:23:43 +08:00
|
|
|
wake_up_q(&wake_q);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
|
2007-10-12 04:11:12 +08:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
2006-01-10 07:59:19 +08:00
|
|
|
/*
|
|
|
|
* Here come the less common (and hence less performance-critical) APIs:
|
|
|
|
* mutex_lock_interruptible() and mutex_trylock().
|
|
|
|
*/
|
2008-02-08 20:19:53 +08:00
|
|
|
static noinline int __sched
|
2013-06-20 19:31:05 +08:00
|
|
|
__mutex_lock_killable_slowpath(struct mutex *lock);
|
2007-12-07 06:37:59 +08:00
|
|
|
|
2008-02-08 20:19:53 +08:00
|
|
|
static noinline int __sched
|
2013-06-20 19:31:05 +08:00
|
|
|
__mutex_lock_interruptible_slowpath(struct mutex *lock);
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2010-09-03 06:48:16 +08:00
|
|
|
/**
|
|
|
|
* mutex_lock_interruptible - acquire the mutex, interruptible
|
2006-01-10 07:59:19 +08:00
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Lock the mutex like mutex_lock(), and return 0 if the mutex has
|
|
|
|
* been acquired or sleep until the mutex becomes available. If a
|
|
|
|
* signal arrives while waiting for the lock then this function
|
|
|
|
* returns -EINTR.
|
|
|
|
*
|
|
|
|
* This function is similar to (but not equivalent to) down_interruptible().
|
|
|
|
*/
|
2008-02-08 20:19:53 +08:00
|
|
|
int __sched mutex_lock_interruptible(struct mutex *lock)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2006-01-11 05:10:36 +08:00
|
|
|
might_sleep();
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
if (__mutex_trylock_fast(lock))
|
2013-06-20 19:31:05 +08:00
|
|
|
return 0;
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
return __mutex_lock_interruptible_slowpath(lock);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(mutex_lock_interruptible);
|
|
|
|
|
2008-02-08 20:19:53 +08:00
|
|
|
int __sched mutex_lock_killable(struct mutex *lock)
|
2007-12-07 06:37:59 +08:00
|
|
|
{
|
|
|
|
might_sleep();
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
if (__mutex_trylock_fast(lock))
|
2013-06-20 19:31:05 +08:00
|
|
|
return 0;
|
2016-08-23 19:36:04 +08:00
|
|
|
|
|
|
|
return __mutex_lock_killable_slowpath(lock);
|
2007-12-07 06:37:59 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mutex_lock_killable);
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
static noinline void __sched
|
|
|
|
__mutex_lock_slowpath(struct mutex *lock)
|
2007-10-12 04:11:12 +08:00
|
|
|
{
|
2013-06-24 16:30:04 +08:00
|
|
|
__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
|
2013-10-17 18:45:29 +08:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2007-10-12 04:11:12 +08:00
|
|
|
}
|
|
|
|
|
2008-02-08 20:19:53 +08:00
|
|
|
static noinline int __sched
|
2013-06-20 19:31:05 +08:00
|
|
|
__mutex_lock_killable_slowpath(struct mutex *lock)
|
2007-12-07 06:37:59 +08:00
|
|
|
{
|
2013-06-24 16:30:04 +08:00
|
|
|
return __mutex_lock_common(lock, TASK_KILLABLE, 0,
|
2013-10-17 18:45:29 +08:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2007-12-07 06:37:59 +08:00
|
|
|
}
|
|
|
|
|
2008-02-08 20:19:53 +08:00
|
|
|
static noinline int __sched
|
2013-06-20 19:31:05 +08:00
|
|
|
__mutex_lock_interruptible_slowpath(struct mutex *lock)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2013-06-24 16:30:04 +08:00
|
|
|
return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
|
2013-10-17 18:45:29 +08:00
|
|
|
NULL, _RET_IP_, NULL, 0);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static noinline int __sched
|
|
|
|
__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
|
2013-10-17 18:45:29 +08:00
|
|
|
NULL, _RET_IP_, ctx, 1);
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
2013-06-24 16:30:04 +08:00
|
|
|
|
|
|
|
static noinline int __sched
|
|
|
|
__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
|
|
|
|
struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
|
2013-10-17 18:45:29 +08:00
|
|
|
NULL, _RET_IP_, ctx, 1);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
|
2007-10-12 04:11:12 +08:00
|
|
|
#endif
|
2006-01-10 07:59:19 +08:00
|
|
|
|
2010-09-03 06:48:16 +08:00
|
|
|
/**
|
|
|
|
* mutex_trylock - try to acquire the mutex, without waiting
|
2006-01-10 07:59:19 +08:00
|
|
|
* @lock: the mutex to be acquired
|
|
|
|
*
|
|
|
|
* Try to acquire the mutex atomically. Returns 1 if the mutex
|
|
|
|
* has been acquired successfully, and 0 on contention.
|
|
|
|
*
|
|
|
|
* NOTE: this function follows the spin_trylock() convention, so
|
2010-09-03 06:48:16 +08:00
|
|
|
* it is negated from the down_trylock() return values! Be careful
|
2006-01-10 07:59:19 +08:00
|
|
|
* about this when converting semaphore users to mutexes.
|
|
|
|
*
|
|
|
|
* This function must not be used in interrupt context. The
|
|
|
|
* mutex must be released by the same task that acquired it.
|
|
|
|
*/
|
2008-02-08 20:19:53 +08:00
|
|
|
int __sched mutex_trylock(struct mutex *lock)
|
2006-01-10 07:59:19 +08:00
|
|
|
{
|
2016-08-23 20:40:16 +08:00
|
|
|
bool locked = __mutex_trylock(lock, false);
|
2009-01-12 21:01:47 +08:00
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
if (locked)
|
|
|
|
mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
2009-01-12 21:01:47 +08:00
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
return locked;
|
2006-01-10 07:59:19 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mutex_trylock);
|
2009-04-30 06:59:58 +08:00
|
|
|
|
2013-06-24 16:30:04 +08:00
|
|
|
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
might_sleep();
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
if (__mutex_trylock_fast(&lock->base)) {
|
2013-06-24 16:30:04 +08:00
|
|
|
ww_mutex_set_context_fastpath(lock, ctx);
|
2016-08-23 19:36:04 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return __ww_mutex_lock_slowpath(lock, ctx);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__ww_mutex_lock);
|
|
|
|
|
|
|
|
int __sched
|
|
|
|
__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
|
|
|
|
{
|
|
|
|
might_sleep();
|
|
|
|
|
2016-08-23 19:36:04 +08:00
|
|
|
if (__mutex_trylock_fast(&lock->base)) {
|
2013-06-24 16:30:04 +08:00
|
|
|
ww_mutex_set_context_fastpath(lock, ctx);
|
2016-08-23 19:36:04 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
|
2013-06-24 16:30:04 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__ww_mutex_lock_interruptible);
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2009-04-30 06:59:58 +08:00
|
|
|
/**
|
|
|
|
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
|
|
|
|
* @cnt: the atomic which we are to dec
|
|
|
|
* @lock: the mutex to return holding if we dec to 0
|
|
|
|
*
|
|
|
|
* return true and hold lock if we dec to 0, return false otherwise
|
|
|
|
*/
|
|
|
|
int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
|
|
|
|
{
|
|
|
|
/* dec if we can't possibly hit 0 */
|
|
|
|
if (atomic_add_unless(cnt, -1, 1))
|
|
|
|
return 0;
|
|
|
|
/* we might hit 0, so take the lock */
|
|
|
|
mutex_lock(lock);
|
|
|
|
if (!atomic_dec_and_test(cnt)) {
|
|
|
|
/* when we actually did the dec, we didn't hit 0 */
|
|
|
|
mutex_unlock(lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
/* we hit 0, and we hold the lock */
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
|