AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

lockdep: Fix lockdep recursion

Browse Source 
Steve reported that lockdep_assert*irq*(), when nested inside lockdep
itself, will trigger a false-positive.

One example is the stack-trace code, as called from inside lockdep,
triggering tracing, which in turn calls RCU, which then uses
lockdep_assert_irqs_disabled().

Fixes: a21ee6055c ("lockdep: Change hardirq{s_enabled,_context} to per-cpu variables")
Reported-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Peter Zijlstra 2020-10-02 11:04:21 +02:00 committed by Ingo Molnar
parent 2bb8945bcc
commit 4d004099a6
2 changed files with 67 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/linux/lockdep.h
View File

@ -534,6 +534,7 @@ do { \
DECLARE_PER_CPU(int, hardirqs_enabled); DECLARE_PER_CPU(int, hardirqs_enabled);
DECLARE_PER_CPU(int, hardirq_context); DECLARE_PER_CPU(int, hardirq_context);
DECLARE_PER_CPU(unsigned int, lockdep_recursion);
/* /*
* The below lockdep_assert_*() macros use raw_cpu_read() to access the above * The below lockdep_assert_*() macros use raw_cpu_read() to access the above
@ -543,25 +544,27 @@ DECLARE_PER_CPU(int, hardirq_context);
* read the value from our previous CPU. * read the value from our previous CPU.
*/ */
#define __lockdep_enabled (debug_locks && !raw_cpu_read(lockdep_recursion))
#define lockdep_assert_irqs_enabled() \ #define lockdep_assert_irqs_enabled() \
do { \ do { \
WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirqs_enabled)); \ WARN_ON_ONCE(__lockdep_enabled && !raw_cpu_read(hardirqs_enabled)); \
} while (0) } while (0)
#define lockdep_assert_irqs_disabled() \ #define lockdep_assert_irqs_disabled() \
do { \ do { \
WARN_ON_ONCE(debug_locks && raw_cpu_read(hardirqs_enabled)); \ WARN_ON_ONCE(__lockdep_enabled && raw_cpu_read(hardirqs_enabled)); \
} while (0) } while (0)
#define lockdep_assert_in_irq() \ #define lockdep_assert_in_irq() \
do { \ do { \
WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirq_context)); \ WARN_ON_ONCE(__lockdep_enabled && !raw_cpu_read(hardirq_context)); \
} while (0) } while (0)
#define lockdep_assert_preemption_enabled() \ #define lockdep_assert_preemption_enabled() \
do { \ do { \
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \ WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
debug_locks && \ __lockdep_enabled && \
(preempt_count() != 0 || \ (preempt_count() != 0 || \
!raw_cpu_read(hardirqs_enabled))); \ !raw_cpu_read(hardirqs_enabled))); \
} while (0) } while (0)
@ -569,7 +572,7 @@ do { \
#define lockdep_assert_preemption_disabled() \ #define lockdep_assert_preemption_disabled() \
do { \ do { \
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \ WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
debug_locks && \ __lockdep_enabled && \
(preempt_count() == 0 && \ (preempt_count() == 0 && \
raw_cpu_read(hardirqs_enabled))); \ raw_cpu_read(hardirqs_enabled))); \
} while (0) } while (0)

99
kernel/locking/lockdep.c
View File

@ -76,6 +76,23 @@ module_param(lock_stat, int, 0644);
#define lock_stat 0 #define lock_stat 0
#endif #endif
DEFINE_PER_CPU(unsigned int, lockdep_recursion);
EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion);
static inline bool lockdep_enabled(void)
{
if (!debug_locks)
return false;
if (raw_cpu_read(lockdep_recursion))
return false;
if (current->lockdep_recursion)
return false;
return true;
}
/* /*
* lockdep_lock: protects the lockdep graph, the hashes and the * lockdep_lock: protects the lockdep graph, the hashes and the
* class/list/hash allocators. * class/list/hash allocators.
@ -93,7 +110,7 @@ static inline void lockdep_lock(void)
arch_spin_lock(&__lock); arch_spin_lock(&__lock);
__owner = current; __owner = current;
current->lockdep_recursion++; __this_cpu_inc(lockdep_recursion);
} }
static inline void lockdep_unlock(void) static inline void lockdep_unlock(void)
@ -101,7 +118,7 @@ static inline void lockdep_unlock(void)
if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current)) if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))
return; return;
current->lockdep_recursion--; __this_cpu_dec(lockdep_recursion);
__owner = NULL; __owner = NULL;
arch_spin_unlock(&__lock); arch_spin_unlock(&__lock);
} }
@ -393,10 +410,15 @@ void lockdep_init_task(struct task_struct *task)
task->lockdep_recursion = 0; task->lockdep_recursion = 0;
} }
static __always_inline void lockdep_recursion_inc(void)
{
__this_cpu_inc(lockdep_recursion);
}
static __always_inline void lockdep_recursion_finish(void) static __always_inline void lockdep_recursion_finish(void)
{ {
if (WARN_ON_ONCE((--current->lockdep_recursion) & LOCKDEP_RECURSION_MASK)) if (WARN_ON_ONCE(__this_cpu_dec_return(lockdep_recursion)))
current->lockdep_recursion = 0; __this_cpu_write(lockdep_recursion, 0);
} }
void lockdep_set_selftest_task(struct task_struct *task) void lockdep_set_selftest_task(struct task_struct *task)
@ -3659,7 +3681,7 @@ void lockdep_hardirqs_on_prepare(unsigned long ip)
if (unlikely(in_nmi())) if (unlikely(in_nmi()))
return; return;
if (unlikely(current->lockdep_recursion & LOCKDEP_RECURSION_MASK)) if (unlikely(__this_cpu_read(lockdep_recursion)))
return; return;
if (unlikely(lockdep_hardirqs_enabled())) { if (unlikely(lockdep_hardirqs_enabled())) {
@ -3695,7 +3717,7 @@ void lockdep_hardirqs_on_prepare(unsigned long ip)
current->hardirq_chain_key = current->curr_chain_key; current->hardirq_chain_key = current->curr_chain_key;
current->lockdep_recursion++; lockdep_recursion_inc();
__trace_hardirqs_on_caller(); __trace_hardirqs_on_caller();
lockdep_recursion_finish(); lockdep_recursion_finish();
} }
@ -3728,7 +3750,7 @@ void noinstr lockdep_hardirqs_on(unsigned long ip)
goto skip_checks; goto skip_checks;
} }
if (unlikely(current->lockdep_recursion & LOCKDEP_RECURSION_MASK)) if (unlikely(__this_cpu_read(lockdep_recursion)))
return; return;
if (lockdep_hardirqs_enabled()) { if (lockdep_hardirqs_enabled()) {
@ -3781,7 +3803,7 @@ void noinstr lockdep_hardirqs_off(unsigned long ip)
if (in_nmi()) { if (in_nmi()) {
if (!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_NMI)) if (!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_NMI))
return; return;
} else if (current->lockdep_recursion & LOCKDEP_RECURSION_MASK) } else if (__this_cpu_read(lockdep_recursion))
return; return;
/* /*
@ -3814,7 +3836,7 @@ void lockdep_softirqs_on(unsigned long ip)
{ {
struct irqtrace_events *trace = &current->irqtrace; struct irqtrace_events *trace = &current->irqtrace;
if (unlikely(!debug_locks || current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
/* /*
@ -3829,7 +3851,7 @@ void lockdep_softirqs_on(unsigned long ip)
return; return;
} }
current->lockdep_recursion++; lockdep_recursion_inc();
/* /*
* We'll do an OFF -> ON transition: * We'll do an OFF -> ON transition:
*/ */
@ -3852,7 +3874,7 @@ void lockdep_softirqs_on(unsigned long ip)
*/ */
void lockdep_softirqs_off(unsigned long ip) void lockdep_softirqs_off(unsigned long ip)
{ {
if (unlikely(!debug_locks || current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
/* /*
@ -4233,11 +4255,11 @@ void lockdep_init_map_waits(struct lockdep_map *lock, const char *name,
if (subclass) { if (subclass) {
unsigned long flags; unsigned long flags;
if (DEBUG_LOCKS_WARN_ON(current->lockdep_recursion)) if (DEBUG_LOCKS_WARN_ON(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
register_lock_class(lock, subclass, 1); register_lock_class(lock, subclass, 1);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -4920,11 +4942,11 @@ void lock_set_class(struct lockdep_map *lock, const char *name,
{ {
unsigned long flags; unsigned long flags;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
check_flags(flags); check_flags(flags);
if (__lock_set_class(lock, name, key, subclass, ip)) if (__lock_set_class(lock, name, key, subclass, ip))
check_chain_key(current); check_chain_key(current);
@ -4937,11 +4959,11 @@ void lock_downgrade(struct lockdep_map *lock, unsigned long ip)
{ {
unsigned long flags; unsigned long flags;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
check_flags(flags); check_flags(flags);
if (__lock_downgrade(lock, ip)) if (__lock_downgrade(lock, ip))
check_chain_key(current); check_chain_key(current);
@ -4979,7 +5001,7 @@ static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock
static bool lockdep_nmi(void) static bool lockdep_nmi(void)
{ {
if (current->lockdep_recursion & LOCKDEP_RECURSION_MASK) if (raw_cpu_read(lockdep_recursion))
return false; return false;
if (!in_nmi()) if (!in_nmi())
@ -5000,7 +5022,10 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip); trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
if (unlikely(current->lockdep_recursion)) { if (!debug_locks)
return;
if (unlikely(!lockdep_enabled())) {
/* XXX allow trylock from NMI ?!? */ /* XXX allow trylock from NMI ?!? */
if (lockdep_nmi() && !trylock) { if (lockdep_nmi() && !trylock) {
struct held_lock hlock; struct held_lock hlock;
@ -5023,7 +5048,7 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
__lock_acquire(lock, subclass, trylock, read, check, __lock_acquire(lock, subclass, trylock, read, check,
irqs_disabled_flags(flags), nest_lock, ip, 0, 0); irqs_disabled_flags(flags), nest_lock, ip, 0, 0);
lockdep_recursion_finish(); lockdep_recursion_finish();
@ -5037,13 +5062,13 @@ void lock_release(struct lockdep_map *lock, unsigned long ip)
trace_lock_release(lock, ip); trace_lock_release(lock, ip);
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
if (__lock_release(lock, ip)) if (__lock_release(lock, ip))
check_chain_key(current); check_chain_key(current);
lockdep_recursion_finish(); lockdep_recursion_finish();
@ -5056,13 +5081,13 @@ noinstr int lock_is_held_type(const struct lockdep_map *lock, int read)
unsigned long flags; unsigned long flags;
int ret = 0; int ret = 0;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return 1; /* avoid false negative lockdep_assert_held() */ return 1; /* avoid false negative lockdep_assert_held() */
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
ret = __lock_is_held(lock, read); ret = __lock_is_held(lock, read);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -5077,13 +5102,13 @@ struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
struct pin_cookie cookie = NIL_COOKIE; struct pin_cookie cookie = NIL_COOKIE;
unsigned long flags; unsigned long flags;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return cookie; return cookie;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
cookie = __lock_pin_lock(lock); cookie = __lock_pin_lock(lock);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -5096,13 +5121,13 @@ void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{ {
unsigned long flags; unsigned long flags;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
__lock_repin_lock(lock, cookie); __lock_repin_lock(lock, cookie);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -5113,13 +5138,13 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{ {
unsigned long flags; unsigned long flags;
if (unlikely(current->lockdep_recursion)) if (unlikely(!lockdep_enabled()))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
__lock_unpin_lock(lock, cookie); __lock_unpin_lock(lock, cookie);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -5249,15 +5274,12 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
trace_lock_acquired(lock, ip); trace_lock_acquired(lock, ip);
if (unlikely(!lock_stat || !debug_locks)) if (unlikely(!lock_stat || !lockdep_enabled()))
return;
if (unlikely(current->lockdep_recursion))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
__lock_contended(lock, ip); __lock_contended(lock, ip);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);
@ -5270,15 +5292,12 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip)
trace_lock_contended(lock, ip); trace_lock_contended(lock, ip);
if (unlikely(!lock_stat || !debug_locks)) if (unlikely(!lock_stat || !lockdep_enabled()))
return;
if (unlikely(current->lockdep_recursion))
return; return;
raw_local_irq_save(flags); raw_local_irq_save(flags);
check_flags(flags); check_flags(flags);
current->lockdep_recursion++; lockdep_recursion_inc();
__lock_acquired(lock, ip); __lock_acquired(lock, ip);
lockdep_recursion_finish(); lockdep_recursion_finish();
raw_local_irq_restore(flags); raw_local_irq_restore(flags);