diff mbox

[tip/core/rcu,01/15] rcu: Add PROVE_RCU_DELAY to provoke difficult races

Message ID 1346352988-32444-1-git-send-email-paulmck@linux.vnet.ibm.com
State Accepted
Commit e3ebfb96f396731ca2d0b108785d5da31b53ab00
Headers show

Commit Message

Paul E. McKenney Aug. 30, 2012, 6:56 p.m. UTC 
From: "Paul E. McKenney" <paul.mckenney@linaro.org>

There have been some recent bugs that were triggered only when
preemptible RCU's __rcu_read_unlock() was preempted just after setting
->rcu_read_lock_nesting to INT_MIN, which is a low-probability event.
Therefore, reproducing those bugs (to say nothing of gaining confidence
in alleged fixes) was quite difficult.  This commit therefore creates
a new debug-only RCU kernel config option that forces a short delay
in __rcu_read_unlock() to increase the probability of those sorts of
bugs occurring.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcupdate.c |    4 ++++
 lib/Kconfig.debug |   14 ++++++++++++++
 2 files changed, 18 insertions(+), 0 deletions(-)

Comments

Josh Triplett Aug. 31, 2012, 4:49 p.m. UTC | #1 
On Thu, Aug 30, 2012 at 11:56:14AM -0700, Paul E. McKenney wrote:
> From: "Paul E. McKenney" <paul.mckenney@linaro.org>
> 
> There have been some recent bugs that were triggered only when
> preemptible RCU's __rcu_read_unlock() was preempted just after setting
> ->rcu_read_lock_nesting to INT_MIN, which is a low-probability event.
> Therefore, reproducing those bugs (to say nothing of gaining confidence
> in alleged fixes) was quite difficult.  This commit therefore creates
> a new debug-only RCU kernel config option that forces a short delay
> in __rcu_read_unlock() to increase the probability of those sorts of
> bugs occurring.
> 
> Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Reviewed-by: Josh Triplett <josh@joshtriplett.org>

If you end up adding more such conditional race-provoking delays
elsewhere in the code, consider creating a prove_rcu_udelay() wrapper
to avoid multiple #ifdefs in the code.

> ---
>  kernel/rcupdate.c |    4 ++++
>  lib/Kconfig.debug |   14 ++++++++++++++
>  2 files changed, 18 insertions(+), 0 deletions(-)
> 
> diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
> index 4e6a61b..29ca1c6 100644
> --- a/kernel/rcupdate.c
> +++ b/kernel/rcupdate.c
> @@ -45,6 +45,7 @@
>  #include <linux/mutex.h>
>  #include <linux/export.h>
>  #include <linux/hardirq.h>
> +#include <linux/delay.h>
>  
>  #define CREATE_TRACE_POINTS
>  #include <trace/events/rcu.h>
> @@ -81,6 +82,9 @@ void __rcu_read_unlock(void)
>  	} else {
>  		barrier();  /* critical section before exit code. */
>  		t->rcu_read_lock_nesting = INT_MIN;
> +#ifdef CONFIG_PROVE_RCU_DELAY
> +		udelay(10); /* Make preemption more probable. */
> +#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */
>  		barrier();  /* assign before ->rcu_read_unlock_special load */
>  		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
>  			rcu_read_unlock_special(t);
> diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> index 2403a63..dacbbe4 100644
> --- a/lib/Kconfig.debug
> +++ b/lib/Kconfig.debug
> @@ -629,6 +629,20 @@ config PROVE_RCU_REPEATEDLY
>  
>  	 Say N if you are unsure.
>  
> +config PROVE_RCU_DELAY
> +	bool "RCU debugging: preemptible RCU race provocation"
> +	depends on DEBUG_KERNEL && PREEMPT_RCU
> +	default n
> +	help
> +	 There is a class of races that involve an unlikely preemption
> +	 of __rcu_read_unlock() just after ->rcu_read_lock_nesting has
> +	 been set to INT_MIN.  This feature inserts a delay at that
> +	 point to increase the probability of these races.
> +
> +	 Say Y to increase probability of preemption of __rcu_read_unlock().
> +
> +	 Say N if you are unsure.
> +
>  config SPARSE_RCU_POINTER
>  	bool "RCU debugging: sparse-based checks for pointer usage"
>  	default n
> -- 
> 1.7.8
>
Paul E. McKenney Sept. 4, 2012, 10:36 p.m. UTC | #2 
On Fri, Aug 31, 2012 at 09:49:36AM -0700, Josh Triplett wrote:
> On Thu, Aug 30, 2012 at 11:56:14AM -0700, Paul E. McKenney wrote:
> > From: "Paul E. McKenney" <paul.mckenney@linaro.org>
> > 
> > There have been some recent bugs that were triggered only when
> > preemptible RCU's __rcu_read_unlock() was preempted just after setting
> > ->rcu_read_lock_nesting to INT_MIN, which is a low-probability event.
> > Therefore, reproducing those bugs (to say nothing of gaining confidence
> > in alleged fixes) was quite difficult.  This commit therefore creates
> > a new debug-only RCU kernel config option that forces a short delay
> > in __rcu_read_unlock() to increase the probability of those sorts of
> > bugs occurring.
> > 
> > Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
> 
> If you end up adding more such conditional race-provoking delays
> elsewhere in the code, consider creating a prove_rcu_udelay() wrapper
> to avoid multiple #ifdefs in the code.

Good point!  In fact, I have added this to my list.

							Thanx, Paul

> > ---
> >  kernel/rcupdate.c |    4 ++++
> >  lib/Kconfig.debug |   14 ++++++++++++++
> >  2 files changed, 18 insertions(+), 0 deletions(-)
> > 
> > diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
> > index 4e6a61b..29ca1c6 100644
> > --- a/kernel/rcupdate.c
> > +++ b/kernel/rcupdate.c
> > @@ -45,6 +45,7 @@
> >  #include <linux/mutex.h>
> >  #include <linux/export.h>
> >  #include <linux/hardirq.h>
> > +#include <linux/delay.h>
> >  
> >  #define CREATE_TRACE_POINTS
> >  #include <trace/events/rcu.h>
> > @@ -81,6 +82,9 @@ void __rcu_read_unlock(void)
> >  	} else {
> >  		barrier();  /* critical section before exit code. */
> >  		t->rcu_read_lock_nesting = INT_MIN;
> > +#ifdef CONFIG_PROVE_RCU_DELAY
> > +		udelay(10); /* Make preemption more probable. */
> > +#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */
> >  		barrier();  /* assign before ->rcu_read_unlock_special load */
> >  		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
> >  			rcu_read_unlock_special(t);
> > diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> > index 2403a63..dacbbe4 100644
> > --- a/lib/Kconfig.debug
> > +++ b/lib/Kconfig.debug
> > @@ -629,6 +629,20 @@ config PROVE_RCU_REPEATEDLY
> >  
> >  	 Say N if you are unsure.
> >  
> > +config PROVE_RCU_DELAY
> > +	bool "RCU debugging: preemptible RCU race provocation"
> > +	depends on DEBUG_KERNEL && PREEMPT_RCU
> > +	default n
> > +	help
> > +	 There is a class of races that involve an unlikely preemption
> > +	 of __rcu_read_unlock() just after ->rcu_read_lock_nesting has
> > +	 been set to INT_MIN.  This feature inserts a delay at that
> > +	 point to increase the probability of these races.
> > +
> > +	 Say Y to increase probability of preemption of __rcu_read_unlock().
> > +
> > +	 Say N if you are unsure.
> > +
> >  config SPARSE_RCU_POINTER
> >  	bool "RCU debugging: sparse-based checks for pointer usage"
> >  	default n
> > -- 
> > 1.7.8
> > 
>
Peter Zijlstra Sept. 6, 2012, 2:38 p.m. UTC | #3 
On Thu, 2012-08-30 at 11:56 -0700, Paul E. McKenney wrote:
> +#ifdef CONFIG_PROVE_RCU_DELAY
> +               udelay(10); /* Make preemption more probable. */
		cond_resched(); /* for extra fun? */
> +#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */
Paul E. McKenney Sept. 6, 2012, 8:51 p.m. UTC | #4 
On Thu, Sep 06, 2012 at 04:38:32PM +0200, Peter Zijlstra wrote:
> On Thu, 2012-08-30 at 11:56 -0700, Paul E. McKenney wrote:
> > +#ifdef CONFIG_PROVE_RCU_DELAY
> > +               udelay(10); /* Make preemption more probable. */
> 		cond_resched(); /* for extra fun? */

The additional fun could include "scheduling while atomic", so I will
pass.  ;-)

(The problem is that __rcu_read_unlock() can be called with interrupts
disabled, among other things.)

							Thanx, Paul

> > +#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */ 
> 
>
Peter Zijlstra Sept. 7, 2012, 6:54 a.m. UTC | #5 
On Thu, 2012-09-06 at 13:51 -0700, Paul E. McKenney wrote:
> On Thu, Sep 06, 2012 at 04:38:32PM +0200, Peter Zijlstra wrote:
> > On Thu, 2012-08-30 at 11:56 -0700, Paul E. McKenney wrote:
> > > +#ifdef CONFIG_PROVE_RCU_DELAY
> > > +               udelay(10); /* Make preemption more probable. */
> > 		cond_resched(); /* for extra fun? */
> 
> The additional fun could include "scheduling while atomic", so I will
> pass.  ;-)
> 
> (The problem is that __rcu_read_unlock() can be called with interrupts
> disabled, among other things.)

Hmm, too bad. Without a preemption point here you're relying on forced
preemption, which of course can only happen on PREEMPT=y kernels.

> > > +#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */ 
> > 
> > 
>
diff mbox

Patch

diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 4e6a61b..29ca1c6 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -45,6 +45,7 @@ 
 #include <linux/mutex.h>
 #include <linux/export.h>
 #include <linux/hardirq.h>
+#include <linux/delay.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/rcu.h>
@@ -81,6 +82,9 @@  void __rcu_read_unlock(void)
 	} else {
 		barrier();  /* critical section before exit code. */
 		t->rcu_read_lock_nesting = INT_MIN;
+#ifdef CONFIG_PROVE_RCU_DELAY
+		udelay(10); /* Make preemption more probable. */
+#endif /* #ifdef CONFIG_PROVE_RCU_DELAY */
 		barrier();  /* assign before ->rcu_read_unlock_special load */
 		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
 			rcu_read_unlock_special(t);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 2403a63..dacbbe4 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -629,6 +629,20 @@  config PROVE_RCU_REPEATEDLY
 
 	 Say N if you are unsure.
 
+config PROVE_RCU_DELAY
+	bool "RCU debugging: preemptible RCU race provocation"
+	depends on DEBUG_KERNEL && PREEMPT_RCU
+	default n
+	help
+	 There is a class of races that involve an unlikely preemption
+	 of __rcu_read_unlock() just after ->rcu_read_lock_nesting has
+	 been set to INT_MIN.  This feature inserts a delay at that
+	 point to increase the probability of these races.
+
+	 Say Y to increase probability of preemption of __rcu_read_unlock().
+
+	 Say N if you are unsure.
+
 config SPARSE_RCU_POINTER
 	bool "RCU debugging: sparse-based checks for pointer usage"
 	default n