diff mbox

[RFC,tip/core/rcu,09/13] rcu: Reduce latency of rcu_prepare_for_idle()

Message ID 1322523185-456-9-git-send-email-paulmck@linux.vnet.ibm.com
State Accepted
Commit 3ad0decf98d97b9039d8ed47cee287366b929cdf
Headers show

Commit Message

Paul E. McKenney Nov. 28, 2011, 11:33 p.m. UTC 
From: Paul E. McKenney <paul.mckenney@linaro.org>

Re-enable interrupts across calls to quiescent-state functions and
also across force_quiescent_state() to reduce latency.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcutree_plugin.h |   12 +++++++-----
 1 files changed, 7 insertions(+), 5 deletions(-)
diff mbox

Patch

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 3d84dbc..42ca5a4 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -2009,7 +2009,6 @@  int rcu_needs_cpu(int cpu)
  */
 static void rcu_prepare_for_idle(int cpu)
 {
-	int c = 0;
 	unsigned long flags;
 
 	local_irq_save(flags);
@@ -2055,27 +2054,30 @@  static void rcu_prepare_for_idle(int cpu)
 	 */
 #ifdef CONFIG_TREE_PREEMPT_RCU
 	if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
+		local_irq_restore(flags);
 		rcu_preempt_qs(cpu);
 		force_quiescent_state(&rcu_preempt_state, 0);
-		c = c || per_cpu(rcu_preempt_data, cpu).nxtlist;
+		local_irq_save(flags);
 	}
 #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 	if (per_cpu(rcu_sched_data, cpu).nxtlist) {
+		local_irq_restore(flags);
 		rcu_sched_qs(cpu);
 		force_quiescent_state(&rcu_sched_state, 0);
-		c = c || per_cpu(rcu_sched_data, cpu).nxtlist;
+		local_irq_save(flags);
 	}
 	if (per_cpu(rcu_bh_data, cpu).nxtlist) {
+		local_irq_restore(flags);
 		rcu_bh_qs(cpu);
 		force_quiescent_state(&rcu_bh_state, 0);
-		c = c || per_cpu(rcu_bh_data, cpu).nxtlist;
+		local_irq_save(flags);
 	}
 
 	/*
 	 * If RCU callbacks are still pending, RCU still needs this CPU.
 	 * So try forcing the callbacks through the grace period.
 	 */
-	if (c) {
+	if (rcu_cpu_has_callbacks(cpu)) {
 		local_irq_restore(flags);
 		trace_rcu_prep_idle("More callbacks");
 		invoke_rcu_core();