@@ -573,6 +573,8 @@ struct softirq_action
asmlinkage void do_softirq(void);
asmlinkage void __do_softirq(void);
+extern void softirq_poll(void);
+
extern void open_softirq(int nr, void (*action)(struct softirq_action *));
extern void softirq_init(void);
extern void __raise_softirq_irqoff(unsigned int nr);
@@ -6,6 +6,7 @@
#include <linux/lockdep.h>
#include <linux/percpu.h>
#include <linux/poll_source.h>
+#include <linux/interrupt.h>
/* The per-cpu list of registered poll sources */
DEFINE_PER_CPU(struct list_head, poll_source_list);
@@ -26,6 +27,8 @@ void poll_source_run_once(void)
list_for_each_entry(src, this_cpu_ptr(&poll_source_list), node)
src->ops->poll(src);
+
+ softirq_poll();
}
/* Called from idle task with TIF_POLLING_NRFLAG set and irqs enabled */
@@ -611,6 +611,20 @@ void irq_enter(void)
irq_enter_rcu();
}
+/**
+ * softirq_poll() - invoke pending softirqs
+ *
+ * Normally it is not necessary to explicitly poll for softirqs, but in the
+ * cpuidle driver a polling function may have raised a softirq with no irq exit
+ * to invoke it. Therefore it is necessary to poll for pending softirqs and
+ * invoke them explicitly.
+ */
+void softirq_poll(void)
+{
+ if (!in_interrupt() && local_softirq_pending())
+ invoke_softirq();
+}
+
static inline void tick_irq_exit(void)
{
#ifdef CONFIG_NO_HZ_COMMON
Normally softirqs are invoked when exiting irqs. When polling in the cpuidle driver there may be no irq. Therefore pending softirqs go unnoticed and polling continues without invoking them. Add a softirq_poll() function to explicitly check for and invoke softirqs. Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> --- This commit is not needed for virtio-blk. I added it when I realized virtio-net's NAPI scheduling might not be detected by the cpuidle busy wait loop because it is unaware of softirqs. However, even after doing this virtio-net's NAPI polling doesn't combine with cpuidle haltpoll. Perhaps this patch is still desirable for cpuidle poll_state in case a softirq is raised? --- include/linux/interrupt.h | 2 ++ drivers/cpuidle/poll_source.c | 3 +++ kernel/softirq.c | 14 ++++++++++++++ 3 files changed, 19 insertions(+)