From a5f1042498465d17e28b80697c2b676e444b7da0 Mon Sep 17 00:00:00 2001
From: Matt Macy <mmacy@FreeBSD.org>
Date: Thu, 17 May 2018 21:39:15 +0000
Subject: [PATCH] epoch: skip poll function call in hardclock unless there are
 callbacks pending

Reported by:	mjg
Approved by:	sbruno
---
 sys/kern/kern_clock.c |  7 +++++--
 sys/kern/subr_epoch.c | 20 ++++++--------------
 sys/sys/epoch.h       |  3 ++-
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 6b432ae84dd1..1d3441529eb0 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/callout.h>
 #include <sys/epoch.h>
+#include <sys/gtaskqueue.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
@@ -468,7 +469,8 @@ hardclock_cpu(int usermode)
 		PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
 #endif
 	callout_process(sbinuptime());
-	epoch_pcpu_poll();
+	if (__predict_false(DPCPU_GET(epoch_cb_count)))
+		GROUPTASK_ENQUEUE(DPCPU_PTR(epoch_cb_task));
 }
 
 /*
@@ -574,7 +576,8 @@ hardclock_cnt(int cnt, int usermode)
 	}
 	if (curcpu == CPU_FIRST())
 		cpu_tick_calibration();
-	epoch_pcpu_poll();
+	if (__predict_false(DPCPU_GET(epoch_cb_count)))
+		GROUPTASK_ENQUEUE(DPCPU_PTR(epoch_cb_task));
 }
 
 void
diff --git a/sys/kern/subr_epoch.c b/sys/kern/subr_epoch.c
index 0a722926d2ba..116bef6a3b9f 100644
--- a/sys/kern/subr_epoch.c
+++ b/sys/kern/subr_epoch.c
@@ -111,8 +111,8 @@ struct epoch {
 
 epoch_t allepochs[MAX_EPOCHS];
 
-static DPCPU_DEFINE(struct grouptask, cb_task);
-static DPCPU_DEFINE(int, cb_count);
+DPCPU_DEFINE(struct grouptask, epoch_cb_task);
+DPCPU_DEFINE(int, epoch_cb_count);
 
 static __read_mostly int domcount[MAXMEMDOM];
 static __read_mostly int domoffsets[MAXMEMDOM];
@@ -157,8 +157,8 @@ epoch_init(void *arg __unused)
 	}
  done:
 	CPU_FOREACH(cpu) {
-		GROUPTASK_INIT(DPCPU_ID_PTR(cpu, cb_task), 0, epoch_call_task, NULL);
-		taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, cb_task), NULL, cpu, -1, "epoch call task");
+		GROUPTASK_INIT(DPCPU_ID_PTR(cpu, epoch_cb_task), 0, epoch_call_task, NULL);
+		taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, -1, "epoch call task");
 	}
 	inited = 1;
 	global_epoch = epoch_alloc();
@@ -533,7 +533,7 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t
 	counter_u64_add(epoch->e_frees, 1);
 
 	critical_enter();
-	*DPCPU_PTR(cb_count) += 1;
+	*DPCPU_PTR(epoch_cb_count) += 1;
 	eps = epoch->e_pcpu[curcpu];
 	ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t*)callback);
 	critical_exit();
@@ -566,7 +566,7 @@ epoch_call_task(void *arg __unused)
 		total += npending - record->n_pending;
 	}
 	epoch_exit_private(&section);
-	*DPCPU_PTR(cb_count) -= total;
+	*DPCPU_PTR(epoch_cb_count) -= total;
 	critical_exit();
 
 	head = ck_stack_batch_pop_npsc(&cb_stack);
@@ -578,14 +578,6 @@ epoch_call_task(void *arg __unused)
 	}
 }
 
-void
-epoch_pcpu_poll(void)
-{
-
-	if (DPCPU_GET(cb_count))
-		GROUPTASK_ENQUEUE(DPCPU_PTR(cb_task));
-}
-
 int
 in_epoch(void)
 {
diff --git a/sys/sys/epoch.h b/sys/sys/epoch.h
index b2999c141777..a60edbab09b8 100644
--- a/sys/sys/epoch.h
+++ b/sys/sys/epoch.h
@@ -36,6 +36,8 @@ struct epoch;
 typedef struct epoch *epoch_t;
 
 extern epoch_t global_epoch;
+DPCPU_DECLARE(int, epoch_cb_count);
+DPCPU_DECLARE(struct grouptask, epoch_cb_task);
 
 struct epoch_context {
 	void *data[2];
@@ -49,7 +51,6 @@ void epoch_enter_internal(epoch_t epoch, struct thread *td);
 void epoch_exit_internal(epoch_t epoch, struct thread *td);
 void epoch_wait(epoch_t epoch);
 void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
-void epoch_pcpu_poll(void);
 int in_epoch(void);
 
 static __inline void