ioat: refcnt users so we can drain them at detach

We only need to borrow a mutex for the drain sleep and the 0->1
transition, so just reuse an existing one for now.

The wchan is arbitrary.  Using refcount itself would have required
__DEVOLATILE(), so use the lock's address instead.

Different uses are tagged by kind, although we only do anything with
that information in INVARIANTS builds.

Sponsored by:	EMC / Isilon Storage Division

sys/dev/ioat/ioat.c

@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include "ioat_internal.h"
 
 #define	IOAT_INTR_TIMO	(hz / 10)
+#define	IOAT_REFLK	(&ioat->submit_lock)
 
 static int ioat_probe(device_t device);
 static int ioat_attach(device_t device);
@@ -69,9 +70,9 @@ static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
 static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
 static void ioat_free_ring_entry(struct ioat_softc *ioat,
     struct ioat_descriptor *desc);
-static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat);
+static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *ioat);
 static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
-static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat,
+static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
     uint32_t index);
 static boolean_t resize_ring(struct ioat_softc *ioat, int order);
 static void ioat_timer_callback(void *arg);
@@ -81,6 +82,12 @@ static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
     int error);
 static int ioat_reset_hw(struct ioat_softc *ioat);
 static void ioat_setup_sysctl(device_t device);
+static inline struct ioat_softc *ioat_get(struct ioat_softc *,
+    enum ioat_ref_kind);
+static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind);
+static inline void ioat_putn(struct ioat_softc *, uint32_t,
+    enum ioat_ref_kind);
+static void ioat_drain(struct ioat_softc *);
 
 #define	ioat_log_message(v, ...) do {					\
 	if ((v) <= g_ioat_debug_level) {				\
@@ -236,10 +243,6 @@ ioat_attach(device_t device)
 		goto err;
 	}
 
-	error = ioat_setup_intr(ioat);
-	if (error != 0)
-		return (error);
-
 	error = ioat3_attach(device);
 	if (error != 0)
 		goto err;
@@ -248,9 +251,13 @@ ioat_attach(device_t device)
 	if (error != 0)
 		goto err;
 
+	error = ioat_setup_intr(ioat);
+	if (error != 0)
+		goto err;
+
 	error = ioat3_selftest(ioat);
 	if (error != 0)
-		return (error);
+		goto err;
 
 	ioat_process_events(ioat);
 	ioat_setup_sysctl(device);
@@ -273,6 +280,7 @@ ioat_detach(device_t device)
 	ioat = DEVICE2SOFTC(device);
 
 	ioat_test_detach();
+	ioat_drain(ioat);
 
 	ioat_teardown_intr(ioat);
 	callout_drain(&ioat->timer);
@@ -579,6 +587,7 @@ ioat_process_events(struct ioat_softc *ioat)
 		if (dmadesc->callback_fn)
 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
 
+		completed++;
 		ioat->tail++;
 		if (desc->hw_desc_bus_addr == status)
 			break;
@@ -594,6 +603,8 @@ ioat_process_events(struct ioat_softc *ioat)
 
 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
 	mtx_unlock(&ioat->cleanup_lock);
+
+	ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
 }
 
 /*
@@ -603,9 +614,18 @@ bus_dmaengine_t
 ioat_get_dmaengine(uint32_t index)
 {
 
-	if (index < ioat_channel_index)
+	if (index >= ioat_channel_index)
-		return (&ioat_channel[index]->dmaengine);
+		return (NULL);
-	return (NULL);
+	return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine);
+}
+
+void
+ioat_put_dmaengine(bus_dmaengine_t dmaengine)
+{
+	struct ioat_softc *ioat;
+
+	ioat = to_ioat_softc(dmaengine);
+	ioat_put(ioat, IOAT_DMAENGINE_REF);
 }
 
 void
@@ -962,6 +982,7 @@ static void
 ioat_submit_single(struct ioat_softc *ioat)
 {
 
+	ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
 	atomic_add_rel_int(&ioat->head, 1);
 
 	if (!ioat->is_completion_pending) {
@@ -1057,3 +1078,71 @@ ioat_setup_sysctl(device_t device)
 	    "tail", CTLFLAG_RD, &ioat->tail,
 	    0, "HW descriptor tail pointer index");
 }
+
+static inline struct ioat_softc *
+ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind)
+{
+	uint32_t old;
+
+	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
+
+	old = atomic_fetchadd_32(&ioat->refcnt, 1);
+	KASSERT(old < UINT32_MAX, ("refcnt overflow"));
+
+#ifdef INVARIANTS
+	old = atomic_fetchadd_32(&ioat->refkinds[kind], 1);
+	KASSERT(old < UINT32_MAX, ("refcnt kind overflow"));
+#endif
+
+	return (ioat);
+}
+
+static inline void
+ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)
+{
+	uint32_t old;
+
+	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
+
+	if (n == 0)
+		return;
+
+#ifdef INVARIANTS
+	old = atomic_fetchadd_32(&ioat->refkinds[kind], -n);
+	KASSERT(old >= n, ("refcnt kind underflow"));
+#endif
+
+	/* Skip acquiring the lock if resulting refcnt > 0. */
+	for (;;) {
+		old = ioat->refcnt;
+		if (old <= n)
+			break;
+		if (atomic_cmpset_32(&ioat->refcnt, old, old - n))
+			return;
+	}
+
+	mtx_lock(IOAT_REFLK);
+	old = atomic_fetchadd_32(&ioat->refcnt, -n);
+	KASSERT(old >= n, ("refcnt error"));
+
+	if (old == n)
+		wakeup(IOAT_REFLK);
+	mtx_unlock(IOAT_REFLK);
+}
+
+static inline void
+ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind)
+{
+
+	ioat_putn(ioat, 1, kind);
+}
+
+static void
+ioat_drain(struct ioat_softc *ioat)
+{
+
+	mtx_lock(IOAT_REFLK);
+	while (ioat->refcnt > 0)
+		msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0);
+	mtx_unlock(IOAT_REFLK);
+}

sys/dev/ioat/ioat.h

@@ -52,6 +52,9 @@ typedef void (*bus_dmaengine_callback_t)(void *arg);
  */
 bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index);
 
+/* Release the DMA channel */
+void ioat_put_dmaengine(bus_dmaengine_t dmaengine);
+
 /*
  * Acquire must be called before issuing an operation to perform. Release is
  * called after. Multiple operations can be issued within the context of one

sys/dev/ioat/ioat_internal.h

@@ -313,6 +313,12 @@ struct ioat_descriptor {
 	bus_addr_t		hw_desc_bus_addr;
 };
 
+enum ioat_ref_kind {
+	IOAT_DMAENGINE_REF = 0,
+	IOAT_ACTIVE_DESCR_REF,
+	IOAT_NUM_REF_KINDS
+};
+
 /* One of these per allocated PCI device. */
 struct ioat_softc {
 	bus_dmaengine_t		dmaengine;
@@ -364,6 +370,10 @@ struct ioat_softc {
 	struct ioat_descriptor	**ring;
 
 	struct mtx		cleanup_lock;
+	volatile uint32_t	refcnt;
+#ifdef INVARIANTS
+	volatile uint32_t	refkinds[IOAT_NUM_REF_KINDS];
+#endif
 };
 
 void ioat_test_attach(void);

sys/dev/ioat/ioat_test.c

@@ -299,7 +299,7 @@ ioat_dma_test(void *arg)
 	rc = ioat_test_prealloc_memory(test, index);
 	if (rc != 0) {
 		ioat_test_log(0, "prealloc_memory: %d\n", rc);
-		return;
+		goto out;
 	}
 	wmb();
 
@@ -330,6 +330,8 @@ ioat_dma_test(void *arg)
 	    ticks - start, ticks - end, (ticks - start) / hz);
 
 	ioat_test_release_memory(test);
+out:
+	ioat_put_dmaengine(dmaengine);
 }
 
 static int