ioat: refcnt users so we can drain them at detach
We only need to borrow a mutex for the drain sleep and the 0->1 transition, so just reuse an existing one for now. The wchan is arbitrary. Using refcount itself would have required __DEVOLATILE(), so use the lock's address instead. Different uses are tagged by kind, although we only do anything with that information in INVARIANTS builds. Sponsored by: EMC / Isilon Storage Division
This commit is contained in:
Conrad Meyer
parent
09f49f249a
commit
466b3540ff
|
|
@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
|
|||
#include "ioat_internal.h"
|
||||
|
||||
#define IOAT_INTR_TIMO (hz / 10)
|
||||
#define IOAT_REFLK (&ioat->submit_lock)
|
||||
|
||||
static int ioat_probe(device_t device);
|
||||
static int ioat_attach(device_t device);
|
||||
|
|
@ -69,9 +70,9 @@ static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
|
|||
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
|
||||
static void ioat_free_ring_entry(struct ioat_softc *ioat,
|
||||
struct ioat_descriptor *desc);
|
||||
static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat);
|
||||
static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *ioat);
|
||||
static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
|
||||
static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat,
|
||||
static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
|
||||
uint32_t index);
|
||||
static boolean_t resize_ring(struct ioat_softc *ioat, int order);
|
||||
static void ioat_timer_callback(void *arg);
|
||||
|
|
@ -81,6 +82,12 @@ static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
|
|||
int error);
|
||||
static int ioat_reset_hw(struct ioat_softc *ioat);
|
||||
static void ioat_setup_sysctl(device_t device);
|
||||
static inline struct ioat_softc *ioat_get(struct ioat_softc *,
|
||||
enum ioat_ref_kind);
|
||||
static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind);
|
||||
static inline void ioat_putn(struct ioat_softc *, uint32_t,
|
||||
enum ioat_ref_kind);
|
||||
static void ioat_drain(struct ioat_softc *);
|
||||
|
||||
#define ioat_log_message(v, ...) do { \
|
||||
if ((v) <= g_ioat_debug_level) { \
|
||||
|
|
@ -236,10 +243,6 @@ ioat_attach(device_t device)
|
|||
goto err;
|
||||
}
|
||||
|
||||
error = ioat_setup_intr(ioat);
|
||||
if (error != 0)
|
||||
return (error);
|
||||
|
||||
error = ioat3_attach(device);
|
||||
if (error != 0)
|
||||
goto err;
|
||||
|
|
@ -248,9 +251,13 @@ ioat_attach(device_t device)
|
|||
if (error != 0)
|
||||
goto err;
|
||||
|
||||
error = ioat_setup_intr(ioat);
|
||||
if (error != 0)
|
||||
goto err;
|
||||
|
||||
error = ioat3_selftest(ioat);
|
||||
if (error != 0)
|
||||
return (error);
|
||||
goto err;
|
||||
|
||||
ioat_process_events(ioat);
|
||||
ioat_setup_sysctl(device);
|
||||
|
|
@ -273,6 +280,7 @@ ioat_detach(device_t device)
|
|||
ioat = DEVICE2SOFTC(device);
|
||||
|
||||
ioat_test_detach();
|
||||
ioat_drain(ioat);
|
||||
|
||||
ioat_teardown_intr(ioat);
|
||||
callout_drain(&ioat->timer);
|
||||
|
|
@ -579,6 +587,7 @@ ioat_process_events(struct ioat_softc *ioat)
|
|||
if (dmadesc->callback_fn)
|
||||
(*dmadesc->callback_fn)(dmadesc->callback_arg);
|
||||
|
||||
completed++;
|
||||
ioat->tail++;
|
||||
if (desc->hw_desc_bus_addr == status)
|
||||
break;
|
||||
|
|
@ -594,6 +603,8 @@ ioat_process_events(struct ioat_softc *ioat)
|
|||
|
||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
|
||||
mtx_unlock(&ioat->cleanup_lock);
|
||||
|
||||
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -603,9 +614,18 @@ bus_dmaengine_t
|
|||
ioat_get_dmaengine(uint32_t index)
|
||||
{
|
||||
|
||||
if (index < ioat_channel_index)
|
||||
return (&ioat_channel[index]->dmaengine);
|
||||
return (NULL);
|
||||
if (index >= ioat_channel_index)
|
||||
return (NULL);
|
||||
return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine);
|
||||
}
|
||||
|
||||
void
|
||||
ioat_put_dmaengine(bus_dmaengine_t dmaengine)
|
||||
{
|
||||
struct ioat_softc *ioat;
|
||||
|
||||
ioat = to_ioat_softc(dmaengine);
|
||||
ioat_put(ioat, IOAT_DMAENGINE_REF);
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -962,6 +982,7 @@ static void
|
|||
ioat_submit_single(struct ioat_softc *ioat)
|
||||
{
|
||||
|
||||
ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
|
||||
atomic_add_rel_int(&ioat->head, 1);
|
||||
|
||||
if (!ioat->is_completion_pending) {
|
||||
|
|
@ -1057,3 +1078,71 @@ ioat_setup_sysctl(device_t device)
|
|||
"tail", CTLFLAG_RD, &ioat->tail,
|
||||
0, "HW descriptor tail pointer index");
|
||||
}
|
||||
|
||||
static inline struct ioat_softc *
|
||||
ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind)
|
||||
{
|
||||
uint32_t old;
|
||||
|
||||
KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
|
||||
|
||||
old = atomic_fetchadd_32(&ioat->refcnt, 1);
|
||||
KASSERT(old < UINT32_MAX, ("refcnt overflow"));
|
||||
|
||||
#ifdef INVARIANTS
|
||||
old = atomic_fetchadd_32(&ioat->refkinds[kind], 1);
|
||||
KASSERT(old < UINT32_MAX, ("refcnt kind overflow"));
|
||||
#endif
|
||||
|
||||
return (ioat);
|
||||
}
|
||||
|
||||
static inline void
|
||||
ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)
|
||||
{
|
||||
uint32_t old;
|
||||
|
||||
KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
|
||||
|
||||
if (n == 0)
|
||||
return;
|
||||
|
||||
#ifdef INVARIANTS
|
||||
old = atomic_fetchadd_32(&ioat->refkinds[kind], -n);
|
||||
KASSERT(old >= n, ("refcnt kind underflow"));
|
||||
#endif
|
||||
|
||||
/* Skip acquiring the lock if resulting refcnt > 0. */
|
||||
for (;;) {
|
||||
old = ioat->refcnt;
|
||||
if (old <= n)
|
||||
break;
|
||||
if (atomic_cmpset_32(&ioat->refcnt, old, old - n))
|
||||
return;
|
||||
}
|
||||
|
||||
mtx_lock(IOAT_REFLK);
|
||||
old = atomic_fetchadd_32(&ioat->refcnt, -n);
|
||||
KASSERT(old >= n, ("refcnt error"));
|
||||
|
||||
if (old == n)
|
||||
wakeup(IOAT_REFLK);
|
||||
mtx_unlock(IOAT_REFLK);
|
||||
}
|
||||
|
||||
static inline void
|
||||
ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind)
|
||||
{
|
||||
|
||||
ioat_putn(ioat, 1, kind);
|
||||
}
|
||||
|
||||
static void
|
||||
ioat_drain(struct ioat_softc *ioat)
|
||||
{
|
||||
|
||||
mtx_lock(IOAT_REFLK);
|
||||
while (ioat->refcnt > 0)
|
||||
msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0);
|
||||
mtx_unlock(IOAT_REFLK);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,9 @@ typedef void (*bus_dmaengine_callback_t)(void *arg);
|
|||
*/
|
||||
bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index);
|
||||
|
||||
/* Release the DMA channel */
|
||||
void ioat_put_dmaengine(bus_dmaengine_t dmaengine);
|
||||
|
||||
/*
|
||||
* Acquire must be called before issuing an operation to perform. Release is
|
||||
* called after. Multiple operations can be issued within the context of one
|
||||
|
|
|
|||
|
|
@ -313,6 +313,12 @@ struct ioat_descriptor {
|
|||
bus_addr_t hw_desc_bus_addr;
|
||||
};
|
||||
|
||||
enum ioat_ref_kind {
|
||||
IOAT_DMAENGINE_REF = 0,
|
||||
IOAT_ACTIVE_DESCR_REF,
|
||||
IOAT_NUM_REF_KINDS
|
||||
};
|
||||
|
||||
/* One of these per allocated PCI device. */
|
||||
struct ioat_softc {
|
||||
bus_dmaengine_t dmaengine;
|
||||
|
|
@ -364,6 +370,10 @@ struct ioat_softc {
|
|||
struct ioat_descriptor **ring;
|
||||
|
||||
struct mtx cleanup_lock;
|
||||
volatile uint32_t refcnt;
|
||||
#ifdef INVARIANTS
|
||||
volatile uint32_t refkinds[IOAT_NUM_REF_KINDS];
|
||||
#endif
|
||||
};
|
||||
|
||||
void ioat_test_attach(void);
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ ioat_dma_test(void *arg)
|
|||
rc = ioat_test_prealloc_memory(test, index);
|
||||
if (rc != 0) {
|
||||
ioat_test_log(0, "prealloc_memory: %d\n", rc);
|
||||
return;
|
||||
goto out;
|
||||
}
|
||||
wmb();
|
||||
|
||||
|
|
@ -330,6 +330,8 @@ ioat_dma_test(void *arg)
|
|||
ticks - start, ticks - end, (ticks - start) / hz);
|
||||
|
||||
ioat_test_release_memory(test);
|
||||
out:
|
||||
ioat_put_dmaengine(dmaengine);
|
||||
}
|
||||
|
||||
static int
|
||||
|
|
|
|||
Loading…
Reference in New Issue