diff --git a/sys/compat/ndis/kern_ndis.c b/sys/compat/ndis/kern_ndis.c index a3fd83094205..a5754b6b6052 100644 --- a/sys/compat/ndis/kern_ndis.c +++ b/sys/compat/ndis/kern_ndis.c @@ -84,7 +84,7 @@ static void ndis_resetdone_func(ndis_handle, ndis_status, uint8_t); static void ndis_sendrsrcavail_func(ndis_handle); static void ndis_intrsetup(kdpc *, device_object *, irp *, struct ndis_softc *); -static void ndis_return(kdpc *, void *, void *, void *); +static void ndis_return(device_object *, void *); static image_patch_table kernndis_functbl[] = { IMPORT_SFUNC(ndis_status_func, 4), @@ -106,6 +106,18 @@ static struct nd_head ndis_devhead; * Note that we call ourselves 'ndisapi' to avoid a namespace * collision with if_ndis.ko, which internally calls itself * 'ndis.' + * + * Note: some of the subsystems depend on each other, so the + * order in which they're started is important. The order of + * importance is: + * + * HAL - spinlocks and IRQL manipulation + * ntoskrnl - DPC and workitem threads, object waiting + * windrv - driver/device registration + * + * The HAL should also be the last thing shut down, since + * the ntoskrnl subsystem will use spinlocks right up until + * the DPC and workitem threads are terminated. */ static int @@ -117,10 +129,10 @@ ndis_modevent(module_t mod, int cmd, void *arg) switch (cmd) { case MOD_LOAD: /* Initialize subsystems */ - windrv_libinit(); hal_libinit(); - ndis_libinit(); ntoskrnl_libinit(); + windrv_libinit(); + ndis_libinit(); usbd_libinit(); patch = kernndis_functbl; @@ -137,11 +149,11 @@ ndis_modevent(module_t mod, int cmd, void *arg) case MOD_SHUTDOWN: if (TAILQ_FIRST(&ndis_devhead) == NULL) { /* Shut down subsystems */ - hal_libfini(); ndis_libfini(); - ntoskrnl_libfini(); usbd_libfini(); windrv_libfini(); + ntoskrnl_libfini(); + hal_libfini(); patch = kernndis_functbl; while (patch->ipt_func != NULL) { @@ -152,11 +164,11 @@ ndis_modevent(module_t mod, int cmd, void *arg) break; case MOD_UNLOAD: /* Shut down subsystems */ - hal_libfini(); ndis_libfini(); - ntoskrnl_libfini(); usbd_libfini(); windrv_libfini(); + ntoskrnl_libfini(); + hal_libfini(); patch = kernndis_functbl; while (patch->ipt_func != NULL) { @@ -441,32 +453,39 @@ ndis_flush_sysctls(arg) } static void -ndis_return(dpc, arg, sysarg1, sysarg2) - kdpc *dpc; +ndis_return(dobj, arg) + device_object *dobj; void *arg; - void *sysarg1; - void *sysarg2; { - struct ndis_softc *sc; + ndis_miniport_block *block; + ndis_miniport_characteristics *ch; ndis_return_handler returnfunc; ndis_handle adapter; ndis_packet *p; uint8_t irql; + list_entry *l; + + block = arg; + ch = IoGetDriverObjectExtension(dobj->do_drvobj, (void *)1); p = arg; - sc = p->np_softc; - adapter = sc->ndis_block->nmb_miniportadapterctx; + adapter = block->nmb_miniportadapterctx; if (adapter == NULL) return; - returnfunc = sc->ndis_chars->nmc_return_packet_func; + returnfunc = ch->nmc_return_packet_func; - if (NDIS_SERIALIZED(sc->ndis_block)) - KeAcquireSpinLock(&sc->ndis_block->nmb_lock, &irql); - MSCALL2(returnfunc, adapter, p); - if (NDIS_SERIALIZED(sc->ndis_block)) - KeReleaseSpinLock(&sc->ndis_block->nmb_lock, irql); + KeAcquireSpinLock(&block->nmb_returnlock, &irql); + while (!IsListEmpty(&block->nmb_returnlist)) { + l = RemoveHeadList((&block->nmb_returnlist)); + p = CONTAINING_RECORD(l, ndis_packet, np_list); + InitializeListHead((&p->np_list)); + KeReleaseSpinLock(&block->nmb_returnlock, irql); + MSCALL2(returnfunc, adapter, p); + KeAcquireSpinLock(&block->nmb_returnlock, &irql); + } + KeReleaseSpinLock(&block->nmb_returnlock, irql); return; } @@ -477,6 +496,7 @@ ndis_return_packet(buf, arg) void *arg; { ndis_packet *p; + ndis_miniport_block *block; if (arg == NULL) return; @@ -490,8 +510,16 @@ ndis_return_packet(buf, arg) if (p->np_refcnt) return; - KeInitializeDpc(&p->np_dpc, kernndis_functbl[7].ipt_wrap, p); - KeInsertQueueDpc(&p->np_dpc, NULL, NULL); + block = ((struct ndis_softc *)p->np_softc)->ndis_block; + + KeAcquireSpinLockAtDpcLevel(&block->nmb_returnlock); + InitializeListHead((&p->np_list)); + InsertHeadList((&block->nmb_returnlist), (&p->np_list)); + KeReleaseSpinLockFromDpcLevel(&block->nmb_returnlock); + + IoQueueWorkItem(block->nmb_returnitem, + (io_workitem_func)kernndis_functbl[7].ipt_wrap, + WORKQUEUE_CRITICAL, block); return; } @@ -621,8 +649,13 @@ ndis_convert_res(arg) case SYS_RES_IRQ: prd->cprd_type = CmResourceTypeInterrupt; prd->cprd_flags = 0; + /* + * Always mark interrupt resources as + * shared, since in our implementation, + * they will be. + */ prd->cprd_sharedisp = - CmResourceShareDeviceExclusive; + CmResourceShareShared; prd->u.cprd_intr.cprd_level = brle->start; prd->u.cprd_intr.cprd_vector = brle->start; prd->u.cprd_intr.cprd_affinity = 0; @@ -1087,8 +1120,12 @@ ndis_halt_nic(arg) #ifdef NDIS_REAP_TIMERS ndis_miniport_timer *t, *n; #endif + ndis_miniport_block *block; + int empty = 0; + uint8_t irql; sc = arg; + block = sc->ndis_block; #ifdef NDIS_REAP_TIMERS /* @@ -1111,6 +1148,19 @@ ndis_halt_nic(arg) if (!cold) KeFlushQueuedDpcs(); + /* + * Wait for all packets to be returned. + */ + + while (1) { + KeAcquireSpinLock(&block->nmb_returnlock, &irql); + empty = IsListEmpty(&block->nmb_returnlist); + KeReleaseSpinLock(&block->nmb_returnlock, irql); + if (empty) + break; + NdisMSleep(1000); + } + NDIS_LOCK(sc); adapter = sc->ndis_block->nmb_miniportadapterctx; if (adapter == NULL) { @@ -1398,6 +1448,17 @@ NdisAddDevice(drv, pdo) ndis_miniport_block *block; struct ndis_softc *sc; uint32_t status; + int error; + + sc = device_get_softc(pdo->do_devext); + + if (sc->ndis_iftype == PCMCIABus || sc->ndis_iftype == PCIBus) { + error = bus_setup_intr(sc->ndis_dev, sc->ndis_irq, + INTR_TYPE_NET | INTR_MPSAFE, + ntoskrnl_intr, NULL, &sc->ndis_intrhand); + if (error) + return(NDIS_STATUS_FAILURE); + } status = IoCreateDevice(drv, sizeof(ndis_miniport_block), NULL, FILE_DEVICE_UNKNOWN, 0, FALSE, &fdo); @@ -1412,17 +1473,19 @@ NdisAddDevice(drv, pdo) block->nmb_physdeviceobj = pdo; block->nmb_nextdeviceobj = IoAttachDeviceToDeviceStack(fdo, pdo); KeInitializeSpinLock(&block->nmb_lock); - InitializeListHead(&block->nmb_parmlist); + KeInitializeSpinLock(&block->nmb_returnlock); KeInitializeEvent(&block->nmb_getevent, EVENT_TYPE_NOTIFY, TRUE); KeInitializeEvent(&block->nmb_setevent, EVENT_TYPE_NOTIFY, TRUE); KeInitializeEvent(&block->nmb_resetevent, EVENT_TYPE_NOTIFY, TRUE); + InitializeListHead(&block->nmb_parmlist); + InitializeListHead(&block->nmb_returnlist); + block->nmb_returnitem = IoAllocateWorkItem(fdo); /* * Stash pointers to the miniport block and miniport * characteristics info in the if_ndis softc so the * UNIX wrapper driver can get to them later. */ - sc = device_get_softc(pdo->do_devext); sc->ndis_block = block; sc->ndis_chars = IoGetDriverObjectExtension(drv, (void *)1); @@ -1471,6 +1534,10 @@ ndis_unload_driver(arg) sc = arg; + if (sc->ndis_intrhand) + bus_teardown_intr(sc->ndis_dev, + sc->ndis_irq, sc->ndis_intrhand); + if (sc->ndis_block->nmb_rlist != NULL) free(sc->ndis_block->nmb_rlist, M_DEVBUF); @@ -1481,6 +1548,7 @@ ndis_unload_driver(arg) if (sc->ndis_chars->nmc_transferdata_func != NULL) NdisFreePacketPool(sc->ndis_block->nmb_rxpool); fdo = sc->ndis_block->nmb_deviceobj; + IoFreeWorkItem(sc->ndis_block->nmb_returnitem); IoDetachDevice(sc->ndis_block->nmb_nextdeviceobj); IoDeleteDevice(fdo); diff --git a/sys/compat/ndis/ndis_var.h b/sys/compat/ndis/ndis_var.h index 5a3d5258d17f..6f3772dbe76b 100644 --- a/sys/compat/ndis/ndis_var.h +++ b/sys/compat/ndis/ndis_var.h @@ -343,6 +343,7 @@ typedef uint8_t ndis_kirql; #define NDIS_80211_NETTYPE_11DS 0x00000001 #define NDIS_80211_NETTYPE_11OFDM5 0x00000002 #define NDIS_80211_NETTYPE_11OFDM24 0x00000003 +#define NDIS_80211_NETTYPE_AUTO 0x00000004 struct ndis_80211_nettype_list { uint32_t ntl_items; @@ -1312,12 +1313,24 @@ struct ndis_packet { void *np_softc; void *np_m0; int np_txidx; - kdpc np_dpc; - kspin_lock np_lock; + list_entry np_list; }; typedef struct ndis_packet ndis_packet; +struct ndis_packet_pool { + slist_header np_head; + int np_dead; + nt_kevent np_event; + kspin_lock np_lock; + int np_cnt; + int np_len; + int np_protrsvd; + void *np_pktmem; +}; + +typedef struct ndis_packet_pool ndis_packet_pool; + /* mbuf ext type for NDIS */ #define EXT_NDIS 0x999 @@ -1617,8 +1630,11 @@ struct ndis_miniport_block { ndis_status nmb_setstat; nt_kevent nmb_setevent; nt_kevent nmb_resetevent; + io_workitem *nmb_returnitem; ndis_miniport_timer *nmb_timerlist; ndis_handle nmb_rxpool; + list_entry nmb_returnlist; + kspin_lock nmb_returnlock; TAILQ_ENTRY(ndis_miniport_block) link; }; @@ -1747,7 +1763,7 @@ extern void NdisAllocatePacket(ndis_status *, ndis_packet **, ndis_handle); extern void NdisFreePacket(ndis_packet *); extern ndis_status NdisScheduleWorkItem(ndis_work_item *); - +extern void NdisMSleep(uint32_t); __END_DECLS #endif /* _NDIS_VAR_H_ */ diff --git a/sys/compat/ndis/ntoskrnl_var.h b/sys/compat/ndis/ntoskrnl_var.h index f954c1470abe..5ec1f54b2864 100644 --- a/sys/compat/ndis/ntoskrnl_var.h +++ b/sys/compat/ndis/ntoskrnl_var.h @@ -35,6 +35,8 @@ #ifndef _NTOSKRNL_VAR_H_ #define _NTOSKRNL_VAR_H_ +#define MTX_NTOSKRNL_SPIN_LOCK "NDIS thread lock" + /* * us_buf is really a wchar_t *, but it's inconvenient to include * all the necessary header goop needed to define it, and it's a @@ -573,7 +575,9 @@ typedef struct custom_extension custom_extension; */ struct kinterrupt { + list_entry ki_list; device_t ki_dev; + int ki_rid; void *ki_cookie; struct resource *ki_irq; kspin_lock ki_lock_priv; @@ -1304,6 +1308,12 @@ extern void ctxsw_wtou(void); extern int ntoskrnl_libinit(void); extern int ntoskrnl_libfini(void); +extern void ntoskrnl_intr(void *); + +extern uint16_t ExQueryDepthSList(slist_header *); +extern slist_entry + *InterlockedPushEntrySList(slist_header *, slist_entry *); +extern slist_entry *InterlockedPopEntrySList(slist_header *); extern uint32_t RtlUnicodeStringToAnsiString(ansi_string *, unicode_string *, uint8_t); extern uint32_t RtlAnsiStringToUnicodeString(unicode_string *, @@ -1342,6 +1352,8 @@ extern void KeAcquireSpinLockAtDpcLevel(kspin_lock *); extern void KeReleaseSpinLockFromDpcLevel(kspin_lock *); #endif extern void KeInitializeSpinLock(kspin_lock *); +extern uint8_t KeAcquireInterruptSpinLock(kinterrupt *); +extern void KeReleaseInterruptSpinLock(kinterrupt *, uint8_t); extern uint8_t KeSynchronizeExecution(kinterrupt *, void *, void *); extern uintptr_t InterlockedExchange(volatile uint32_t *, uintptr_t); diff --git a/sys/compat/ndis/subr_ndis.c b/sys/compat/ndis/subr_ndis.c index 78135150b35c..8e4d640ffec9 100644 --- a/sys/compat/ndis/subr_ndis.c +++ b/sys/compat/ndis/subr_ndis.c @@ -112,7 +112,6 @@ __FBSDID("$FreeBSD$"); #include static char ndis_filepath[MAXPATHLEN]; -extern struct nd_head ndis_devhead; SYSCTL_STRING(_hw, OID_AUTO, ndis_filepath, CTLFLAG_RW, ndis_filepath, MAXPATHLEN, "Path used by NdisOpenFile() to search for files"); @@ -238,7 +237,6 @@ static void NdisGetBufferPhysicalArraySize(ndis_buffer *, uint32_t *); static void NdisQueryBufferOffset(ndis_buffer *, uint32_t *, uint32_t *); -static void NdisMSleep(uint32_t); static uint32_t NdisReadPcmciaAttributeMemory(ndis_handle, uint32_t, void *, uint32_t); static uint32_t NdisWritePcmciaAttributeMemory(ndis_handle, @@ -421,7 +419,6 @@ NdisMRegisterMiniport(handle, characteristics, len) if (IoAllocateDriverObjectExtension(drv, (void *)1, sizeof(ndis_miniport_characteristics), (void **)&ch) != STATUS_SUCCESS) { - printf("register error\n"); return(NDIS_STATUS_RESOURCES); } @@ -1846,27 +1843,43 @@ NdisAllocatePacketPool(status, pool, descnum, protrsvdlen) uint32_t descnum; uint32_t protrsvdlen; { - ndis_packet *cur; + ndis_packet_pool *p; + ndis_packet *packets; int i; - *pool = malloc((sizeof(ndis_packet) + protrsvdlen) * - ((descnum + NDIS_POOL_EXTRA) + 1), - M_DEVBUF, M_NOWAIT|M_ZERO); - - if (*pool == NULL) { + p = ExAllocatePoolWithTag(NonPagedPool, sizeof(ndis_packet_pool), 0); + if (p == NULL) { *status = NDIS_STATUS_RESOURCES; return; } - cur = (ndis_packet *)*pool; - KeInitializeSpinLock(&cur->np_lock); - cur->np_private.npp_flags = 0x1; /* mark the head of the list */ - cur->np_private.npp_totlen = 0; /* init deletetion flag */ - for (i = 0; i < (descnum + NDIS_POOL_EXTRA); i++) { - cur->np_private.npp_head = (ndis_handle)(cur + 1); - cur++; + p->np_cnt = descnum + NDIS_POOL_EXTRA; + p->np_protrsvd = protrsvdlen; + p->np_len = sizeof(ndis_packet) + protrsvdlen; + + packets = ExAllocatePoolWithTag(NonPagedPool, p->np_cnt * + p->np_len, 0); + + + if (packets == NULL) { + ExFreePool(p); + *status = NDIS_STATUS_RESOURCES; + return; } + p->np_pktmem = packets; + + for (i = 0; i < p->np_cnt; i++) + InterlockedPushEntrySList(&p->np_head, + (struct slist_entry *)&packets[i]); + +#ifdef NDIS_DEBUG_PACKETS + p->np_dead = 0; + KeInitializeSpinLock(&p->np_lock); + KeInitializeEvent(&p->np_event, EVENT_TYPE_NOTIFY, TRUE); +#endif + + *pool = p; *status = NDIS_STATUS_SUCCESS; return; } @@ -1887,41 +1900,42 @@ uint32_t NdisPacketPoolUsage(pool) ndis_handle pool; { - ndis_packet *head; - uint8_t irql; - uint32_t cnt; + ndis_packet_pool *p; - head = (ndis_packet *)pool; - KeAcquireSpinLock(&head->np_lock, &irql); - cnt = head->np_private.npp_count; - KeReleaseSpinLock(&head->np_lock, irql); - - return(cnt); + p = (ndis_packet_pool *)pool; + return(p->np_cnt - ExQueryDepthSList(&p->np_head)); } void NdisFreePacketPool(pool) ndis_handle pool; { - ndis_packet *head; + ndis_packet_pool *p; + int usage; +#ifdef NDIS_DEBUG_PACKETS uint8_t irql; +#endif - head = pool; + p = (ndis_packet_pool *)pool; - /* Mark this pool as 'going away.' */ +#ifdef NDIS_DEBUG_PACKETS + KeAcquireSpinLock(&p->np_lock, &irql); +#endif - KeAcquireSpinLock(&head->np_lock, &irql); - head->np_private.npp_totlen = 1; + usage = NdisPacketPoolUsage(pool); - /* If there are no buffers loaned out, destroy the pool. */ +#ifdef NDIS_DEBUG_PACKETS + if (usage) { + p->np_dead = 1; + KeResetEvent(&p->np_event); + KeReleaseSpinLock(&p->np_lock, irql); + KeWaitForSingleObject(&p->np_event, 0, 0, FALSE, NULL); + } else + KeReleaseSpinLock(&p->np_lock, irql); +#endif - if (head->np_private.npp_count == 0) { - KeReleaseSpinLock(&head->np_lock, irql); - free(pool, M_DEVBUF); - } else { - printf("NDIS: buggy driver deleting active packet pool!\n"); - KeReleaseSpinLock(&head->np_lock, irql); - } + ExFreePool(p->np_pktmem); + ExFreePool(p); return; } @@ -1932,42 +1946,41 @@ NdisAllocatePacket(status, packet, pool) ndis_packet **packet; ndis_handle pool; { - ndis_packet *head, *pkt; + ndis_packet_pool *p; + ndis_packet *pkt; +#ifdef NDIS_DEBUG_PACKETS uint8_t irql; +#endif - head = (ndis_packet *)pool; - KeAcquireSpinLock(&head->np_lock, &irql); + p = (ndis_packet_pool *)pool; - if (head->np_private.npp_flags != 0x1) { - *status = NDIS_STATUS_FAILURE; - KeReleaseSpinLock(&head->np_lock, irql); +#ifdef NDIS_DEBUG_PACKETS + KeAcquireSpinLock(&p->np_lock, &irql); + if (p->np_dead) { + KeReleaseSpinLock(&p->np_lock, irql); + printf("NDIS: tried to allocate packet from dead pool %p\n", + pool); + *status = NDIS_STATUS_RESOURCES; return; } +#endif - /* - * If this pool is marked as 'going away' don't allocate any - * more packets out of it. - */ + pkt = (ndis_packet *)InterlockedPopEntrySList(&p->np_head); - if (head->np_private.npp_totlen) { - *status = NDIS_STATUS_FAILURE; - KeReleaseSpinLock(&head->np_lock, irql); - return; - } - - pkt = (ndis_packet *)head->np_private.npp_head; +#ifdef NDIS_DEBUG_PACKETS + KeReleaseSpinLock(&p->np_lock, irql); +#endif if (pkt == NULL) { *status = NDIS_STATUS_RESOURCES; - KeReleaseSpinLock(&head->np_lock, irql); return; } - head->np_private.npp_head = pkt->np_private.npp_head; - pkt->np_private.npp_head = pkt->np_private.npp_tail = NULL; + bzero((char *)pkt, sizeof(ndis_packet)); + /* Save pointer to the pool. */ - pkt->np_private.npp_pool = head; + pkt->np_private.npp_pool = pool; /* Set the oob offset pointer. Lots of things expect this. */ pkt->np_private.npp_packetooboffset = offsetof(ndis_packet, np_oob); @@ -1983,11 +1996,8 @@ NdisAllocatePacket(status, packet, pool) *packet = pkt; - head->np_private.npp_count++; *status = NDIS_STATUS_SUCCESS; - KeReleaseSpinLock(&head->np_lock, irql); - return; } @@ -1995,34 +2005,26 @@ void NdisFreePacket(packet) ndis_packet *packet; { - ndis_packet *head; + ndis_packet_pool *p; +#ifdef NDIS_DEBUG_PACKETS uint8_t irql; +#endif - if (packet == NULL || packet->np_private.npp_pool == NULL) - return; + p = (ndis_packet_pool *)packet->np_private.npp_pool; - head = packet->np_private.npp_pool; - KeAcquireSpinLock(&head->np_lock, &irql); +#ifdef NDIS_DEBUG_PACKETS + KeAcquireSpinLock(&p->np_lock, &irql); +#endif - if (head->np_private.npp_flags != 0x1) { - KeReleaseSpinLock(&head->np_lock, irql); - return; + InterlockedPushEntrySList(&p->np_head, (slist_entry *)packet); + +#ifdef NDIS_DEBUG_PACKETS + if (p->np_dead) { + if (ExQueryDepthSList(&p->np_head) == p->np_cnt) + KeSetEvent(&p->np_event, IO_NO_INCREMENT, FALSE); } - - packet->np_private.npp_head = head->np_private.npp_head; - head->np_private.npp_head = (ndis_buffer *)packet; - head->np_private.npp_count--; - - /* - * If the pool has been marked for deletion and there are - * no more packets outstanding, nuke the pool. - */ - - if (head->np_private.npp_totlen && head->np_private.npp_count == 0) { - KeReleaseSpinLock(&head->np_lock, irql); - free(head, M_DEVBUF); - } else - KeReleaseSpinLock(&head->np_lock, irql); + KeReleaseSpinLock(&p->np_lock, irql); +#endif return; } @@ -2255,7 +2257,7 @@ static void NdisSetEvent(event) ndis_event *event; { - KeSetEvent(&event->ne_event, 0, 0); + KeSetEvent(&event->ne_event, IO_NO_INCREMENT, FALSE); return; } @@ -2276,7 +2278,7 @@ NdisWaitEvent(event, msecs) uint32_t rval; duetime = ((int64_t)msecs * -10000); - rval = KeWaitForSingleObject((nt_dispatch_header *)event, + rval = KeWaitForSingleObject(event, 0, 0, TRUE, msecs ? & duetime : NULL); if (rval == STATUS_TIMEOUT) @@ -2479,8 +2481,7 @@ NdisMDeregisterInterrupt(intr) IoDisconnectInterrupt(intr->ni_introbj); - KeWaitForSingleObject((nt_dispatch_header *)&intr->ni_dpcevt, - 0, 0, FALSE, NULL); + KeWaitForSingleObject(&intr->ni_dpcevt, 0, 0, FALSE, NULL); KeResetEvent(&intr->ni_dpcevt); return; @@ -2569,7 +2570,7 @@ NdisQueryBufferOffset(buf, off, len) return; } -static void +void NdisMSleep(usecs) uint32_t usecs; { @@ -2586,9 +2587,9 @@ NdisMSleep(usecs) else { KeInitializeTimer(&timer); KeSetTimer(&timer, ((int64_t)usecs * -10), NULL); - KeWaitForSingleObject((nt_dispatch_header *)&timer, - 0, 0, FALSE, NULL); + KeWaitForSingleObject(&timer, 0, 0, FALSE, NULL); } + return; } diff --git a/sys/compat/ndis/subr_ntoskrnl.c b/sys/compat/ndis/subr_ntoskrnl.c index 023526de6530..f0564663ea5c 100644 --- a/sys/compat/ndis/subr_ntoskrnl.c +++ b/sys/compat/ndis/subr_ntoskrnl.c @@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -78,12 +79,20 @@ __FBSDID("$FreeBSD$"); #include #include +#ifdef NTOSKRNL_DEBUG_TIMERS +static int sysctl_show_timers(SYSCTL_HANDLER_ARGS); + +SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLFLAG_RW, 0, 0, + sysctl_show_timers, "I", "Show ntoskrnl timer stats"); +#endif + struct kdpc_queue { list_entry kq_disp; struct thread *kq_td; int kq_cpu; int kq_exit; - struct mtx kq_lock; + int kq_running; + kspin_lock kq_lock; nt_kevent kq_proc; nt_kevent kq_done; nt_kevent kq_dead; @@ -99,9 +108,25 @@ struct wb_ext { typedef struct wb_ext wb_ext; #define NTOSKRNL_TIMEOUTS 256 -struct callout ntoskrnl_callout[NTOSKRNL_TIMEOUTS]; -int ntoskrnl_callidx; -#define CALLOUT_INC(i) (i) = ((i) + 1) % NTOSKRNL_TIMEOUTS +#ifdef NTOSKRNL_DEBUG_TIMERS +static uint64_t ntoskrnl_timer_fires; +static uint64_t ntoskrnl_timer_sets; +static uint64_t ntoskrnl_timer_reloads; +static uint64_t ntoskrnl_timer_cancels; +#endif + +struct callout_entry { + struct callout ce_callout; + list_entry ce_list; +}; + +typedef struct callout_entry callout_entry; + +static struct list_entry ntoskrnl_calllist; +static struct mtx ntoskrnl_calllock; + +static struct list_entry ntoskrnl_intlist; +static kspin_lock ntoskrnl_intlock; static uint8_t RtlEqualUnicodeString(unicode_string *, unicode_string *, uint8_t); @@ -126,8 +151,12 @@ static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t); static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *); static void ntoskrnl_satisfy_multiple_waits(wait_block *); static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *); +static void ntoskrnl_insert_timer(ktimer *, int); +static void ntoskrnl_remove_timer(ktimer *); +#ifdef NTOSKRNL_DEBUG_TIMERS +static void ntoskrnl_show_timers(void); +#endif static void ntoskrnl_timercall(void *); -static void ntoskrnl_run_dpc(void *); static void ntoskrnl_dpc_thread(void *); static void ntoskrnl_destroy_dpc_threads(void); static void ntoskrnl_destroy_workitem_threads(void); @@ -136,9 +165,6 @@ static void ntoskrnl_workitem(device_object *, void *); static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int); static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int); static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *); -static device_t ntoskrnl_finddev(device_t, uint32_t, - uint8_t, uint8_t, struct resource **); -static void ntoskrnl_intr(void *); static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t); static uint16_t READ_REGISTER_USHORT(uint16_t *); static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t); @@ -165,15 +191,11 @@ static void ExInitializeNPagedLookasideList(npaged_lookaside_list *, lookaside_alloc_func *, lookaside_free_func *, uint32_t, size_t, uint32_t, uint16_t); static void ExDeleteNPagedLookasideList(npaged_lookaside_list *); -static slist_entry - *InterlockedPushEntrySList(slist_header *, slist_entry *); -static slist_entry *InterlockedPopEntrySList(slist_header *); static slist_entry *ExInterlockedPushEntrySList(slist_header *, slist_entry *, kspin_lock *); static slist_entry *ExInterlockedPopEntrySList(slist_header *, kspin_lock *); -static uint16_t ExQueryDepthSList(slist_header *); static uint32_t InterlockedIncrement(volatile uint32_t *); static uint32_t InterlockedDecrement(volatile uint32_t *); static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t); @@ -215,7 +237,7 @@ static void DbgBreakPoint(void); static void dummy(void); static struct mtx ntoskrnl_dispatchlock; -static kspin_lock ntoskrnl_global; +static struct mtx ntoskrnl_interlock; static kspin_lock ntoskrnl_cancellock; static int ntoskrnl_kth = 0; static struct nt_objref_head ntoskrnl_reflist; @@ -232,15 +254,21 @@ ntoskrnl_libinit() int error; struct proc *p; kdpc_queue *kq; + callout_entry *e; int i; char name[64]; mtx_init(&ntoskrnl_dispatchlock, "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE); - KeInitializeSpinLock(&ntoskrnl_global); + mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); KeInitializeSpinLock(&ntoskrnl_cancellock); + KeInitializeSpinLock(&ntoskrnl_intlock); TAILQ_INIT(&ntoskrnl_reflist); + InitializeListHead(&ntoskrnl_calllist); + InitializeListHead(&ntoskrnl_intlist); + mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN); + kq_queues = ExAllocatePoolWithTag(NonPagedPool, sizeof(kdpc_queue) * mp_ncpus, 0); @@ -291,8 +319,15 @@ ntoskrnl_libinit() patch++; } - for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) - callout_init(&ntoskrnl_callout[i], CALLOUT_MPSAFE); + for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) { + e = ExAllocatePoolWithTag(NonPagedPool, + sizeof(callout_entry), 0); + if (e == NULL) + panic("failed to allocate timeouts"); + mtx_lock_spin(&ntoskrnl_calllock); + InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); + mtx_unlock_spin(&ntoskrnl_calllock); + } /* * MDLs are supposed to be variable size (they describe @@ -320,6 +355,8 @@ int ntoskrnl_libfini() { image_patch_table *patch; + callout_entry *e; + list_entry *l; patch = ntoskrnl_functbl; while (patch->ipt_func != NULL) { @@ -338,7 +375,19 @@ ntoskrnl_libfini() uma_zdestroy(mdl_zone); uma_zdestroy(iw_zone); + mtx_lock_spin(&ntoskrnl_calllock); + while(!IsListEmpty(&ntoskrnl_calllist)) { + l = RemoveHeadList(&ntoskrnl_calllist); + e = CONTAINING_RECORD(l, callout_entry, ce_list); + mtx_unlock_spin(&ntoskrnl_calllock); + ExFreePool(e); + mtx_lock_spin(&ntoskrnl_calllock); + } + mtx_unlock_spin(&ntoskrnl_calllock); + mtx_destroy(&ntoskrnl_dispatchlock); + mtx_destroy(&ntoskrnl_interlock); + mtx_destroy(&ntoskrnl_calllock); return(0); } @@ -1132,100 +1181,41 @@ IofCompleteRequest(ip, prioboost) return; } -static device_t -ntoskrnl_finddev(dev, vector, irql, shared, res) - device_t dev; - uint32_t vector; - uint8_t irql; - uint8_t shared; - struct resource **res; -{ - device_t *children; - device_t matching_dev; - int childcnt; - struct resource *r; - struct resource_list *rl; - struct resource_list_entry *rle; - uint32_t flags; - int i; - - /* We only want devices that have been successfully probed. */ - - if (device_is_alive(dev) == FALSE) - return(NULL); - - device_get_children(dev, &children, &childcnt); - - /* - * If this device has no children, it's a leaf: we can - * examine its resources. If the interrupt resource we - * want isn't here, we return NULL, otherwise we return - * the device to terminate the recursive search. - */ - - if (childcnt == 0) { - rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); - if (rl == NULL) - return(NULL); -#if __FreeBSD_version < 600022 - SLIST_FOREACH(rle, rl, link) { -#else - STAILQ_FOREACH(rle, rl, link) { -#endif - r = rle->res; - - if (r == NULL) - continue; - - flags = rman_get_flags(r); - - if (!(flags & RF_ACTIVE)) - continue; - - if (shared == TRUE && !(flags & RF_SHAREABLE)) - continue; - - if (rle->type == SYS_RES_IRQ && - rman_get_start(r) == irql) { - *res = r; - return(dev); - } - } - /* No match. */ - return (NULL); - } - - /* - * If this device has children, do another - * level of recursion to inspect them. - */ - - for (i = 0; i < childcnt; i++) { - matching_dev = ntoskrnl_finddev(children[i], - vector, irql, shared, res); - if (matching_dev != NULL) { - free(children, M_TEMP); - return(matching_dev); - } - } - - free(children, M_TEMP); - return(NULL); -} - -static void +void ntoskrnl_intr(arg) void *arg; { kinterrupt *iobj; uint8_t irql; + list_entry *l; - iobj = arg; + KeAcquireSpinLock(&ntoskrnl_intlock, &irql); + l = ntoskrnl_intlist.nle_flink; + while (l != &ntoskrnl_intlist) { + iobj = CONTAINING_RECORD(l, kinterrupt, ki_list); + MSCALL1(iobj->ki_svcfunc, iobj->ki_svcctx); + l = l->nle_flink; + } + KeReleaseSpinLock(&ntoskrnl_intlock, irql); - KeAcquireSpinLock(iobj->ki_lock, &irql); - MSCALL1(iobj->ki_svcfunc, iobj->ki_svcctx); - KeReleaseSpinLock(iobj->ki_lock, irql); + return; +} +uint8_t +KeAcquireInterruptSpinLock(iobj) + kinterrupt *iobj; +{ + uint8_t irql; + KeAcquireSpinLock(&ntoskrnl_intlock, &irql); + return(irql); +} + +void +KeReleaseInterruptSpinLock(iobj, irql) + kinterrupt *iobj; + uint8_t irql; +{ + KeReleaseSpinLock(&ntoskrnl_intlock, irql); return; } @@ -1237,18 +1227,27 @@ KeSynchronizeExecution(iobj, syncfunc, syncctx) { uint8_t irql; - KeAcquireSpinLock(iobj->ki_lock, &irql); + KeAcquireSpinLock(&ntoskrnl_intlock, &irql); MSCALL1(syncfunc, syncctx); - KeReleaseSpinLock(iobj->ki_lock, irql); + KeReleaseSpinLock(&ntoskrnl_intlock, irql); return(TRUE); } /* - * This routine is a pain because the only thing we get passed - * here is the interrupt request level and vector, but bus_setup_intr() - * needs the device too. We can hack around this for now, but it's - * awkward. + * IoConnectInterrupt() is passed only the interrupt vector and + * irql that a device wants to use, but no device-specific tag + * of any kind. This conflicts rather badly with FreeBSD's + * bus_setup_intr(), which needs the device_t for the device + * requesting interrupt delivery. In order to bypass this + * inconsistency, we implement a second level of interrupt + * dispatching on top of bus_setup_intr(). All devices use + * ntoskrnl_intr() as their ISR, and any device requesting + * interrupts will be registered with ntoskrnl_intr()'s interrupt + * dispatch list. When an interrupt arrives, we walk the list + * and invoke all the registered ISRs. This effectively makes all + * interrupts shared, but it's the only way to duplicate the + * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly. */ uint32_t @@ -1266,35 +1265,12 @@ IoConnectInterrupt(iobj, svcfunc, svcctx, lock, vector, irql, uint32_t affinity; uint8_t savefloat; { - devclass_t nexus_class; - device_t *nexus_devs, devp; - int nexus_count = 0; - device_t matching_dev = NULL; - struct resource *res; - int i, error; - - nexus_class = devclass_find("nexus"); - devclass_get_devices(nexus_class, &nexus_devs, &nexus_count); - - for (i = 0; i < nexus_count; i++) { - devp = nexus_devs[i]; - matching_dev = ntoskrnl_finddev(devp, vector, - irql, shared, &res); - if (matching_dev) - break; - } - - free(nexus_devs, M_TEMP); - - if (matching_dev == NULL) - return(STATUS_INVALID_PARAMETER); + uint8_t curirql; *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0); if (*iobj == NULL) return(STATUS_INSUFFICIENT_RESOURCES); - (*iobj)->ki_dev = matching_dev; - (*iobj)->ki_irq = res; (*iobj)->ki_svcfunc = svcfunc; (*iobj)->ki_svcctx = svcctx; @@ -1304,13 +1280,9 @@ IoConnectInterrupt(iobj, svcfunc, svcctx, lock, vector, irql, } else (*iobj)->ki_lock = lock; - error = bus_setup_intr(matching_dev, res, INTR_TYPE_NET | INTR_MPSAFE, - ntoskrnl_intr, *iobj, &(*iobj)->ki_cookie); - - if (error) { - ExFreePool(iobj); - return (STATUS_INVALID_PARAMETER); - } + KeAcquireSpinLock(&ntoskrnl_intlock, &curirql); + InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list)); + KeReleaseSpinLock(&ntoskrnl_intlock, curirql); return(STATUS_SUCCESS); } @@ -1319,10 +1291,15 @@ void IoDisconnectInterrupt(iobj) kinterrupt *iobj; { + uint8_t irql; + if (iobj == NULL) return; - bus_teardown_intr(iobj->ki_dev, iobj->ki_irq, iobj->ki_cookie); + KeAcquireSpinLock(&ntoskrnl_intlock, &irql); + RemoveEntryList((&iobj->ki_list)); + KeReleaseSpinLock(&ntoskrnl_intlock, irql); + ExFreePool(iobj); return; @@ -1537,6 +1514,7 @@ ntoskrnl_waittest(obj, increment) if (satisfied == TRUE) cv_broadcastpri(&we->we_cv, w->wb_oldpri - (increment * 4)); + e = e->nle_flink; } @@ -2227,25 +2205,29 @@ ExDeleteNPagedLookasideList(lookaside) return; } -static slist_entry * +slist_entry * InterlockedPushEntrySList(head, entry) slist_header *head; slist_entry *entry; { slist_entry *oldhead; - oldhead = ExInterlockedPushEntrySList(head, entry, &ntoskrnl_global); + mtx_lock_spin(&ntoskrnl_interlock); + oldhead = ntoskrnl_pushsl(head, entry); + mtx_unlock_spin(&ntoskrnl_interlock); return(oldhead); } -static slist_entry * +slist_entry * InterlockedPopEntrySList(head) slist_header *head; { slist_entry *first; - first = ExInterlockedPopEntrySList(head, &ntoskrnl_global); + mtx_lock_spin(&ntoskrnl_interlock); + first = ntoskrnl_popsl(head); + mtx_unlock_spin(&ntoskrnl_interlock); return(first); } @@ -2256,14 +2238,7 @@ ExInterlockedPushEntrySList(head, entry, lock) slist_entry *entry; kspin_lock *lock; { - slist_entry *oldhead; - uint8_t irql; - - KeAcquireSpinLock(lock, &irql); - oldhead = ntoskrnl_pushsl(head, entry); - KeReleaseSpinLock(lock, irql); - - return(oldhead); + return(InterlockedPushEntrySList(head, entry)); } static slist_entry * @@ -2271,37 +2246,22 @@ ExInterlockedPopEntrySList(head, lock) slist_header *head; kspin_lock *lock; { - slist_entry *first; - uint8_t irql; - - KeAcquireSpinLock(lock, &irql); - first = ntoskrnl_popsl(head); - KeReleaseSpinLock(lock, irql); - - return(first); + return(InterlockedPopEntrySList(head)); } -static uint16_t +uint16_t ExQueryDepthSList(head) slist_header *head; { uint16_t depth; - uint8_t irql; - KeAcquireSpinLock(&ntoskrnl_global, &irql); + mtx_lock_spin(&ntoskrnl_interlock); depth = head->slh_list.slh_depth; - KeReleaseSpinLock(&ntoskrnl_global, irql); + mtx_unlock_spin(&ntoskrnl_interlock); return(depth); } -/* - * The KeInitializeSpinLock(), KefAcquireSpinLockAtDpcLevel() - * and KefReleaseSpinLockFromDpcLevel() appear to be analagous - * to splnet()/splx() in their use. We can't create a new mutex - * lock here because there is no complimentary KeFreeSpinLock() - * function. Instead, we grab a mutex from the mutex pool. - */ void KeInitializeSpinLock(lock) kspin_lock *lock; @@ -2316,8 +2276,18 @@ void KefAcquireSpinLockAtDpcLevel(lock) kspin_lock *lock; { - while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) +#ifdef NTOSKRNL_DEBUG_SPINLOCKS + int i = 0; +#endif + + while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) { /* sit and spin */; +#ifdef NTOSKRNL_DEBUG_SPINLOCKS + i++; + if (i > 200000000) + panic("DEADLOCK!"); +#endif + } return; } @@ -2368,13 +2338,12 @@ InterlockedExchange(dst, val) volatile uint32_t *dst; uintptr_t val; { - uint8_t irql; uintptr_t r; - KeAcquireSpinLock(&ntoskrnl_global, &irql); + mtx_lock_spin(&ntoskrnl_interlock); r = *dst; *dst = val; - KeReleaseSpinLock(&ntoskrnl_global, irql); + mtx_unlock_spin(&ntoskrnl_interlock); return(r); } @@ -2400,11 +2369,9 @@ ExInterlockedAddLargeStatistic(addend, inc) uint64_t *addend; uint32_t inc; { - uint8_t irql; - - KeAcquireSpinLock(&ntoskrnl_global, &irql); + mtx_lock_spin(&ntoskrnl_interlock); *addend += inc; - KeReleaseSpinLock(&ntoskrnl_global, irql); + mtx_unlock_spin(&ntoskrnl_interlock); return; }; @@ -2579,23 +2546,24 @@ ntoskrnl_workitem_thread(arg) kdpc_queue *kq; list_entry *l; io_workitem *iw; + uint8_t irql; kq = arg; InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; - mtx_init(&kq->kq_lock, "NDIS thread lock", NULL, MTX_SPIN); + KeInitializeSpinLock(&kq->kq_lock); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_dead, EVENT_TYPE_SYNC, FALSE); while (1) { KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLock(&kq->kq_lock, &irql); if (kq->kq_exit) { - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_dead, IO_NO_INCREMENT, FALSE); break; } @@ -2607,16 +2575,14 @@ ntoskrnl_workitem_thread(arg) InitializeListHead((&iw->iw_listentry)); if (iw->iw_func == NULL) continue; - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx); - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLock(&kq->kq_lock, &irql); } - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); } - mtx_destroy(&kq->kq_lock); - #if __FreeBSD_version < 502113 mtx_lock(&Giant); #endif @@ -2679,13 +2645,11 @@ IoQueueWorkItem(iw, iw_func, qtype, ctx) kdpc_queue *kq; list_entry *l; io_workitem *cur; - - iw->iw_func = iw_func; - iw->iw_ctx = ctx; + uint8_t irql; kq = wq_queues + iw->iw_idx; - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLock(&kq->kq_lock, &irql); /* * Traverse the list and make sure this workitem hasn't @@ -2698,14 +2662,17 @@ IoQueueWorkItem(iw, iw_func, qtype, ctx) cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); if (cur == iw) { /* Already queued -- do nothing. */ - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); return; } l = l->nle_flink; } + iw->iw_func = iw_func; + iw->iw_ctx = ctx; + InsertTailList((&kq->kq_disp), (&iw->iw_listentry)); - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE); @@ -2765,6 +2732,7 @@ ExQueueWorkItem(w, qtype) kdpc_queue *kq; list_entry *l; io_workitem *cur; + uint8_t irql; /* @@ -2777,18 +2745,18 @@ ExQueueWorkItem(w, qtype) */ kq = wq_queues + WORKITEM_LEGACY_THREAD; - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLock(&kq->kq_lock, &irql); l = kq->kq_disp.nle_flink; while (l != &kq->kq_disp) { cur = CONTAINING_RECORD(l, io_workitem, iw_listentry); if (cur->iw_dobj == (device_object *)w) { /* Already queued -- do nothing. */ - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); return; } l = l->nle_flink; } - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); iw = IoAllocateWorkItem((device_object *)w); if (iw == NULL) @@ -3362,12 +3330,28 @@ ntoskrnl_timercall(arg) { ktimer *timer; struct timeval tv; + kdpc *dpc; mtx_lock(&ntoskrnl_dispatchlock); timer = arg; - callout_init(timer->k_callout, CALLOUT_MPSAFE); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_timer_fires++; +#endif + ntoskrnl_remove_timer(timer); + + /* + * This should never happen, but complain + * if it does. + */ + + if (timer->k_header.dh_inserted == FALSE) { + mtx_unlock(&ntoskrnl_dispatchlock); + printf("NTOS: timer %p fired even though " + "it was canceled\n", timer); + return; + } /* Mark the timer as no longer being on the timer queue. */ @@ -3391,18 +3375,111 @@ ntoskrnl_timercall(arg) tv.tv_sec = 0; tv.tv_usec = timer->k_period * 1000; timer->k_header.dh_inserted = TRUE; - timer->k_callout = &ntoskrnl_callout[ntoskrnl_callidx]; - CALLOUT_INC(ntoskrnl_callidx); - callout_reset(timer->k_callout, tvtohz(&tv), - ntoskrnl_timercall, timer); + ntoskrnl_insert_timer(timer, tvtohz(&tv)); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_timer_reloads++; +#endif } + dpc = timer->k_dpc; + + mtx_unlock(&ntoskrnl_dispatchlock); + /* If there's a DPC associated with the timer, queue it up. */ - if (timer->k_dpc != NULL) - KeInsertQueueDpc(timer->k_dpc, NULL, NULL); + if (dpc != NULL) + KeInsertQueueDpc(dpc, NULL, NULL); - mtx_unlock(&ntoskrnl_dispatchlock); + return; +} + +#ifdef NTOSKRNL_DEBUG_TIMERS +static int +sysctl_show_timers(SYSCTL_HANDLER_ARGS) +{ + int ret; + + ret = 0; + ntoskrnl_show_timers(); + return (sysctl_handle_int(oidp, &ret, 0, req)); +} + +static void +ntoskrnl_show_timers() +{ + int i = 0; + list_entry *l; + + mtx_lock_spin(&ntoskrnl_calllock); + l = ntoskrnl_calllist.nle_flink; + while(l != &ntoskrnl_calllist) { + i++; + l = l->nle_flink; + } + mtx_unlock_spin(&ntoskrnl_calllock); + + printf("\n"); + printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS); + printf("timer sets: %qu\n", ntoskrnl_timer_sets); + printf("timer reloads: %qu\n", ntoskrnl_timer_reloads); + printf("timer cancels: %qu\n", ntoskrnl_timer_cancels); + printf("timer fires: %qu\n", ntoskrnl_timer_fires); + printf("\n"); + + return; +} +#endif + +/* + * Must be called with dispatcher lock held. + */ + +static void +ntoskrnl_insert_timer(timer, ticks) + ktimer *timer; + int ticks; +{ + callout_entry *e; + list_entry *l; + struct callout *c; + + /* + * Try and allocate a timer. + */ + mtx_lock_spin(&ntoskrnl_calllock); + if (IsListEmpty(&ntoskrnl_calllist)) { + mtx_unlock_spin(&ntoskrnl_calllock); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_show_timers(); +#endif + panic("out of timers!"); + } + l = RemoveHeadList(&ntoskrnl_calllist); + mtx_unlock_spin(&ntoskrnl_calllock); + + e = CONTAINING_RECORD(l, callout_entry, ce_list); + c = &e->ce_callout; + + timer->k_callout = c; + + callout_init(c, CALLOUT_MPSAFE); + callout_reset(c, ticks, ntoskrnl_timercall, timer); + + return; +} + +static void +ntoskrnl_remove_timer(timer) + ktimer *timer; +{ + callout_entry *e; + + e = (callout_entry *)timer->k_callout; + callout_stop(timer->k_callout); + + mtx_lock_spin(&ntoskrnl_calllock); + InsertHeadList((&ntoskrnl_calllist), (&e->ce_list)); + mtx_unlock_spin(&ntoskrnl_calllock); return; } @@ -3465,13 +3542,15 @@ ntoskrnl_dpc_thread(arg) kdpc_queue *kq; kdpc *d; list_entry *l; + uint8_t irql; kq = arg; InitializeListHead(&kq->kq_disp); kq->kq_td = curthread; kq->kq_exit = 0; - mtx_init(&kq->kq_lock, "NDIS thread lock", NULL, MTX_SPIN); + kq->kq_running = FALSE; + KeInitializeSpinLock(&kq->kq_lock); KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE); KeInitializeEvent(&kq->kq_dead, EVENT_TYPE_SYNC, FALSE); @@ -3493,31 +3572,33 @@ ntoskrnl_dpc_thread(arg) while (1) { KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL); - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLock(&kq->kq_lock, &irql); if (kq->kq_exit) { - mtx_unlock_spin(&kq->kq_lock); + KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_dead, IO_NO_INCREMENT, FALSE); break; } + kq->kq_running = TRUE; + while (!IsListEmpty(&kq->kq_disp)) { l = RemoveHeadList((&kq->kq_disp)); d = CONTAINING_RECORD(l, kdpc, k_dpclistentry); InitializeListHead((&d->k_dpclistentry)); - d->k_lock = NULL; - mtx_unlock_spin(&kq->kq_lock); - ntoskrnl_run_dpc(d); - mtx_lock_spin(&kq->kq_lock); + KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); + MSCALL4(d->k_deferedfunc, d, d->k_deferredctx, + d->k_sysarg1, d->k_sysarg2); + KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); } - mtx_unlock_spin(&kq->kq_lock); + kq->kq_running = FALSE; + + KeReleaseSpinLock(&kq->kq_lock, irql); KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE); - } - mtx_destroy(&kq->kq_lock); #if __FreeBSD_version < 502113 mtx_lock(&Giant); #endif @@ -3525,33 +3606,6 @@ ntoskrnl_dpc_thread(arg) return; /* notreached */ } - -/* - * This is a wrapper for Windows deferred procedure calls that - * have been placed on an NDIS thread work queue. We need it - * since the DPC could be a _stdcall function. Also, as far as - * I can tell, defered procedure calls must run at DISPATCH_LEVEL. - */ -static void -ntoskrnl_run_dpc(arg) - void *arg; -{ - kdpc_func dpcfunc; - kdpc *dpc; - uint8_t irql; - - dpc = arg; - dpcfunc = dpc->k_deferedfunc; - if (dpcfunc == NULL) - return; - irql = KeRaiseIrql(DISPATCH_LEVEL); - MSCALL4(dpcfunc, dpc, dpc->k_deferredctx, - dpc->k_sysarg1, dpc->k_sysarg2); - KeLowerIrql(irql); - - return; -} - static void ntoskrnl_destroy_dpc_threads(void) { @@ -3589,10 +3643,10 @@ ntoskrnl_insert_dpc(head, dpc) l = l->nle_flink; } - if (dpc->k_importance == KDPC_IMPORTANCE_HIGH) - InsertHeadList((head), (&dpc->k_dpclistentry)); - else + if (dpc->k_importance == KDPC_IMPORTANCE_LOW) InsertTailList((head), (&dpc->k_dpclistentry)); + else + InsertHeadList((head), (&dpc->k_dpclistentry)); return (TRUE); } @@ -3611,11 +3665,6 @@ KeInitializeDpc(dpc, dpcfunc, dpcctx) dpc->k_deferredctx = dpcctx; dpc->k_num = KDPC_CPU_DEFAULT; dpc->k_importance = KDPC_IMPORTANCE_MEDIUM; - /* - * In case someone tries to dequeue a DPC that - * hasn't been queued yet. - */ - dpc->k_lock = NULL; InitializeListHead((&dpc->k_dpclistentry)); return; @@ -3629,12 +3678,12 @@ KeInsertQueueDpc(dpc, sysarg1, sysarg2) { kdpc_queue *kq; uint8_t r; + uint8_t irql; if (dpc == NULL) return(FALSE); - dpc->k_sysarg1 = sysarg1; - dpc->k_sysarg2 = sysarg2; + irql = KeRaiseIrql(DISPATCH_LEVEL); /* * By default, the DPC is queued to run on the same CPU @@ -3647,16 +3696,13 @@ KeInsertQueueDpc(dpc, sysarg1, sysarg2) else kq += dpc->k_num; - /* - * Also by default, we put the DPC on the medium - * priority queue. - */ - - mtx_lock_spin(&kq->kq_lock); + KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc); - if (r == TRUE) - dpc->k_lock = &kq->kq_lock; - mtx_unlock_spin(&kq->kq_lock); + if (r == TRUE) { + dpc->k_sysarg1 = sysarg1; + dpc->k_sysarg2 = sysarg2; + } + KeReleaseSpinLock(&kq->kq_lock, irql); if (r == FALSE) return(r); @@ -3670,26 +3716,27 @@ uint8_t KeRemoveQueueDpc(dpc) kdpc *dpc; { - struct mtx *lock; + kdpc_queue *kq; + uint8_t irql; if (dpc == NULL) return(FALSE); - lock = dpc->k_lock; + irql = KeRaiseIrql(DISPATCH_LEVEL); - if (lock == NULL) - return(FALSE); + kq = kq_queues + dpc->k_num; - mtx_lock_spin(lock); - dpc->k_lock = NULL; + KeAcquireSpinLockAtDpcLevel(&kq->kq_lock); if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) { - mtx_unlock_spin(lock); + KeReleaseSpinLockFromDpcLevel(&kq->kq_lock); + KeLowerIrql(irql); return(FALSE); } RemoveEntryList((&dpc->k_dpclistentry)); InitializeListHead((&dpc->k_dpclistentry)); - mtx_unlock_spin(lock); + + KeReleaseSpinLock(&kq->kq_lock, irql); return(TRUE); } @@ -3763,7 +3810,10 @@ KeSetTimerEx(timer, duetime, period, dpc) mtx_lock(&ntoskrnl_dispatchlock); if (timer->k_header.dh_inserted == TRUE) { - callout_stop(timer->k_callout); + ntoskrnl_remove_timer(timer); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_timer_cancels++; +#endif timer->k_header.dh_inserted = FALSE; pending = TRUE; } else @@ -3790,10 +3840,10 @@ KeSetTimerEx(timer, duetime, period, dpc) } timer->k_header.dh_inserted = TRUE; - timer->k_callout = &ntoskrnl_callout[ntoskrnl_callidx]; - CALLOUT_INC(ntoskrnl_callidx); - callout_reset(timer->k_callout, tvtohz(&tv), - ntoskrnl_timercall, timer); + ntoskrnl_insert_timer(timer, tvtohz(&tv)); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_timer_sets++; +#endif mtx_unlock(&ntoskrnl_dispatchlock); @@ -3830,7 +3880,10 @@ KeCancelTimer(timer) if (timer->k_header.dh_inserted == TRUE) { timer->k_header.dh_inserted = FALSE; - callout_stop(timer->k_callout); + ntoskrnl_remove_timer(timer); +#ifdef NTOSKRNL_DEBUG_TIMERS + ntoskrnl_timer_cancels++; +#endif } mtx_unlock(&ntoskrnl_dispatchlock); @@ -3905,6 +3958,8 @@ image_patch_table ntoskrnl_functbl[] = { IMPORT_SFUNC(IoMakeAssociatedIrp, 2), IMPORT_SFUNC(IoFreeIrp, 1), IMPORT_SFUNC(IoInitializeIrp, 3), + IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1), + IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2), IMPORT_SFUNC(KeSynchronizeExecution, 3), IMPORT_SFUNC(KeWaitForSingleObject, 5), IMPORT_SFUNC(KeWaitForMultipleObjects, 8), diff --git a/sys/dev/if_ndis/if_ndis.c b/sys/dev/if_ndis/if_ndis.c index 5ec5dc802849..417612eaaa48 100644 --- a/sys/dev/if_ndis/if_ndis.c +++ b/sys/dev/if_ndis/if_ndis.c @@ -120,12 +120,14 @@ static funcptr ndis_linksts_done_wrap; static funcptr ndis_ticktask_wrap; static funcptr ndis_starttask_wrap; static funcptr ndis_resettask_wrap; +static funcptr ndis_inputtask_wrap; static void ndis_tick (void *); static void ndis_ticktask (device_object *, void *); static void ndis_start (struct ifnet *); static void ndis_starttask (device_object *, void *); static void ndis_resettask (device_object *, void *); +static void ndis_inputtask (device_object *, void *); static int ndis_ioctl (struct ifnet *, u_long, caddr_t); static int ndis_wi_ioctl_get (struct ifnet *, u_long, caddr_t); static int ndis_wi_ioctl_set (struct ifnet *, u_long, caddr_t); @@ -199,6 +201,8 @@ ndisdrv_modevent(mod, cmd, arg) 2, WINDRV_WRAP_STDCALL); windrv_wrap((funcptr)ndis_resettask, &ndis_resettask_wrap, 2, WINDRV_WRAP_STDCALL); + windrv_wrap((funcptr)ndis_inputtask, &ndis_inputtask_wrap, + 2, WINDRV_WRAP_STDCALL); break; case MOD_UNLOAD: ndisdrv_loaded--; @@ -217,6 +221,7 @@ ndisdrv_modevent(mod, cmd, arg) windrv_unwrap(ndis_ticktask_wrap); windrv_unwrap(ndis_starttask_wrap); windrv_unwrap(ndis_resettask_wrap); + windrv_unwrap(ndis_inputtask_wrap); break; default: error = EINVAL; @@ -488,6 +493,7 @@ ndis_attach(dev) ifp->if_softc = sc; KeInitializeSpinLock(&sc->ndis_spinlock); + KeInitializeSpinLock(&sc->ndis_rxlock); InitializeListHead(&sc->ndis_shlist); if (sc->ndis_iftype == PCMCIABus) { @@ -571,6 +577,14 @@ ndis_attach(dev) goto fail; } + /* + * If this is a deserialized miniport, we don't have + * to honor the OID_GEN_MAXIMUM_SEND_PACKETS result. + */ + + if (!NDIS_SERIALIZED(sc->ndis_block)) + sc->ndis_maxpkts = NDIS_TXPKTS; + /* Enforce some sanity, just in case. */ if (sc->ndis_maxpkts == 0) @@ -582,7 +596,7 @@ ndis_attach(dev) /* Allocate a pool of ndis_packets for TX encapsulation. */ NdisAllocatePacketPool(&i, &sc->ndis_txpool, - sc->ndis_maxpkts, PROTOCOL_RESERVED_SIZE_IN_PACKET); + NDIS_TXPKTS, PROTOCOL_RESERVED_SIZE_IN_PACKET); if (i != NDIS_STATUS_SUCCESS) { sc->ndis_txpool = NULL; @@ -616,6 +630,14 @@ ndis_attach(dev) /* Check for task offload support. */ ndis_probe_offload(sc); +#if __FreeBSD_version < 502109 + /* + * An NDIS device was detected. Inform the world. + */ + device_printf(dev, "%s address: %6D\n", + sc->ndis_80211 ? "802.11" : "Ethernet", eaddr, ":"); +#endif + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_mtu = ETHERMTU; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; @@ -888,6 +910,7 @@ ndis_attach(dev) sc->ndis_tickitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj); sc->ndis_startitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj); sc->ndis_resetitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj); + sc->ndis_inputitem = IoAllocateWorkItem(sc->ndis_block->nmb_deviceobj); KeInitializeDpc(&sc->ndis_rxdpc, ndis_rxeof_xfr_wrap, sc->ndis_block); @@ -941,6 +964,8 @@ ndis_detach(dev) IoFreeWorkItem(sc->ndis_startitem); if (sc->ndis_resetitem != NULL) IoFreeWorkItem(sc->ndis_resetitem); + if (sc->ndis_inputitem != NULL) + IoFreeWorkItem(sc->ndis_inputitem); bus_generic_detach(dev); @@ -1037,6 +1062,9 @@ ndis_resume(dev) /* * The following bunch of routines are here to support drivers that * use the NdisMEthIndicateReceive()/MiniportTransferData() mechanism. + * The NdisMEthIndicateReceive() handler runs at DISPATCH_LEVEL for + * serialized miniports, or IRQL <= DISPATCH_LEVEL for deserialized + * miniports. */ static void @@ -1103,14 +1131,19 @@ ndis_rxeof_eth(adapter, ctx, addr, hdr, hdrlen, lookahead, lookaheadlen, pktlen) KeAcquireSpinLock(&block->nmb_lock, &irql); - InsertTailList((&block->nmb_packetlist), - ((list_entry *)&p->u.np_clrsvd.np_miniport_rsvd)); + InsertTailList((&block->nmb_packetlist), (&p->np_list)); KeReleaseSpinLock(&block->nmb_lock, irql); return; } +/* + * NdisMEthIndicateReceiveComplete() handler, runs at DISPATCH_LEVEL + * for serialized miniports, or IRQL <= DISPATCH_LEVEL for deserialized + * miniports. + */ + static void ndis_rxeof_done(adapter) ndis_handle adapter; @@ -1130,7 +1163,7 @@ ndis_rxeof_done(adapter) } /* - * Runs at DISPATCH_LEVEL. + * MiniportTransferData() handler, runs at DISPATCH_LEVEL. */ static void ndis_rxeof_xfr(dpc, adapter, sysarg1, sysarg2) @@ -1157,8 +1190,8 @@ ndis_rxeof_xfr(dpc, adapter, sysarg1, sysarg2) l = block->nmb_packetlist.nle_flink; while(!IsListEmpty(&block->nmb_packetlist)) { l = RemoveHeadList((&block->nmb_packetlist)); - p = CONTAINING_RECORD(l, ndis_packet, - u.np_clrsvd.np_miniport_rsvd); + p = CONTAINING_RECORD(l, ndis_packet, np_list); + InitializeListHead((&p->np_list)); priv = (ndis_ethpriv *)&p->np_protocolreserved; m = p->np_m0; @@ -1185,8 +1218,12 @@ ndis_rxeof_xfr(dpc, adapter, sysarg1, sysarg2) if (status == NDIS_STATUS_SUCCESS) { IoFreeMdl(p->np_private.npp_head); NdisFreePacket(p); - ifp->if_ipackets++; - (*ifp->if_input)(ifp, m); + KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock); + _IF_ENQUEUE(&sc->ndis_rxqueue, m); + KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock); + IoQueueWorkItem(sc->ndis_inputitem, + (io_workitem_func)ndis_inputtask_wrap, + WORKQUEUE_CRITICAL, ifp); } if (status == NDIS_STATUS_FAILURE) @@ -1201,6 +1238,9 @@ ndis_rxeof_xfr(dpc, adapter, sysarg1, sysarg2) return; } +/* + * NdisMTransferDataComplete() handler, runs at DISPATCH_LEVEL. + */ static void ndis_rxeof_xfr_done(adapter, packet, status, len) ndis_handle adapter; @@ -1228,8 +1268,13 @@ ndis_rxeof_xfr_done(adapter, packet, status, len) m->m_len = m->m_pkthdr.len; m->m_pkthdr.rcvif = ifp; - ifp->if_ipackets++; - (*ifp->if_input)(ifp, m); + KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock); + _IF_ENQUEUE(&sc->ndis_rxqueue, m); + KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock); + IoQueueWorkItem(sc->ndis_inputitem, + (io_workitem_func)ndis_inputtask_wrap, + WORKQUEUE_CRITICAL, ifp); + return; } /* @@ -1311,7 +1356,6 @@ ndis_rxeof(adapter, packets, pktcnt) } m0 = m; m0->m_pkthdr.rcvif = ifp; - ifp->if_ipackets++; /* Deal with checksum offload. */ @@ -1333,13 +1377,55 @@ ndis_rxeof(adapter, packets, pktcnt) } } - (*ifp->if_input)(ifp, m0); + KeAcquireSpinLockAtDpcLevel(&sc->ndis_rxlock); + _IF_ENQUEUE(&sc->ndis_rxqueue, m); + KeReleaseSpinLockFromDpcLevel(&sc->ndis_rxlock); + IoQueueWorkItem(sc->ndis_inputitem, + (io_workitem_func)ndis_inputtask_wrap, + WORKQUEUE_CRITICAL, ifp); } } return; } +/* + * This routine is run at PASSIVE_LEVEL. We use this routine to pass + * packets into the stack in order to avoid calling (*ifp->if_input)() + * with any locks held (at DISPATCH_LEVEL, we'll be holding the + * 'dispatch level' per-cpu sleep lock). + */ + +static void +ndis_inputtask(dobj, arg) + device_object *dobj; + void *arg; +{ + ndis_miniport_block *block; + struct ifnet *ifp; + struct ndis_softc *sc; + struct mbuf *m; + uint8_t irql; + + ifp = arg; + sc = ifp->if_softc; + block = dobj->do_devext; + + KeAcquireSpinLock(&sc->ndis_rxlock, &irql); + while(1) { + _IF_DEQUEUE(&sc->ndis_rxqueue, m); + if (m == NULL) + break; + KeReleaseSpinLock(&sc->ndis_rxlock, irql); + ifp->if_ipackets++; + (*ifp->if_input)(ifp, m); + KeAcquireSpinLock(&sc->ndis_rxlock, &irql); + } + KeReleaseSpinLock(&sc->ndis_rxlock, irql); + + return; +} + /* * A frame was downloaded to the chip. It's safe for us to clean up * the list buffers. diff --git a/sys/dev/if_ndis/if_ndisvar.h b/sys/dev/if_ndis/if_ndisvar.h index b595dd76571e..1eecc59e412f 100644 --- a/sys/dev/if_ndis/if_ndisvar.h +++ b/sys/dev/if_ndis/if_ndisvar.h @@ -85,8 +85,9 @@ TAILQ_HEAD(nch, ndis_cfglist); #define NDIS_INITIALIZED(sc) (sc->ndis_block->nmb_devicectx != NULL) +#define NDIS_TXPKTS 64 #define NDIS_INC(x) \ - (x)->ndis_txidx = ((x)->ndis_txidx + 1) % (x)->ndis_maxpkts + (x)->ndis_txidx = ((x)->ndis_txidx + 1) % NDIS_TXPKTS #if __FreeBSD_version < 600000 #define arpcom ic.ic_ac @@ -159,11 +160,9 @@ struct ndis_softc { io_workitem *ndis_tickitem; io_workitem *ndis_startitem; io_workitem *ndis_resetitem; + io_workitem *ndis_inputitem; kdpc ndis_rxdpc; bus_dma_tag_t ndis_parent_tag; -/* - struct ndis_shmem *ndis_shlist; -*/ list_entry ndis_shlist; bus_dma_tag_t ndis_mtag; bus_dma_tag_t ndis_ttag; @@ -173,6 +172,8 @@ struct ndis_softc { struct ndis_evt ndis_evt[NDIS_EVENTS]; int ndis_evtpidx; int ndis_evtcidx; + struct ifqueue ndis_rxqueue; + kspin_lock ndis_rxlock; }; #define NDIS_LOCK(_sc) KeAcquireSpinLock(&(_sc)->ndis_spinlock, \