From 60f1e276256cf82ca08af26bbcd05ad24bb6c4e3 Mon Sep 17 00:00:00 2001 From: Kip Macy Date: Tue, 15 Jan 2008 03:27:42 +0000 Subject: [PATCH] - move WR_LEN in to cxgb_adapter.h add PIO_LEN to make intent clearer - move cxgb_tx_common in to cxgb_multiq.c and rename to cxgb_tx - move cxgb_tx_common dependencies - further simplify cxgb_dequeue_packet for the non-multiqueue case - only launch one service thread per port in the non-multiq case - remove dead cleaning code from cxgb_sge.c - simplify PIO case substantially in by returning directly from mbuf collapse and just using m_copydata - remove gratuitous m_gethdr in the rx path - clarify freeing of mbufs in collapse --- sys/dev/cxgb/cxgb_adapter.h | 15 +++- sys/dev/cxgb/cxgb_main.c | 73 --------------- sys/dev/cxgb/cxgb_multiq.c | 166 ++++++++++++++++++++++++++--------- sys/dev/cxgb/cxgb_sge.c | 125 +++++++++----------------- sys/dev/cxgb/sys/mvec.h | 4 +- sys/dev/cxgb/sys/uipc_mvec.c | 59 +++---------- 6 files changed, 192 insertions(+), 250 deletions(-) diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h index df8aa2b1831b..6910ba95a476 100644 --- a/sys/dev/cxgb/cxgb_adapter.h +++ b/sys/dev/cxgb/cxgb_adapter.h @@ -149,13 +149,24 @@ enum { /* adapter flags */ #define FL_Q_SIZE 4096 #define JUMBO_Q_SIZE 1024 #define RSPQ_Q_SIZE 1024 +#if 0 #define TX_ETH_Q_SIZE 1024 +#else +#define TX_ETH_Q_SIZE 64 +#endif enum { TXQ_ETH = 0, TXQ_OFLD = 1, TXQ_CTRL = 2, }; +/* + * work request size in bytes + */ +#define WR_LEN (WR_FLITS * 8) +#define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt)) + + /* careful, the following are set on priv_flags and must not collide with * IFF_ flags! */ @@ -288,7 +299,6 @@ struct sge_txq { struct mtx lock; struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1]; bus_dma_segment_t txq_segs[TX_MAX_SEGS]; - struct mbuf *txq_m_vec[TX_WR_COUNT_MAX]; #define TXQ_NAME_LEN 32 char lockbuf[TXQ_NAME_LEN]; }; @@ -610,10 +620,7 @@ void cxgb_pcpu_shutdown_threads(struct adapter *sc); void cxgb_pcpu_startup_threads(struct adapter *sc); int process_responses(adapter_t *adap, struct sge_qset *qs, int budget); -int cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax); void t3_free_qset(adapter_t *sc, struct sge_qset *q); -int cxgb_dequeue_packet(struct ifnet *, struct sge_txq *, struct mbuf **); void cxgb_start(struct ifnet *ifp); void refill_fl_service(adapter_t *adap, struct sge_fl *fl); -int reclaim_completed_tx(struct sge_txq *q, int reclaim_min); #endif diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c index c808bcae69b1..e5057585a85e 100644 --- a/sys/dev/cxgb/cxgb_main.c +++ b/sys/dev/cxgb/cxgb_main.c @@ -281,21 +281,6 @@ struct cxgb_ident { static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset); -static __inline void -check_pkt_coalesce(struct sge_qset *qs) -{ - struct adapter *sc; - struct sge_txq *txq; - - txq = &qs->txq[TXQ_ETH]; - sc = qs->port->adapter; - - if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) - sc->tunq_fill[qs->idx] = 0; - else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) - sc->tunq_fill[qs->idx] = 1; -} - static __inline char t3rev2char(struct adapter *adapter) { @@ -1865,64 +1850,6 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) return (error); } -int -cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax) -{ - struct sge_txq *txq; - int err, in_use_init, count; - struct mbuf **m_vec; - - txq = &qs->txq[TXQ_ETH]; - m_vec = txq->txq_m_vec; - in_use_init = txq->in_use; - err = 0; - while ((txq->in_use - in_use_init < txmax) && - (txq->size > txq->in_use + TX_MAX_DESC)) { - check_pkt_coalesce(qs); - count = cxgb_dequeue_packet(ifp, txq, m_vec); - if (count == 0) - break; - ETHER_BPF_MTAP(ifp, m_vec[0]); - - if ((err = t3_encap(qs, m_vec, count)) != 0) - break; - txq->txq_enqueued += count; - } -#if 0 /* !MULTIQ */ - if (__predict_false(err)) { - if (err == ENOMEM) { - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - IFQ_LOCK(&ifp->if_snd); - IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]); - IFQ_UNLOCK(&ifp->if_snd); - } - } - if (err == 0 && m_vec[0] == NULL) { - err = ENOBUFS; - } - else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && - (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { - setbit(&qs->txq_stopped, TXQ_ETH); - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - err = ENOSPC; - } -#else - if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) { - err = ENOSPC; - setbit(&qs->txq_stopped, TXQ_ETH); - } - if (err == ENOMEM) { - int i; - /* - * Sub-optimal :-/ - */ - for (i = 0; i < count; i++) - m_freem(m_vec[i]); - } -#endif - return (err); -} - static int cxgb_media_change(struct ifnet *ifp) { diff --git a/sys/dev/cxgb/cxgb_multiq.c b/sys/dev/cxgb/cxgb_multiq.c index fc180aa8e755..537b1db8681d 100644 --- a/sys/dev/cxgb/cxgb_multiq.c +++ b/sys/dev/cxgb/cxgb_multiq.c @@ -115,6 +115,9 @@ static void cxgb_pcpu_start_proc(void *arg); #ifdef IFNET_MULTIQUEUE static int cxgb_pcpu_cookie_to_qidx(struct port_info *, uint32_t cookie); #endif +static int cxgb_tx(struct sge_qset *qs, uint32_t txmax); + + static inline int cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m) { @@ -124,7 +127,6 @@ cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m) #ifndef IFNET_MULTIQUEUE panic("not expecting enqueue without multiqueue"); #endif - KASSERT(m != NULL, ("null mbuf")); KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type)); if (qs->qs_flags & QS_EXITING) { @@ -164,8 +166,8 @@ cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m) return (err); } -int -cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_vec) +static int +cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec) { struct mbuf *m; struct sge_qset *qs; @@ -176,10 +178,16 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v if (txq->immpkt != NULL) panic("immediate packet set"); -#endif - mtx_assert(&txq->lock, MA_OWNED); + IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m); + if (m == NULL) + return (0); + + m_vec[0] = m; + return (1); +#endif + coalesced = count = size = 0; qs = txq_to_qset(txq, TXQ_ETH); if (qs->qs_flags & QS_EXITING) @@ -193,39 +201,20 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v } sc = qs->port->adapter; -#ifndef IFNET_MULTIQUEUE - /* - * This is terrible from a cache and locking efficiency standpoint - * but then again ... so is ifnet. - */ - while (((qs->qs_flags & QS_EXITING) == 0) && !IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd) && !buf_ring_full(&txq->txq_mr)) { - - struct mbuf *m = NULL; - - IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m); - if (m) { - KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type)); - if (buf_ring_enqueue(&txq->txq_mr, m)) - panic("ring full"); - } else - break; - } -#endif m = buf_ring_dequeue(&txq->txq_mr); - if (m == NULL) + if (m == NULL) return (0); - buf_ring_scan(&txq->txq_mr, m, __FILE__, __LINE__); - KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type)); - m_vec[0] = m; - if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX || m->m_next != NULL || - (cxgb_pcpu_tx_coalesce == 0)) { - return (1); - } -#ifndef IFNET_MULTIQUEUE - panic("coalesce not supported yet"); -#endif count = 1; + KASSERT(m->m_type == MT_DATA, + ("m=%p is bad mbuf type %d from ring cons=%d prod=%d", m, + m->m_type, txq->txq_mr.br_cons, txq->txq_mr.br_prod)); + m_vec[0] = m; + if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX || + m->m_next != NULL || (cxgb_pcpu_tx_coalesce == 0)) { + return (count); + } + size = m->m_pkthdr.len; for (m = buf_ring_peek(&txq->txq_mr); m != NULL; m = buf_ring_peek(&txq->txq_mr)) { @@ -381,13 +370,15 @@ cxgb_pcpu_free(struct sge_qset *qs) { struct mbuf *m; struct sge_txq *txq = &qs->txq[TXQ_ETH]; - + + mtx_lock(&txq->lock); while ((m = mbufq_dequeue(&txq->sendq)) != NULL) m_freem(m); while ((m = buf_ring_dequeue(&txq->txq_mr)) != NULL) m_freem(m); t3_free_tx_desc_all(txq); + mtx_unlock(&txq->lock); } static int @@ -400,6 +391,7 @@ cxgb_pcpu_reclaim_tx(struct sge_txq *txq) KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d", qs->qs_cpuid, curcpu)); #endif + mtx_assert(&txq->lock, MA_OWNED); reclaimable = desc_reclaimable(txq); if (reclaimable == 0) @@ -429,6 +421,8 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush) sc = pi->adapter; txq = &qs->txq[TXQ_ETH]; + mtx_assert(&txq->lock, MA_OWNED); + retry: if (!pi->link_config.link_ok) initerr = ENXIO; @@ -474,10 +468,12 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush) DPRINTF("stopped=%d flush=%d max_desc=%d\n", stopped, flush, max_desc); - err = flush ? cxgb_tx_common(qs->port->ifp, qs, max_desc) : ENOSPC; + err = flush ? cxgb_tx(qs, max_desc) : ENOSPC; - if ((tx_flush && flush && err == 0) && !buf_ring_empty(&txq->txq_mr)) { + if ((tx_flush && flush && err == 0) && + (!buf_ring_empty(&txq->txq_mr) || + !IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) { struct thread *td = curthread; if (++i > 1) { @@ -525,7 +521,8 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt) txq = &qs->txq[TXQ_ETH]; - if (((sc->tunq_coalesce == 0) || (buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) || + if (((sc->tunq_coalesce == 0) || + (buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) || (cxgb_pcpu_tx_coalesce == 0)) && mtx_trylock(&txq->lock)) { if (cxgb_debug) printf("doing immediate transmit\n"); @@ -658,12 +655,18 @@ cxgb_pcpu_cookie_to_qidx(struct port_info *pi, uint32_t cookie) void cxgb_pcpu_startup_threads(struct adapter *sc) { - int i, j; + int i, j, nqsets; struct proc *p; - + + for (i = 0; i < (sc)->params.nports; ++i) { struct port_info *pi = adap2pinfo(sc, i); - + +#ifdef IFNET_MULTIQUEUE + nqsets = pi->nqsets; +#else + nqsets = 1; +#endif for (j = 0; j < pi->nqsets; ++j) { struct sge_qset *qs; @@ -701,3 +704,84 @@ cxgb_pcpu_shutdown_threads(struct adapter *sc) } } } + +static __inline void +check_pkt_coalesce(struct sge_qset *qs) +{ + struct adapter *sc; + struct sge_txq *txq; + + txq = &qs->txq[TXQ_ETH]; + sc = qs->port->adapter; + + if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2)))) + sc->tunq_fill[qs->idx] = 0; + else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2)))) + sc->tunq_fill[qs->idx] = 1; +} + +static int +cxgb_tx(struct sge_qset *qs, uint32_t txmax) +{ + struct sge_txq *txq; + struct ifnet *ifp = qs->port->ifp; + int i, err, in_use_init, count; + struct mbuf *m_vec[TX_WR_COUNT_MAX]; + + txq = &qs->txq[TXQ_ETH]; + ifp = qs->port->ifp; + in_use_init = txq->in_use; + err = 0; + + for (i = 0; i < TX_WR_COUNT_MAX; i++) + m_vec[i] = NULL; + + mtx_assert(&txq->lock, MA_OWNED); + while ((txq->in_use - in_use_init < txmax) && + (txq->size > txq->in_use + TX_MAX_DESC)) { + check_pkt_coalesce(qs); + count = cxgb_dequeue_packet(txq, m_vec); + if (count == 0) { + err = ENOBUFS; + break; + } + ETHER_BPF_MTAP(ifp, m_vec[0]); + + if ((err = t3_encap(qs, m_vec, count)) != 0) + break; + txq->txq_enqueued += count; + m_vec[0] = NULL; + } +#if 0 /* !MULTIQ */ + if (__predict_false(err)) { + if (err == ENOMEM) { + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + IFQ_LOCK(&ifp->if_snd); + IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]); + IFQ_UNLOCK(&ifp->if_snd); + } + } + else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) && + (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { + setbit(&qs->txq_stopped, TXQ_ETH); + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + err = ENOSPC; + } +#else + if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) { + err = ENOSPC; + setbit(&qs->txq_stopped, TXQ_ETH); + } + if (err == ENOMEM) { + int i; + /* + * Sub-optimal :-/ + */ + printf("ENOMEM!!!"); + for (i = 0; i < count; i++) + m_freem(m_vec[i]); + } +#endif + return (err); +} + diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index b1e56aaa0e70..c854d9d15837 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -26,6 +26,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ +#define DEBUG_BUFRING + #include __FBSDID("$FreeBSD$"); @@ -72,7 +74,7 @@ __FBSDID("$FreeBSD$"); int txq_fills = 0; static int bogus_imm = 0; -static int recycle_enable = 1; +static int recycle_enable = 0; extern int cxgb_txq_buf_ring_size; int cxgb_cached_allocations; int cxgb_cached; @@ -92,10 +94,6 @@ extern int cxgb_use_16k_clusters; */ #define TX_RECLAIM_PERIOD (hz >> 1) -/* - * work request size in bytes - */ -#define WR_LEN (WR_FLITS * 8) /* * Values for sge_txq.flags @@ -218,13 +216,6 @@ reclaim_completed_tx_(struct sge_txq *q, int reclaim_min) return (reclaim); } -int -reclaim_completed_tx(struct sge_txq *q, int reclaim_min) -{ - return reclaim_completed_tx_(q, reclaim_min); -} - - /** * should_restart_tx - are there enough resources to restart a Tx queue? * @q: the Tx queue @@ -672,8 +663,7 @@ alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, if (sw_size) { len = nelem * sw_size; - s = malloc(len, M_DEVBUF, M_WAITOK); - bzero(s, len); + s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); *(void **)sdesc = s; } if (parent_entry_tag == NULL) @@ -736,17 +726,16 @@ sge_timer_cb(void *arg) struct sge_qset *qs; struct sge_txq *txq; int i, j; - int reclaim_eth, reclaim_ofl, refill_rx; + int reclaim_ofl, refill_rx; for (i = 0; i < sc->params.nports; i++) for (j = 0; j < sc->port[i].nqsets; j++) { qs = &sc->sge.qs[i + j]; txq = &qs->txq[0]; - reclaim_eth = txq[TXQ_ETH].processed - txq[TXQ_ETH].cleaned; reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || (qs->fl[1].credits < qs->fl[1].size)); - if (reclaim_eth || reclaim_ofl || refill_rx) { + if (reclaim_ofl || refill_rx) { pi = &sc->port[i]; taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task); break; @@ -834,31 +823,14 @@ refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) static __inline void sge_txq_reclaim_(struct sge_txq *txq, int force) { - int reclaimable, n; - struct port_info *pi; - pi = txq->port; -reclaim_more: - n = 0; - if ((reclaimable = desc_reclaimable(txq)) < 16) + if (desc_reclaimable(txq) < 16) return; if (mtx_trylock(&txq->lock) == 0) return; - n = reclaim_completed_tx_(txq, 16); + reclaim_completed_tx_(txq, 16); mtx_unlock(&txq->lock); - if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE && - txq->size - txq->in_use >= TX_START_MAX_DESC) { - struct sge_qset *qs = txq_to_qset(txq, TXQ_ETH); - - txq_fills++; - pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - clrbit(&qs->txq_stopped, TXQ_ETH); - taskqueue_enqueue(pi->tq, &pi->start_task); - } - - if (n) - goto reclaim_more; } static void @@ -886,8 +858,6 @@ sge_timer_reclaim(void *arg, int ncount) #endif for (i = 0; i < nqsets; i++) { qs = &sc->sge.qs[i]; - txq = &qs->txq[TXQ_ETH]; - sge_txq_reclaim_(txq, FALSE); txq = &qs->txq[TXQ_OFLD]; sge_txq_reclaim_(txq, FALSE); @@ -1184,7 +1154,7 @@ write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs * is freed all clusters will be freed * with it */ - txsd->mi.mi_base = NULL; + KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", txsd->mi.mi_base)); wrp = (struct work_request_hdr *)txd; wrp->wr_hi = htonl(V_WR_DATATYPE(1) | V_WR_SGLSFLT(1)) | wr_hi; @@ -1244,6 +1214,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count) DPRINTF("t3_encap cpu=%d ", curcpu); + mi = NULL; pi = qs->port; sc = pi->adapter; txq = &qs->txq[TXQ_ETH]; @@ -1252,9 +1223,13 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count) sgl = txq->txq_sgl; segs = txq->txq_segs; m0 = *m; + DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); - + if (cxgb_debug) + printf("mi_base=%p cidx=%d pidx=%d\n\n", txsd->mi.mi_base, txq->cidx, txq->pidx); + + mtx_assert(&txq->lock, MA_OWNED); cntrl = V_TXPKT_INTF(pi->txpkt_intf); /* * XXX need to add VLAN support for 6.x @@ -1263,7 +1238,9 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count) if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); #endif - + KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", + txsd->mi.mi_base)); + if (count > 1) { panic("count > 1 not support in CVS\n"); if ((err = busdma_map_sg_vec(m, &m0, segs, count))) @@ -1276,13 +1253,10 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count) } KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d count=%d", nsegs, count)); - if (m0->m_type == MT_DATA) - DPRINTF("mbuf type=%d tags:%d head=%p", m0->m_type, !SLIST_EMPTY(&m0->m_pkthdr.tags), - SLIST_FIRST(&m0->m_pkthdr.tags)); - - mi_collapse_mbuf(&txsd->mi, m0); - mi = &txsd->mi; - + if (m0->m_pkthdr.len > PIO_LEN) { + mi_collapse_mbuf(&txsd->mi, m0); + mi = &txsd->mi; + } if (count > 1) { struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; int i, fidx; @@ -1373,25 +1347,11 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count) mlen = m0->m_pkthdr.len; cpl->len = htonl(mlen | 0x80000000); - if (mlen <= WR_LEN - sizeof(*cpl)) { + if (mlen <= PIO_LEN) { txq_prod(txq, 1, &txqs); - - DPRINTF("mlen==%d max=%ld\n", mlen, (WR_LEN - sizeof(*cpl))); - if (mi->mi_type != MT_IOVEC && - mi->mi_type != MT_CLIOVEC) - memcpy(&txd->flit[2], mi->mi_data, mlen); - else { - /* - * XXX mbuf_iovec - */ -#if 0 - m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); -#endif - printf("bailing on m_copydata\n"); - } - m_freem_iovec(&txsd->mi); - txsd->mi.mi_base = NULL; - + m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); + m_freem(m0); + m0 = NULL; flits = (mlen + 7) / 8 + 2; cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | @@ -1797,13 +1757,18 @@ t3_sge_stop(adapter_t *sc) for (nqsets = i = 0; i < (sc)->params.nports; i++) nqsets += sc->port[i].nqsets; - +#ifdef notyet + /* + * + * XXX + */ for (i = 0; i < nqsets; ++i) { struct sge_qset *qs = &sc->sge.qs[i]; taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); } +#endif } /** @@ -1839,7 +1804,8 @@ t3_free_tx_desc(struct sge_txq *q, int reclaimable) bus_dmamap_unload(q->entry_tag, txsd->map); txsd->flags &= ~TX_SW_DESC_MAPPED; } - m_freem_iovec(&txsd->mi); + m_freem_iovec(&txsd->mi); + buf_ring_scan(&q->txq_mr, txsd->mi.mi_base, __FILE__, __LINE__); txsd->mi.mi_base = NULL; #if defined(DIAGNOSTIC) && 0 @@ -2513,7 +2479,7 @@ init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone) static int get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, - struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m) + struct t3_mbuf_hdr *mh, struct rsp_desc *r) { unsigned int len_cq = ntohl(r->len_cq); @@ -2522,6 +2488,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, uint32_t len = G_RSPD_LEN(len_cq); uint32_t flags = ntohl(r->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); + struct mbuf *m; uint32_t *ref; int ret = 0; @@ -2536,13 +2503,13 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, cl = mtod(m0, void *); memcpy(cl, sd->data, len); recycle_rx_buf(adap, fl, fl->cidx); - *m = m0; + m = m0; } else { skip_recycle: int flags = 0; bus_dmamap_unload(fl->entry_tag, sd->map); cl = sd->rxsd_cl; - *m = m0 = (struct mbuf *)cl; + m = m0 = (struct mbuf *)cl; m0->m_len = len; if ((sopeop == RSPQ_SOP_EOP) || @@ -2561,8 +2528,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, case RSPQ_NSOP_NEOP: DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); if (mh->mh_tail == NULL) { - if (cxgb_debug) - printf("discarding intermediate descriptor entry\n"); + printf("discarding intermediate descriptor entry\n"); m_freem(m); break; } @@ -2798,18 +2764,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; #ifdef DISABLE_MBUF_IOVEC - struct mbuf *m; - m = m_gethdr(M_DONTWAIT, MT_DATA); - - if (m == NULL) { - log(LOG_WARNING, "failed to get mbuf for packet\n"); - budget_left--; - break; - } else { - m->m_next = m->m_nextpkt = NULL; - } - - eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m); + eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r); #else eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r); #ifdef IFNET_MULTIQUEUE diff --git a/sys/dev/cxgb/sys/mvec.h b/sys/dev/cxgb/sys/mvec.h index f873f567cd5a..57a6478cd530 100644 --- a/sys/dev/cxgb/sys/mvec.h +++ b/sys/dev/cxgb/sys/mvec.h @@ -232,13 +232,15 @@ m_freem_iovec(struct mbuf_iovec *mi) struct mbuf *m; switch (mi->mi_type) { + case EXT_MBUF: + m_free_fast((struct mbuf *)mi->mi_base); + break; case EXT_IOVEC: case EXT_CLIOVEC: case EXT_JMPIOVEC: m = (struct mbuf *)mi->mi_base; m_free_iovec(m, mi->mi_type); break; - case EXT_MBUF: case EXT_CLUSTER: case EXT_JUMBOP: case EXT_JUMBO9: diff --git a/sys/dev/cxgb/sys/uipc_mvec.c b/sys/dev/cxgb/sys/uipc_mvec.c index 3f6ecf631346..9dc902b69663 100644 --- a/sys/dev/cxgb/sys/uipc_mvec.c +++ b/sys/dev/cxgb/sys/uipc_mvec.c @@ -162,7 +162,7 @@ _mcl_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m) } KASSERT(mi->mi_len != 0, ("miov has len 0")); KASSERT(mi->mi_type > 0, ("mi_type is invalid")); - + KASSERT(mi->mi_base, ("mi_base is invalid")); return (n); } @@ -204,40 +204,15 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs) if (n->m_flags & M_PKTHDR && !SLIST_EMPTY(&n->m_pkthdr.tags)) m_tag_delete_chain(n, NULL); + if (n->m_pkthdr.len <= PIO_LEN) + return (0); retry: seg_count = 0; if (n->m_next == NULL) { busdma_map_mbuf_fast(n, segs); *nsegs = 1; - return (0); } - - if (n->m_pkthdr.len <= 104) { - caddr_t data; - - if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) - return (ENOMEM); - - data = m0->m_data; - memcpy(m0, n, sizeof(struct m_hdr) + sizeof(struct pkthdr)); - m0->m_data = data; - m0->m_len = n->m_pkthdr.len; - m0->m_flags &= ~M_EXT; - m0->m_next = NULL; - m0->m_type = n->m_type; - n->m_flags &= ~M_PKTHDR; - while (n) { - memcpy(data, n->m_data, n->m_len); - data += n->m_len; - n = n->m_next; - } - m_freem(*m); - n = m0; - *m = n; - goto retry; - } - while (n && seg_count < TX_MAX_SEGS) { marray[seg_count] = n; @@ -249,18 +224,6 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs) n = n->m_next; } -#if 0 - /* - * XXX needs more careful consideration - */ - if (__predict_false(seg_count == 1)) { - n = marray[0]; - if (n != *m) - - /* XXX */ - goto retry; - } -#endif if (seg_count == 0) { if (cxgb_debug) printf("empty segment chain\n"); @@ -302,16 +265,20 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs) } n = *m; while (n) { - if (((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT) && - (n->m_len > 0) && (n->m_ext.ext_type != EXT_PACKET) ) + if (n->m_ext.ext_type == EXT_PACKET) + goto skip; + else if (n->m_len == 0) + /* do nothing */; + else if ((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT) n->m_flags &= ~M_EXT; - else if ((n->m_len > 0) || (n->m_ext.ext_type == EXT_PACKET)) { - n = n->m_next; - continue; - } + else + goto skip; mhead = n->m_next; m_free(n); n = mhead; + continue; + skip: + n = n->m_next; } *nsegs = seg_count; *m = m0;