From da403aea11bda66779e16f5e850d5d1c6d22a4b8 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Tue, 16 May 2017 19:53:38 +0000
Subject: [PATCH 01/73] Pretend that there is some security when executing in
 direct mode.

Do not allow direct exec if we the process is suid. Try to follow Unix
permission checks for DACs, ignore ACLs.

Reviewed by:	emaste
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
Differential revision:	https://reviews.freebsd.org/D10750
---
 libexec/rtld-elf/rtld.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index e1eeba7d64d1..e14a41a09689 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -345,12 +345,14 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     const Elf_Phdr *phdr;
     Objlist initlist;
     RtldLockState lockstate;
+    struct stat st;
     Elf_Addr *argcp;
     char **argv, *argv0, **env, **envp, *kexecpath, *library_path_rpath;
     caddr_t imgentry;
     char buf[MAXPATHLEN];
     int argc, fd, i, mib[2], phnum;
     size_t len;
+    bool dir_enable;
 
     /*
      * On entry, the dynamic linker itself has not been relocated yet.
@@ -419,6 +421,11 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	assert(aux_info[AT_PHDR] != NULL);
 	phdr = (const Elf_Phdr *)aux_info[AT_PHDR]->a_un.a_ptr;
 	if (phdr == obj_rtld.phdr) {
+	    if (!trust) {
+		rtld_printf("Tainted process refusing to run binary %s\n",
+		  argv0);
+		rtld_die();
+	    }
 	    dbg("opening main program in direct exec mode");
 	    if (argc >= 2) {
 		argv0 = argv[1];
@@ -428,6 +435,37 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 		      rtld_strerror(errno));
 		    rtld_die();
 		}
+		if (fstat(fd, &st) == -1) {
+		    rtld_printf("Stat %s: %s\n", argv0,
+		      rtld_strerror(errno));
+		    rtld_die();
+		}
+
+		/*
+		 * Rough emulation of the permission checks done by
+		 * execve(2), only Unix DACs are checked, ACLs are
+		 * ignored.  Preserve the semantic of disabling owner
+		 * to execute if owner x bit is cleared, even if
+		 * others x bit is enabled.
+		 * mmap(2) does not allow to mmap with PROT_EXEC if
+		 * binary' file comes from noexec mount.  We cannot
+		 * set VV_TEXT on the binary.
+		 */
+		dir_enable = false;
+		if (st.st_uid == geteuid()) {
+		    if ((st.st_mode & S_IXUSR) != 0)
+			dir_enable = true;
+		} else if (st.st_gid == getegid()) {
+		    if ((st.st_mode & S_IXGRP) != 0)
+			dir_enable = true;
+		} else if ((st.st_mode & S_IXOTH) != 0) {
+		    dir_enable = true;
+		}
+		if (!dir_enable) {
+		    rtld_printf("No execute permission for binary %s\n",
+		      argv0);
+		    rtld_die();
+		}
 
 		/*
 		 * For direct exec mode, argv[0] is the interpreter

From 77388ed2f3d0687c8ea97c6d3379396d0556ee06 Mon Sep 17 00:00:00 2001
From: David C Somayajulu <davidcs@FreeBSD.org>
Date: Tue, 16 May 2017 21:34:40 +0000
Subject: [PATCH 02/73] 1. Move Rx Processing to fp_taskqueue(). With this CPU
 utilization for processing interrupts drops to around 1% for 100G and under
 1% for other speeds. 2. Use sysctls for TRACE_LRO_CNT and TRACE_TSO_PKT_LEN
 3. remove unused mtx tx_lock 4. bind taskqueue kernel thread to the
 appropriate cpu core 5. when tx_ring is full, stop further transmits till at
 least 1/16th of the Tx Ring is empty. In our case 1K entries. Also if there
 are rx_pkts to process, put the taskqueue thread to sleep for 100ms, before
 enabling interrupts. 6. Use rx_pkt_threshold of 128.

MFC after:3 days
---
 sys/dev/qlnx/qlnxe/qlnx_def.h   |   4 +-
 sys/dev/qlnx/qlnxe/qlnx_os.c    | 245 ++++++++++++++++++--------------
 sys/dev/qlnx/qlnxe/qlnx_os.h    |   5 +-
 sys/modules/qlnx/qlnxe/Makefile |   2 -
 4 files changed, 142 insertions(+), 114 deletions(-)

diff --git a/sys/dev/qlnx/qlnxe/qlnx_def.h b/sys/dev/qlnx/qlnxe/qlnx_def.h
index f62931aadd46..e517a4c5799e 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_def.h
+++ b/sys/dev/qlnx/qlnxe/qlnx_def.h
@@ -191,6 +191,7 @@ struct qlnx_fastpath {
 	struct mtx		tx_mtx;
 	char			tx_mtx_name[32];
 	struct buf_ring		*tx_br;
+	uint32_t		tx_ring_full;
 
 	struct task		fp_task;
 	struct taskqueue	*fp_taskqueue;
@@ -364,6 +365,8 @@ struct qlnx_host {
 	/* debug */
 
 	uint32_t                dbg_level;
+	uint32_t                dbg_trace_lro_cnt;
+	uint32_t                dbg_trace_tso_pkt_len;
 	uint32_t                dp_level;
 	uint32_t                dp_module;
 
@@ -386,7 +389,6 @@ struct qlnx_host {
 
 	/* tx related */
 	struct callout		tx_callout;
-	struct mtx		tx_lock;
 	uint32_t		txr_idx;
 
 	/* rx related */
diff --git a/sys/dev/qlnx/qlnxe/qlnx_os.c b/sys/dev/qlnx/qlnxe/qlnx_os.c
index 4560a90984ed..5d6d1fd3cfd2 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_os.c
+++ b/sys/dev/qlnx/qlnxe/qlnx_os.c
@@ -382,16 +382,77 @@ qlnx_fp_taskqueue(void *context, int pending)
         struct ifnet		*ifp;
         struct mbuf		*mp;
         int			ret;
+	int			lro_enable, tc;
+	int			rx_int = 0, total_rx_count = 0;
+	struct thread		*cthread;
 
         fp = context;
 
         if (fp == NULL)
                 return;
 
+	cthread = curthread;
+
+	thread_lock(cthread);
+
+	if (!sched_is_bound(cthread))
+		sched_bind(cthread, fp->rss_id);
+
+	thread_unlock(cthread);
+
         ha = (qlnx_host_t *)fp->edev;
 
         ifp = ha->ifp;
 
+	lro_enable = ha->ifp->if_capenable & IFCAP_LRO;
+
+	rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold, lro_enable);
+
+	if (rx_int) {
+		fp->rx_pkts += rx_int;
+		total_rx_count += rx_int;
+	}
+
+#ifdef QLNX_SOFT_LRO
+	{
+		struct lro_ctrl *lro;
+
+		lro = &fp->rxq->lro;
+
+		if (lro_enable && total_rx_count) {
+
+#if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
+
+			if (ha->dbg_trace_lro_cnt) {
+				if (lro->lro_mbuf_count & ~1023)
+					fp->lro_cnt_1024++;
+				else if (lro->lro_mbuf_count & ~511)
+					fp->lro_cnt_512++;
+				else if (lro->lro_mbuf_count & ~255)
+					fp->lro_cnt_256++;
+				else if (lro->lro_mbuf_count & ~127)
+					fp->lro_cnt_128++;
+				else if (lro->lro_mbuf_count & ~63)
+					fp->lro_cnt_64++;
+			}
+			tcp_lro_flush_all(lro);
+
+#else
+			struct lro_entry *queued;
+
+			while ((!SLIST_EMPTY(&lro->lro_active))) {
+				queued = SLIST_FIRST(&lro->lro_active);
+				SLIST_REMOVE_HEAD(&lro->lro_active, next);
+				tcp_lro_flush(lro, queued);
+			}
+#endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
+		}
+	}
+#endif /* #ifdef QLNX_SOFT_LRO */
+
+	ecore_sb_update_sb_idx(fp->sb_info);
+	rmb();
+
         mtx_lock(&fp->tx_mtx);
 
         if (((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
@@ -401,13 +462,19 @@ qlnx_fp_taskqueue(void *context, int pending)
                 goto qlnx_fp_taskqueue_exit;
         }
 
-        (void)qlnx_tx_int(ha, fp, fp->txq[0]);
+	for (tc = 0; tc < ha->num_tc; tc++) {
+		(void)qlnx_tx_int(ha, fp, fp->txq[tc]);
+	}
 
         mp = drbr_peek(ifp, fp->tx_br);
 
         while (mp != NULL) {
 
-                ret = qlnx_send(ha, fp, &mp);
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			ret = qlnx_send(ha, fp, &mp);
+		} else {
+			ret = -1;
+		}
 
                 if (ret) {
 
@@ -428,14 +495,28 @@ qlnx_fp_taskqueue(void *context, int pending)
                         fp->tx_pkts_processed++;
                 }
 
+		if (fp->tx_ring_full)
+			break;
+
                 mp = drbr_peek(ifp, fp->tx_br);
         }
 
-        (void)qlnx_tx_int(ha, fp, fp->txq[0]);
+	for (tc = 0; tc < ha->num_tc; tc++) {
+		(void)qlnx_tx_int(ha, fp, fp->txq[tc]);
+	}
 
         mtx_unlock(&fp->tx_mtx);
 
 qlnx_fp_taskqueue_exit:
+	if (rx_int) {
+		if (fp->fp_taskqueue != NULL)
+			taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
+	} else {
+		if (fp->tx_ring_full) {
+			qlnx_mdelay(__func__, 100);
+		}
+		ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
+	}
 
         QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = %d\n", __func__, ret));
         return;
@@ -504,7 +585,9 @@ qlnx_drain_fp_taskqueues(qlnx_host_t *ha)
                 fp = &ha->fp_array[i];
 
 		if (fp->fp_taskqueue != NULL) {
+			QLNX_UNLOCK(ha);
 			taskqueue_drain(fp->fp_taskqueue, &fp->fp_task);
+			QLNX_LOCK(ha);
 		}
 	}
 	return;
@@ -540,7 +623,6 @@ qlnx_pci_attach(device_t dev)
         ha->pci_dev = dev;
 
 	mtx_init(&ha->hw_lock, "qlnx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
-        mtx_init(&ha->tx_lock, "qlnx_tx_lock", MTX_NETWORK_LOCK, MTX_DEF);
 
         ha->flags.lock_init = 1;
 
@@ -944,7 +1026,6 @@ qlnx_release(qlnx_host_t *ha)
                 pci_release_msi(dev);
 
         if (ha->flags.lock_init) {
-                mtx_destroy(&ha->tx_lock);
                 mtx_destroy(&ha->hw_lock);
         }
 
@@ -1226,7 +1307,6 @@ qlnx_add_fp_stats_sysctls(qlnx_host_t *ha)
 			CTLFLAG_RD, &ha->fp_array[i].err_tx_cons_idx_conflict,
 			"err_tx_cons_idx_conflict");
 
-#ifdef QLNX_TRACE_LRO_CNT
 		SYSCTL_ADD_QUAD(ctx, node_children,
 			OID_AUTO, "lro_cnt_64",
 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_64,
@@ -1251,7 +1331,6 @@ qlnx_add_fp_stats_sysctls(qlnx_host_t *ha)
 			OID_AUTO, "lro_cnt_1024",
 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_1024,
 			"lro_cnt_1024");
-#endif /* #ifdef QLNX_TRACE_LRO_CNT */
 
 		/* Rx Related */
 
@@ -1710,6 +1789,18 @@ qlnx_add_sysctls(qlnx_host_t *ha)
                 OID_AUTO, "dp_level", CTLFLAG_RW,
                 &ha->dp_level, ha->dp_level, "DP Level");
 
+        ha->dbg_trace_lro_cnt = 0;
+        SYSCTL_ADD_UINT(ctx, children,
+                OID_AUTO, "dbg_trace_lro_cnt", CTLFLAG_RW,
+                &ha->dbg_trace_lro_cnt, ha->dbg_trace_lro_cnt,
+		"Trace LRO Counts");
+
+        ha->dbg_trace_tso_pkt_len = 0;
+        SYSCTL_ADD_UINT(ctx, children,
+                OID_AUTO, "dbg_trace_tso_pkt_len", CTLFLAG_RW,
+                &ha->dbg_trace_tso_pkt_len, ha->dbg_trace_tso_pkt_len,
+		"Trace TSO packet lengths");
+
         ha->dp_module = 0;
         SYSCTL_ADD_UINT(ctx, children,
                 OID_AUTO, "dp_module", CTLFLAG_RW,
@@ -1755,7 +1846,7 @@ qlnx_add_sysctls(qlnx_host_t *ha)
                 &ha->tx_coalesce_usecs, ha->tx_coalesce_usecs,
 		"tx_coalesce_usecs");
 
-	ha->rx_pkt_threshold = 32;
+	ha->rx_pkt_threshold = 128;
         SYSCTL_ADD_UINT(ctx, children,
                 OID_AUTO, "rx_pkt_threshold", CTLFLAG_RW,
                 &ha->rx_pkt_threshold, ha->rx_pkt_threshold,
@@ -2162,7 +2253,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			ifp->if_mtu = ifr->ifr_mtu;
 			ha->max_frame_size =
 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
-			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				qlnx_init_locked(ha);
 			}
 
@@ -2178,7 +2269,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		QLNX_LOCK(ha);
 
 		if (ifp->if_flags & IFF_UP) {
-			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				if ((ifp->if_flags ^ ha->if_flags) &
 					IFF_PROMISC) {
 					ret = qlnx_set_promisc(ha);
@@ -2712,6 +2803,16 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 	tx_data_bd	= NULL;
 
 	txq = fp->txq[0];
+
+	if (fp->tx_ring_full) {
+		elem_left = ecore_chain_get_elem_left(&txq->tx_pbl);
+
+		if (elem_left < (TX_RING_SIZE >> 4)) 
+			return (-1);
+		else 
+			fp->tx_ring_full = 0;
+	}
+
 	idx = txq->sw_tx_prod;
 
 	map = txq->sw_tx_ring[idx].map;
@@ -2720,20 +2821,18 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 	ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
 			BUS_DMA_NOWAIT);
 
-#ifdef QLNX_TRACE_TSO_PKT_LEN
-
-	if (!fp->tx_tso_min_pkt_len) {
-		fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
-		fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
-	} else {
-		if (fp->tx_tso_min_pkt_len > m_head->m_pkthdr.len)
+	if (ha->dbg_trace_tso_pkt_len) {
+		if (!fp->tx_tso_min_pkt_len) {
 			fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
-		if (fp->tx_tso_max_pkt_len < m_head->m_pkthdr.len)
-			fp->tx_tso_max_pkt_len = m_head->m_pkthdr.len;
+			fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
+		} else {
+			if (fp->tx_tso_min_pkt_len > m_head->m_pkthdr.len)
+				fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
+			if (fp->tx_tso_max_pkt_len < m_head->m_pkthdr.len)
+				fp->tx_tso_max_pkt_len = m_head->m_pkthdr.len;
+		}
 	}
 
-#endif /* #ifdef QLNX_TRACE_TSO_PKT_LEN */
-
 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO)
 		offset = qlnx_tcp_offset(ha, m_head);
 
@@ -2815,14 +2914,12 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 	QL_ASSERT(ha, (nsegs != 0), ("qlnx_send: empty packet"));
 
-#ifdef QLNX_TRACE_TSO_PKT_LEN
-
-	if (nsegs < QLNX_FP_MAX_SEGS)
-		fp->tx_pkts[(nsegs - 1)]++;
-	else
-		fp->tx_pkts[(QLNX_FP_MAX_SEGS - 1)]++; 
-
-#endif /* #ifdef QLNX_TRACE_TSO_PKT_LEN */
+	if (ha->dbg_trace_tso_pkt_len) {
+		if (nsegs < QLNX_FP_MAX_SEGS)
+			fp->tx_pkts[(nsegs - 1)]++;
+		else
+			fp->tx_pkts[(QLNX_FP_MAX_SEGS - 1)]++; 
+	}
 
 	if ((nsegs + QLNX_TX_ELEM_RESERVE) >
 		(int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) {
@@ -2843,6 +2940,7 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 				__func__, nsegs, elem_left, fp->rss_id));
 
 			fp->err_tx_nsegs_gt_elem_left++;
+			fp->tx_ring_full = 1;
 			ha->storm_stats_enable = 1;
 			return (ENOBUFS);
 		}
@@ -3051,15 +3149,13 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 	first_bd->data.nbds = nbd;
 
-#ifdef QLNX_TRACE_TSO_PKT_LEN
+	if (ha->dbg_trace_tso_pkt_len) {
+		if (fp->tx_tso_max_nsegs < nsegs)
+			fp->tx_tso_max_nsegs = nsegs;
 
-	if (fp->tx_tso_max_nsegs < nsegs)
-		fp->tx_tso_max_nsegs = nsegs;
-
-	if ((nsegs < fp->tx_tso_min_nsegs) || (!fp->tx_tso_min_nsegs))
-		fp->tx_tso_min_nsegs = nsegs;
-
-#endif /* #ifdef QLNX_TRACE_TSO_PKT_LEN */
+		if ((nsegs < fp->tx_tso_min_nsegs) || (!fp->tx_tso_min_nsegs))
+			fp->tx_tso_min_nsegs = nsegs;
+	}
 
 	txq->sw_tx_ring[idx].nsegs = nsegs;
 	txq->sw_tx_prod = (txq->sw_tx_prod + 1) & (TX_RING_SIZE - 1);
@@ -4188,11 +4284,9 @@ qlnx_fp_isr(void *arg)
         qlnx_ivec_t		*ivec = arg;
         qlnx_host_t		*ha;
         struct qlnx_fastpath	*fp = NULL;
-        int			idx, lro_enable, tc;
-        int			rx_int = 0, total_rx_count = 0;
+        int			idx;
 
         ha = ivec->ha;
-        lro_enable = ha->ifp->if_capenable & IFCAP_LRO;
 
         if (ha->state != QLNX_STATE_OPEN) {
                 return;
@@ -4214,73 +4308,8 @@ qlnx_fp_isr(void *arg)
                 ha->err_fp_null++;
         } else {
                 ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
-
-                do {
-                        for (tc = 0; tc < ha->num_tc; tc++) {
-                                if (mtx_trylock(&fp->tx_mtx)) {
-                                        qlnx_tx_int(ha, fp, fp->txq[tc]);
-                                        mtx_unlock(&fp->tx_mtx);
-                                }
-                        }
-
-                        rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold,
-                                        lro_enable);
-
-                        if (rx_int) {
-                                fp->rx_pkts += rx_int;
-				total_rx_count += rx_int;
-			}
-
-                } while (rx_int);
-
-
-#ifdef QLNX_SOFT_LRO
-                {
-                        struct lro_ctrl *lro;
-
-                        lro = &fp->rxq->lro;
-
-                        if (lro_enable && total_rx_count) {
-
-#if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
-
-#ifdef QLNX_TRACE_LRO_CNT
-				if (lro->lro_mbuf_count & ~1023)
-					fp->lro_cnt_1024++;
-				else if (lro->lro_mbuf_count & ~511)
-					fp->lro_cnt_512++;
-				else if (lro->lro_mbuf_count & ~255)
-					fp->lro_cnt_256++;
-				else if (lro->lro_mbuf_count & ~127)
-					fp->lro_cnt_128++;
-				else if (lro->lro_mbuf_count & ~63)
-					fp->lro_cnt_64++;
-#endif /* #ifdef QLNX_TRACE_LRO_CNT */
-
-                                tcp_lro_flush_all(lro);
-
-#else
-                                struct lro_entry *queued;
-
-                                while ((!SLIST_EMPTY(&lro->lro_active))) {
-                                        queued = SLIST_FIRST(&lro->lro_active);
-                                        SLIST_REMOVE_HEAD(&lro->lro_active, \
-						next);
-                                        tcp_lro_flush(lro, queued);
-                                }
-#endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
-                        }
-                }
-#endif /* #ifdef QLNX_SOFT_LRO */
-
-                if (fp->fp_taskqueue != NULL)
-                        taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
-
-                ecore_sb_update_sb_idx(fp->sb_info);
-                rmb();
-                ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
-
-                return;
+		if (fp->fp_taskqueue != NULL)
+			taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
         }
 
         return;
@@ -5150,6 +5179,8 @@ qlnx_init_fp(qlnx_host_t *ha)
 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", qlnx_name_str,
 			rss_id);
 
+		fp->tx_ring_full = 0;
+
 		/* reset all the statistics counters */
 
 		fp->tx_pkts_processed = 0;
diff --git a/sys/dev/qlnx/qlnxe/qlnx_os.h b/sys/dev/qlnx/qlnxe/qlnx_os.h
index 55321f271bd2..5c0f5b763f12 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_os.h
+++ b/sys/dev/qlnx/qlnxe/qlnx_os.h
@@ -92,6 +92,7 @@
 #include <sys/kthread.h>
 #include <sys/libkern.h>
 #include <sys/smp.h>
+#include <sys/sched.h>
 
 static __inline int qlnx_ms_to_hz(int ms)
 {
@@ -138,10 +139,6 @@ MALLOC_DECLARE(M_QLNXBUF);
 #define QLNX_LOCK(ha)		mtx_lock(&ha->hw_lock)
 #define QLNX_UNLOCK(ha)		mtx_unlock(&ha->hw_lock)
 
- 
-#define QLNX_TX_LOCK(ha)	mtx_lock(&ha->tx_lock);
-#define QLNX_TX_UNLOCK(ha)	mtx_unlock(&ha->tx_lock);
-
 /*
  * structure encapsulating a DMA buffer
  */
diff --git a/sys/modules/qlnx/qlnxe/Makefile b/sys/modules/qlnx/qlnxe/Makefile
index c217a0a33eaf..2b91540daa9a 100644
--- a/sys/modules/qlnx/qlnxe/Makefile
+++ b/sys/modules/qlnx/qlnxe/Makefile
@@ -65,8 +65,6 @@ CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include
 #CFLAGS += -DQLNX_SOFT_LRO
 #CFLAGS += -DQLNX_QSORT_LRO
 #CFLAGS += -DQLNX_MAX_COALESCE
-#CFLAGS += -DQLNX_TRACE_LRO_CNT
-#CFLAGS += -DQLNX_TRACE_TSO_PKT_LEN
 
 
 .include <bsd.kmod.mk>

From 7a377fbeb1b2e2fcfb726d2c9fca568dbbc9c2be Mon Sep 17 00:00:00 2001
From: David C Somayajulu <davidcs@FreeBSD.org>
Date: Tue, 16 May 2017 21:46:30 +0000
Subject: [PATCH 03/73] QL_DPRINT macro modfied to handle multiple args; print
 line#.

Submitted by:Shminderjit.Singh@cavium.com
MFC after:3 days
---
 sys/dev/qlnx/qlnxe/qlnx_def.h   | 158 ++++++--
 sys/dev/qlnx/qlnxe/qlnx_ioctl.c |  93 ++---
 sys/dev/qlnx/qlnxe/qlnx_os.c    | 642 ++++++++++++++------------------
 3 files changed, 452 insertions(+), 441 deletions(-)

diff --git a/sys/dev/qlnx/qlnxe/qlnx_def.h b/sys/dev/qlnx/qlnxe/qlnx_def.h
index e517a4c5799e..dca025bcdcbf 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_def.h
+++ b/sys/dev/qlnx/qlnxe/qlnx_def.h
@@ -483,35 +483,141 @@ typedef struct qlnx_host qlnx_host_t;
 
 #ifdef QLNX_DEBUG
 
-#define QL_DPRINT1(ha, x)       if (ha->dbg_level & 0x0001) device_printf x
-#define QL_DPRINT2(ha, x)       if (ha->dbg_level & 0x0002) device_printf x
-#define QL_DPRINT3(ha, x)       if (ha->dbg_level & 0x0004) device_printf x
-#define QL_DPRINT4(ha, x)       if (ha->dbg_level & 0x0008) device_printf x
-#define QL_DPRINT5(ha, x)       if (ha->dbg_level & 0x0010) device_printf x
-#define QL_DPRINT6(ha, x)       if (ha->dbg_level & 0x0020) device_printf x
-#define QL_DPRINT7(ha, x)       if (ha->dbg_level & 0x0040) device_printf x
-#define QL_DPRINT8(ha, x)       if (ha->dbg_level & 0x0080) device_printf x
-#define QL_DPRINT9(ha, x)       if (ha->dbg_level & 0x0100) device_printf x
-#define QL_DPRINT11(ha, x)      if (ha->dbg_level & 0x0400) device_printf x
-#define QL_DPRINT12(ha, x)      if (ha->dbg_level & 0x0800) device_printf x
-#define QL_DPRINT13(ha, x)      if (ha->dbg_level & 0x1000) device_printf x
-#define QL_DPRINT14(ha, x)      if (ha->dbg_level & 0x2000) device_printf x
+#define QL_DPRINT1(ha, x, ...) 					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0001) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT2(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0002) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT3(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0004) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT4(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0008) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT5(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0010) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT6(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0020) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT7(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0040) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT8(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0080) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT9(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0100) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT11(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0400) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT12(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x0800) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
+#define QL_DPRINT13(ha, x, ...)					\
+	do { 							\
+		if ((ha)->dbg_level & 0x1000) {			\
+			device_printf ((ha)->pci_dev,		\
+				"[%s:%d]" x,			\
+				__func__, __LINE__,		\
+				## __VA_ARGS__);		\
+		}						\
+	} while (0)
+
 
 #else
 
-#define QL_DPRINT1(ha, x)
-#define QL_DPRINT2(ha, x)
-#define QL_DPRINT3(ha, x)
-#define QL_DPRINT4(ha, x)
-#define QL_DPRINT5(ha, x)
-#define QL_DPRINT6(ha, x)
-#define QL_DPRINT7(ha, x)
-#define QL_DPRINT8(ha, x)
-#define QL_DPRINT9(ha, x)
-#define QL_DPRINT11(ha, x)
-#define QL_DPRINT12(ha, x)
-#define QL_DPRINT13(ha, x)
-#define QL_DPRINT14(ha, x)
+#define QL_DPRINT1(ha, x, ...)
+#define QL_DPRINT2(ha, x, ...)
+#define QL_DPRINT3(ha, x, ...)
+#define QL_DPRINT4(ha, x, ...)
+#define QL_DPRINT5(ha, x, ...)
+#define QL_DPRINT6(ha, x, ...)
+#define QL_DPRINT7(ha, x, ...)
+#define QL_DPRINT8(ha, x, ...)
+#define QL_DPRINT9(ha, x, ...)
+#define QL_DPRINT11(ha, x, ...)
+#define QL_DPRINT12(ha, x, ...)
+#define QL_DPRINT13(ha, x, ...)
 
 #endif /* #ifdef QLNX_DEBUG */
 
diff --git a/sys/dev/qlnx/qlnxe/qlnx_ioctl.c b/sys/dev/qlnx/qlnxe/qlnx_ioctl.c
index f7a1262af5a7..41a8f8d98592 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_ioctl.c
+++ b/sys/dev/qlnx/qlnxe/qlnx_ioctl.c
@@ -121,8 +121,7 @@ qlnx_grc_dump(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
 	if (!p_ptt) {
-		QL_DPRINT1(ha, (ha->pci_dev, "%s : ecore_ptt_acquire failed\n",
-			__func__));
+		QL_DPRINT1(ha,"ecore_ptt_acquire failed\n");
 		return (rval);
 	}
 
@@ -133,9 +132,8 @@ qlnx_grc_dump(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index)
 	 	rval = 0;	
 		ha->grcdump_taken = 1;
 	} else
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s : ecore_dbg_grc_dump failed [%d, 0x%x]\n",
-			__func__, hwfn_index, rval));
+		QL_DPRINT1(ha,"ecore_dbg_grc_dump failed [%d, 0x%x]\n",
+			   hwfn_index, rval);
 
 	ecore_ptt_release(p_hwfn, p_ptt);
 
@@ -177,8 +175,7 @@ qlnx_get_grc_dump(qlnx_host_t *ha, qlnx_grcdump_t *grcdump)
 
 		grcdump->grcdump_dwords[i] = dwords;
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: grcdump_dwords[%d] = 0x%x\n",
-			__func__, i, dwords));
+		QL_DPRINT1(ha,"grcdump_dwords[%d] = 0x%x\n", i, dwords);
 
 		rval = copyout(ha->grcdump[i], grcdump->grcdump[i],
 				ha->grcdump_size[i]);
@@ -213,8 +210,7 @@ qlnx_idle_chk(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
 	if (!p_ptt) {
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s : ecore_ptt_acquire failed\n", __func__));
+		QL_DPRINT1(ha,"ecore_ptt_acquire failed\n");
 		return (rval);
 	}
 
@@ -225,9 +221,8 @@ qlnx_idle_chk(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index)
 	 	rval = 0;	
 		ha->idle_chk_taken = 1;
 	} else
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s : ecore_dbg_idle_chk_dump failed [%d, 0x%x]\n",
-			__func__, hwfn_index, rval));
+		QL_DPRINT1(ha,"ecore_dbg_idle_chk_dump failed [%d, 0x%x]\n",
+			   hwfn_index, rval);
 
 	ecore_ptt_release(p_hwfn, p_ptt);
 
@@ -271,8 +266,7 @@ qlnx_get_idle_chk(qlnx_host_t *ha, qlnx_idle_chk_t *idle_chk)
 		
 		idle_chk->idle_chk_dwords[i] = dwords;
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: idle_chk_dwords[%d] = 0x%x\n",
-			__func__, i, dwords));
+		QL_DPRINT1(ha,"idle_chk_dwords[%d] = 0x%x\n", i, dwords);
 
                	rval = copyout(ha->idle_chk[i], idle_chk->idle_chk[i],
 				ha->idle_chk_size[i]);
@@ -299,9 +293,8 @@ qlnx_get_trace_cmd_size(qlnx_host_t *ha, int hwfn_index, uint16_t cmd)
         p_ptt = ecore_ptt_acquire(p_hwfn);
 
         if (!p_ptt) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: ecore_ptt_acquire [%d, 0x%x]failed\n",
-                        __func__, hwfn_index, cmd));
+                QL_DPRINT1(ha, "ecore_ptt_acquire [%d, 0x%x]failed\n",
+                           hwfn_index, cmd);
                 return (0);
         }
 
@@ -334,8 +327,7 @@ qlnx_get_trace_cmd_size(qlnx_host_t *ha, int hwfn_index, uint16_t cmd)
 	}
 
         if (rval != DBG_STATUS_OK) {
-                QL_DPRINT1(ha, (ha->pci_dev, "%s : cmd = 0x%x failed [0x%x]\n",
-                        __func__, cmd, rval));
+                QL_DPRINT1(ha,"cmd = 0x%x failed [0x%x]\n", cmd, rval);
 		num_dwords = 0;
         }
 
@@ -369,9 +361,8 @@ qlnx_get_trace(qlnx_host_t *ha, int hwfn_index, qlnx_trace_t *trace)
 
 	buffer = qlnx_zalloc(trace->size[hwfn_index]);
 	if (buffer == NULL) { 
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: qlnx_zalloc [%d, 0x%x]failed\n",
-                        __func__, hwfn_index, trace->cmd));
+                QL_DPRINT1(ha,"qlnx_zalloc [%d, 0x%x]failed\n",
+                           hwfn_index, trace->cmd);
                 return (ENXIO);
 	}
 	ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver());
@@ -380,9 +371,8 @@ qlnx_get_trace(qlnx_host_t *ha, int hwfn_index, qlnx_trace_t *trace)
         p_ptt = ecore_ptt_acquire(p_hwfn);
 
         if (!p_ptt) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: ecore_ptt_acquire [%d, 0x%x]failed\n",
-                        __func__, hwfn_index, trace->cmd));
+                QL_DPRINT1(ha, "ecore_ptt_acquire [%d, 0x%x]failed\n",
+                           hwfn_index, trace->cmd);
                 return (ENXIO);
         }
 
@@ -420,8 +410,7 @@ qlnx_get_trace(qlnx_host_t *ha, int hwfn_index, qlnx_trace_t *trace)
 	}
 
         if (rval != DBG_STATUS_OK) {
-                QL_DPRINT1(ha, (ha->pci_dev, "%s : cmd = 0x%x failed [0x%x]\n",
-                        __func__, trace->cmd, rval));
+                QL_DPRINT1(ha,"cmd = 0x%x failed [0x%x]\n", trace->cmd, rval);
 		num_dwords = 0;
         }
 
@@ -609,21 +598,18 @@ qlnx_write_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram, uint32_t cmd)
 
 	ret = copyin(nvram->data, buf, nvram->data_len);
 
-	QL_DPRINT9(ha,
-		(ha->pci_dev, "%s: issue cmd = 0x%x data = %p "
-		" data_len = 0x%x ret = 0x%x exit\n", __func__,
-		cmd, nvram->data, nvram->data_len, ret));
+	QL_DPRINT9(ha, "issue cmd = 0x%x data = %p \
+		 data_len = 0x%x ret = 0x%x exit\n",
+		cmd, nvram->data, nvram->data_len, ret);
 
 	if (ret == 0) {
 		ret = ecore_mcp_nvm_write(&ha->cdev, cmd,
 			nvram->offset, buf, nvram->data_len);
 	}
 
-	QL_DPRINT9(ha,
-		(ha->pci_dev, "%s: cmd = 0x%x data = %p "
-		" data_len = 0x%x resp = 0x%x ret = 0x%x exit\n",
-		__func__, cmd, nvram->data, nvram->data_len,
-		ha->cdev.mcp_nvm_resp, ret));
+	QL_DPRINT9(ha, "cmd = 0x%x data = %p \
+		 data_len = 0x%x resp = 0x%x ret = 0x%x exit\n",
+		cmd, nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret);
 
 	free(buf, M_QLNXBUF);
 
@@ -644,10 +630,9 @@ qlnx_read_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram)
 	ret = ecore_mcp_nvm_read(&ha->cdev, nvram->offset, buf,
 		nvram->data_len);
 
-	QL_DPRINT9(ha, (ha->pci_dev, "%s: data = %p data_len = 0x%x "
-		" resp = 0x%x ret = 0x%x exit\n", __func__,
-		nvram->data, nvram->data_len,
-		ha->cdev.mcp_nvm_resp, ret));
+	QL_DPRINT9(ha, " data = %p data_len = 0x%x \
+		 resp = 0x%x ret = 0x%x exit\n",
+		nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret);
 
 	if (ret == 0) {
 		ret = copyout(buf, nvram->data, nvram->data_len);
@@ -672,10 +657,9 @@ qlnx_get_nvram_resp(qlnx_host_t *ha, qlnx_nvram_t *nvram)
 
 	ret = ecore_mcp_nvm_resp(&ha->cdev, buf);
 
-	QL_DPRINT9(ha, (ha->pci_dev, "%s: data = %p data_len = 0x%x "
-		" resp = 0x%x ret = 0x%x exit\n", __func__,
-		nvram->data, nvram->data_len,
-		ha->cdev.mcp_nvm_resp, ret));
+	QL_DPRINT9(ha, "data = %p data_len = 0x%x \
+		 resp = 0x%x ret = 0x%x exit\n",
+		nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret);
 
 	if (ret == 0) {
 		ret = copyout(buf, nvram->data, nvram->data_len);
@@ -708,28 +692,25 @@ qlnx_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram)
 	case QLNX_NVRAM_CMD_SET_SECURE_MODE:
 		ret = ecore_mcp_nvm_set_secure_mode(&ha->cdev, nvram->offset);
 
-		QL_DPRINT9(ha, (ha->pci_dev,
-			"%s: QLNX_NVRAM_CMD_SET_SECURE_MODE "
-			" resp = 0x%x ret = 0x%x exit\n", __func__,
-			ha->cdev.mcp_nvm_resp, ret));
+		QL_DPRINT9(ha, "QLNX_NVRAM_CMD_SET_SECURE_MODE \
+			 resp = 0x%x ret = 0x%x exit\n",
+			 ha->cdev.mcp_nvm_resp, ret);
 		break;
 
 	case QLNX_NVRAM_CMD_DEL_FILE:
 		ret = ecore_mcp_nvm_del_file(&ha->cdev, nvram->offset);
 
-		QL_DPRINT9(ha, (ha->pci_dev,
-			"%s: QLNX_NVRAM_CMD_DEL_FILE "
-			" resp = 0x%x ret = 0x%x exit\n", __func__,
-			ha->cdev.mcp_nvm_resp, ret));
+		QL_DPRINT9(ha, "QLNX_NVRAM_CMD_DEL_FILE \
+			 resp = 0x%x ret = 0x%x exit\n",
+			ha->cdev.mcp_nvm_resp, ret);
 		break;
 
 	case QLNX_NVRAM_CMD_PUT_FILE_BEGIN:
 		ret = ecore_mcp_nvm_put_file_begin(&ha->cdev, nvram->offset);
 
-		QL_DPRINT9(ha, (ha->pci_dev,
-			"%s: QLNX_NVRAM_CMD_PUT_FILE_BEGIN "
-			" resp = 0x%x ret = 0x%x exit\n", __func__,
-			ha->cdev.mcp_nvm_resp, ret));
+		QL_DPRINT9(ha, "QLNX_NVRAM_CMD_PUT_FILE_BEGIN \
+			 resp = 0x%x ret = 0x%x exit\n",
+			ha->cdev.mcp_nvm_resp, ret);
 		break;
 
 	case QLNX_NVRAM_CMD_GET_NVRAM_RESP:
diff --git a/sys/dev/qlnx/qlnxe/qlnx_os.c b/sys/dev/qlnx/qlnxe/qlnx_os.c
index 5d6d1fd3cfd2..90ac191ad240 100644
--- a/sys/dev/qlnx/qlnxe/qlnx_os.c
+++ b/sys/dev/qlnx/qlnxe/qlnx_os.c
@@ -303,7 +303,7 @@ qlnx_sp_intr(void *arg)
 
 	ha = (qlnx_host_t *)p_hwfn->p_dev;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
 		if (&ha->cdev.hwfns[i] == p_hwfn) {
@@ -311,7 +311,7 @@ qlnx_sp_intr(void *arg)
 			break;
 		}
 	}
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 	
 	return;
 }
@@ -353,8 +353,7 @@ qlnx_create_sp_taskqueues(qlnx_host_t *ha)
 		taskqueue_start_threads(&ha->sp_taskqueue[i], 1, PI_NET, "%s",
 			tq_name);
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: %p\n", __func__,
-			ha->sp_taskqueue[i]));
+		QL_DPRINT1(ha, "%p\n", ha->sp_taskqueue[i]);
 	}
 
 	return (0);
@@ -518,7 +517,7 @@ qlnx_fp_taskqueue(void *context, int pending)
 		ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
 	}
 
-        QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = %d\n", __func__, ret));
+        QL_DPRINT2(ha, "exit ret = %d\n", ret);
         return;
 }
 
@@ -548,8 +547,7 @@ qlnx_create_fp_taskqueues(qlnx_host_t *ha)
 		taskqueue_start_threads(&fp->fp_taskqueue, 1, PI_NET, "%s",
 			tq_name);
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: %p\n", __func__,
-			fp->fp_taskqueue));
+		QL_DPRINT1(ha, "%p\n",fp->fp_taskqueue);
 	}
 
 	return (0);
@@ -711,15 +709,14 @@ qlnx_pci_attach(device_t dev)
 	else
 		ha->num_rss = ha->msix_count - ha->cdev.num_hwfns;
 
-	QL_DPRINT1(ha, (dev, "%s:\n\t\t\tpci_reg [%p, 0x%08x 0x%08x]"
+	QL_DPRINT1(ha, "\n\t\t\tpci_reg [%p, 0x%08x 0x%08x]"
 		"\n\t\t\tdbells [%p, 0x%08x 0x%08x]"
 		"\n\t\t\tmsix [%p, 0x%08x 0x%08x 0x%x 0x%x]"
 		"\n\t\t\t[ncpus = %d][num_rss = 0x%x] [num_tc = 0x%x]\n",
-		__func__, ha->pci_reg, rsrc_len_reg,
+		 ha->pci_reg, rsrc_len_reg,
 		ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid,
 		ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev),
-		ha->msix_count, mp_ncpus, ha->num_rss, ha->num_tc));
-
+		ha->msix_count, mp_ncpus, ha->num_rss, ha->num_tc);
         if (pci_alloc_msix(dev, &ha->msix_count)) {
                 device_printf(dev, "%s: pci_alloc_msix[%d] failed\n", __func__,
                         ha->msix_count);
@@ -755,9 +752,9 @@ qlnx_pci_attach(device_t dev)
 			goto qlnx_pci_attach_err;
 		}
 
-		QL_DPRINT1(ha, (dev, "%s: p_hwfn [%p] sp_irq_rid %d"
-			" sp_irq %p sp_handle %p\n", __func__, p_hwfn,
-			ha->sp_irq_rid[i], ha->sp_irq[i], ha->sp_handle[i]));
+		QL_DPRINT1(ha, "p_hwfn [%p] sp_irq_rid %d"
+			" sp_irq %p sp_handle %p\n", p_hwfn,
+			ha->sp_irq_rid[i], ha->sp_irq[i], ha->sp_handle[i]);
 
 	}
 
@@ -800,8 +797,8 @@ qlnx_pci_attach(device_t dev)
 			goto qlnx_pci_attach_err;
 
 		ha->grcdump_size[i] = ha->grcdump_size[i] << 2;
-		QL_DPRINT1(ha, (dev, "grcdump_size[%d] = 0x%08x\n",
-			i, ha->grcdump_size[i]));
+		QL_DPRINT1(ha, "grcdump_size[%d] = 0x%08x\n",
+			i, ha->grcdump_size[i]);
 
 		ha->grcdump[i] = qlnx_zalloc(ha->grcdump_size[i]);
 		if (ha->grcdump[i] == NULL) {
@@ -815,8 +812,8 @@ qlnx_pci_attach(device_t dev)
 			goto qlnx_pci_attach_err;
 
 		ha->idle_chk_size[i] = ha->idle_chk_size[i] << 2;
-		QL_DPRINT1(ha, (dev, "idle_chk_size[%d] = 0x%08x\n",
-			i, ha->idle_chk_size[i]));
+		QL_DPRINT1(ha, "idle_chk_size[%d] = 0x%08x\n",
+			i, ha->idle_chk_size[i]);
 
 		ha->idle_chk[i] = qlnx_zalloc(ha->idle_chk_size[i]);
 
@@ -855,8 +852,8 @@ qlnx_pci_attach(device_t dev)
 		FW_MAJOR_VERSION, FW_MINOR_VERSION, FW_REVISION_VERSION,
 		FW_ENGINEERING_VERSION);
 
-	QL_DPRINT1(ha, (dev, "%s: STORM_FW version %s MFW version %s\n",
-		__func__, ha->stormfw_ver, ha->mfw_ver));
+	QL_DPRINT1(ha, "STORM_FW version %s MFW version %s\n",
+		 ha->stormfw_ver, ha->mfw_ver);
 
 	qlnx_init_ifnet(dev, ha);
 
@@ -874,7 +871,7 @@ qlnx_pci_attach(device_t dev)
                 goto qlnx_pci_attach_err;
         }
 
-	QL_DPRINT2(ha, (dev, "%s: success\n", __func__));
+	QL_DPRINT2(ha, "success\n");
 
         return (0);
 
@@ -958,7 +955,7 @@ qlnx_release(qlnx_host_t *ha)
 
         dev = ha->pci_dev;
 
-	QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	for (i = 0; i < QLNX_MAX_HW_FUNCS; i++) {
 		if (ha->idle_chk[i] != NULL) {
@@ -1041,7 +1038,7 @@ qlnx_release(qlnx_host_t *ha)
                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->msix_rid,
                                 ha->msix_bar);
 
-	QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 	return;
 }
 
@@ -1053,14 +1050,14 @@ qlnx_trigger_dump(qlnx_host_t *ha)
 	if (ha->ifp != NULL)
 		ha->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: start\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
 		qlnx_grc_dump(ha, &ha->grcdump_dwords[i], i);
 		qlnx_idle_chk(ha, &ha->idle_chk_dwords[i], i);
 	}
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: end\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 
 	return;
 }
@@ -1779,12 +1776,11 @@ qlnx_add_sysctls(qlnx_host_t *ha)
 		"\tpersonality = 6 => Default in Shared Memory\n");
 
         ha->dbg_level = 0;
-
         SYSCTL_ADD_UINT(ctx, children,
                 OID_AUTO, "debug", CTLFLAG_RW,
                 &ha->dbg_level, ha->dbg_level, "Debug Level");
 
-        ha->dp_level = 0;
+        ha->dp_level = 0x01;
         SYSCTL_ADD_UINT(ctx, children,
                 OID_AUTO, "dp_level", CTLFLAG_RW,
                 &ha->dp_level, ha->dp_level, "DP Level");
@@ -1999,7 +1995,7 @@ qlnx_init_ifnet(device_t dev, qlnx_host_t *ha)
 
         ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
 
-        QL_DPRINT2(ha, (dev, "%s: exit\n", __func__));
+        QL_DPRINT2(ha, "exit\n");
 
         return;
 }
@@ -2009,6 +2005,8 @@ qlnx_init_locked(qlnx_host_t *ha)
 {
 	struct ifnet	*ifp = ha->ifp;
 
+	QL_DPRINT1(ha, "Driver Initialization start \n");
+
 	qlnx_stop(ha);
 
 	if (qlnx_load(ha) == 0) {
@@ -2026,13 +2024,13 @@ qlnx_init(void *arg)
 
 	ha = (qlnx_host_t *)arg;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	QLNX_LOCK(ha);
 	qlnx_init_locked(ha);
 	QLNX_UNLOCK(ha);
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 
 	return;
 }
@@ -2221,8 +2219,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 	switch (cmd) {
 	case SIOCSIFADDR:
-		QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFADDR (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx)\n", cmd);
 
 		if (ifa->ifa_addr->sa_family == AF_INET) {
 			ifp->if_flags |= IFF_UP;
@@ -2231,10 +2228,8 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 				qlnx_init_locked(ha);
 				QLNX_UNLOCK(ha);
 			}
-			QL_DPRINT4(ha, (ha->pci_dev,
-				"%s: SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
-				__func__, cmd,
-				ntohl(IA_SIN(ifa)->sin_addr.s_addr)));
+			QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
+				   cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr));
 
 			arp_ifinit(ifp, ifa);
 		} else {
@@ -2243,8 +2238,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		break;
 
 	case SIOCSIFMTU:
-		QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFMTU (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "SIOCSIFMTU (0x%lx)\n", cmd);
 
 		if (ifr->ifr_mtu > QLNX_MAX_MTU) {
 			ret = EINVAL;
@@ -2263,8 +2257,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		break;
 
 	case SIOCSIFFLAGS:
-		QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFFLAGS (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "SIOCSIFFLAGS (0x%lx)\n", cmd);
 
 		QLNX_LOCK(ha);
 
@@ -2292,8 +2285,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		break;
 
 	case SIOCADDMULTI:
-		QL_DPRINT4(ha, (ha->pci_dev,
-			"%s: %s (0x%lx)\n", __func__, "SIOCADDMULTI", cmd));
+		QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCADDMULTI", cmd);
 
 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 			if (qlnx_set_multi(ha, 1))
@@ -2302,8 +2294,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		break;
 
 	case SIOCDELMULTI:
-		QL_DPRINT4(ha, (ha->pci_dev,
-			"%s: %s (0x%lx)\n", __func__, "SIOCDELMULTI", cmd));
+		QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCDELMULTI", cmd);
 
 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 			if (qlnx_set_multi(ha, 0))
@@ -2313,9 +2304,8 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
-		QL_DPRINT4(ha, (ha->pci_dev,
-			"%s: SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n", cmd);
+
 		ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
 		break;
 
@@ -2323,8 +2313,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		
 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 
-		QL_DPRINT4(ha, (ha->pci_dev, "%s: SIOCSIFCAP (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "SIOCSIFCAP (0x%lx)\n", cmd);
 
 		if (mask & IFCAP_HWCSUM)
 			ifp->if_capenable ^= IFCAP_HWCSUM;
@@ -2367,8 +2356,7 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		p_ptt = ecore_ptt_acquire(p_hwfn);
 
 		if (!p_ptt) {
-			QL_DPRINT1(ha, (ha->pci_dev, "%s :"
-				" ecore_ptt_acquire failed\n", __func__));
+			QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
 			ret = -1;
 			break;
 		}
@@ -2386,20 +2374,19 @@ qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 		ret = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
 
-		QL_DPRINT8(ha, (ha->pci_dev, "SIOCGI2C copyout ret = %d"
-			" len = %d addr = 0x%02x offset = 0x%04x"
-			" data[0..7]=0x%02x 0x%02x 0x%02x 0x%02x 0x%02x"
-			" 0x%02x 0x%02x 0x%02x\n",
+		QL_DPRINT8(ha, "SIOCGI2C copyout ret = %d \
+			 len = %d addr = 0x%02x offset = 0x%04x \
+			 data[0..7]=0x%02x 0x%02x 0x%02x 0x%02x 0x%02x \
+			 0x%02x 0x%02x 0x%02x\n",
 			ret, i2c.len, i2c.dev_addr, i2c.offset,
 			i2c.data[0], i2c.data[1], i2c.data[2], i2c.data[3],
-			i2c.data[4], i2c.data[5], i2c.data[6], i2c.data[7]));
+			i2c.data[4], i2c.data[5], i2c.data[6], i2c.data[7]);
 		break;
 	}
 #endif /* #if (__FreeBSD_version >= 1100101) */
 
 	default:
-		QL_DPRINT4(ha, (ha->pci_dev, "%s: default (0x%lx)\n",
-			__func__, cmd));
+		QL_DPRINT4(ha, "default (0x%lx)\n", cmd);
 		ret = ether_ioctl(ifp, cmd, data);
 		break;
 	}
@@ -2416,14 +2403,14 @@ qlnx_media_change(struct ifnet *ifp)
 
 	ha = (qlnx_host_t *)ifp->if_softc;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	ifm = &ha->media;
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 		ret = EINVAL;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 
 	return (ret);
 }
@@ -2435,7 +2422,7 @@ qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 
 	ha = (qlnx_host_t *)ifp->if_softc;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
@@ -2451,8 +2438,7 @@ qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 				(IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
 	}
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit (%s)\n", __func__,
-		(ha->link_up ? "link_up" : "link_down")));
+	QL_DPRINT2(ha, "exit (%s)\n", (ha->link_up ? "link_up" : "link_down"));
 
 	return;
 }
@@ -2478,20 +2464,19 @@ qlnx_free_tx_pkt(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 		QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL);
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: (mp == NULL) "
+		QL_DPRINT1(ha, "(mp == NULL) "
 			" tx_idx = 0x%x"
 			" ecore_prod_idx = 0x%x"
 			" ecore_cons_idx = 0x%x"
 			" hw_bd_cons = 0x%x"
 			" txq_db_last = 0x%x"
 			" elem_left = 0x%x\n",
-			__func__,
 			fp->rss_id,
 			ecore_chain_get_prod_idx(&txq->tx_pbl),
 			ecore_chain_get_cons_idx(&txq->tx_pbl),
 			le16toh(*txq->hw_cons_ptr),
 			txq->tx_db.raw,
-			ecore_chain_get_elem_left(&txq->tx_pbl)));
+			ecore_chain_get_elem_left(&txq->tx_pbl));
 
 		fp->err_tx_free_pkt_null++;
 
@@ -2552,20 +2537,20 @@ qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 			QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF);
 
-			QL_DPRINT1(ha, (ha->pci_dev, "%s: (diff = 0x%x) "
+			QL_DPRINT1(ha, "(diff = 0x%x) "
 				" tx_idx = 0x%x"
 				" ecore_prod_idx = 0x%x"
 				" ecore_cons_idx = 0x%x"
 				" hw_bd_cons = 0x%x"
 				" txq_db_last = 0x%x"
 				" elem_left = 0x%x\n",
-				__func__, diff,
+				diff,
 				fp->rss_id,
 				ecore_chain_get_prod_idx(&txq->tx_pbl),
 				ecore_chain_get_cons_idx(&txq->tx_pbl),
 				le16toh(*txq->hw_cons_ptr),
 				txq->tx_db.raw,
-				ecore_chain_get_elem_left(&txq->tx_pbl)));
+				ecore_chain_get_elem_left(&txq->tx_pbl));
 
 			fp->err_tx_cons_idx_conflict++;
 
@@ -2587,7 +2572,7 @@ qlnx_transmit(struct ifnet *ifp, struct mbuf  *mp)
         struct qlnx_fastpath	*fp;
         int			rss_id = 0, ret = 0;
 
-        QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+        QL_DPRINT2(ha, "enter\n");
 
 #if __FreeBSD_version >= 1100000
         if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE)
@@ -2615,7 +2600,7 @@ qlnx_transmit(struct ifnet *ifp, struct mbuf  *mp)
 
 qlnx_transmit_exit:
 
-        QL_DPRINT2(ha, (ha->pci_dev, "%s: exit ret = %d\n", __func__, ret));
+        QL_DPRINT2(ha, "exit ret = %d\n", ret);
         return ret;
 }
 
@@ -2629,7 +2614,7 @@ qlnx_qflush(struct ifnet *ifp)
 
 	ha = (qlnx_host_t *)ifp->if_softc;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	for (rss_id = 0; rss_id < ha->num_rss; rss_id++) {
 
@@ -2648,7 +2633,7 @@ qlnx_qflush(struct ifnet *ifp)
 			mtx_unlock(&fp->tx_mtx);
 		}
 	}
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 
 	return;
 }
@@ -2792,7 +2777,7 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 	uint32_t		nbds_in_hdr = 0;
 	uint32_t		offset = 0;
 
-	QL_DPRINT8(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT8(ha, "enter\n");
 
 	if (!ha->link_up)
 		return (-1);
@@ -2844,8 +2829,7 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 		struct mbuf *m;
 
-		QL_DPRINT8(ha, (ha->pci_dev, "%s: EFBIG [%d]\n", __func__,
-			m_head->m_pkthdr.len));
+		QL_DPRINT8(ha, "EFBIG [%d]\n", m_head->m_pkthdr.len);
 
 		fp->tx_defrag++;
 
@@ -2855,9 +2839,7 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 			fp->tx_pkts_freed++;
 			m_freem(m_head);
 			*m_headp = NULL;
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: m_defrag() = NULL [%d]\n",
-				__func__, ret));
+			QL_DPRINT1(ha, "m_defrag() = NULL [%d]\n", ret);
 			return (ENOBUFS);
 		}
 
@@ -2869,9 +2851,9 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 			fp->err_tx_defrag_dmamap_load++;
 
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: bus_dmamap_load_mbuf_sg failed0[%d, %d]\n",
-				__func__, ret, m_head->m_pkthdr.len));
+			QL_DPRINT1(ha,
+				"bus_dmamap_load_mbuf_sg failed0 [%d, %d]\n",
+				ret, m_head->m_pkthdr.len);
 
 			fp->tx_pkts_freed++;
 			m_freem(m_head);
@@ -2885,9 +2867,9 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 			fp->err_tx_non_tso_max_seg++;
 
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: (%d) nsegs too many for non-TSO[%d, %d]\n",
-				__func__, ret, nsegs, m_head->m_pkthdr.len));
+			QL_DPRINT1(ha,
+				"(%d) nsegs too many for non-TSO [%d, %d]\n",
+				ret, nsegs, m_head->m_pkthdr.len);
 
 			fp->tx_pkts_freed++;
 			m_freem(m_head);
@@ -2902,10 +2884,8 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 		fp->err_tx_dmamap_load++;
 
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: bus_dmamap_load_mbuf_sg failed1[%d, %d]\n",
-			__func__, ret, m_head->m_pkthdr.len));
-
+		QL_DPRINT1(ha, "bus_dmamap_load_mbuf_sg failed1 [%d, %d]\n",
+			   ret, m_head->m_pkthdr.len);
 		fp->tx_pkts_freed++;
 		m_freem(m_head);
 		*m_headp = NULL;
@@ -2924,9 +2904,9 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 	if ((nsegs + QLNX_TX_ELEM_RESERVE) >
 		(int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) {
 
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: (%d, 0x%x) insuffient BDs"
-			"in chain[%d] trying to free packets\n",
-			__func__, nsegs, elem_left, fp->rss_id));
+		QL_DPRINT1(ha, "(%d, 0x%x) insuffient BDs"
+			" in chain[%d] trying to free packets\n",
+			nsegs, elem_left, fp->rss_id);
 
 		fp->tx_nsegs_gt_elem_left++;
 
@@ -2935,9 +2915,9 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 		if ((nsegs + QLNX_TX_ELEM_RESERVE) > (int)(elem_left =
 			ecore_chain_get_elem_left(&txq->tx_pbl))) {
 
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: (%d, 0x%x) insuffient BDs in chain[%d]\n",
-				__func__, nsegs, elem_left, fp->rss_id));
+			QL_DPRINT1(ha,
+				"(%d, 0x%x) insuffient BDs in chain[%d]\n",
+				nsegs, elem_left, fp->rss_id);
 
 			fp->err_tx_nsegs_gt_elem_left++;
 			fp->tx_ring_full = 1;
@@ -3165,7 +3145,7 @@ qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
 
 	qlnx_txq_doorbell_wr32(ha, txq->doorbell_addr, txq->tx_db.raw);
    
-	QL_DPRINT8(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT8(ha, "exit\n");
 	return (0);
 }
 
@@ -3185,6 +3165,8 @@ qlnx_stop(qlnx_host_t *ha)
 	 * propagate the if_drv_flags
 	 * state to each tx thread
 	 */
+        QL_DPRINT1(ha, "QLNX STATE = %d\n",ha->state);
+
 	if (ha->state == QLNX_STATE_OPEN) {
         	for (i = 0; i < ha->num_rss; i++) {
 			struct qlnx_fastpath *fp = &ha->fp_array[i];
@@ -3277,8 +3259,7 @@ qlnx_rx_jumbo_chain(qlnx_host_t *ha, struct qlnx_fastpath *fp,
                 mp = sw_rx_data->data;
 
 		if (mp == NULL) {
-                	QL_DPRINT1(ha, (ha->pci_dev, "%s: mp = NULL\n",
-				__func__));
+                	QL_DPRINT1(ha, "mp = NULL\n");
 			fp->err_rx_mp_null++;
         		rxq->sw_rx_cons  =
 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
@@ -3293,10 +3274,8 @@ qlnx_rx_jumbo_chain(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
                 if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
 
-                        QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: New buffer allocation failed, dropping"
-				" incoming packet and reusing its buffer\n",
-				__func__));
+                        QL_DPRINT1(ha, "New buffer allocation failed, dropping"
+				" incoming packet and reusing its buffer\n");
 
                         qlnx_reuse_rx_data(rxq);
                         fp->err_rx_alloc_errors++;
@@ -3356,29 +3335,29 @@ qlnx_tpa_start(qlnx_host_t *ha,
 	dev = ha->pci_dev;
 	agg_index = cqe->tpa_agg_index;
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: enter\n "
-                "\t type = 0x%x\n"
-                "\t bitfields = 0x%x\n"
-                "\t seg_len = 0x%x\n"
-                "\t pars_flags = 0x%x\n"
-                "\t vlan_tag = 0x%x\n"
-                "\t rss_hash = 0x%x\n"
-                "\t len_on_first_bd = 0x%x\n"
-                "\t placement_offset = 0x%x\n"
-                "\t tpa_agg_index = 0x%x\n"
-                "\t header_len = 0x%x\n"
-                "\t ext_bd_len_list[0] = 0x%x\n"
-                "\t ext_bd_len_list[1] = 0x%x\n"
-                "\t ext_bd_len_list[2] = 0x%x\n"
-                "\t ext_bd_len_list[3] = 0x%x\n"
-                "\t ext_bd_len_list[4] = 0x%x\n",
-                __func__, fp->rss_id, cqe->type, cqe->bitfields, cqe->seg_len,
+        QL_DPRINT7(ha, "[rss_id = %d]: enter\n \
+                \t type = 0x%x\n \
+                \t bitfields = 0x%x\n \
+                \t seg_len = 0x%x\n \
+                \t pars_flags = 0x%x\n \
+                \t vlan_tag = 0x%x\n \
+                \t rss_hash = 0x%x\n \
+                \t len_on_first_bd = 0x%x\n \
+                \t placement_offset = 0x%x\n \
+                \t tpa_agg_index = 0x%x\n \
+                \t header_len = 0x%x\n \
+                \t ext_bd_len_list[0] = 0x%x\n \
+                \t ext_bd_len_list[1] = 0x%x\n \
+                \t ext_bd_len_list[2] = 0x%x\n \
+                \t ext_bd_len_list[3] = 0x%x\n \
+                \t ext_bd_len_list[4] = 0x%x\n",
+                fp->rss_id, cqe->type, cqe->bitfields, cqe->seg_len,
                 cqe->pars_flags.flags, cqe->vlan_tag,
                 cqe->rss_hash, cqe->len_on_first_bd, cqe->placement_offset,
                 cqe->tpa_agg_index, cqe->header_len,
                 cqe->ext_bd_len_list[0], cqe->ext_bd_len_list[1],
                 cqe->ext_bd_len_list[2], cqe->ext_bd_len_list[3],
-                cqe->ext_bd_len_list[4]));
+                cqe->ext_bd_len_list[4]);
 
 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
 		fp->err_rx_tpa_invalid_agg_num++;
@@ -3389,11 +3368,10 @@ qlnx_tpa_start(qlnx_host_t *ha,
 	bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD);
 	mp = sw_rx_data->data;
 
-	QL_DPRINT7(ha, (dev, "%s[%d]: mp = %p \n ", __func__, fp->rss_id, mp));
+	QL_DPRINT7(ha, "[rss_id = %d]: mp = %p \n ", fp->rss_id, mp);
 
 	if (mp == NULL) {
-               	QL_DPRINT7(ha, (dev, "%s[%d]: mp = NULL\n", __func__,
-			fp->rss_id));
+               	QL_DPRINT7(ha, "[%d]: mp = NULL\n", fp->rss_id);
 		fp->err_rx_mp_null++;
        		rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
 
@@ -3402,10 +3380,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	if ((le16toh(cqe->pars_flags.flags)) & CQE_FLAGS_ERR) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: CQE in CONS = %u has error,"
-			" flags = %x, dropping incoming packet\n", __func__,
-			fp->rss_id, rxq->sw_rx_cons,
-			le16toh(cqe->pars_flags.flags)));
+		QL_DPRINT7(ha, "[%d]: CQE in CONS = %u has error,"
+			" flags = %x, dropping incoming packet\n", fp->rss_id,
+			rxq->sw_rx_cons, le16toh(cqe->pars_flags.flags));
 
 		fp->err_rx_hw_errors++;
 
@@ -3418,9 +3395,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: New buffer allocation failed,"
+		QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
 			" dropping incoming packet and reusing its buffer\n",
-			__func__, fp->rss_id));
+			fp->rss_id);
 
 		fp->err_rx_alloc_errors++;
 		QLNX_INC_IQDROPS(ifp);
@@ -3472,9 +3449,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: invalid aggregation state,"
+		QL_DPRINT7(ha, "[%d]: invalid aggregation state,"
 			" dropping incoming packet and reusing its buffer\n",
-			__func__, fp->rss_id));
+			fp->rss_id);
 
 		QLNX_INC_IQDROPS(ifp);
 
@@ -3511,7 +3488,7 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 4\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 4\n ", fp->rss_id);
 
 		if (cqe->ext_bd_len_list[i] == 0)
 			break;
@@ -3523,8 +3500,7 @@ qlnx_tpa_start(qlnx_host_t *ha,
 		mpc = sw_rx_data->data;
 
 		if (mpc == NULL) {
-			QL_DPRINT7(ha, (ha->pci_dev, "%s[%d]: mpc = NULL\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
 			fp->err_rx_mp_null++;
 			if (mpf != NULL)
 				m_freem(mpf);
@@ -3538,10 +3514,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 		}
 
 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
-			QL_DPRINT7(ha, (dev,
-				"%s[%d]: New buffer allocation failed, dropping"
-				" incoming packet and reusing its buffer\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
+				" dropping incoming packet and reusing its"
+				" buffer\n", fp->rss_id);
 
 			qlnx_reuse_rx_data(rxq);
 
@@ -3579,9 +3554,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: invalid aggregation state,"
-			" dropping incoming packet and reusing its buffer\n",
-			__func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: invalid aggregation state, dropping"
+			" incoming packet and reusing its buffer\n",
+			fp->rss_id);
 
 		QLNX_INC_IQDROPS(ifp);
 
@@ -3661,10 +3636,9 @@ qlnx_tpa_start(qlnx_host_t *ha,
 
 	rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_START;
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: 5\n" "\tagg_state = %d\n"
-                "\t mpf = %p mpl = %p\n", __func__, fp->rss_id,
-                rxq->tpa_info[agg_index].agg_state,
-                rxq->tpa_info[agg_index].mpf, rxq->tpa_info[agg_index].mpl));
+        QL_DPRINT7(ha, "[%d]: 5\n\tagg_state = %d\n\t mpf = %p mpl = %p\n",
+		fp->rss_id, rxq->tpa_info[agg_index].agg_state,
+                rxq->tpa_info[agg_index].mpf, rxq->tpa_info[agg_index].mpl);
 
 	return;
 }
@@ -3683,23 +3657,23 @@ qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 	dev = ha->pci_dev;
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: enter\n "
-                "\t type = 0x%x\n"
-                "\t tpa_agg_index = 0x%x\n"
-                "\t len_list[0] = 0x%x\n"
-                "\t len_list[1] = 0x%x\n"
-                "\t len_list[2] = 0x%x\n"
-                "\t len_list[3] = 0x%x\n"
-                "\t len_list[4] = 0x%x\n"
-                "\t len_list[5] = 0x%x\n",
-                __func__, fp->rss_id, cqe->type, cqe->tpa_agg_index,
+        QL_DPRINT7(ha, "[%d]: enter\n \
+                \t type = 0x%x\n \
+                \t tpa_agg_index = 0x%x\n \
+                \t len_list[0] = 0x%x\n \
+                \t len_list[1] = 0x%x\n \
+                \t len_list[2] = 0x%x\n \
+                \t len_list[3] = 0x%x\n \
+                \t len_list[4] = 0x%x\n \
+                \t len_list[5] = 0x%x\n",
+                fp->rss_id, cqe->type, cqe->tpa_agg_index,
                 cqe->len_list[0], cqe->len_list[1], cqe->len_list[2],
-                cqe->len_list[3], cqe->len_list[4], cqe->len_list[5]));
+                cqe->len_list[3], cqe->len_list[4], cqe->len_list[5]);
 
 	agg_index = cqe->tpa_agg_index;
 
 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
-		QL_DPRINT7(ha, (dev, "%s[%d]: 0\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id);
 		fp->err_rx_tpa_invalid_agg_num++;
 		return;
 	}
@@ -3707,7 +3681,7 @@ qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 	for (i = 0; i < ETH_TPA_CQE_CONT_LEN_LIST_SIZE; i++) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 1\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id);
 
 		if (cqe->len_list[i] == 0)
 			break;
@@ -3726,8 +3700,7 @@ qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 		if (mpc == NULL) {
 
-			QL_DPRINT7(ha, (dev, "%s[%d]: mpc = NULL\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
 
 			fp->err_rx_mp_null++;
 			if (mpf != NULL)
@@ -3743,10 +3716,9 @@ qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
 
-			QL_DPRINT7(ha, (dev,
-				"%s[%d]: New buffer allocation failed, dropping"
-				" incoming packet and reusing its buffer\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
+				" dropping incoming packet and reusing its"
+				" buffer\n", fp->rss_id);
 
 			qlnx_reuse_rx_data(rxq);
 
@@ -3782,8 +3754,8 @@ qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 			(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
 	}
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: 2\n" "\tmpf = %p mpl = %p\n",
-                __func__, fp->rss_id, mpf, mpl));
+        QL_DPRINT7(ha, "[%d]: 2\n" "\tmpf = %p mpl = %p\n",
+                  fp->rss_id, mpf, mpl);
 
 	if (mpf != NULL) {
 		mp = rxq->tpa_info[agg_index].mpl;
@@ -3811,29 +3783,29 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 	dev = ha->pci_dev;
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: enter\n "
-                "\t type = 0x%x\n"
-                "\t tpa_agg_index = 0x%x\n"
-                "\t total_packet_len = 0x%x\n"
-                "\t num_of_bds = 0x%x\n"
-                "\t end_reason = 0x%x\n"
-                "\t num_of_coalesced_segs = 0x%x\n"
-                "\t ts_delta = 0x%x\n"
-                "\t len_list[0] = 0x%x\n"
-                "\t len_list[1] = 0x%x\n"
-                "\t len_list[2] = 0x%x\n"
-                "\t len_list[3] = 0x%x\n",
-                __func__, fp->rss_id, cqe->type, cqe->tpa_agg_index,
+        QL_DPRINT7(ha, "[%d]: enter\n \
+                \t type = 0x%x\n \
+                \t tpa_agg_index = 0x%x\n \
+                \t total_packet_len = 0x%x\n \
+                \t num_of_bds = 0x%x\n \
+                \t end_reason = 0x%x\n \
+                \t num_of_coalesced_segs = 0x%x\n \
+                \t ts_delta = 0x%x\n \
+                \t len_list[0] = 0x%x\n \
+                \t len_list[1] = 0x%x\n \
+                \t len_list[2] = 0x%x\n \
+                \t len_list[3] = 0x%x\n",
+                 fp->rss_id, cqe->type, cqe->tpa_agg_index,
                 cqe->total_packet_len, cqe->num_of_bds,
                 cqe->end_reason, cqe->num_of_coalesced_segs, cqe->ts_delta,
                 cqe->len_list[0], cqe->len_list[1], cqe->len_list[2],
-                cqe->len_list[3]));
+                cqe->len_list[3]);
 
 	agg_index = cqe->tpa_agg_index;
 
 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 0\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id);
 
 		fp->err_rx_tpa_invalid_agg_num++;
 		return (0);
@@ -3842,7 +3814,7 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 	for (i = 0; i < ETH_TPA_CQE_END_LEN_LIST_SIZE; i++) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 1\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id);
 
 		if (cqe->len_list[i] == 0)
 			break;
@@ -3850,8 +3822,7 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 		if (rxq->tpa_info[agg_index].agg_state != 
 			QLNX_AGG_STATE_START) {
 
-			QL_DPRINT7(ha, (dev, "%s[%d]: 2\n ", __func__,
-				fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: 2\n ", fp->rss_id);
 	
 			qlnx_reuse_rx_data(rxq);
 			continue;
@@ -3865,8 +3836,7 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 		if (mpc == NULL) {
 
-			QL_DPRINT7(ha, (dev, "%s[%d]: mpc = NULL\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
 
 			fp->err_rx_mp_null++;
 			if (mpf != NULL)
@@ -3881,10 +3851,9 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 		}
 
 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
-			QL_DPRINT7(ha, (dev,
-				"%s[%d]: New buffer allocation failed, dropping"
-				" incoming packet and reusing its buffer\n",
-				__func__, fp->rss_id));
+			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
+				" dropping incoming packet and reusing its"
+				" buffer\n", fp->rss_id);
 
 			qlnx_reuse_rx_data(rxq);
 
@@ -3920,11 +3889,11 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 			(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
 	}
 
-	QL_DPRINT7(ha, (dev, "%s[%d]: 5\n ", __func__, fp->rss_id));
+	QL_DPRINT7(ha, "[%d]: 5\n ", fp->rss_id);
 
 	if (mpf != NULL) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 6\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 6\n ", fp->rss_id);
 
 		mp = rxq->tpa_info[agg_index].mpl;
 		mp->m_len = ha->rx_buf_size;
@@ -3933,7 +3902,7 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 
 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) {
 
-		QL_DPRINT7(ha, (dev, "%s[%d]: 7\n ", __func__, fp->rss_id));
+		QL_DPRINT7(ha, "[%d]: 7\n ", fp->rss_id);
 
 		if (rxq->tpa_info[agg_index].mpf != NULL)
 			m_freem(rxq->tpa_info[agg_index].mpf);
@@ -3966,10 +3935,10 @@ qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 	QLNX_INC_IPACKETS(ifp);
 	QLNX_INC_IBYTES(ifp, (cqe->total_packet_len));
 
-        QL_DPRINT7(ha, (dev, "%s[%d]: 8 csum_data = 0x%x csum_flags = 0x%lx\n "
-		"m_len = 0x%x m_pkthdr_len = 0x%x\n",
-                __func__, fp->rss_id, mp->m_pkthdr.csum_data,
-                mp->m_pkthdr.csum_flags, mp->m_len, mp->m_pkthdr.len));
+        QL_DPRINT7(ha, "[%d]: 8 csum_data = 0x%x csum_flags = 0x%lx\n \
+		m_len = 0x%x m_pkthdr_len = 0x%x\n",
+                fp->rss_id, mp->m_pkthdr.csum_data,
+                mp->m_pkthdr.csum_flags, mp->m_len, mp->m_pkthdr.len);
 
 	(*ifp->if_input)(ifp, mp);
 
@@ -4027,7 +3996,7 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
                 cqe_type = cqe->fast_path_regular.type;
 
                 if (cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH) {
-                        QL_DPRINT3(ha, (ha->pci_dev, "Got a slowath CQE\n"));
+                        QL_DPRINT3(ha, "Got a slowath CQE\n");
 
                         ecore_eth_cqe_completion(p_hwfn,
                                         (struct eth_slow_path_rx_cqe *)cqe);
@@ -4068,8 +4037,7 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
                 mp = sw_rx_data->data;
 
 		if (mp == NULL) {
-                	QL_DPRINT1(ha, (ha->pci_dev, "%s: mp = NULL\n",
-				__func__));
+                	QL_DPRINT1(ha, "mp = NULL\n");
 			fp->err_rx_mp_null++;
         		rxq->sw_rx_cons  =
 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
@@ -4083,12 +4051,11 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
                 len =  le16toh(fp_cqe->pkt_len);
                 pad = fp_cqe->placement_offset;
 
-		QL_DPRINT3(ha,
-			(ha->pci_dev, "CQE type = %x, flags = %x, vlan = %x,"
+		QL_DPRINT3(ha, "CQE type = %x, flags = %x, vlan = %x,"
 			" len %u, parsing flags = %d pad  = %d\n",
 			cqe_type, fp_cqe->bitfields,
 			le16toh(fp_cqe->vlan_tag),
-			len, le16toh(fp_cqe->pars_flags.flags), pad));
+			len, le16toh(fp_cqe->pars_flags.flags), pad);
 
 		data = mtod(mp, uint8_t *);
 		data = data + pad;
@@ -4106,11 +4073,9 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
 		if ((le16toh(cqe->fast_path_regular.pars_flags.flags)) &
 			CQE_FLAGS_ERR) {
 
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"CQE in CONS = %u has error, flags = %x,"
+			QL_DPRINT1(ha, "CQE in CONS = %u has error, flags = %x,"
 				" dropping incoming packet\n", sw_comp_cons,
-			le16toh(cqe->fast_path_regular.pars_flags.flags)));
-
+			le16toh(cqe->fast_path_regular.pars_flags.flags));
 			fp->err_rx_hw_errors++;
 
                         qlnx_reuse_rx_data(rxq);
@@ -4122,10 +4087,8 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
 
                 if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
 
-                        QL_DPRINT1(ha, (ha->pci_dev,
-				"New buffer allocation failed, dropping"
-				" incoming packet and reusing its buffer\n"));
-
+                        QL_DPRINT1(ha, "New buffer allocation failed, dropping"
+				" incoming packet and reusing its buffer\n");
                         qlnx_reuse_rx_data(rxq);
 
                         fp->err_rx_alloc_errors++;
@@ -4141,10 +4104,8 @@ qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
 		m_adj(mp, pad);
 		mp->m_pkthdr.len = len;
 
-		QL_DPRINT1(ha,
-			(ha->pci_dev, "%s: len = %d len_on_first_bd = %d\n",
-			__func__, len, len_on_first_bd));
-
+		QL_DPRINT1(ha, "len = %d len_on_first_bd = %d\n",
+			   len, len_on_first_bd);
 		if ((len > 60 ) && (len > len_on_first_bd)) {
 
 			mp->m_len = len_on_first_bd;
@@ -4295,16 +4256,13 @@ qlnx_fp_isr(void *arg)
         idx = ivec->rss_idx;
 
         if ((idx = ivec->rss_idx) >= ha->num_rss) {
-                QL_DPRINT1(ha, (ha->pci_dev, "%s: illegal interrupt[%d]\n",
-                        __func__, idx));
+                QL_DPRINT1(ha, "illegal interrupt[%d]\n", idx);
                 ha->err_illegal_intr++;
                 return;
         }
         fp = &ha->fp_array[idx];
 
         if (fp == NULL) {
-                QL_DPRINT1(ha, (ha->pci_dev, "%s: fp_array[%d] NULL\n",
-                        __func__, idx));
                 ha->err_fp_null++;
         } else {
                 ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
@@ -4332,11 +4290,11 @@ qlnx_sp_isr(void *arg)
 
 	ha->sp_interrupts++;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	ecore_int_sp_dpc(p_hwfn);
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 	
 	return;
 }
@@ -4384,8 +4342,7 @@ qlnx_alloc_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf)
                         &dma_buf->dma_tag);
 
         if (ret) {
-                QL_DPRINT1(ha,
-			(dev, "%s: could not create dma tag\n", __func__));
+                QL_DPRINT1(ha, "could not create dma tag\n");
                 goto qlnx_alloc_dmabuf_exit;
         }
         ret = bus_dmamem_alloc(dma_buf->dma_tag,
@@ -4394,8 +4351,7 @@ qlnx_alloc_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf)
                         &dma_buf->dma_map);
         if (ret) {
                 bus_dma_tag_destroy(dma_buf->dma_tag);
-                QL_DPRINT1(ha,
-			(dev, "%s: bus_dmamem_alloc failed\n", __func__));
+                QL_DPRINT1(ha, "bus_dmamem_alloc failed\n");
                 goto qlnx_alloc_dmabuf_exit;
         }
 
@@ -4457,11 +4413,11 @@ qlnx_dma_alloc_coherent(void *ecore_dev, bus_addr_t *phys, uint32_t size)
 	dma_p = (qlnx_dma_t *)((uint8_t *)dma_buf.dma_b + size);
 
 	memcpy(dma_p, &dma_buf, sizeof(qlnx_dma_t));
-
-	QL_DPRINT5(ha, (dev, "%s: [%p %p %p %p 0x%08x ]\n", __func__,
+/*
+	QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n",
 		(void *)dma_buf.dma_map, (void *)dma_buf.dma_tag,
-		dma_buf.dma_b, (void *)dma_buf.dma_addr, size));
-
+		dma_buf.dma_b, (void *)dma_buf.dma_addr, size);
+*/
 	return (dma_buf.dma_b);
 }
 
@@ -4482,11 +4438,11 @@ qlnx_dma_free_coherent(void *ecore_dev, void *v_addr, bus_addr_t phys,
 	size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
 
 	dma_p = (qlnx_dma_t *)((uint8_t *)v_addr + size);
-
-	QL_DPRINT5(ha, (dev, "%s: [%p %p %p %p 0x%08x ]\n", __func__,
+/*
+	QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n",
 		(void *)dma_p->dma_map, (void *)dma_p->dma_tag,
-		dma_p->dma_b, (void *)dma_p->dma_addr, size));
-
+		dma_p->dma_b, (void *)dma_p->dma_addr, size);
+*/
 	dma_buf = *dma_p;
 
 	qlnx_free_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf);
@@ -4518,8 +4474,7 @@ qlnx_alloc_parent_dma_tag(qlnx_host_t *ha)
                         &ha->parent_tag);
 
         if (ret) {
-                QL_DPRINT1(ha, (dev, "%s: could not create parent dma tag\n",
-                        __func__));
+                QL_DPRINT1(ha, "could not create parent dma tag\n");
                 return (-1);
         }
 
@@ -4554,8 +4509,7 @@ qlnx_alloc_tx_dma_tag(qlnx_host_t *ha)
                 NULL,    /* lockfuncarg */
                 &ha->tx_tag)) {
 
-                QL_DPRINT1(ha, (ha->pci_dev, "%s: tx_tag alloc failed\n",
-                        __func__));
+                QL_DPRINT1(ha, "tx_tag alloc failed\n");
                 return (-1);
         }
 
@@ -4588,8 +4542,7 @@ qlnx_alloc_rx_dma_tag(qlnx_host_t *ha)
                         NULL,    /* lockfuncarg */
                         &ha->rx_tag)) {
 
-                QL_DPRINT1(ha, (ha->pci_dev, "%s: rx_tag alloc failed\n",
-                        __func__));
+                QL_DPRINT1(ha, " rx_tag alloc failed\n");
 
                 return (-1);
         }
@@ -4679,15 +4632,15 @@ qlnx_pci_write_config_dword(void *ecore_dev, uint32_t pci_reg,
 int
 qlnx_pci_find_capability(void *ecore_dev, int cap)
 {
-	int reg;
+	int		reg;
+	qlnx_host_t	*ha;
 
-	if (pci_find_cap(((qlnx_host_t *)ecore_dev)->pci_dev, PCIY_EXPRESS,
-			&reg) == 0)
+	ha = ecore_dev;
+
+	if (pci_find_cap(ha->pci_dev, PCIY_EXPRESS, &reg) == 0)
 		return reg;
 	else {
-		QL_DPRINT1(((qlnx_host_t *)ecore_dev),
-			(((qlnx_host_t *)ecore_dev)->pci_dev,
-			"%s: failed\n", __func__));
+		QL_DPRINT1(ha, "failed\n");
 		return 0;
 	}
 }
@@ -5074,13 +5027,14 @@ qlnx_get_protocol_stats(void *cdev, int proto_type, void *proto_stats)
 	enum ecore_mcp_protocol_type	type;
 	union ecore_mcp_protocol_stats	*stats;
 	struct ecore_eth_stats		eth_stats;
-	device_t			dev;
+	qlnx_host_t			*ha;
 
-	dev = ((qlnx_host_t *)cdev)->pci_dev;
+	ha = cdev;
 	stats = proto_stats;
 	type = proto_type;
 
         switch (type) {
+
         case ECORE_MCP_LAN_STATS:
                 ecore_get_vport_stats((struct ecore_dev *)cdev, &eth_stats);
                 stats->lan_stats.ucast_rx_pkts = eth_stats.common.rx_ucast_pkts;
@@ -5089,11 +5043,9 @@ qlnx_get_protocol_stats(void *cdev, int proto_type, void *proto_stats)
                 break;
 
 	default:
-		((qlnx_host_t *)cdev)->err_get_proto_invalid_type++;
+		ha->err_get_proto_invalid_type++;
 
-		QL_DPRINT1(((qlnx_host_t *)cdev),
-			(dev, "%s: invalid protocol type 0x%x\n", __func__,
-			type));
+		QL_DPRINT1(ha, "invalid protocol type 0x%x\n", type);
 		break;
 	}
 	return;
@@ -5109,8 +5061,7 @@ qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
 	if (p_ptt ==  NULL) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-                        "%s : ecore_ptt_acquire failed\n", __func__));
+                QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
                 return (-1);
 	}
 	ecore_mcp_get_mfw_ver(p_hwfn, p_ptt, mfw_ver, NULL);
@@ -5130,8 +5081,7 @@ qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
 	if (p_ptt ==  NULL) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-                        "%s : ecore_ptt_acquire failed\n", __func__));
+                QL_DPRINT1(ha,"ecore_ptt_acquire failed\n");
                 return (-1);
 	}
 	ecore_mcp_get_flash_size(p_hwfn, p_ptt, flash_size);
@@ -5234,11 +5184,11 @@ qlnx_sb_init(struct ecore_dev *cdev, struct ecore_sb_info *sb_info,
         p_hwfn = &cdev->hwfns[hwfn_index];
         rel_sb_id = sb_id / cdev->num_hwfns;
 
-        QL_DPRINT2(((qlnx_host_t *)cdev), (((qlnx_host_t *)cdev)->pci_dev,
-                "%s: hwfn_index = %d p_hwfn = %p sb_id = 0x%x rel_sb_id = 0x%x "
-                "sb_info = %p sb_virt_addr = %p sb_phy_addr = %p\n",
-                __func__, hwfn_index, p_hwfn, sb_id, rel_sb_id, sb_info,
-                sb_virt_addr, (void *)sb_phy_addr));
+        QL_DPRINT2(((qlnx_host_t *)cdev), 
+                "hwfn_index = %d p_hwfn = %p sb_id = 0x%x rel_sb_id = 0x%x \
+                sb_info = %p sb_virt_addr = %p sb_phy_addr = %p\n",
+                hwfn_index, p_hwfn, sb_id, rel_sb_id, sb_info,
+                sb_virt_addr, (void *)sb_phy_addr);
 
         rc = ecore_int_sb_init(p_hwfn, p_hwfn->p_main_ptt, sb_info,
                              sb_virt_addr, sb_phy_addr, rel_sb_id);
@@ -5262,14 +5212,12 @@ qlnx_alloc_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info, u16 sb_id)
 	sb_virt = OSAL_DMA_ALLOC_COHERENT(cdev, (&sb_phys), size);
 
         if (!sb_virt) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: Status block allocation failed\n", __func__));
+                QL_DPRINT1(ha, "Status block allocation failed\n");
                 return -ENOMEM;
         }
 
         rc = qlnx_sb_init(cdev, sb_info, sb_virt, sb_phys, sb_id);
         if (rc) {
-                QL_DPRINT1(ha, (ha->pci_dev, "%s: failed\n", __func__));
                 OSAL_DMA_FREE_COHERENT(cdev, sb_virt, sb_phys, size);
         }
 
@@ -5365,8 +5313,7 @@ qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
 	mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size);
 
         if (mp == NULL) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s : Failed to allocate Rx data\n", __func__));
+                QL_DPRINT1(ha, "Failed to allocate Rx data\n");
                 return -ENOMEM;
         }
 
@@ -5380,10 +5327,8 @@ qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
 
 	if (ret || !dma_addr || (nsegs != 1)) {
 		m_freem(mp);
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
-                        __func__, ret, (long long unsigned int)dma_addr,
-                        nsegs));
+		QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
+                           ret, (long long unsigned int)dma_addr, nsegs);
 		return -ENOMEM;
 	}
 
@@ -5418,8 +5363,7 @@ qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size,
 	mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size);
 
         if (mp == NULL) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s : Failed to allocate Rx data\n", __func__));
+                QL_DPRINT1(ha, "Failed to allocate Rx data\n");
                 return -ENOMEM;
         }
 
@@ -5433,10 +5377,8 @@ qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size,
 
 	if (ret || !dma_addr || (nsegs != 1)) {
 		m_freem(mp);
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
-                        __func__, ret, (long long unsigned int)dma_addr,
-                        nsegs));
+		QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
+			ret, (long long unsigned int)dma_addr, nsegs);
 		return -ENOMEM;
 	}
 
@@ -5533,13 +5475,11 @@ qlnx_alloc_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
         }
         num_allocated = i;
         if (!num_allocated) {
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: Rx buffers allocation failed\n", __func__));
+		QL_DPRINT1(ha, "Rx buffers allocation failed\n");
                 goto err;
         } else if (num_allocated < rxq->num_rx_buffers) {
-		QL_DPRINT1(ha, (ha->pci_dev,
-                	"%s: Allocated less buffers than"
-			" desired (%d allocated)\n", __func__, num_allocated));
+		QL_DPRINT1(ha, "Allocated less buffers than"
+			" desired (%d allocated)\n", num_allocated);
         }
 
 #ifdef QLNX_SOFT_LRO
@@ -5551,16 +5491,14 @@ qlnx_alloc_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
 
 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
 		if (tcp_lro_init_args(lro, ifp, 0, rxq->num_rx_buffers)) {
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: tcp_lro_init[%d] failed\n",
-				__func__, rxq->rxq_id));
+			QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n",
+				   rxq->rxq_id);
 			goto err;
 		}
 #else
 		if (tcp_lro_init(lro)) {
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: tcp_lro_init[%d] failed\n",
-				__func__, rxq->rxq_id));
+			QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n",
+				   rxq->rxq_id);
 			goto err;
 		}
 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
@@ -5684,9 +5622,9 @@ qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp)
         fp->tx_br = buf_ring_alloc(TX_RING_SIZE, M_DEVBUF,
                                    M_NOWAIT, &fp->tx_mtx);
         if (fp->tx_br == NULL) {
-            QL_DPRINT1(ha, (ha->pci_dev, "buf_ring_alloc failed for "
-		" fp[%d, %d]\n", ha->dev_unit, fp->rss_id));
-            return -ENOMEM;
+		QL_DPRINT1(ha, "buf_ring_alloc failed for fp[%d, %d]\n",
+			ha->dev_unit, fp->rss_id);
+		return -ENOMEM;
         }
 	return 0;
 }
@@ -5793,7 +5731,7 @@ qlnx_start_vport(struct ecore_dev *cdev,
 	vport_start_params.mtu = mtu;
 
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: setting mtu to %d\n", __func__, mtu));
+	QL_DPRINT2(ha, "Setting mtu to %d and VPORT ID = %d\n", mtu, vport_id);
 
         for_each_hwfn(cdev, i) {
                 struct ecore_hwfn *p_hwfn = &cdev->hwfns[i];
@@ -5804,17 +5742,15 @@ qlnx_start_vport(struct ecore_dev *cdev,
                 rc = ecore_sp_vport_start(p_hwfn, &vport_start_params);
 
                 if (rc) {
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: Failed to start VPORT V-PORT %d "
-				"with MTU %d\n", __func__, vport_id, mtu));
+			QL_DPRINT1(ha, "Failed to start VPORT V-PORT %d"
+				" with MTU %d\n" , vport_id, mtu);
                         return -ENOMEM;
                 }
 
                 ecore_hw_start_fastpath(p_hwfn);
 
-		QL_DPRINT2(ha, (ha->pci_dev,
-			"%s: Started V-PORT %d with MTU %d\n",
-			__func__, vport_id, mtu));
+		QL_DPRINT2(ha, "Started V-PORT %d with MTU %d\n",
+			vport_id, mtu);
         }
         return 0;
 }
@@ -5852,8 +5788,10 @@ qlnx_update_vport(struct ecore_dev *cdev,
         /* RSS - is a bit tricky, since upper-layer isn't familiar with hwfns.
          * We need to re-fix the rss values per engine for CMT.
          */
-
+	if (params->rss_params->update_rss_config)
         sp_params.rss_params = params->rss_params;
+	else
+		sp_params.rss_params =  NULL;
 
         for_each_hwfn(cdev, i) {
 
@@ -5875,8 +5813,7 @@ qlnx_update_vport(struct ecore_dev *cdev,
 			}
 
 			for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE;) {
-				QL_DPRINT3(ha, (ha->pci_dev,
-					"%p %p %p %p %p %p %p %p \n",
+				QL_DPRINT3(ha, "%p %p %p %p %p %p %p %p \n",
 					rss->rss_ind_table[j],
 					rss->rss_ind_table[j+1],
 					rss->rss_ind_table[j+2],
@@ -5884,28 +5821,28 @@ qlnx_update_vport(struct ecore_dev *cdev,
 					rss->rss_ind_table[j+4],
 					rss->rss_ind_table[j+5],
 					rss->rss_ind_table[j+6],
-					rss->rss_ind_table[j+7]));
+					rss->rss_ind_table[j+7]);
 					j += 8;
 			}
 		}
 
                 sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid;
+
+		QL_DPRINT1(ha, "Update sp vport ID=%d\n", params->vport_id);
+
                 rc = ecore_sp_vport_update(p_hwfn, &sp_params,
                                            ECORE_SPQ_MODE_EBLOCK, NULL);
                 if (rc) {
-			QL_DPRINT1(ha, (ha->pci_dev,
-				"%s:Failed to update VPORT\n", __func__));
+			QL_DPRINT1(ha, "Failed to update VPORT\n");
                         return rc;
                 }
 
-                QL_DPRINT2(ha, (ha->pci_dev,
-			"%s: Updated V-PORT %d: tx_active_flag %d,"
-			"rx_active_flag %d [tx_update %d], [rx_update %d]\n",
-			__func__,
+                QL_DPRINT2(ha, "Updated V-PORT %d: tx_active_flag %d, \
+			rx_active_flag %d [tx_update %d], [rx_update %d]\n",
 			params->vport_id, params->vport_active_tx_flg,
 			params->vport_active_rx_flg,
 			params->update_vport_active_tx_flg,
-			params->update_vport_active_rx_flg));
+			params->update_vport_active_rx_flg);
         }
 
         return 0;
@@ -5999,10 +5936,11 @@ qlnx_start_queues(qlnx_host_t *ha)
 
 	ifp = ha->ifp;
 
+	QL_DPRINT1(ha, "Num RSS = %d\n", ha->num_rss);
+
         if (!ha->num_rss) {
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: Cannot update V-VPORT as active as there"
-			" are no Rx queues\n", __func__));
+		QL_DPRINT1(ha, "Cannot update V-VPORT as active as there"
+			" are no Rx queues\n");
                 return -EINVAL;
         }
 
@@ -6014,15 +5952,13 @@ qlnx_start_queues(qlnx_host_t *ha)
 			vlan_removal_en, tx_switching, hw_lro_enable);
 
         if (rc) {
-                QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: Start V-PORT failed %d\n", __func__, rc));
+                QL_DPRINT1(ha, "Start V-PORT failed %d\n", rc);
                 return rc;
         }
 
-	QL_DPRINT2(ha, (ha->pci_dev,
-		"%s: Start vport ramrod passed,"
-		" vport_id = %d, MTU = %d, vlan_removal_en = %d\n", __func__,
-		vport_id, (int)(ifp->if_mtu + 0xe), vlan_removal_en));
+	QL_DPRINT2(ha, "Start vport ramrod passed, "
+		"vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
+		vport_id, (int)(ifp->if_mtu + 0xe), vlan_removal_en);
 
         for_each_rss(i) {
 		struct ecore_rxq_start_ret_params rx_ret_params;
@@ -6055,9 +5991,7 @@ qlnx_start_queues(qlnx_host_t *ha)
 			&rx_ret_params);
 
                 if (rc) {
-                	QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: Start RXQ #%d failed %d\n", __func__,
-				i, rc));
+                	QL_DPRINT1(ha, "Start RXQ #%d failed %d\n", i, rc);
                         return rc;
                 }
 
@@ -6091,9 +6025,8 @@ qlnx_start_queues(qlnx_host_t *ha)
 				&tx_ret_params);
 
                         if (rc) {
-                		QL_DPRINT1(ha, (ha->pci_dev,
-					"%s: Start TXQ #%d failed %d\n",
-					__func__, txq->index, rc));
+                		QL_DPRINT1(ha, "Start TXQ #%d failed %d\n",
+					   txq->index, rc);
                                 return rc;
                         }
 
@@ -6173,8 +6106,7 @@ qlnx_start_queues(qlnx_host_t *ha)
 
         rc = qlnx_update_vport(cdev, &vport_update_params);
         if (rc) {
-		QL_DPRINT1(ha, (ha->pci_dev,
-			"%s: Update V-PORT failed %d\n", __func__, rc));
+		QL_DPRINT1(ha, "Update V-PORT failed %d\n", rc);
                 return rc;
         }
 
@@ -6188,7 +6120,7 @@ qlnx_drain_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 	uint16_t	hw_bd_cons;
 	uint16_t	ecore_cons_idx;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
 	hw_bd_cons = le16toh(*txq->hw_cons_ptr);
 
@@ -6206,8 +6138,7 @@ qlnx_drain_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp,
 		hw_bd_cons = le16toh(*txq->hw_cons_ptr);
 	}
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s[%d, %d]: done\n", __func__,
-		fp->rss_id, txq->index));
+	QL_DPRINT2(ha, "[%d, %d]: done\n", fp->rss_id, txq->index);
 
         return 0;
 }
@@ -6237,10 +6168,11 @@ qlnx_stop_queues(qlnx_host_t *ha)
         vport_update_params.update_inner_vlan_removal_flg = 0;
         vport_update_params.inner_vlan_removal_flg = 0;
 
+	QL_DPRINT1(ha, "Update vport ID= %d\n", vport_update_params.vport_id);
+
         rc = qlnx_update_vport(cdev, &vport_update_params);
         if (rc) {
-		QL_DPRINT1(ha, (ha->pci_dev, "%s:Failed to update vport\n",
-			__func__));
+		QL_DPRINT1(ha, "Failed to update vport\n");
                 return rc;
         }
 
@@ -6273,9 +6205,8 @@ qlnx_stop_queues(qlnx_host_t *ha)
 					fp->txq[tc]->handle);
 					
                         if (rc) {
-				QL_DPRINT1(ha, (ha->pci_dev,
-					"%s: Failed to stop TXQ #%d\n",
-					__func__, tx_queue_id));
+				QL_DPRINT1(ha, "Failed to stop TXQ #%d\n",
+					   tx_queue_id);
                                 return rc;
                         }
                 }
@@ -6284,8 +6215,7 @@ qlnx_stop_queues(qlnx_host_t *ha)
 		rc = ecore_eth_rx_queue_stop(p_hwfn, fp->rxq->handle, false,
 				false);
                 if (rc) {
-                        QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: Failed to stop RXQ #%d\n", __func__, i));
+                        QL_DPRINT1(ha, "Failed to stop RXQ #%d\n", i);
                         return rc;
                 }
         }
@@ -6298,8 +6228,7 @@ qlnx_stop_queues(qlnx_host_t *ha)
 		rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, 0);
 
 		if (rc) {
-                        QL_DPRINT1(ha, (ha->pci_dev,
-				"%s: Failed to stop VPORT\n", __func__));
+                        QL_DPRINT1(ha, "Failed to stop VPORT\n");
 			return rc;
 		}
 	}
@@ -6578,7 +6507,7 @@ qlnx_load(qlnx_host_t *ha)
 	cdev = &ha->cdev;
         dev = ha->pci_dev;
 
-	QL_DPRINT2(ha, (dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
 
         rc = qlnx_alloc_mem_arrays(ha);
         if (rc)
@@ -6590,8 +6519,8 @@ qlnx_load(qlnx_host_t *ha)
         if (rc)
                 goto qlnx_load_exit1;
 
-        QL_DPRINT2(ha, (dev, "%s: Allocated %d RSS queues on %d TC/s\n",
-		__func__, ha->num_rss, ha->num_tc));
+        QL_DPRINT2(ha, "Allocated %d RSS queues on %d TC/s\n",
+		   ha->num_rss, ha->num_tc);
 
 	for (i = 0; i < ha->num_rss; i++) {
 
@@ -6600,15 +6529,14 @@ qlnx_load(qlnx_host_t *ha)
                         NULL, qlnx_fp_isr, &ha->irq_vec[i],
                         &ha->irq_vec[i].handle))) {
 
-                        QL_DPRINT1(ha, (dev, "could not setup interrupt\n"));
-
+                        QL_DPRINT1(ha, "could not setup interrupt\n");
                         goto qlnx_load_exit2;
 		}
 
-		QL_DPRINT2(ha, (dev, "%s: rss_id = %d irq_rid %d"
-			" irq %p handle %p\n", __func__, i,
+		QL_DPRINT2(ha, "rss_id = %d irq_rid %d \
+			 irq %p handle %p\n", i,
 			ha->irq_vec[i].irq_rid,
-			ha->irq_vec[i].irq, ha->irq_vec[i].handle));
+			ha->irq_vec[i].irq, ha->irq_vec[i].handle);
 
 		bus_bind_intr(dev, ha->irq_vec[i].irq, (i % mp_ncpus));
 	}
@@ -6617,8 +6545,7 @@ qlnx_load(qlnx_host_t *ha)
         if (rc)
                 goto qlnx_load_exit2;
 
-        QL_DPRINT2(ha, (dev, "%s: Start VPORT, RXQ and TXQ succeeded\n",
-		__func__));
+        QL_DPRINT2(ha, "Start VPORT, RXQ and TXQ succeeded\n");
 
         /* Add primary mac and set Rx filters */
         rc = qlnx_set_rx_mode(ha);
@@ -6644,7 +6571,7 @@ qlnx_load(qlnx_host_t *ha)
         ha->num_rss = 0;
 
 qlnx_load_exit0:
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit [%d]\n", __func__, rc));
+	QL_DPRINT2(ha, "exit [%d]\n", rc);
         return rc;
 }
 
@@ -6701,7 +6628,8 @@ qlnx_unload(qlnx_host_t *ha)
 	cdev = &ha->cdev;
         dev = ha->pci_dev;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: enter\n", __func__));
+	QL_DPRINT2(ha, "enter\n");
+        QL_DPRINT1(ha, " QLNX STATE = %d\n",ha->state);
 
 	if (ha->state == QLNX_STATE_OPEN) {
 
@@ -6731,7 +6659,7 @@ qlnx_unload(qlnx_host_t *ha)
 
         ha->state = QLNX_STATE_CLOSED;
 
-	QL_DPRINT2(ha, (ha->pci_dev, "%s: exit\n", __func__));
+	QL_DPRINT2(ha, "exit\n");
 	return;
 }
 
@@ -6748,8 +6676,7 @@ qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
         if (!p_ptt) {
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: ecore_ptt_acquire failed\n",
-			__func__));
+		QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
                 return (rval);
         }
 
@@ -6758,9 +6685,8 @@ qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
 	if (rval == DBG_STATUS_OK)
                 rval = 0;
         else {
-		QL_DPRINT1(ha, (ha->pci_dev,
-                	"%s : ecore_dbg_grc_get_dump_buf_size failed [0x%x]\n",
-			__func__, rval));
+		QL_DPRINT1(ha, "ecore_dbg_grc_get_dump_buf_size failed"
+			"[0x%x]\n", rval);
 	}
 
         ecore_ptt_release(p_hwfn, p_ptt);
@@ -6781,8 +6707,7 @@ qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
 	p_ptt = ecore_ptt_acquire(p_hwfn);
 
         if (!p_ptt) {
-		QL_DPRINT1(ha, (ha->pci_dev, "%s: ecore_ptt_acquire failed\n",
-			__func__));
+		QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
                 return (rval);
         }
 
@@ -6791,9 +6716,8 @@ qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
 	if (rval == DBG_STATUS_OK)
                 rval = 0;
         else {
-		QL_DPRINT1(ha, (ha->pci_dev, "%s : "
-			"ecore_dbg_idle_chk_get_dump_buf_size failed [0x%x]\n",
-			__func__, rval));
+		QL_DPRINT1(ha, "ecore_dbg_idle_chk_get_dump_buf_size failed"
+			" [0x%x]\n", rval);
 	}
 
         ecore_ptt_release(p_hwfn, p_ptt);

From 5183ddf2ed56519b669320095ed00528a71a9393 Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Tue, 16 May 2017 21:54:51 +0000
Subject: [PATCH 04/73] sh: Simplify output buffering.

Similarly to how STPUTC was changed, change struct output to store the
pointer just past the end of the available space instead of the size of the
available space, so after writing a character it is only necessary to
increment a pointer and not to decrement a counter.
---
 bin/sh/eval.c   |  7 +++++--
 bin/sh/output.c | 17 ++++++++---------
 bin/sh/output.h |  4 ++--
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/bin/sh/eval.c b/bin/sh/eval.c
index e09549295f66..f637cfab5fd9 100644
--- a/bin/sh/eval.c
+++ b/bin/sh/eval.c
@@ -1080,8 +1080,8 @@ evalcommand(union node *cmd, int flags, struct backcmd *backcmd)
 #endif
 		mode = (cmdentry.u.index == EXECCMD)? 0 : REDIR_PUSH;
 		if (flags == EV_BACKCMD) {
-			memout.nleft = 0;
 			memout.nextc = memout.buf;
+			memout.bufend = memout.buf;
 			memout.bufsize = 64;
 			mode |= REDIR_BACKQ;
 		}
@@ -1134,8 +1134,11 @@ evalcommand(union node *cmd, int flags, struct backcmd *backcmd)
 			exitshell(exitstatus);
 		if (flags == EV_BACKCMD) {
 			backcmd->buf = memout.buf;
-			backcmd->nleft = memout.nextc - memout.buf;
+			backcmd->nleft = memout.buf != NULL ?
+			    memout.nextc - memout.buf : 0;
 			memout.buf = NULL;
+			memout.nextc = NULL;
+			memout.bufend = NULL;
 		}
 		if (cmdentry.u.index != EXECCMD)
 			popredir();
diff --git a/bin/sh/output.c b/bin/sh/output.c
index 2f9fc925d88c..64d311f0c2d6 100644
--- a/bin/sh/output.c
+++ b/bin/sh/output.c
@@ -71,9 +71,9 @@ __FBSDID("$FreeBSD$");
 
 static int doformat_wr(void *, const char *, int);
 
-struct output output = {NULL, 0, NULL, OUTBUFSIZ, 1, 0};
-struct output errout = {NULL, 0, NULL, 256, 2, 0};
-struct output memout = {NULL, 0, NULL, 0, MEM_OUT, 0};
+struct output output = {NULL, NULL, NULL, OUTBUFSIZ, 1, 0};
+struct output errout = {NULL, NULL, NULL, 256, 2, 0};
+struct output memout = {NULL, NULL, NULL, 0, MEM_OUT, 0};
 struct output *out1 = &output;
 struct output *out2 = &errout;
 
@@ -214,20 +214,19 @@ emptyoutbuf(struct output *dest)
 		INTOFF;
 		dest->buf = ckmalloc(dest->bufsize);
 		dest->nextc = dest->buf;
-		dest->nleft = dest->bufsize;
+		dest->bufend = dest->buf + dest->bufsize;
 		INTON;
 	} else if (dest->fd == MEM_OUT) {
-		offset = dest->bufsize;
+		offset = dest->nextc - dest->buf;
 		INTOFF;
 		dest->bufsize <<= 1;
 		dest->buf = ckrealloc(dest->buf, dest->bufsize);
-		dest->nleft = dest->bufsize - offset;
+		dest->bufend = dest->buf + dest->bufsize;
 		dest->nextc = dest->buf + offset;
 		INTON;
 	} else {
 		flushout(dest);
 	}
-	dest->nleft--;
 }
 
 
@@ -248,7 +247,6 @@ flushout(struct output *dest)
 	if (xwrite(dest->fd, dest->buf, dest->nextc - dest->buf) < 0)
 		dest->flags |= OUTPUT_ERR;
 	dest->nextc = dest->buf;
-	dest->nleft = dest->bufsize;
 }
 
 
@@ -258,8 +256,9 @@ freestdout(void)
 	INTOFF;
 	if (output.buf) {
 		ckfree(output.buf);
+		output.nextc = NULL;
 		output.buf = NULL;
-		output.nleft = 0;
+		output.bufend = NULL;
 	}
 	INTON;
 }
diff --git a/bin/sh/output.h b/bin/sh/output.h
index 21cbba1efa58..f43a60cc2aaa 100644
--- a/bin/sh/output.h
+++ b/bin/sh/output.h
@@ -40,7 +40,7 @@
 
 struct output {
 	char *nextc;
-	int nleft;
+	char *bufend;
 	char *buf;
 	int bufsize;
 	short fd;
@@ -75,7 +75,7 @@ void fmtstr(char *, int, const char *, ...) __printflike(3, 4);
 void doformat(struct output *, const char *, va_list) __printflike(2, 0);
 int xwrite(int, const char *, int);
 
-#define outc(c, file)	(--(file)->nleft < 0? (emptyoutbuf(file), *(file)->nextc++ = (c)) : (*(file)->nextc++ = (c)))
+#define outc(c, file)	((file)->nextc == (file)->bufend ? (emptyoutbuf(file), *(file)->nextc++ = (c)) : (*(file)->nextc++ = (c)))
 #define out1c(c)	outc(c, out1);
 #define out2c(c)	outcslow(c, out2);
 

From 04794d246a91235a077093b2db976d93701871e7 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Tue, 16 May 2017 23:15:40 +0000
Subject: [PATCH 05/73] Note that the first release of FreeBSD/alpha was 3.2.

Alpha is explicitly mentioned in the release announcements beginning with
3.2.
---
 share/man/man7/arch.7 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/share/man/man7/arch.7 b/share/man/man7/arch.7
index a6e052f46f39..63f64ad7361d 100644
--- a/share/man/man7/arch.7
+++ b/share/man/man7/arch.7
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 5, 2017
+.Dd May 16, 2017
 .Dt ARCH 7
 .Os
 .Sh NAME
@@ -90,7 +90,7 @@ architectures, the final release.
 .Pp
 .Bl -column -offset indent "Sy Architecture" "Sy Initial Release" "Sy Final Release"
 .It Sy Architecture Ta Sy Initial Release Ta Sy Final Release
-.It alpha       Ta 3.x   Ta 6.4
+.It alpha       Ta 3.2   Ta 6.4
 .It amd64       Ta 5.1
 .It arm         Ta 6.0
 .It armeb       Ta 8.0

From 720e7bb69c26b445287b7dd12d2f1e709b227953 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Tue, 16 May 2017 23:18:50 +0000
Subject: [PATCH 06/73] Add support for child devices that aren't ports.

Invoke any identify routines of child drivers during attach before attaching
children, and delete any remaining devices after deleting ports.

MFC after:	1 month
Sponsored by:	Chelsio Communications
---
 sys/dev/cxgbe/t4_main.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 9b5e4ff09a11..cbc8e8c6a27f 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -1186,6 +1186,12 @@ t4_attach(device_t dev)
 		goto done;
 	}
 
+	rc = bus_generic_probe(dev);
+	if (rc != 0) {
+		device_printf(dev, "failed to probe child drivers: %d\n", rc);
+		goto done;
+	}
+
 	rc = bus_generic_attach(dev);
 	if (rc != 0) {
 		device_printf(dev,
@@ -1339,6 +1345,8 @@ t4_detach_common(device_t dev)
 		}
 	}
 
+	device_delete_children(dev);
+
 	if (sc->flags & FULL_INIT_DONE)
 		adapter_full_uninit(sc);
 

From 00f6cd3f568295e501deae63e38b34d1137a6eb9 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Tue, 16 May 2017 23:31:52 +0000
Subject: [PATCH 07/73] Add sglist_append_sglist().

This function permits a range of one scatter/gather list to be appended to
another sglist.  This can be used to construct a scatter/gather list that
reorders or duplicates ranges from one or more existing scatter/gather
lists.

Sponsored by:	Chelsio Communications
---
 share/man/man9/Makefile |  1 +
 share/man/man9/sglist.9 | 19 +++++++++++++++++-
 sys/kern/subr_sglist.c  | 43 +++++++++++++++++++++++++++++++++++++++++
 sys/sys/sglist.h        |  2 ++
 4 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index 0307bf259671..7ce3ee711d43 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -1572,6 +1572,7 @@ MLINKS+=sglist.9 sglist_alloc.9 \
 	sglist.9 sglist_append_bio.9 \
 	sglist.9 sglist_append_mbuf.9 \
 	sglist.9 sglist_append_phys.9 \
+	sglist.9 sglist_append_sglist.9 \
 	sglist.9 sglist_append_uio.9 \
 	sglist.9 sglist_append_user.9 \
 	sglist.9 sglist_append_vmpages.9 \
diff --git a/share/man/man9/sglist.9 b/share/man/man9/sglist.9
index f45d261495a7..408f3194e4b4 100644
--- a/share/man/man9/sglist.9
+++ b/share/man/man9/sglist.9
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 12, 2014
+.Dd May 16, 2017
 .Dt SGLIST 9
 .Os
 .Sh NAME
@@ -36,6 +36,7 @@
 .Nm sglist_append_bio ,
 .Nm sglist_append_mbuf ,
 .Nm sglist_append_phys ,
+.Nm sglist_append_sglist ,
 .Nm sglist_append_uio ,
 .Nm sglist_append_user ,
 .Nm sglist_append_vmpages ,
@@ -67,6 +68,8 @@
 .Ft int
 .Fn sglist_append_phys "struct sglist *sg" "vm_paddr_t paddr" "size_t len"
 .Ft int
+.Fn sglist_append_sglist "struct sglist *sg" "struct sglist *source" "size_t offset" "size_t len"
+.Ft int
 .Fn sglist_append_uio "struct sglist *sg" "struct uio *uio"
 .Ft int
 .Fn sglist_append_user "struct sglist *sg" "void *buf" "size_t len" "struct thread *td"
@@ -252,6 +255,20 @@ and is
 bytes long.
 .Pp
 The
+.Nm sglist_append_sglist
+function appends physical address ranges described by the scatter/gather list
+.Fa source
+to the scatter/gather list
+.Fa sg .
+The physical address ranges start at offset
+.Fa offset
+within
+.Fa source
+and continue for
+.Fa len
+bytes.
+.Pp
+The
 .Nm sglist_append_uio
 function appends the physical address ranges described by a
 .Xr uio 9
diff --git a/sys/kern/subr_sglist.c b/sys/kern/subr_sglist.c
index 0d371a48ded3..dbc2a5e72fee 100644
--- a/sys/kern/subr_sglist.c
+++ b/sys/kern/subr_sglist.c
@@ -412,6 +412,49 @@ sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td)
 	return (error);
 }
 
+/*
+ * Append a subset of an existing scatter/gather list 'source' to a
+ * the scatter/gather list 'sg'.  If there are insufficient segments,
+ * then this fails with EFBIG.
+ */
+int
+sglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset,
+    size_t length)
+{
+	struct sgsave save;
+	struct sglist_seg *ss;
+	size_t seglen;
+	int error, i;
+
+	if (sg->sg_maxseg == 0 || length == 0)
+		return (EINVAL);
+	SGLIST_SAVE(sg, save);
+	error = EINVAL;
+	ss = &sg->sg_segs[sg->sg_nseg - 1];
+	for (i = 0; i < source->sg_nseg; i++) {
+		if (offset >= source->sg_segs[i].ss_len) {
+			offset -= source->sg_segs[i].ss_len;
+			continue;
+		}
+		seglen = source->sg_segs[i].ss_len - offset;
+		if (seglen > length)
+			seglen = length;
+		error = _sglist_append_range(sg, &ss,
+		    source->sg_segs[i].ss_paddr + offset, seglen);
+		if (error)
+			break;
+		offset = 0;
+		length -= seglen;
+		if (length == 0)
+			break;
+	}
+	if (length != 0)
+		error = EINVAL;
+	if (error)
+		SGLIST_RESTORE(sg, save);
+	return (error);
+}
+
 /*
  * Append the segments that describe a single uio to a scatter/gather
  * list.  If there are insufficient segments, then this fails with
diff --git a/sys/sys/sglist.h b/sys/sys/sglist.h
index 1c0985803ebd..7d22e28ecc84 100644
--- a/sys/sys/sglist.h
+++ b/sys/sys/sglist.h
@@ -88,6 +88,8 @@ int	sglist_append_bio(struct sglist *sg, struct bio *bp);
 int	sglist_append_mbuf(struct sglist *sg, struct mbuf *m0);
 int	sglist_append_phys(struct sglist *sg, vm_paddr_t paddr,
 	    size_t len);
+int	sglist_append_sglist(struct sglist *sg, struct sglist *source,
+	    size_t offset, size_t length);
 int	sglist_append_uio(struct sglist *sg, struct uio *uio);
 int	sglist_append_user(struct sglist *sg, void *buf, size_t len,
 	    struct thread *td);

From 3e85b721d653d65a5b6ff655ed1551113f0e0d5b Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 17 May 2017 00:34:34 +0000
Subject: [PATCH 08/73] Remove register keyword from sys/ and ANSIfy prototypes

A long long time ago the register keyword told the compiler to store
the corresponding variable in a CPU register, but it is not relevant
for any compiler used in the FreeBSD world today.

ANSIfy related prototypes while here.

Reviewed by:	cem, jhb
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D10193
---
 sys/amd64/amd64/sys_machdep.c               |   4 +-
 sys/amd64/amd64/vm_machdep.c                |   8 +-
 sys/arm/arm/sys_machdep.c                   |   4 +-
 sys/arm/arm/vm_machdep.c                    |   3 +-
 sys/arm/include/atomic-v4.h                 |   2 +-
 sys/crypto/des/des_enc.c                    |  20 +-
 sys/crypto/des/des_setkey.c                 |   8 +-
 sys/ddb/db_access.c                         |   6 +-
 sys/ddb/db_output.c                         |   4 +-
 sys/ddb/db_sym.c                            |  11 +-
 sys/dev/cs/if_cs.c                          |   2 +-
 sys/dev/ixgb/if_ixgb.c                      |   2 +-
 sys/dev/lge/if_lge.c                        |   8 +-
 sys/dev/mse/mse_isa.c                       |   2 +-
 sys/dev/my/if_my.c                          |   4 +-
 sys/dev/pcn/if_pcn.c                        |   2 +-
 sys/dev/ppbus/immio.c                       |   4 +-
 sys/dev/ppbus/vpoio.c                       |   4 +-
 sys/dev/ppc/ppc.c                           |  10 +-
 sys/dev/qlxgb/qla_os.c                      |   2 +-
 sys/dev/qlxgbe/ql_os.c                      |   2 +-
 sys/dev/qlxge/qls_os.c                      |   2 +-
 sys/dev/rl/if_rl.c                          |   8 +-
 sys/dev/sound/pci/fm801.c                   |   2 +-
 sys/dev/speaker/spkr.c                      |   4 +-
 sys/dev/tl/if_tl.c                          |   6 +-
 sys/dev/usb/usb_busdma.c                    |   2 +-
 sys/dev/wb/if_wb.c                          |   8 +-
 sys/dev/xl/if_xl.c                          |   6 +-
 sys/fs/fifofs/fifo_vnops.c                  |   2 +-
 sys/fs/nandfs/nandfs_vnops.c                |   4 +-
 sys/i386/i386/in_cksum.c                    |   6 +-
 sys/i386/i386/k6_mem.c                      |   2 +-
 sys/i386/i386/sys_machdep.c                 |   4 +-
 sys/i386/i386/vm_machdep.c                  |   2 +-
 sys/i386/ibcs2/ibcs2_misc.c                 | 160 ++-----
 sys/i386/ibcs2/ibcs2_other.c                |   2 +-
 sys/i386/ibcs2/ibcs2_signal.c               |  28 +-
 sys/i386/ibcs2/ibcs2_socksys.c              |  12 +-
 sys/i386/isa/ccbque.h                       |   6 +-
 sys/i386/isa/elink.c                        |   4 +-
 sys/kern/inflate.c                          |  32 +-
 sys/kern/kern_clock.c                       |  18 +-
 sys/kern/kern_exec.c                        |  24 +-
 sys/kern/kern_prot.c                        |  26 +-
 sys/kern/kern_resource.c                    |  16 +-
 sys/kern/kern_sig.c                         |   6 +-
 sys/kern/kern_timeout.c                     |   2 +-
 sys/kern/kern_xxx.c                         |  20 +-
 sys/kern/sched_4bsd.c                       |   2 +-
 sys/kern/sysv_msg.c                         |  76 ++--
 sys/kern/vfs_export.c                       |  10 +-
 sys/kern/vfs_mount.c                        |  36 +-
 sys/kern/vfs_syscalls.c                     | 453 ++++----------------
 sys/kern/vfs_vnops.c                        |  34 +-
 sys/libkern/zlib.c                          |  36 +-
 sys/mips/cavium/cryptocteon/cavium_crypto.c |  20 +-
 sys/mips/mips/vm_machdep.c                  |   5 +-
 sys/net/altq/altq_rio.c                     |   2 +-
 sys/net/altq/altq_rmclass.h                 |   4 +-
 sys/net/bpf_filter.c                        |  10 +-
 sys/net/if_llatbl.c                         |   2 +-
 sys/net/if_media.c                          |   2 +-
 sys/net/slcompress.c                        |  64 ++-
 sys/netinet/in.c                            |  14 +-
 sys/netinet/in_cksum.c                      |   6 +-
 sys/netinet/ip_icmp.c                       |  16 +-
 sys/netinet6/in6_pcb.c                      |  14 +-
 sys/netinet6/raw_ip6.c                      |   4 +-
 sys/netipsec/ipsec_mbuf.c                   |   4 +-
 sys/rpc/clnt.h                              |   2 +-
 sys/sparc64/include/pcpu.h                  |   4 +-
 72 files changed, 434 insertions(+), 912 deletions(-)

diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c
index 24009dbaed5e..7480bcd87fd7 100644
--- a/sys/amd64/amd64/sys_machdep.c
+++ b/sys/amd64/amd64/sys_machdep.c
@@ -169,9 +169,7 @@ update_gdt_fsbase(struct thread *td, uint32_t base)
 }
 
 int
-sysarch(td, uap)
-	struct thread *td;
-	register struct sysarch_args *uap;
+sysarch(struct thread *td, struct sysarch_args *uap)
 {
 	int error = 0;
 	struct pcb *pcb = curthread->td_pcb;
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index 20c7ccea8099..b8b06c62207e 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -148,13 +148,9 @@ alloc_fpusave(int flags)
  * ready to run and return to user mode.
  */
 void
-cpu_fork(td1, p2, td2, flags)
-	register struct thread *td1;
-	register struct proc *p2;
-	struct thread *td2;
-	int flags;
+cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 {
-	register struct proc *p1;
+	struct proc *p1;
 	struct pcb *pcb2;
 	struct mdproc *mdp1, *mdp2;
 	struct proc_ldt *pldt;
diff --git a/sys/arm/arm/sys_machdep.c b/sys/arm/arm/sys_machdep.c
index 087afbdca6ee..af34e30930b6 100644
--- a/sys/arm/arm/sys_machdep.c
+++ b/sys/arm/arm/sys_machdep.c
@@ -188,9 +188,7 @@ arm32_get_tp(struct thread *td, void *args)
 }
 
 int
-sysarch(td, uap)
-	struct thread *td;
-	register struct sysarch_args *uap;
+sysarch(struct thread *td, struct sysarch_args *uap)
 {
 	int error;
 
diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index a55e61d16b4e..bebae6c703af 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -93,8 +93,7 @@ uint32_t initial_fpscr = VFPSCR_DN | VFPSCR_FZ;
  * ready to run and return to user mode.
  */
 void
-cpu_fork(register struct thread *td1, register struct proc *p2,
-    struct thread *td2, int flags)
+cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 {
 	struct pcb *pcb2;
 	struct trapframe *tf;
diff --git a/sys/arm/include/atomic-v4.h b/sys/arm/include/atomic-v4.h
index 7edebff26b62..71587f4211f0 100644
--- a/sys/arm/include/atomic-v4.h
+++ b/sys/arm/include/atomic-v4.h
@@ -299,7 +299,7 @@ atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
 static __inline u_int32_t
 atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval)
 {
-	register int done, ras_start = ARM_RAS_START;
+	int done, ras_start = ARM_RAS_START;
 
 	__asm __volatile("1:\n"
 	    "adr	%1, 1b\n"
diff --git a/sys/crypto/des/des_enc.c b/sys/crypto/des/des_enc.c
index c8d484a0d48c..f703b3099bc8 100644
--- a/sys/crypto/des/des_enc.c
+++ b/sys/crypto/des/des_enc.c
@@ -69,14 +69,14 @@ extern	const DES_LONG des_SPtrans[8][64];
 
 void des_encrypt1(DES_LONG *data, des_key_schedule ks, int enc)
 {
-	register DES_LONG l,r,t,u;
+	DES_LONG l,r,t,u;
 #ifdef DES_PTR
-	register const unsigned char *des_SP=(const unsigned char *)des_SPtrans;
+	const unsigned char *des_SP=(const unsigned char *)des_SPtrans;
 #endif
 #ifndef DES_UNROLL
-	register int i;
+	int i;
 #endif
-	register DES_LONG *s;
+	DES_LONG *s;
 
 	r=data[0];
 	l=data[1];
@@ -167,14 +167,14 @@ void des_encrypt1(DES_LONG *data, des_key_schedule ks, int enc)
 
 void des_encrypt2(DES_LONG *data, des_key_schedule ks, int enc)
 {
-	register DES_LONG l,r,t,u;
+	DES_LONG l,r,t,u;
 #ifdef DES_PTR
-	register const unsigned char *des_SP=(const unsigned char *)des_SPtrans;
+	const unsigned char *des_SP=(const unsigned char *)des_SPtrans;
 #endif
 #ifndef DES_UNROLL
-	register int i;
+	int i;
 #endif
-	register DES_LONG *s;
+	DES_LONG *s;
 
 	r=data[0];
 	l=data[1];
@@ -259,7 +259,7 @@ void des_encrypt2(DES_LONG *data, des_key_schedule ks, int enc)
 void des_encrypt3(DES_LONG *data, des_key_schedule ks1, des_key_schedule ks2,
 	     des_key_schedule ks3)
 {
-	register DES_LONG l,r;
+	DES_LONG l,r;
 
 	l=data[0];
 	r=data[1];
@@ -279,7 +279,7 @@ void des_encrypt3(DES_LONG *data, des_key_schedule ks1, des_key_schedule ks2,
 void des_decrypt3(DES_LONG *data, des_key_schedule ks1, des_key_schedule ks2,
 	     des_key_schedule ks3)
 {
-	register DES_LONG l,r;
+	DES_LONG l,r;
 
 	l=data[0];
 	r=data[1];
diff --git a/sys/crypto/des/des_setkey.c b/sys/crypto/des/des_setkey.c
index 7d1fb506a22d..6fefa27fb8a0 100644
--- a/sys/crypto/des/des_setkey.c
+++ b/sys/crypto/des/des_setkey.c
@@ -172,10 +172,10 @@ int des_set_key_checked(des_cblock *key, des_key_schedule schedule)
 void des_set_key_unchecked(des_cblock *key, des_key_schedule schedule)
 {
 	static int shifts2[16]={0,0,1,1,1,1,1,1,0,1,1,1,1,1,1,0};
-	register DES_LONG c,d,t,s,t2;
-	register const unsigned char *in;
-	register DES_LONG *k;
-	register int i;
+	DES_LONG c,d,t,s,t2;
+	const unsigned char *in;
+	DES_LONG *k;
+	int i;
 
 	k = &schedule->ks.deslong[0];
 	in = &(*key)[0];
diff --git a/sys/ddb/db_access.c b/sys/ddb/db_access.c
index 7001e7264a12..ec2107352f1d 100644
--- a/sys/ddb/db_access.c
+++ b/sys/ddb/db_access.c
@@ -57,8 +57,8 @@ db_expr_t
 db_get_value(db_addr_t addr, int size, bool is_signed)
 {
 	char		data[sizeof(u_int64_t)];
-	register db_expr_t value;
-	register int	i;
+	db_expr_t	value;
+	int		i;
 
 	if (db_read_bytes(addr, size, data) != 0) {
 		db_printf("*** error reading from address %llx ***\n",
@@ -87,7 +87,7 @@ void
 db_put_value(db_addr_t addr, int size, db_expr_t value)
 {
 	char		data[sizeof(int)];
-	register int	i;
+	int		i;
 
 #if	BYTE_MSF
 	for (i = size - 1; i >= 0; i--)
diff --git a/sys/ddb/db_output.c b/sys/ddb/db_output.c
index 109c7a50e1d9..c0dc58bc599b 100644
--- a/sys/ddb/db_output.c
+++ b/sys/ddb/db_output.c
@@ -92,7 +92,7 @@ static void	db_pager(void);
 void
 db_force_whitespace(void)
 {
-	register int last_print, next_tab;
+	int last_print, next_tab;
 
 	last_print = db_last_non_space;
 	while (last_print < db_output_position) {
@@ -355,7 +355,7 @@ db_iprintf(const char *fmt,...)
 	char bufr[DDB_BUFR_SIZE];
 #endif
 	struct dbputchar_arg dca;
-	register int i;
+	int i;
 	va_list listp;
 
 	for (i = db_indent; i >= 8; i -= 8)
diff --git a/sys/ddb/db_sym.c b/sys/ddb/db_sym.c
index c21445c520ba..d3f9554aa90b 100644
--- a/sys/ddb/db_sym.c
+++ b/sys/ddb/db_sym.c
@@ -286,10 +286,10 @@ static c_db_sym_t
 db_lookup(const char *symstr)
 {
 	c_db_sym_t sp;
-	register int i;
+	int i;
 	int symtab_start = 0;
 	int symtab_end = db_nsymtab;
-	register const char *cp;
+	const char *cp;
 
 	/*
 	 * Look for, remove, and remember any symbol table specifier.
@@ -343,8 +343,8 @@ static bool
 db_symbol_is_ambiguous(c_db_sym_t sym)
 {
 	const char	*sym_name;
-	register int	i;
-	register bool	found_once = false;
+	int		i;
+	bool		found_once = false;
 
 	if (!db_qualify_ambiguous_names)
 		return (false);
@@ -367,10 +367,9 @@ db_symbol_is_ambiguous(c_db_sym_t sym)
 c_db_sym_t
 db_search_symbol(db_addr_t val, db_strategy_t strategy, db_expr_t *offp)
 {
-	register
 	unsigned int	diff;
 	size_t		newdiff;
-	register int	i;
+	int		i;
 	c_db_sym_t	ret = C_DB_SYM_NULL, sym;
 
 	newdiff = diff = val;
diff --git a/sys/dev/cs/if_cs.c b/sys/dev/cs/if_cs.c
index 3c52d892ab52..2756aec1d598 100644
--- a/sys/dev/cs/if_cs.c
+++ b/sys/dev/cs/if_cs.c
@@ -1038,7 +1038,7 @@ cs_setmode(struct cs_softc *sc)
 }
 
 static int
-cs_ioctl(register struct ifnet *ifp, u_long command, caddr_t data)
+cs_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 {
 	struct cs_softc *sc=ifp->if_softc;
 	struct ifreq *ifr = (struct ifreq *)data;
diff --git a/sys/dev/ixgb/if_ixgb.c b/sys/dev/ixgb/if_ixgb.c
index 21f463eebc11..430c13c72d10 100644
--- a/sys/dev/ixgb/if_ixgb.c
+++ b/sys/dev/ixgb/if_ixgb.c
@@ -1777,7 +1777,7 @@ static int
 ixgb_get_buf(int i, struct adapter * adapter,
 	     struct mbuf * nmp)
 {
-	struct mbuf *mp = nmp;
+	struct mbuf    *mp = nmp;
 	struct ixgb_buffer *rx_buffer;
 	struct ifnet   *ifp;
 	bus_addr_t      paddr;
diff --git a/sys/dev/lge/if_lge.c b/sys/dev/lge/if_lge.c
index 721aee8974e0..262aab35522b 100644
--- a/sys/dev/lge/if_lge.c
+++ b/sys/dev/lge/if_lge.c
@@ -216,7 +216,7 @@ lge_eeprom_getword(sc, addr, dest)
 	int			addr;
 	u_int16_t		*dest;
 {
-	register int		i;
+	int			i;
 	u_int32_t		val;
 
 	CSR_WRITE_4(sc, LGE_EECTL, LGE_EECTL_CMD_READ|
@@ -413,7 +413,7 @@ static void
 lge_reset(sc)
 	struct lge_softc	*sc;
 {
-	register int		i;
+	int			i;
 
 	LGE_SETBIT(sc, LGE_MODE1, LGE_MODE1_SETRST_CTL0|LGE_MODE1_SOFTRST);
 
@@ -756,7 +756,7 @@ lge_alloc_jumbo_mem(sc)
 	struct lge_softc	*sc;
 {
 	caddr_t			ptr;
-	register int		i;
+	int			i;
 	struct lge_jpool_entry   *entry;
 
 	/* Grab a big chunk o' storage. */
@@ -1524,7 +1524,7 @@ static void
 lge_stop(sc)
 	struct lge_softc	*sc;
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp;
 
 	LGE_LOCK_ASSERT(sc);
diff --git a/sys/dev/mse/mse_isa.c b/sys/dev/mse/mse_isa.c
index 8cc85babecd2..354ad636a82f 100644
--- a/sys/dev/mse/mse_isa.c
+++ b/sys/dev/mse/mse_isa.c
@@ -309,7 +309,7 @@ mse_disablelogi(struct resource *port)
 static void
 mse_getlogi(struct resource *port, int *dx, int *dy, int *but)
 {
-	register char x, y;
+	char x, y;
 
 	bus_write_1(port, MSE_PORTC, MSE_HOLD | MSE_RXLOW);
 	x = bus_read_1(port, MSE_PORTA);
diff --git a/sys/dev/my/if_my.c b/sys/dev/my/if_my.c
index e2430c1a919d..14dda5090f20 100644
--- a/sys/dev/my/if_my.c
+++ b/sys/dev/my/if_my.c
@@ -751,7 +751,7 @@ my_setcfg(struct my_softc * sc, int bmcr)
 static void
 my_reset(struct my_softc * sc)
 {
-	register int    i;
+	int    i;
 
 	MY_LOCK_ASSERT(sc);
 	MY_SETBIT(sc, MY_BCR, MY_SWR);
@@ -1717,7 +1717,7 @@ my_watchdog(void *arg)
 static void
 my_stop(struct my_softc * sc)
 {
-	register int    i;
+	int    i;
 	struct ifnet   *ifp;
 
 	MY_LOCK_ASSERT(sc);
diff --git a/sys/dev/pcn/if_pcn.c b/sys/dev/pcn/if_pcn.c
index d63d4aaeb34f..a81bd0178b7d 100644
--- a/sys/dev/pcn/if_pcn.c
+++ b/sys/dev/pcn/if_pcn.c
@@ -1453,7 +1453,7 @@ pcn_watchdog(struct pcn_softc *sc)
 static void
 pcn_stop(struct pcn_softc *sc)
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp;
 
 	PCN_LOCK_ASSERT(sc);
diff --git a/sys/dev/ppbus/immio.c b/sys/dev/ppbus/immio.c
index 8b0c8de62c70..6d460b758905 100644
--- a/sys/dev/ppbus/immio.c
+++ b/sys/dev/ppbus/immio.c
@@ -674,10 +674,10 @@ imm_do_scsi(struct vpoio_data *vpo, int host, int target, char *command,
 		int *ret)
 {
 	device_t ppbus = device_get_parent(vpo->vpo_dev);
-	register char r;
+	char r;
 	char l, h = 0;
 	int len, error = 0, not_connected = 0;
-	register int k;
+	int k;
 	int negociated = 0;
 
 	/*
diff --git a/sys/dev/ppbus/vpoio.c b/sys/dev/ppbus/vpoio.c
index a989e8d10339..57129d78e28e 100644
--- a/sys/dev/ppbus/vpoio.c
+++ b/sys/dev/ppbus/vpoio.c
@@ -680,10 +680,10 @@ vpoio_do_scsi(struct vpoio_data *vpo, int host, int target, char *command,
 		int *ret)
 {
 	device_t ppbus = device_get_parent(vpo->vpo_dev);
-	register char r;
+	char r;
 	char l, h = 0;
 	int len, error = 0;
-	register int k;
+	int k;
 
 	/*
 	 * enter disk state, allocate the ppbus
diff --git a/sys/dev/ppc/ppc.c b/sys/dev/ppc/ppc.c
index cd9cf2148762..4186aadb5f74 100644
--- a/sys/dev/ppc/ppc.c
+++ b/sys/dev/ppc/ppc.c
@@ -285,7 +285,7 @@ ppc_detect_port(struct ppc_data *ppc)
 static void
 ppc_reset_epp_timeout(struct ppc_data *ppc)
 {
-	register char r;
+	char r;
 
 	r = r_str(ppc);
 	w_str(ppc, r | 0x1);
@@ -1321,10 +1321,10 @@ ppc_exec_microseq(device_t dev, struct ppb_microseq **p_msq)
 	int i, iter, len;
 	int error;
 
-	register int reg;
-	register char mask;
-	register int accum = 0;
-	register char *ptr = NULL;
+	int reg;
+	char mask;
+	int accum = 0;
+	char *ptr = NULL;
 
 	struct ppb_microseq *stack = NULL;
 
diff --git a/sys/dev/qlxgb/qla_os.c b/sys/dev/qlxgb/qla_os.c
index da1e34f0c5b9..8e3d8f46bc22 100644
--- a/sys/dev/qlxgb/qla_os.c
+++ b/sys/dev/qlxgb/qla_os.c
@@ -1401,7 +1401,7 @@ int
 qla_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp,
 	uint32_t jumbo)
 {
-	register struct mbuf *mp = nmp;
+	struct mbuf *mp = nmp;
 	struct ifnet   *ifp;
 	int             ret = 0;
 	uint32_t	offset;
diff --git a/sys/dev/qlxgbe/ql_os.c b/sys/dev/qlxgbe/ql_os.c
index 735035d3d572..5bcc41f3c3e4 100644
--- a/sys/dev/qlxgbe/ql_os.c
+++ b/sys/dev/qlxgbe/ql_os.c
@@ -1797,7 +1797,7 @@ qla_free_rcv_bufs(qla_host_t *ha)
 int
 ql_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp)
 {
-	register struct mbuf *mp = nmp;
+	struct mbuf *mp = nmp;
 	struct ifnet   		*ifp;
 	int            		ret = 0;
 	uint32_t		offset;
diff --git a/sys/dev/qlxge/qls_os.c b/sys/dev/qlxge/qls_os.c
index 878a0e635b81..6b6d74c273e8 100644
--- a/sys/dev/qlxge/qls_os.c
+++ b/sys/dev/qlxge/qls_os.c
@@ -1396,7 +1396,7 @@ qls_alloc_rcv_bufs(qla_host_t *ha)
 int
 qls_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp)
 {
-	register struct mbuf *mp = nmp;
+	struct mbuf *mp = nmp;
 	struct ifnet   		*ifp;
 	int            		ret = 0;
 	uint32_t		offset;
diff --git a/sys/dev/rl/if_rl.c b/sys/dev/rl/if_rl.c
index 356da24f2e1e..7866677923f4 100644
--- a/sys/dev/rl/if_rl.c
+++ b/sys/dev/rl/if_rl.c
@@ -276,7 +276,7 @@ DRIVER_MODULE(miibus, rl, miibus_driver, miibus_devclass, 0, 0);
 static void
 rl_eeprom_putbyte(struct rl_softc *sc, int addr)
 {
-	register int		d, i;
+	int			d, i;
 
 	d = addr | sc->rl_eecmd_read;
 
@@ -303,7 +303,7 @@ rl_eeprom_putbyte(struct rl_softc *sc, int addr)
 static void
 rl_eeprom_getword(struct rl_softc *sc, int addr, uint16_t *dest)
 {
-	register int		i;
+	int			i;
 	uint16_t		word = 0;
 
 	/* Enter EEPROM access mode. */
@@ -561,7 +561,7 @@ rl_rxfilter(struct rl_softc *sc)
 static void
 rl_reset(struct rl_softc *sc)
 {
-	register int		i;
+	int			i;
 
 	RL_LOCK_ASSERT(sc);
 
@@ -1912,7 +1912,7 @@ rl_watchdog(struct rl_softc *sc)
 static void
 rl_stop(struct rl_softc *sc)
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp = sc->rl_ifp;
 
 	RL_LOCK_ASSERT(sc);
diff --git a/sys/dev/sound/pci/fm801.c b/sys/dev/sound/pci/fm801.c
index c1b6393cd754..a8e330216101 100644
--- a/sys/dev/sound/pci/fm801.c
+++ b/sys/dev/sound/pci/fm801.c
@@ -394,7 +394,7 @@ fm801ch_setspeed(kobj_t obj, void *data, u_int32_t speed)
 {
 	struct fm801_chinfo *ch = data;
 	struct fm801_info *fm801 = ch->parent;
-	register int i;
+	int i;
 
 
 	for (i = 0; i < 10 && fm801_rates[i].limit <= speed; i++) ;
diff --git a/sys/dev/speaker/spkr.c b/sys/dev/speaker/spkr.c
index ec890ff4b666..9b5746d6cf92 100644
--- a/sys/dev/speaker/spkr.c
+++ b/sys/dev/speaker/spkr.c
@@ -201,7 +201,7 @@ playinit()
 static void
 playtone(int pitch, int value, int sustain)
 {
-	register int sound, silence, snum = 1, sdenom = 1;
+	int sound, silence, snum = 1, sdenom = 1;
 
 	/* this weirdness avoids floating-point arithmetic */
 	for (; sustain; sustain--) {
@@ -243,7 +243,7 @@ playstring(char *cp, size_t slen)
 				{v = v * 10 + (*++cp - '0'); slen--;}
 	for (; slen--; cp++) {
 		int sustain, timeval, tempo;
-		register char c = toupper(*cp);
+		char c = toupper(*cp);
 
 #ifdef DEBUG
 		(void) printf("playstring: %c (%x)\n", c, c);
diff --git a/sys/dev/tl/if_tl.c b/sys/dev/tl/if_tl.c
index b80e81f7af1b..5d218e794f5e 100644
--- a/sys/dev/tl/if_tl.c
+++ b/sys/dev/tl/if_tl.c
@@ -534,7 +534,7 @@ static u_int8_t tl_eeprom_putbyte(sc, byte)
 	struct tl_softc		*sc;
 	int			byte;
 {
-	register int		i, ack = 0;
+	int			i, ack = 0;
 
 	/*
 	 * Make sure we're in TX mode.
@@ -579,7 +579,7 @@ static u_int8_t tl_eeprom_getbyte(sc, addr, dest)
 	int			addr;
 	u_int8_t		*dest;
 {
-	register int		i;
+	int			i;
 	u_int8_t		byte = 0;
 	device_t		tl_dev = sc->tl_dev;
 
@@ -2199,7 +2199,7 @@ static void
 tl_stop(sc)
 	struct tl_softc		*sc;
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp;
 
 	TL_LOCK_ASSERT(sc);
diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c
index dc681aa7a75a..d0cf5636c531 100644
--- a/sys/dev/usb/usb_busdma.c
+++ b/sys/dev/usb/usb_busdma.c
@@ -232,7 +232,7 @@ struct usb_m_copy_in_arg {
 static int
 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
 {
-	register struct usb_m_copy_in_arg *ua = arg;
+	struct usb_m_copy_in_arg *ua = arg;
 
 	usbd_copy_in(ua->cache, ua->dst_offset, src, count);
 	ua->dst_offset += count;
diff --git a/sys/dev/wb/if_wb.c b/sys/dev/wb/if_wb.c
index 4215d9e6aa71..ca66226dd898 100644
--- a/sys/dev/wb/if_wb.c
+++ b/sys/dev/wb/if_wb.c
@@ -256,7 +256,7 @@ wb_eeprom_putbyte(sc, addr)
 	struct wb_softc		*sc;
 	int			addr;
 {
-	register int		d, i;
+	int			d, i;
 
 	d = addr | WB_EECMD_READ;
 
@@ -286,7 +286,7 @@ wb_eeprom_getword(sc, addr, dest)
 	int			addr;
 	u_int16_t		*dest;
 {
-	register int		i;
+	int			i;
 	u_int16_t		word = 0;
 
 	/* Enter EEPROM access mode. */
@@ -507,7 +507,7 @@ static void
 wb_reset(sc)
 	struct wb_softc		*sc;
 {
-	register int		i;
+	int			i;
 	struct mii_data		*mii;
 	struct mii_softc	*miisc;
 
@@ -1574,7 +1574,7 @@ static void
 wb_stop(sc)
 	struct wb_softc		*sc;
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp;
 
 	WB_LOCK_ASSERT(sc);
diff --git a/sys/dev/xl/if_xl.c b/sys/dev/xl/if_xl.c
index e82fe1c0228e..4c1c238981d1 100644
--- a/sys/dev/xl/if_xl.c
+++ b/sys/dev/xl/if_xl.c
@@ -353,7 +353,7 @@ xl_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 static void
 xl_wait(struct xl_softc *sc)
 {
-	register int		i;
+	int			i;
 
 	for (i = 0; i < XL_TIMEOUT; i++) {
 		if ((CSR_READ_2(sc, XL_STATUS) & XL_STAT_CMDBUSY) == 0)
@@ -836,7 +836,7 @@ xl_setmode(struct xl_softc *sc, int media)
 static void
 xl_reset(struct xl_softc *sc)
 {
-	register int		i;
+	int			i;
 
 	XL_LOCK_ASSERT(sc);
 
@@ -3153,7 +3153,7 @@ xl_watchdog(struct xl_softc *sc)
 static void
 xl_stop(struct xl_softc *sc)
 {
-	register int		i;
+	int			i;
 	struct ifnet		*ifp = sc->xl_ifp;
 
 	XL_LOCK_ASSERT(sc);
diff --git a/sys/fs/fifofs/fifo_vnops.c b/sys/fs/fifofs/fifo_vnops.c
index 5b8d9d39d807..024adee34cbb 100644
--- a/sys/fs/fifofs/fifo_vnops.c
+++ b/sys/fs/fifofs/fifo_vnops.c
@@ -313,7 +313,7 @@ int
 fifo_printinfo(vp)
 	struct vnode *vp;
 {
-	register struct fifoinfo *fip = vp->v_fifoinfo;
+	struct fifoinfo *fip = vp->v_fifoinfo;
 
 	if (fip == NULL){
 		printf(", NULL v_fifoinfo");
diff --git a/sys/fs/nandfs/nandfs_vnops.c b/sys/fs/nandfs/nandfs_vnops.c
index 1ca6eac926bb..147336db9ff6 100644
--- a/sys/fs/nandfs/nandfs_vnops.c
+++ b/sys/fs/nandfs/nandfs_vnops.c
@@ -129,8 +129,8 @@ nandfs_reclaim(struct vop_reclaim_args *ap)
 static int
 nandfs_read(struct vop_read_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
-	register struct nandfs_node *node = VTON(vp);
+	struct vnode *vp = ap->a_vp;
+	struct nandfs_node *node = VTON(vp);
 	struct nandfs_device *nandfsdev = node->nn_nandfsdev;
 	struct uio *uio = ap->a_uio;
 	struct buf *bp;
diff --git a/sys/i386/i386/in_cksum.c b/sys/i386/i386/in_cksum.c
index 32fdf7ebf983..99fa4448b778 100644
--- a/sys/i386/i386/in_cksum.c
+++ b/sys/i386/i386/in_cksum.c
@@ -263,9 +263,9 @@ in_cksum_skip(m, len, skip)
 	int len;
 	int skip;
 {
-	register u_short *w;
-	register unsigned sum = 0;
-	register int mlen = 0;
+	u_short *w;
+	unsigned sum = 0;
+	int mlen = 0;
 	int byte_swapped = 0;
 	union { char	c[2]; u_short	s; } su;
 
diff --git a/sys/i386/i386/k6_mem.c b/sys/i386/i386/k6_mem.c
index c99cf277489a..091b189a3ea5 100644
--- a/sys/i386/i386/k6_mem.c
+++ b/sys/i386/i386/k6_mem.c
@@ -78,7 +78,7 @@ static __inline int
 k6_mrmake(struct mem_range_desc *desc, u_int32_t *mtrr)
 {
 	u_int32_t len = 0, wc, uc;
-	register int bit;
+	int bit;
 
 	if (desc->mr_base &~ 0xfffe0000)
 		return (EINVAL);
diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c
index bbf631781d1d..3afed49c7b3b 100644
--- a/sys/i386/i386/sys_machdep.c
+++ b/sys/i386/i386/sys_machdep.c
@@ -98,9 +98,7 @@ struct sysarch_args {
 #endif
 
 int
-sysarch(td, uap)
-	struct thread *td;
-	register struct sysarch_args *uap;
+sysarch(struct thread *td, struct sysarch_args *uap)
 {
 	int error;
 	union descriptor *lp;
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 1896bbd5bf11..a0a9473b3bdf 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -166,7 +166,7 @@ alloc_fpusave(int flags)
 void
 cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 {
-	register struct proc *p1;
+	struct proc *p1;
 	struct pcb *pcb2;
 	struct mdproc *mdp2;
 
diff --git a/sys/i386/ibcs2/ibcs2_misc.c b/sys/i386/ibcs2/ibcs2_misc.c
index ccf5190b0a71..6e1b43ffdf27 100644
--- a/sys/i386/ibcs2/ibcs2_misc.c
+++ b/sys/i386/ibcs2/ibcs2_misc.c
@@ -93,9 +93,7 @@ __FBSDID("$FreeBSD$");
 #include <security/mac/mac_framework.h>
 
 int
-ibcs2_ulimit(td, uap)
-	struct thread *td;
-	struct ibcs2_ulimit_args *uap;
+ibcs2_ulimit(struct thread *td, struct ibcs2_ulimit_args *uap)
 {
 	struct rlimit rl;
 	int error;
@@ -134,9 +132,7 @@ ibcs2_ulimit(td, uap)
 #define IBCS2_WSTOPPED       0177
 #define IBCS2_STOPCODE(sig)  ((sig) << 8 | IBCS2_WSTOPPED)
 int
-ibcs2_wait(td, uap)
-	struct thread *td;
-	struct ibcs2_wait_args *uap;
+ibcs2_wait(struct thread *td, struct ibcs2_wait_args *uap)
 {
 	int error, options, status;
 	int *statusp;
@@ -185,9 +181,7 @@ ibcs2_wait(td, uap)
 }
 
 int
-ibcs2_execv(td, uap)
-	struct thread *td;
-	struct ibcs2_execv_args *uap;
+ibcs2_execv(struct thread *td, struct ibcs2_execv_args *uap)
 {
 	struct image_args eargs;
 	struct vmspace *oldvmspace;
@@ -210,9 +204,7 @@ ibcs2_execv(td, uap)
 }
 
 int
-ibcs2_execve(td, uap) 
-        struct thread *td;
-        struct ibcs2_execve_args *uap;
+ibcs2_execve(struct thread *td, struct ibcs2_execve_args *uap)
 {
 	struct image_args eargs;
 	struct vmspace *oldvmspace;
@@ -236,9 +228,7 @@ ibcs2_execve(td, uap)
 }
 
 int
-ibcs2_umount(td, uap)
-	struct thread *td;
-	struct ibcs2_umount_args *uap;
+ibcs2_umount(struct thread *td, struct ibcs2_umount_args *uap)
 {
 	struct unmount_args um;
 
@@ -248,9 +238,7 @@ ibcs2_umount(td, uap)
 }
 
 int
-ibcs2_mount(td, uap)
-	struct thread *td;
-	struct ibcs2_mount_args *uap;
+ibcs2_mount(struct thread *td, struct ibcs2_mount_args *uap)
 {
 #ifdef notyet
 	int oflags = uap->flags, nflags, error;
@@ -321,15 +309,13 @@ ibcs2_mount(td, uap)
  */
 
 int
-ibcs2_getdents(td, uap)
-	struct thread *td;
-	register struct ibcs2_getdents_args *uap;
+ibcs2_getdents(struct thread *td, struct ibcs2_getdents_args *uap)
 {
-	register struct vnode *vp;
-	register caddr_t inp, buf;	/* BSD-format */
-	register int len, reclen;	/* BSD-format */
-	register caddr_t outp;		/* iBCS2-format */
-	register int resid;		/* iBCS2-format */
+	struct vnode *vp;
+	caddr_t inp, buf;		/* BSD-format */
+	int len, reclen;		/* BSD-format */
+	caddr_t outp;			/* iBCS2-format */
+	int resid;			/* iBCS2-format */
 	cap_rights_t rights;
 	struct file *fp;
 	struct uio auio;
@@ -475,15 +461,13 @@ ibcs2_getdents(td, uap)
 }
 
 int
-ibcs2_read(td, uap)
-	struct thread *td;
-	struct ibcs2_read_args *uap;
+ibcs2_read(struct thread *td, struct ibcs2_read_args *uap)
 {
-	register struct vnode *vp;
-	register caddr_t inp, buf;	/* BSD-format */
-	register int len, reclen;	/* BSD-format */
-	register caddr_t outp;		/* iBCS2-format */
-	register int resid;		/* iBCS2-format */
+	struct vnode *vp;
+	caddr_t inp, buf;		/* BSD-format */
+	int len, reclen;		/* BSD-format */
+	caddr_t outp;			/* iBCS2-format */
+	int resid;			/* iBCS2-format */
 	cap_rights_t rights;
 	struct file *fp;
 	struct uio auio;
@@ -640,9 +624,7 @@ ibcs2_read(td, uap)
 }
 
 int
-ibcs2_mknod(td, uap)
-	struct thread *td;
-	struct ibcs2_mknod_args *uap;
+ibcs2_mknod(struct thread *td, struct ibcs2_mknod_args *uap)
 {
 	char *path;
 	int error;
@@ -660,9 +642,7 @@ ibcs2_mknod(td, uap)
 }
 
 int
-ibcs2_getgroups(td, uap)
-	struct thread *td;
-	struct ibcs2_getgroups_args *uap;
+ibcs2_getgroups(struct thread *td, struct ibcs2_getgroups_args *uap)
 {
 	struct ucred *cred;
 	ibcs2_gid_t *iset;
@@ -690,9 +670,7 @@ ibcs2_getgroups(td, uap)
 }
 
 int
-ibcs2_setgroups(td, uap)
-	struct thread *td;
-	struct ibcs2_setgroups_args *uap;
+ibcs2_setgroups(struct thread *td, struct ibcs2_setgroups_args *uap)
 {
 	ibcs2_gid_t *iset;
 	gid_t *gp;
@@ -722,9 +700,7 @@ ibcs2_setgroups(td, uap)
 }
 
 int
-ibcs2_setuid(td, uap)
-	struct thread *td;
-	struct ibcs2_setuid_args *uap;
+ibcs2_setuid(struct thread *td, struct ibcs2_setuid_args *uap)
 {
 	struct setuid_args sa;
 
@@ -733,9 +709,7 @@ ibcs2_setuid(td, uap)
 }
 
 int
-ibcs2_setgid(td, uap)
-	struct thread *td;
-	struct ibcs2_setgid_args *uap;
+ibcs2_setgid(struct thread *td, struct ibcs2_setgid_args *uap)
 {
 	struct setgid_args sa;
 
@@ -744,9 +718,7 @@ ibcs2_setgid(td, uap)
 }
 
 int
-ibcs2_time(td, uap)
-	struct thread *td;
-	struct ibcs2_time_args *uap;
+ibcs2_time(struct thread *td, struct ibcs2_time_args *uap)
 {
 	struct timeval tv;
 
@@ -760,9 +732,7 @@ ibcs2_time(td, uap)
 }
 
 int
-ibcs2_pathconf(td, uap)
-	struct thread *td;
-	struct ibcs2_pathconf_args *uap;
+ibcs2_pathconf(struct thread *td, struct ibcs2_pathconf_args *uap)
 {
 	char *path;
 	int error;
@@ -775,18 +745,14 @@ ibcs2_pathconf(td, uap)
 }
 
 int
-ibcs2_fpathconf(td, uap)
-	struct thread *td;
-	struct ibcs2_fpathconf_args *uap;
+ibcs2_fpathconf(struct thread *td, struct ibcs2_fpathconf_args *uap)
 {
 	uap->name++;	/* iBCS2 _PC_* defines are offset by one */
         return sys_fpathconf(td, (struct fpathconf_args *)uap);
 }
 
 int
-ibcs2_sysconf(td, uap)
-	struct thread *td;
-	struct ibcs2_sysconf_args *uap;
+ibcs2_sysconf(struct thread *td, struct ibcs2_sysconf_args *uap)
 {
 	int mib[2], value, len, error;
 
@@ -845,9 +811,7 @@ ibcs2_sysconf(td, uap)
 }
 
 int
-ibcs2_alarm(td, uap)
-	struct thread *td;
-	struct ibcs2_alarm_args *uap;
+ibcs2_alarm(struct thread *td, struct ibcs2_alarm_args *uap)
 {
 	struct itimerval itv, oitv;
 	int error;
@@ -865,9 +829,7 @@ ibcs2_alarm(td, uap)
 }
 
 int
-ibcs2_times(td, uap)
-	struct thread *td;
-	struct ibcs2_times_args *uap;
+ibcs2_times(struct thread *td, struct ibcs2_times_args *uap)
 {
 	struct rusage ru;
 	struct timeval t;
@@ -895,9 +857,7 @@ ibcs2_times(td, uap)
 }
 
 int
-ibcs2_stime(td, uap)
-	struct thread *td;
-	struct ibcs2_stime_args *uap;
+ibcs2_stime(struct thread *td, struct ibcs2_stime_args *uap)
 {
 	struct timeval tv;
 	long secs;
@@ -915,9 +875,7 @@ ibcs2_stime(td, uap)
 }
 
 int
-ibcs2_utime(td, uap)
-	struct thread *td;
-	struct ibcs2_utime_args *uap;
+ibcs2_utime(struct thread *td, struct ibcs2_utime_args *uap)
 {
 	struct ibcs2_utimbuf ubuf;
 	struct timeval tbuf[2], *tp;
@@ -944,9 +902,7 @@ ibcs2_utime(td, uap)
 }
 
 int
-ibcs2_nice(td, uap)
-	struct thread *td;
-	struct ibcs2_nice_args *uap;
+ibcs2_nice(struct thread *td, struct ibcs2_nice_args *uap)
 {
 	int error;
 	struct setpriority_args sa;
@@ -965,9 +921,7 @@ ibcs2_nice(td, uap)
  */
 
 int
-ibcs2_pgrpsys(td, uap)
-	struct thread *td;
-	struct ibcs2_pgrpsys_args *uap;
+ibcs2_pgrpsys(struct thread *td, struct ibcs2_pgrpsys_args *uap)
 {
 	struct proc *p = td->td_proc;
 	switch (uap->type) {
@@ -1012,9 +966,7 @@ ibcs2_pgrpsys(td, uap)
  */
 
 int
-ibcs2_plock(td, uap)
-	struct thread *td;
-	struct ibcs2_plock_args *uap;
+ibcs2_plock(struct thread *td, struct ibcs2_plock_args *uap)
 {
 	int error;
 #define IBCS2_UNLOCK	0
@@ -1044,9 +996,7 @@ ibcs2_plock(td, uap)
 }
 
 int
-ibcs2_uadmin(td, uap)
-	struct thread *td;
-	struct ibcs2_uadmin_args *uap;
+ibcs2_uadmin(struct thread *td, struct ibcs2_uadmin_args *uap)
 {
 #define SCO_A_REBOOT        1
 #define SCO_A_SHUTDOWN      2
@@ -1093,9 +1043,7 @@ ibcs2_uadmin(td, uap)
 }
 
 int
-ibcs2_sysfs(td, uap)
-	struct thread *td;
-	struct ibcs2_sysfs_args *uap;
+ibcs2_sysfs(struct thread *td, struct ibcs2_sysfs_args *uap)
 {
 #define IBCS2_GETFSIND        1
 #define IBCS2_GETFSTYP        2
@@ -1111,9 +1059,7 @@ ibcs2_sysfs(td, uap)
 }
 
 int
-ibcs2_unlink(td, uap)
-	struct thread *td;
-	struct ibcs2_unlink_args *uap;
+ibcs2_unlink(struct thread *td, struct ibcs2_unlink_args *uap)
 {
 	char *path;
 	int error;
@@ -1125,9 +1071,7 @@ ibcs2_unlink(td, uap)
 }
 
 int
-ibcs2_chdir(td, uap)
-	struct thread *td;
-	struct ibcs2_chdir_args *uap;
+ibcs2_chdir(struct thread *td, struct ibcs2_chdir_args *uap)
 {
 	char *path;
 	int error;
@@ -1139,9 +1083,7 @@ ibcs2_chdir(td, uap)
 }
 
 int
-ibcs2_chmod(td, uap)
-	struct thread *td;
-	struct ibcs2_chmod_args *uap;
+ibcs2_chmod(struct thread *td, struct ibcs2_chmod_args *uap)
 {
 	char *path;
 	int error;
@@ -1153,9 +1095,7 @@ ibcs2_chmod(td, uap)
 }
 
 int
-ibcs2_chown(td, uap)
-	struct thread *td;
-	struct ibcs2_chown_args *uap;
+ibcs2_chown(struct thread *td, struct ibcs2_chown_args *uap)
 {
 	char *path;
 	int error;
@@ -1168,9 +1108,7 @@ ibcs2_chown(td, uap)
 }
 
 int
-ibcs2_rmdir(td, uap)
-	struct thread *td;
-	struct ibcs2_rmdir_args *uap;
+ibcs2_rmdir(struct thread *td, struct ibcs2_rmdir_args *uap)
 {
 	char *path;
 	int error;
@@ -1182,9 +1120,7 @@ ibcs2_rmdir(td, uap)
 }
 
 int
-ibcs2_mkdir(td, uap)
-	struct thread *td;
-	struct ibcs2_mkdir_args *uap;
+ibcs2_mkdir(struct thread *td, struct ibcs2_mkdir_args *uap)
 {
 	char *path;
 	int error;
@@ -1196,9 +1132,7 @@ ibcs2_mkdir(td, uap)
 }
 
 int
-ibcs2_symlink(td, uap)
-	struct thread *td;
-	struct ibcs2_symlink_args *uap;
+ibcs2_symlink(struct thread *td, struct ibcs2_symlink_args *uap)
 {
 	char *path, *link;
 	int error;
@@ -1221,9 +1155,7 @@ ibcs2_symlink(td, uap)
 }
 
 int
-ibcs2_rename(td, uap)
-	struct thread *td;
-	struct ibcs2_rename_args *uap;
+ibcs2_rename(struct thread *td, struct ibcs2_rename_args *uap)
 {
 	char *from, *to;
 	int error;
@@ -1246,9 +1178,7 @@ ibcs2_rename(td, uap)
 }
 
 int
-ibcs2_readlink(td, uap)
-	struct thread *td;
-	struct ibcs2_readlink_args *uap;
+ibcs2_readlink(struct thread *td, struct ibcs2_readlink_args *uap)
 {
 	char *path;
 	int error;
diff --git a/sys/i386/ibcs2/ibcs2_other.c b/sys/i386/ibcs2/ibcs2_other.c
index b49e605eb232..80f3c5ac17ef 100644
--- a/sys/i386/ibcs2/ibcs2_other.c
+++ b/sys/i386/ibcs2/ibcs2_other.c
@@ -68,7 +68,7 @@ ibcs2_secure(struct thread *td, struct ibcs2_secure_args *uap)
 }
 
 int
-ibcs2_lseek(struct thread *td, register struct ibcs2_lseek_args *uap)
+ibcs2_lseek(struct thread *td, struct ibcs2_lseek_args *uap)
 {
 	struct lseek_args largs;
 	int error;
diff --git a/sys/i386/ibcs2/ibcs2_signal.c b/sys/i386/ibcs2/ibcs2_signal.c
index 6776df19e29e..4c17aa9bc977 100644
--- a/sys/i386/ibcs2/ibcs2_signal.c
+++ b/sys/i386/ibcs2/ibcs2_signal.c
@@ -190,9 +190,7 @@ bsd_to_ibcs2_sigaction(bsa, isa)
 }
 
 int
-ibcs2_sigaction(td, uap)
-	register struct thread *td;
-	struct ibcs2_sigaction_args *uap;
+ibcs2_sigaction(struct thread *td, struct ibcs2_sigaction_args *uap)
 {
 	struct ibcs2_sigaction isa;
 	struct sigaction nbsa, obsa;
@@ -218,9 +216,7 @@ ibcs2_sigaction(td, uap)
 }
 
 int
-ibcs2_sigsys(td, uap)
-	register struct thread *td;
-	struct ibcs2_sigsys_args *uap;
+ibcs2_sigsys(struct thread *td, struct ibcs2_sigsys_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct sigaction sa;
@@ -341,9 +337,7 @@ ibcs2_sigsys(td, uap)
 }
 
 int
-ibcs2_sigprocmask(td, uap)
-	register struct thread *td;
-	struct ibcs2_sigprocmask_args *uap;
+ibcs2_sigprocmask(struct thread *td, struct ibcs2_sigprocmask_args *uap)
 {
 	ibcs2_sigset_t iss;
 	sigset_t oss, nss;
@@ -379,9 +373,7 @@ ibcs2_sigprocmask(td, uap)
 }
 
 int
-ibcs2_sigpending(td, uap)
-	register struct thread *td;
-	struct ibcs2_sigpending_args *uap;
+ibcs2_sigpending(struct thread *td, struct ibcs2_sigpending_args *uap)
 {
 	struct proc *p = td->td_proc;
 	sigset_t bss;
@@ -398,9 +390,7 @@ ibcs2_sigpending(td, uap)
 }
 
 int
-ibcs2_sigsuspend(td, uap)
-	register struct thread *td;
-	struct ibcs2_sigsuspend_args *uap;
+ibcs2_sigsuspend(struct thread *td, struct ibcs2_sigsuspend_args *uap)
 {
 	ibcs2_sigset_t sss;
 	sigset_t bss;
@@ -414,9 +404,7 @@ ibcs2_sigsuspend(td, uap)
 }
 
 int
-ibcs2_pause(td, uap)
-	register struct thread *td;
-	struct ibcs2_pause_args *uap;
+ibcs2_pause(struct thread *td, struct ibcs2_pause_args *uap)
 {
 	sigset_t mask;
 
@@ -427,9 +415,7 @@ ibcs2_pause(td, uap)
 }
 
 int
-ibcs2_kill(td, uap)
-	register struct thread *td;
-	struct ibcs2_kill_args *uap;
+ibcs2_kill(struct thread *td, struct ibcs2_kill_args *uap)
 {
 	struct kill_args ka;
 
diff --git a/sys/i386/ibcs2/ibcs2_socksys.c b/sys/i386/ibcs2/ibcs2_socksys.c
index 80b121614b8c..3b5a7e53551d 100644
--- a/sys/i386/ibcs2/ibcs2_socksys.c
+++ b/sys/i386/ibcs2/ibcs2_socksys.c
@@ -58,9 +58,7 @@ static int ibcs2_setipdomainname(struct thread *,
  */
 
 int
-ibcs2_socksys(td, uap)
-	register struct thread *td;
-	register struct ibcs2_socksys_args *uap;
+ibcs2_socksys(struct thread *td, struct ibcs2_socksys_args *uap)
 {
 	int error;
 	int realargs[7]; /* 1 for command, 6 for recvfrom */
@@ -142,9 +140,7 @@ ibcs2_socksys(td, uap)
 
 /* ARGSUSED */
 static int
-ibcs2_getipdomainname(td, uap)
-        struct thread *td;
-        struct getipdomainname_args *uap;
+ibcs2_getipdomainname(struct thread *td, struct getipdomainname_args *uap)
 {
 	char hname[MAXHOSTNAMELEN], *dptr;
 	int len;
@@ -167,9 +163,7 @@ ibcs2_getipdomainname(td, uap)
 
 /* ARGSUSED */
 static int
-ibcs2_setipdomainname(td, uap)
-        struct thread *td;
-        struct setipdomainname_args *uap;
+ibcs2_setipdomainname(struct thread *td, struct setipdomainname_args *uap)
 {
 	char hname[MAXHOSTNAMELEN], *ptr;
 	int error, sctl[2], hlen;
diff --git a/sys/i386/isa/ccbque.h b/sys/i386/isa/ccbque.h
index 72375b087db1..5a053f12c94e 100644
--- a/sys/i386/isa/ccbque.h
+++ b/sys/i386/isa/ccbque.h
@@ -51,7 +51,7 @@ struct CCBTYPE##que {							\
 									\
 void DEV##_init_ccbque(int);					\
 struct CCBTYPE *DEV##_get_ccb(void);				\
-void DEV##_free_ccb(register struct CCBTYPE *);
+void DEV##_free_ccb(struct CCBTYPE *);
 
 /* (II)  static allocated memory */
 #define GENERIC_CCB_STATIC_ALLOC(DEV, CCBTYPE)				\
@@ -72,7 +72,7 @@ DEV##_init_ccbque(count)						\
 struct CCBTYPE *							\
 DEV##_get_ccb()								\
 {									\
-	register struct CCBTYPE *cb;					\
+	struct CCBTYPE *cb;						\
 	int s = splcam();						\
 									\
 	if (CCBTYPE##que.count < CCBTYPE##que.maxccb)			\
@@ -105,7 +105,7 @@ out:									\
 									\
 void									\
 DEV##_free_ccb(cb)							\
-	register struct CCBTYPE *cb;					\
+	struct CCBTYPE *cb;						\
 {									\
 	int s = splcam();						\
 									\
diff --git a/sys/i386/isa/elink.c b/sys/i386/isa/elink.c
index 5fbfd363192b..e3e4755dee99 100644
--- a/sys/i386/isa/elink.c
+++ b/sys/i386/isa/elink.c
@@ -70,8 +70,8 @@ elink_reset()
 void
 elink_idseq(u_char p)
 {
-	register int i;
-	register u_char c;
+	int i;
+	u_char c;
 
 	c = 0xff;
 	for (i = 255; i; i--) {
diff --git a/sys/kern/inflate.c b/sys/kern/inflate.c
index e842c8e80108..d7a480bb97d1 100644
--- a/sys/kern/inflate.c
+++ b/sys/kern/inflate.c
@@ -431,17 +431,17 @@ huft_build(struct inflate *glbl, unsigned *b, unsigned n, unsigned s,
 	unsigned        f;	/* i repeats in table every f entries */
 	int             g;	/* maximum code length */
 	int             h;	/* table level */
-	register unsigned i;	/* counter, current code */
-	register unsigned j;	/* counter */
-	register int    k;	/* number of bits in current code */
+	unsigned i;		/* counter, current code */
+	unsigned j;		/* counter */
+	int    k;		/* number of bits in current code */
 	int             lx[BMAX + 1];	/* memory for l[-1..BMAX-1] */
 	int            *l = lx + 1;	/* stack of bits per table */
-	register unsigned *p;	/* pointer into c[], b[], or v[] */
-	register struct huft *q;/* points to current table */
+	unsigned *p;		/* pointer into c[], b[], or v[] */
+	struct huft *q;		/* points to current table */
 	struct huft     r;	/* table entry for structure assignment */
 	struct huft    *u[BMAX];/* table stack */
 	unsigned        v[N_MAX];	/* values in order of bit length */
-	register int    w;	/* bits before this table == (l * h) */
+	int    w;		/* bits before this table == (l * h) */
 	unsigned        x[BMAX + 1];	/* bit offsets, then code stack */
 	unsigned       *xp;	/* pointer into x */
 	int             y;	/* number of dummy codes added */
@@ -627,7 +627,7 @@ huft_free(struct inflate *glbl, struct huft *t)
    list of the tables it made, with the links in a dummy first entry of
    each table. */
 {
-	register struct huft *p, *q;
+	struct huft *p, *q;
 
 	/* Go through linked list, freeing from the malloced (t[-1]) address. */
 	p = t;
@@ -650,13 +650,13 @@ static int
 inflate_codes(struct inflate *glbl, struct huft *tl, struct huft*td, int bl,
     int bd)
 {
-	register unsigned e;	/* table entry flag/number of extra bits */
+	unsigned e;		/* table entry flag/number of extra bits */
 	unsigned        n, d;	/* length and index for copy */
 	unsigned        w;	/* current window position */
 	struct huft    *t;	/* pointer to table entry */
 	unsigned        ml, md;	/* masks for bl and bd bits */
-	register ulg    b;	/* bit buffer */
-	register unsigned k;	/* number of bits in bit buffer */
+	ulg    b;		/* bit buffer */
+	unsigned k;		/* number of bits in bit buffer */
 
 	/* make local copies of globals */
 	b = glbl->gz_bb;			/* initialize bit buffer */
@@ -745,8 +745,8 @@ inflate_stored(struct inflate *glbl)
 {
 	unsigned        n;	/* number of bytes in block */
 	unsigned        w;	/* current window position */
-	register ulg    b;	/* bit buffer */
-	register unsigned k;	/* number of bits in bit buffer */
+	ulg    b;		/* bit buffer */
+	unsigned k;		/* number of bits in bit buffer */
 
 	/* make local copies of globals */
 	b = glbl->gz_bb;			/* initialize bit buffer */
@@ -849,8 +849,8 @@ inflate_dynamic(struct inflate *glbl)
 	unsigned        ll[286 + 30];	/* literal/length and distance code
 					 * lengths */
 #endif
-	register ulg    b;	/* bit buffer */
-	register unsigned k;	/* number of bits in bit buffer */
+	ulg    b;		/* bit buffer */
+	unsigned k;		/* number of bits in bit buffer */
 
 	/* make local bit buffer */
 	b = glbl->gz_bb;
@@ -980,8 +980,8 @@ static int
 inflate_block(struct inflate *glbl, int *e)
 {
 	unsigned        t;	/* block type */
-	register ulg    b;	/* bit buffer */
-	register unsigned k;	/* number of bits in bit buffer */
+	ulg    b;		/* bit buffer */
+	unsigned k;		/* number of bits in bit buffer */
 
 	/* make local bit buffer */
 	b = glbl->gz_bb;
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index e6c925bbc0c4..e88fcd14c2b4 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -390,10 +390,9 @@ static int devpoll_run = 0;
  */
 /* ARGSUSED*/
 static void
-initclocks(dummy)
-	void *dummy;
+initclocks(void *dummy)
 {
-	register int i;
+	int i;
 
 	/*
 	 * Set divisors to 1 (normal case) and let the machine-specific
@@ -581,11 +580,10 @@ hardclock_sync(int cpu)
  * Compute number of ticks in the specified amount of time.
  */
 int
-tvtohz(tv)
-	struct timeval *tv;
+tvtohz(struct timeval *tv)
 {
-	register unsigned long ticks;
-	register long sec, usec;
+	unsigned long ticks;
+	long sec, usec;
 
 	/*
 	 * If the number of usecs in the whole seconds part of the time
@@ -642,8 +640,7 @@ tvtohz(tv)
  * keeps the profile clock running constantly.
  */
 void
-startprofclock(p)
-	register struct proc *p;
+startprofclock(struct proc *p)
 {
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
@@ -662,8 +659,7 @@ startprofclock(p)
  * Stop profiling on a process.
  */
 void
-stopprofclock(p)
-	register struct proc *p;
+stopprofclock(struct proc *p)
 {
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 7b5b0786deca..fe9638a064d7 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -353,10 +353,7 @@ kern_execve(struct thread *td, struct image_args *args, struct mac *mac_p)
  * userspace pointers from the passed thread.
  */
 static int
-do_execve(td, args, mac_p)
-	struct thread *td;
-	struct image_args *args;
-	struct mac *mac_p;
+do_execve(struct thread *td, struct image_args *args, struct mac *mac_p)
 {
 	struct proc *p = td->td_proc;
 	struct nameidata nd;
@@ -1042,8 +1039,7 @@ exec_map_first_page(imgp)
 }
 
 void
-exec_unmap_first_page(imgp)
-	struct image_params *imgp;
+exec_unmap_first_page(struct image_params *imgp)
 {
 	vm_page_t m;
 
@@ -1063,9 +1059,7 @@ exec_unmap_first_page(imgp)
  *	automatically in trap.c.
  */
 int
-exec_new_vmspace(imgp, sv)
-	struct image_params *imgp;
-	struct sysentvec *sv;
+exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv)
 {
 	int error;
 	struct proc *p = imgp->proc;
@@ -1460,8 +1454,7 @@ exec_free_args(struct image_args *args)
  * as the initial stack pointer.
  */
 register_t *
-exec_copyout_strings(imgp)
-	struct image_params *imgp;
+exec_copyout_strings(struct image_params *imgp)
 {
 	int argc, envc;
 	char **vectp;
@@ -1617,8 +1610,7 @@ exec_copyout_strings(imgp)
  *	Return 0 for success or error code on failure.
  */
 int
-exec_check_permissions(imgp)
-	struct image_params *imgp;
+exec_check_permissions(struct image_params *imgp)
 {
 	struct vnode *vp = imgp->vp;
 	struct vattr *attr = imgp->attr;
@@ -1688,8 +1680,7 @@ exec_check_permissions(imgp)
  * Exec handler registration
  */
 int
-exec_register(execsw_arg)
-	const struct execsw *execsw_arg;
+exec_register(const struct execsw *execsw_arg)
 {
 	const struct execsw **es, **xs, **newexecsw;
 	int count = 2;	/* New slot and trailing NULL */
@@ -1711,8 +1702,7 @@ exec_register(execsw_arg)
 }
 
 int
-exec_unregister(execsw_arg)
-	const struct execsw *execsw_arg;
+exec_unregister(const struct execsw *execsw_arg)
 {
 	const struct execsw **es, **xs, **newexecsw;
 	int count = 1;
diff --git a/sys/kern/kern_prot.c b/sys/kern/kern_prot.c
index 16c8e2180b76..68f4f75f054a 100644
--- a/sys/kern/kern_prot.c
+++ b/sys/kern/kern_prot.c
@@ -295,7 +295,7 @@ struct getgroups_args {
 };
 #endif
 int
-sys_getgroups(struct thread *td, register struct getgroups_args *uap)
+sys_getgroups(struct thread *td, struct getgroups_args *uap)
 {
 	struct ucred *cred;
 	u_int ngrp;
@@ -324,7 +324,7 @@ struct setsid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setsid(register struct thread *td, struct setsid_args *uap)
+sys_setsid(struct thread *td, struct setsid_args *uap)
 {
 	struct pgrp *pgrp;
 	int error;
@@ -382,11 +382,11 @@ struct setpgid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setpgid(struct thread *td, register struct setpgid_args *uap)
+sys_setpgid(struct thread *td, struct setpgid_args *uap)
 {
 	struct proc *curp = td->td_proc;
-	register struct proc *targp;	/* target process */
-	register struct pgrp *pgrp;	/* target pgrp */
+	struct proc *targp;	/* target process */
+	struct pgrp *pgrp;	/* target pgrp */
 	int error;
 	struct pgrp *newpgrp;
 
@@ -877,7 +877,7 @@ struct setreuid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setreuid(register struct thread *td, struct setreuid_args *uap)
+sys_setreuid(struct thread *td, struct setreuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -947,7 +947,7 @@ struct setregid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setregid(register struct thread *td, struct setregid_args *uap)
+sys_setregid(struct thread *td, struct setregid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1012,7 +1012,7 @@ struct setresuid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setresuid(register struct thread *td, struct setresuid_args *uap)
+sys_setresuid(struct thread *td, struct setresuid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1094,7 +1094,7 @@ struct setresgid_args {
 #endif
 /* ARGSUSED */
 int
-sys_setresgid(register struct thread *td, struct setresgid_args *uap)
+sys_setresgid(struct thread *td, struct setresgid_args *uap)
 {
 	struct proc *p = td->td_proc;
 	struct ucred *newcred, *oldcred;
@@ -1161,7 +1161,7 @@ struct getresuid_args {
 #endif
 /* ARGSUSED */
 int
-sys_getresuid(register struct thread *td, struct getresuid_args *uap)
+sys_getresuid(struct thread *td, struct getresuid_args *uap)
 {
 	struct ucred *cred;
 	int error1 = 0, error2 = 0, error3 = 0;
@@ -1188,7 +1188,7 @@ struct getresgid_args {
 #endif
 /* ARGSUSED */
 int
-sys_getresgid(register struct thread *td, struct getresgid_args *uap)
+sys_getresgid(struct thread *td, struct getresgid_args *uap)
 {
 	struct ucred *cred;
 	int error1 = 0, error2 = 0, error3 = 0;
@@ -1213,7 +1213,7 @@ struct issetugid_args {
 #endif
 /* ARGSUSED */
 int
-sys_issetugid(register struct thread *td, struct issetugid_args *uap)
+sys_issetugid(struct thread *td, struct issetugid_args *uap)
 {
 	struct proc *p = td->td_proc;
 
@@ -1778,7 +1778,7 @@ p_canwait(struct thread *td, struct proc *p)
 struct ucred *
 crget(void)
 {
-	register struct ucred *cr;
+	struct ucred *cr;
 
 	cr = malloc(sizeof(*cr), M_CRED, M_WAITOK | M_ZERO);
 	refcount_init(&cr->cr_ref, 1);
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 85e8d01cc1be..4cb6f4299600 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -90,7 +90,7 @@ struct getpriority_args {
 };
 #endif
 int
-sys_getpriority(struct thread *td, register struct getpriority_args *uap)
+sys_getpriority(struct thread *td, struct getpriority_args *uap)
 {
 	struct proc *p;
 	struct pgrp *pg;
@@ -367,7 +367,7 @@ struct rtprio_args {
 };
 #endif
 int
-sys_rtprio(struct thread *td, register struct rtprio_args *uap)
+sys_rtprio(struct thread *td, struct rtprio_args *uap)
 {
 	struct proc *p;
 	struct thread *tdp;
@@ -533,7 +533,7 @@ struct osetrlimit_args {
 };
 #endif
 int
-osetrlimit(struct thread *td, register struct osetrlimit_args *uap)
+osetrlimit(struct thread *td, struct osetrlimit_args *uap)
 {
 	struct orlimit olim;
 	struct rlimit lim;
@@ -554,7 +554,7 @@ struct ogetrlimit_args {
 };
 #endif
 int
-ogetrlimit(struct thread *td, register struct ogetrlimit_args *uap)
+ogetrlimit(struct thread *td, struct ogetrlimit_args *uap)
 {
 	struct orlimit olim;
 	struct rlimit rl;
@@ -587,7 +587,7 @@ struct __setrlimit_args {
 };
 #endif
 int
-sys_setrlimit(struct thread *td, register struct __setrlimit_args *uap)
+sys_setrlimit(struct thread *td, struct __setrlimit_args *uap)
 {
 	struct rlimit alim;
 	int error;
@@ -645,7 +645,7 @@ kern_proc_setrlimit(struct thread *td, struct proc *p, u_int which,
     struct rlimit *limp)
 {
 	struct plimit *newlim, *oldlim;
-	register struct rlimit *alimp;
+	struct rlimit *alimp;
 	struct rlimit oldssiz;
 	int error;
 
@@ -775,7 +775,7 @@ struct __getrlimit_args {
 #endif
 /* ARGSUSED */
 int
-sys_getrlimit(struct thread *td, register struct __getrlimit_args *uap)
+sys_getrlimit(struct thread *td, struct __getrlimit_args *uap)
 {
 	struct rlimit rlim;
 	int error;
@@ -945,7 +945,7 @@ struct getrusage_args {
 };
 #endif
 int
-sys_getrusage(register struct thread *td, register struct getrusage_args *uap)
+sys_getrusage(struct thread *td, struct getrusage_args *uap)
 {
 	struct rusage ru;
 	int error;
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index d2596985090e..7b6e97877c1d 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -929,7 +929,7 @@ osigreturn(struct thread *td, struct osigreturn_args *uap)
 void
 siginit(struct proc *p)
 {
-	register int i;
+	int i;
 	struct sigacts *ps;
 
 	PROC_LOCK(p);
@@ -2392,7 +2392,7 @@ static void
 tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval)
 {
 	struct proc *p = td->td_proc;
-	register int prop;
+	int prop;
 	int wakeup_swapper;
 
 	wakeup_swapper = 0;
@@ -2979,7 +2979,7 @@ thread_stopped(struct proc *p)
  */
 int
 postsig(sig)
-	register int sig;
+	int sig;
 {
 	struct thread *td = curthread;
 	struct proc *p = td->td_proc;
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index f4ff571eff2d..9c46011f55dc 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -1475,7 +1475,7 @@ _callout_init_lock(struct callout *c, struct lock_object *lock, int flags)
 void
 adjust_timeout_calltodo(struct timeval *time_change)
 {
-	register struct callout *p;
+	struct callout *p;
 	unsigned long delta_ticks;
 
 	/* 
diff --git a/sys/kern/kern_xxx.c b/sys/kern/kern_xxx.c
index 68f94e84de8f..b37e4b07692d 100644
--- a/sys/kern/kern_xxx.c
+++ b/sys/kern/kern_xxx.c
@@ -58,9 +58,7 @@ struct gethostname_args {
 #endif
 /* ARGSUSED */
 int
-ogethostname(td, uap)
-	struct thread *td;
-	struct gethostname_args *uap;
+ogethostname(struct thread *td, struct gethostname_args *uap)
 {
 	int name[2];
 	size_t len = uap->len;
@@ -79,9 +77,7 @@ struct sethostname_args {
 #endif
 /* ARGSUSED */
 int
-osethostname(td, uap)
-	struct thread *td;
-	register struct sethostname_args *uap;
+osethostname(struct thread *td, struct sethostname_args *uap)
 {
 	int name[2];
 
@@ -98,9 +94,7 @@ struct ogethostid_args {
 #endif
 /* ARGSUSED */
 int
-ogethostid(td, uap)
-	struct thread *td;
-	struct ogethostid_args *uap;
+ogethostid(struct thread *td, struct ogethostid_args *uap)
 {
 	size_t len = sizeof(long);
 	int name[2];
@@ -120,9 +114,7 @@ struct osethostid_args {
 #endif
 /* ARGSUSED */
 int
-osethostid(td, uap)
-	struct thread *td;
-	struct osethostid_args *uap;
+osethostid(struct thread *td, struct osethostid_args *uap)
 {
 	int name[2];
 
@@ -133,9 +125,7 @@ osethostid(td, uap)
 }
 
 int
-oquota(td, uap)
-	struct thread *td;
-	struct oquota_args *uap;
+oquota(struct thread *td, struct oquota_args *uap)
 {
 
 	return (ENOSYS);
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index a561c2eb8eaa..c4d54d6f3402 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -456,7 +456,7 @@ SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, "");
 static void
 schedcpu(void)
 {
-	register fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
+	fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
 	struct thread *td;
 	struct proc *p;
 	struct td_sched *ts;
diff --git a/sys/kern/sysv_msg.c b/sys/kern/sysv_msg.c
index 80102fa25742..207fb1f5d7bc 100644
--- a/sys/kern/sysv_msg.c
+++ b/sys/kern/sysv_msg.c
@@ -480,9 +480,7 @@ struct msgctl_args {
 };
 #endif
 int
-sys_msgctl(td, uap)
-	struct thread *td;
-	register struct msgctl_args *uap;
+sys_msgctl(struct thread *td, struct msgctl_args *uap)
 {
 	int msqid = uap->msqid;
 	int cmd = uap->cmd;
@@ -507,7 +505,7 @@ kern_msgctl(td, msqid, cmd, msqbuf)
 	struct msqid_ds *msqbuf;
 {
 	int rval, error, msqix;
-	register struct msqid_kernel *msqkptr;
+	struct msqid_kernel *msqkptr;
 	struct prison *rpr;
 
 	rpr = msg_find_prison(td->td_ucred);
@@ -644,15 +642,13 @@ struct msgget_args {
 #endif
 
 int
-sys_msgget(td, uap)
-	struct thread *td;
-	register struct msgget_args *uap;
+sys_msgget(struct thread *td, struct msgget_args *uap)
 {
 	int msqid, error = 0;
 	int key = uap->key;
 	int msgflg = uap->msgflg;
 	struct ucred *cred = td->td_ucred;
-	register struct msqid_kernel *msqkptr = NULL;
+	struct msqid_kernel *msqkptr = NULL;
 
 	DPRINTF(("msgget(0x%x, 0%o)\n", key, msgflg));
 
@@ -764,23 +760,18 @@ sys_msgget(td, uap)
 #ifndef _SYS_SYSPROTO_H_
 struct msgsnd_args {
 	int	msqid;
-	const void	*msgp;
+	const void	*msgp;	/* XXX msgp is actually mtext. */
 	size_t	msgsz;
 	int	msgflg;
 };
 #endif
 int
-kern_msgsnd(td, msqid, msgp, msgsz, msgflg, mtype)
-	struct thread *td;
-	int msqid;
-	const void *msgp;	/* XXX msgp is actually mtext. */
-	size_t msgsz;
-	int msgflg;
-	long mtype;
+kern_msgsnd(struct thread *td, int msqid, const void *msgp,
+    size_t msgsz, int msgflg, long mtype)
 {
 	int msqix, segs_needed, error = 0;
-	register struct msqid_kernel *msqkptr;
-	register struct msg *msghdr;
+	struct msqid_kernel *msqkptr;
+	struct msg *msghdr;
 	struct prison *rpr;
 	short next;
 #ifdef RACCT
@@ -1117,9 +1108,7 @@ kern_msgsnd(td, msqid, msgp, msgsz, msgflg, mtype)
 }
 
 int
-sys_msgsnd(td, uap)
-	struct thread *td;
-	register struct msgsnd_args *uap;
+sys_msgsnd(struct thread *td, struct msgsnd_args *uap)
 {
 	int error;
 	long mtype;
@@ -1145,19 +1134,14 @@ struct msgrcv_args {
 	int	msgflg;
 };
 #endif
+/* XXX msgp is actually mtext. */
 int
-kern_msgrcv(td, msqid, msgp, msgsz, msgtyp, msgflg, mtype)
-	struct thread *td;
-	int msqid;
-	void *msgp;	/* XXX msgp is actually mtext. */
-	size_t msgsz;
-	long msgtyp;
-	int msgflg;
-	long *mtype;
+kern_msgrcv(struct thread *td, int msqid, void *msgp, size_t msgsz, long msgtyp,
+    int msgflg, long *mtype)
 {
 	size_t len;
-	register struct msqid_kernel *msqkptr;
-	register struct msg *msghdr;
+	struct msqid_kernel *msqkptr;
+	struct msg *msghdr;
 	struct prison *rpr;
 	int msqix, error = 0;
 	short next;
@@ -1411,9 +1395,7 @@ kern_msgrcv(td, msqid, msgp, msgsz, msgtyp, msgflg, mtype)
 }
 
 int
-sys_msgrcv(td, uap)
-	struct thread *td;
-	register struct msgrcv_args *uap;
+sys_msgrcv(struct thread *td, struct msgrcv_args *uap)
 {
 	int error;
 	long mtype;
@@ -1812,19 +1794,19 @@ static sy_call_t *msgcalls[] = {
 
 /*
  * Entry point for all MSG calls.
+ *
+ * XXX actually varargs.
+ * struct msgsys_args {
+ *		int	which;
+ *		int	a2;
+ *		int	a3;
+ *		int	a4;
+ *		int	a5;
+ *		int	a6;
+ *	} *uap;
  */
 int
-sys_msgsys(td, uap)
-	struct thread *td;
-	/* XXX actually varargs. */
-	struct msgsys_args /* {
-		int	which;
-		int	a2;
-		int	a3;
-		int	a4;
-		int	a5;
-		int	a6;
-	} */ *uap;
+sys_msgsys(struct thread *td, struct msgsys_args *uap)
 {
 	int error;
 
@@ -1847,9 +1829,7 @@ struct freebsd7_msgctl_args {
 };
 #endif
 int
-freebsd7_msgctl(td, uap)
-	struct thread *td;
-	struct freebsd7_msgctl_args *uap;
+freebsd7_msgctl(struct thread *td, struct freebsd7_msgctl_args *uap)
 {
 	struct msqid_ds_old msqold;
 	struct msqid_ds msqbuf;
diff --git a/sys/kern/vfs_export.c b/sys/kern/vfs_export.c
index 77a353b3cadb..03fa23e85e78 100644
--- a/sys/kern/vfs_export.c
+++ b/sys/kern/vfs_export.c
@@ -100,9 +100,9 @@ static int
 vfs_hang_addrlist(struct mount *mp, struct netexport *nep,
     struct export_args *argp)
 {
-	register struct netcred *np;
-	register struct radix_node_head *rnh;
-	register int i;
+	struct netcred *np;
+	struct radix_node_head *rnh;
+	int i;
 	struct radix_node *rn;
 	struct sockaddr *saddr, *smask = NULL;
 #if defined(INET6) || defined(INET)
@@ -448,8 +448,8 @@ static struct netcred *
 vfs_export_lookup(struct mount *mp, struct sockaddr *nam)
 {
 	struct netexport *nep;
-	register struct netcred *np;
-	register struct radix_node_head *rnh;
+	struct netcred *np;
+	struct radix_node_head *rnh;
 	struct sockaddr *saddr;
 
 	nep = mp->mnt_export;
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 34bf1aaf456c..117ad95438d2 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -366,14 +366,15 @@ vfs_mergeopts(struct vfsoptlist *toopts, struct vfsoptlist *oldopts)
 /*
  * Mount a filesystem.
  */
+#ifndef _SYS_SYSPROTO_H_
+struct nmount_args {
+	struct iovec *iovp;
+	unsigned int iovcnt;
+	int flags;
+};
+#endif
 int
-sys_nmount(td, uap)
-	struct thread *td;
-	struct nmount_args /* {
-		struct iovec *iovp;
-		unsigned int iovcnt;
-		int flags;
-	} */ *uap;
+sys_nmount(struct thread *td, struct nmount_args *uap)
 {
 	struct uio *auio;
 	int error;
@@ -706,14 +707,7 @@ struct mount_args {
 #endif
 /* ARGSUSED */
 int
-sys_mount(td, uap)
-	struct thread *td;
-	struct mount_args /* {
-		char *type;
-		char *path;
-		int flags;
-		caddr_t data;
-	} */ *uap;
+sys_mount(struct thread *td, struct mount_args *uap)
 {
 	char *fstype;
 	struct vfsconf *vfsp = NULL;
@@ -1544,11 +1538,7 @@ vfs_filteropt(struct vfsoptlist *opts, const char **legal)
  * with the address of the option.
  */
 int
-vfs_getopt(opts, name, buf, len)
-	struct vfsoptlist *opts;
-	const char *name;
-	void **buf;
-	int *len;
+vfs_getopt(struct vfsoptlist *opts, const char *name, void **buf, int *len)
 {
 	struct vfsopt *opt;
 
@@ -1761,11 +1751,7 @@ vfs_setopts(struct vfsoptlist *opts, const char *name, const char *value)
  * Returns ENOENT if the option is not found.
  */
 int
-vfs_copyopt(opts, name, dest, len)
-	struct vfsoptlist *opts;
-	const char *name;
-	void *dest;
-	int len;
+vfs_copyopt(struct vfsoptlist *opts, const char *name, void *dest, int len)
 {
 	struct vfsopt *opt;
 
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 7bb9e378576d..7671e1b84ac2 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -115,9 +115,7 @@ struct sync_args {
 #endif
 /* ARGSUSED */
 int
-sys_sync(td, uap)
-	struct thread *td;
-	struct sync_args *uap;
+sys_sync(struct thread *td, struct sync_args *uap)
 {
 	struct mount *mp, *nmp;
 	int save;
@@ -156,14 +154,7 @@ struct quotactl_args {
 };
 #endif
 int
-sys_quotactl(td, uap)
-	struct thread *td;
-	register struct quotactl_args /* {
-		char *path;
-		int cmd;
-		int uid;
-		caddr_t arg;
-	} */ *uap;
+sys_quotactl(struct thread *td, struct quotactl_args *uap)
 {
 	struct mount *mp;
 	struct nameidata nd;
@@ -291,12 +282,7 @@ struct statfs_args {
 };
 #endif
 int
-sys_statfs(td, uap)
-	struct thread *td;
-	register struct statfs_args /* {
-		char *path;
-		struct statfs *buf;
-	} */ *uap;
+sys_statfs(struct thread *td, struct statfs_args *uap)
 {
 	struct statfs *sfp;
 	int error;
@@ -339,12 +325,7 @@ struct fstatfs_args {
 };
 #endif
 int
-sys_fstatfs(td, uap)
-	struct thread *td;
-	register struct fstatfs_args /* {
-		int fd;
-		struct statfs *buf;
-	} */ *uap;
+sys_fstatfs(struct thread *td, struct fstatfs_args *uap)
 {
 	struct statfs *sfp;
 	int error;
@@ -394,13 +375,7 @@ struct getfsstat_args {
 };
 #endif
 int
-sys_getfsstat(td, uap)
-	struct thread *td;
-	register struct getfsstat_args /* {
-		struct statfs *buf;
-		long bufsize;
-		int mode;
-	} */ *uap;
+sys_getfsstat(struct thread *td, struct getfsstat_args *uap)
 {
 	size_t count;
 	int error;
@@ -559,12 +534,7 @@ struct freebsd4_statfs_args {
 };
 #endif
 int
-freebsd4_statfs(td, uap)
-	struct thread *td;
-	struct freebsd4_statfs_args /* {
-		char *path;
-		struct ostatfs *buf;
-	} */ *uap;
+freebsd4_statfs(struct thread *td, struct freebsd4_statfs_args *uap)
 {
 	struct ostatfs osb;
 	struct statfs *sfp;
@@ -590,12 +560,7 @@ struct freebsd4_fstatfs_args {
 };
 #endif
 int
-freebsd4_fstatfs(td, uap)
-	struct thread *td;
-	struct freebsd4_fstatfs_args /* {
-		int fd;
-		struct ostatfs *buf;
-	} */ *uap;
+freebsd4_fstatfs(struct thread *td, struct freebsd4_fstatfs_args *uap)
 {
 	struct ostatfs osb;
 	struct statfs *sfp;
@@ -622,13 +587,7 @@ struct freebsd4_getfsstat_args {
 };
 #endif
 int
-freebsd4_getfsstat(td, uap)
-	struct thread *td;
-	register struct freebsd4_getfsstat_args /* {
-		struct ostatfs *buf;
-		long bufsize;
-		int mode;
-	} */ *uap;
+freebsd4_getfsstat(struct thread *td, struct freebsd4_getfsstat_args *uap)
 {
 	struct statfs *buf, *sp;
 	struct ostatfs osb;
@@ -668,12 +627,7 @@ struct freebsd4_fhstatfs_args {
 };
 #endif
 int
-freebsd4_fhstatfs(td, uap)
-	struct thread *td;
-	struct freebsd4_fhstatfs_args /* {
-		struct fhandle *u_fhp;
-		struct ostatfs *buf;
-	} */ *uap;
+freebsd4_fhstatfs(struct thread *td, struct freebsd4_fhstatfs_args *uap)
 {
 	struct ostatfs osb;
 	struct statfs *sfp;
@@ -697,9 +651,7 @@ freebsd4_fhstatfs(td, uap)
  * Convert a new format statfs structure to an old format statfs structure.
  */
 static void
-cvtstatfs(nsp, osp)
-	struct statfs *nsp;
-	struct ostatfs *osp;
+cvtstatfs(struct statfs *nsp, struct ostatfs *osp)
 {
 
 	statfs_scale_blocks(nsp, LONG_MAX);
@@ -737,11 +689,7 @@ struct fchdir_args {
 };
 #endif
 int
-sys_fchdir(td, uap)
-	struct thread *td;
-	struct fchdir_args /* {
-		int fd;
-	} */ *uap;
+sys_fchdir(struct thread *td, struct fchdir_args *uap)
 {
 	struct vnode *vp, *tdp;
 	struct mount *mp;
@@ -788,11 +736,7 @@ struct chdir_args {
 };
 #endif
 int
-sys_chdir(td, uap)
-	struct thread *td;
-	struct chdir_args /* {
-		char *path;
-	} */ *uap;
+sys_chdir(struct thread *td, struct chdir_args *uap)
 {
 
 	return (kern_chdir(td, uap->path, UIO_USERSPACE));
@@ -828,11 +772,7 @@ struct chroot_args {
 };
 #endif
 int
-sys_chroot(td, uap)
-	struct thread *td;
-	struct chroot_args /* {
-		char *path;
-	} */ *uap;
+sys_chroot(struct thread *td, struct chroot_args *uap)
 {
 	struct nameidata nd;
 	int error;
@@ -870,9 +810,7 @@ sys_chroot(td, uap)
  * instance.
  */
 int
-change_dir(vp, td)
-	struct vnode *vp;
-	struct thread *td;
+change_dir(struct vnode *vp, struct thread *td)
 {
 #ifdef MAC
 	int error;
@@ -936,13 +874,7 @@ struct open_args {
 };
 #endif
 int
-sys_open(td, uap)
-	struct thread *td;
-	register struct open_args /* {
-		char *path;
-		int flags;
-		int mode;
-	} */ *uap;
+sys_open(struct thread *td, struct open_args *uap)
 {
 
 	return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -1116,12 +1048,7 @@ struct ocreat_args {
 };
 #endif
 int
-ocreat(td, uap)
-	struct thread *td;
-	register struct ocreat_args /* {
-		char *path;
-		int mode;
-	} */ *uap;
+ocreat(struct thread *td, struct ocreat_args *uap)
 {
 
 	return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -1140,13 +1067,7 @@ struct mknod_args {
 };
 #endif
 int
-sys_mknod(td, uap)
-	struct thread *td;
-	register struct mknod_args /* {
-		char *path;
-		int mode;
-		int dev;
-	} */ *uap;
+sys_mknod(struct thread *td, struct mknod_args *uap)
 {
 
 	return (kern_mknodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -1283,12 +1204,7 @@ struct mkfifo_args {
 };
 #endif
 int
-sys_mkfifo(td, uap)
-	struct thread *td;
-	register struct mkfifo_args /* {
-		char *path;
-		int mode;
-	} */ *uap;
+sys_mkfifo(struct thread *td, struct mkfifo_args *uap)
 {
 
 	return (kern_mkfifoat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -1375,12 +1291,7 @@ struct link_args {
 };
 #endif
 int
-sys_link(td, uap)
-	struct thread *td;
-	register struct link_args /* {
-		char *path;
-		char *link;
-	} */ *uap;
+sys_link(struct thread *td, struct link_args *uap)
 {
 
 	return (kern_linkat(td, AT_FDCWD, AT_FDCWD, uap->path, uap->link,
@@ -1544,12 +1455,7 @@ struct symlink_args {
 };
 #endif
 int
-sys_symlink(td, uap)
-	struct thread *td;
-	register struct symlink_args /* {
-		char *path;
-		char *link;
-	} */ *uap;
+sys_symlink(struct thread *td, struct symlink_args *uap)
 {
 
 	return (kern_symlinkat(td, uap->path, AT_FDCWD, uap->link,
@@ -1641,12 +1547,13 @@ kern_symlinkat(struct thread *td, char *path1, int fd, char *path2,
 /*
  * Delete a whiteout from the filesystem.
  */
+#ifndef _SYS_SYSPROTO_H_
+struct undelete_args {
+	char *path;
+};
+#endif
 int
-sys_undelete(td, uap)
-	struct thread *td;
-	register struct undelete_args /* {
-		char *path;
-	} */ *uap;
+sys_undelete(struct thread *td, struct undelete_args *uap)
 {
 	struct mount *mp;
 	struct nameidata nd;
@@ -1693,11 +1600,7 @@ struct unlink_args {
 };
 #endif
 int
-sys_unlink(td, uap)
-	struct thread *td;
-	struct unlink_args /* {
-		char *path;
-	} */ *uap;
+sys_unlink(struct thread *td, struct unlink_args *uap)
 {
 
 	return (kern_unlinkat(td, AT_FDCWD, uap->path, UIO_USERSPACE, 0));
@@ -1862,11 +1765,8 @@ freebsd6_lseek(struct thread *td, struct freebsd6_lseek_args *uap)
  * Check access permissions using passed credentials.
  */
 static int
-vn_access(vp, user_flags, cred, td)
-	struct vnode	*vp;
-	int		user_flags;
-	struct ucred	*cred;
-	struct thread	*td;
+vn_access(struct vnode *vp, int user_flags, struct ucred *cred,
+     struct thread *td)
 {
 	accmode_t accmode;
 	int error;
@@ -1902,12 +1802,7 @@ struct access_args {
 };
 #endif
 int
-sys_access(td, uap)
-	struct thread *td;
-	register struct access_args /* {
-		char *path;
-		int amode;
-	} */ *uap;
+sys_access(struct thread *td, struct access_args *uap)
 {
 
 	return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -1988,12 +1883,7 @@ struct eaccess_args {
 };
 #endif
 int
-sys_eaccess(td, uap)
-	struct thread *td;
-	register struct eaccess_args /* {
-		char *path;
-		int amode;
-	} */ *uap;
+sys_eaccess(struct thread *td, struct eaccess_args *uap)
 {
 
 	return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2011,12 +1901,7 @@ struct ostat_args {
 };
 #endif
 int
-ostat(td, uap)
-	struct thread *td;
-	register struct ostat_args /* {
-		char *path;
-		struct ostat *ub;
-	} */ *uap;
+ostat(struct thread *td, struct ostat_args *uap)
 {
 	struct stat sb;
 	struct ostat osb;
@@ -2040,12 +1925,7 @@ struct olstat_args {
 };
 #endif
 int
-olstat(td, uap)
-	struct thread *td;
-	register struct olstat_args /* {
-		char *path;
-		struct ostat *ub;
-	} */ *uap;
+olstat(struct thread *td, struct olstat_args *uap)
 {
 	struct stat sb;
 	struct ostat osb;
@@ -2063,9 +1943,7 @@ olstat(td, uap)
  * Convert from an old to a new stat structure.
  */
 void
-cvtstat(st, ost)
-	struct stat *st;
-	struct ostat *ost;
+cvtstat(struct stat *st, struct ostat *ost)
 {
 
 	bzero(ost, sizeof(*ost));
@@ -2100,12 +1978,7 @@ struct stat_args {
 };
 #endif
 int
-sys_stat(td, uap)
-	struct thread *td;
-	register struct stat_args /* {
-		char *path;
-		struct stat *ub;
-	} */ *uap;
+sys_stat(struct thread *td, struct stat_args *uap)
 {
 	struct stat sb;
 	int error;
@@ -2187,12 +2060,7 @@ struct lstat_args {
 };
 #endif
 int
-sys_lstat(td, uap)
-	struct thread *td;
-	register struct lstat_args /* {
-		char *path;
-		struct stat *ub;
-	} */ *uap;
+sys_lstat(struct thread *td, struct lstat_args *uap)
 {
 	struct stat sb;
 	int error;
@@ -2208,9 +2076,7 @@ sys_lstat(td, uap)
  * Implementation of the NetBSD [l]stat() functions.
  */
 void
-cvtnstat(sb, nsb)
-	struct stat *sb;
-	struct nstat *nsb;
+cvtnstat( struct stat *sb, struct nstat *nsb)
 {
 
 	bzero(nsb, sizeof *nsb);
@@ -2239,12 +2105,7 @@ struct nstat_args {
 };
 #endif
 int
-sys_nstat(td, uap)
-	struct thread *td;
-	register struct nstat_args /* {
-		char *path;
-		struct nstat *ub;
-	} */ *uap;
+sys_nstat(struct thread *td, struct nstat_args *uap)
 {
 	struct stat sb;
 	struct nstat nsb;
@@ -2268,12 +2129,7 @@ struct lstat_args {
 };
 #endif
 int
-sys_nlstat(td, uap)
-	struct thread *td;
-	register struct nlstat_args /* {
-		char *path;
-		struct nstat *ub;
-	} */ *uap;
+sys_nlstat(struct thread *td, struct nlstat_args *uap)
 {
 	struct stat sb;
 	struct nstat nsb;
@@ -2297,12 +2153,7 @@ struct pathconf_args {
 };
 #endif
 int
-sys_pathconf(td, uap)
-	struct thread *td;
-	register struct pathconf_args /* {
-		char *path;
-		int name;
-	} */ *uap;
+sys_pathconf(struct thread *td, struct pathconf_args *uap)
 {
 
 	return (kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name, FOLLOW));
@@ -2315,12 +2166,7 @@ struct lpathconf_args {
 };
 #endif
 int
-sys_lpathconf(td, uap)
-	struct thread *td;
-	register struct lpathconf_args /* {
-		char *path;
-		int name;
-	} */ *uap;
+sys_lpathconf(struct thread *td, struct lpathconf_args *uap)
 {
 
 	return (kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name,
@@ -2356,13 +2202,7 @@ struct readlink_args {
 };
 #endif
 int
-sys_readlink(td, uap)
-	struct thread *td;
-	register struct readlink_args /* {
-		char *path;
-		char *buf;
-		size_t count;
-	} */ *uap;
+sys_readlink(struct thread *td, struct readlink_args *uap)
 {
 
 	return (kern_readlinkat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2434,10 +2274,7 @@ kern_readlinkat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
  * Common implementation code for chflags() and fchflags().
  */
 static int
-setfflags(td, vp, flags)
-	struct thread *td;
-	struct vnode *vp;
-	u_long flags;
+setfflags(struct thread *td, struct vnode *vp, u_long flags)
 {
 	struct mount *mp;
 	struct vattr vattr;
@@ -2484,12 +2321,7 @@ struct chflags_args {
 };
 #endif
 int
-sys_chflags(td, uap)
-	struct thread *td;
-	register struct chflags_args /* {
-		const char *path;
-		u_long flags;
-	} */ *uap;
+sys_chflags(struct thread *td, struct chflags_args *uap)
 {
 
 	return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2521,13 +2353,14 @@ sys_chflagsat(struct thread *td, struct chflagsat_args *uap)
 /*
  * Same as chflags() but doesn't follow symlinks.
  */
+#ifndef _SYS_SYSPROTO_H_
+struct lchflags_args {
+	const char *path;
+	u_long flags;
+};
+#endif
 int
-sys_lchflags(td, uap)
-	struct thread *td;
-	register struct lchflags_args /* {
-		const char *path;
-		u_long flags;
-	} */ *uap;
+sys_lchflags(struct thread *td, struct lchflags_args *uap)
 {
 
 	return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2564,12 +2397,7 @@ struct fchflags_args {
 };
 #endif
 int
-sys_fchflags(td, uap)
-	struct thread *td;
-	register struct fchflags_args /* {
-		int fd;
-		u_long flags;
-	} */ *uap;
+sys_fchflags(struct thread *td, struct fchflags_args *uap)
 {
 	struct file *fp;
 	cap_rights_t rights;
@@ -2595,11 +2423,7 @@ sys_fchflags(td, uap)
  * Common implementation code for chmod(), lchmod() and fchmod().
  */
 int
-setfmode(td, cred, vp, mode)
-	struct thread *td;
-	struct ucred *cred;
-	struct vnode *vp;
-	int mode;
+setfmode(struct thread *td, struct ucred *cred, struct vnode *vp, int mode)
 {
 	struct mount *mp;
 	struct vattr vattr;
@@ -2630,12 +2454,7 @@ struct chmod_args {
 };
 #endif
 int
-sys_chmod(td, uap)
-	struct thread *td;
-	register struct chmod_args /* {
-		char *path;
-		int mode;
-	} */ *uap;
+sys_chmod(struct thread *td, struct chmod_args *uap)
 {
 
 	return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2674,12 +2493,7 @@ struct lchmod_args {
 };
 #endif
 int
-sys_lchmod(td, uap)
-	struct thread *td;
-	register struct lchmod_args /* {
-		char *path;
-		int mode;
-	} */ *uap;
+sys_lchmod(struct thread *td, struct lchmod_args *uap)
 {
 
 	return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2737,12 +2551,8 @@ sys_fchmod(struct thread *td, struct fchmod_args *uap)
  * Common implementation for chown(), lchown(), and fchown()
  */
 int
-setfown(td, cred, vp, uid, gid)
-	struct thread *td;
-	struct ucred *cred;
-	struct vnode *vp;
-	uid_t uid;
-	gid_t gid;
+setfown(struct thread *td, struct ucred *cred, struct vnode *vp, uid_t uid,
+    gid_t gid)
 {
 	struct mount *mp;
 	struct vattr vattr;
@@ -2776,13 +2586,7 @@ struct chown_args {
 };
 #endif
 int
-sys_chown(td, uap)
-	struct thread *td;
-	register struct chown_args /* {
-		char *path;
-		int uid;
-		int gid;
-	} */ *uap;
+sys_chown(struct thread *td, struct chown_args *uap)
 {
 
 	return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE, uap->uid,
@@ -2843,13 +2647,7 @@ struct lchown_args {
 };
 #endif
 int
-sys_lchown(td, uap)
-	struct thread *td;
-	register struct lchown_args /* {
-		char *path;
-		int uid;
-		int gid;
-	} */ *uap;
+sys_lchown(struct thread *td, struct lchown_args *uap)
 {
 
 	return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -2867,13 +2665,7 @@ struct fchown_args {
 };
 #endif
 int
-sys_fchown(td, uap)
-	struct thread *td;
-	register struct fchown_args /* {
-		int fd;
-		int uid;
-		int gid;
-	} */ *uap;
+sys_fchown(struct thread *td, struct fchown_args *uap)
 {
 	struct file *fp;
 	cap_rights_t rights;
@@ -2893,10 +2685,8 @@ sys_fchown(td, uap)
  * Common implementation code for utimes(), lutimes(), and futimes().
  */
 static int
-getutimes(usrtvp, tvpseg, tsp)
-	const struct timeval *usrtvp;
-	enum uio_seg tvpseg;
-	struct timespec *tsp;
+getutimes(const struct timeval *usrtvp, enum uio_seg tvpseg,
+    struct timespec *tsp)
 {
 	struct timeval tv[2];
 	const struct timeval *tvp;
@@ -2973,12 +2763,8 @@ getutimens(const struct timespec *usrtsp, enum uio_seg tspseg,
  * and utimensat().
  */
 static int
-setutimes(td, vp, ts, numtimes, nullflag)
-	struct thread *td;
-	struct vnode *vp;
-	const struct timespec *ts;
-	int numtimes;
-	int nullflag;
+setutimes(struct thread *td, struct vnode *vp, const struct timespec *ts,
+    int numtimes, int nullflag)
 {
 	struct mount *mp;
 	struct vattr vattr;
@@ -3021,12 +2807,7 @@ struct utimes_args {
 };
 #endif
 int
-sys_utimes(td, uap)
-	struct thread *td;
-	register struct utimes_args /* {
-		char *path;
-		struct timeval *tptr;
-	} */ *uap;
+sys_utimes(struct thread *td, struct utimes_args *uap)
 {
 
 	return (kern_utimesat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -3080,12 +2861,7 @@ struct lutimes_args {
 };
 #endif
 int
-sys_lutimes(td, uap)
-	struct thread *td;
-	register struct lutimes_args /* {
-		char *path;
-		struct timeval *tptr;
-	} */ *uap;
+sys_lutimes(struct thread *td, struct lutimes_args *uap)
 {
 
 	return (kern_lutimes(td, uap->path, UIO_USERSPACE, uap->tptr,
@@ -3121,12 +2897,7 @@ struct futimes_args {
 };
 #endif
 int
-sys_futimes(td, uap)
-	struct thread *td;
-	register struct futimes_args /* {
-		int  fd;
-		struct timeval *tptr;
-	} */ *uap;
+sys_futimes(struct thread *td, struct futimes_args *uap)
 {
 
 	return (kern_futimes(td, uap->fd, uap->tptr, UIO_USERSPACE));
@@ -3244,13 +3015,7 @@ struct truncate_args {
 };
 #endif
 int
-sys_truncate(td, uap)
-	struct thread *td;
-	register struct truncate_args /* {
-		char *path;
-		int pad;
-		off_t length;
-	} */ *uap;
+sys_truncate(struct thread *td, struct truncate_args *uap)
 {
 
 	return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
@@ -3409,12 +3174,7 @@ struct rename_args {
 };
 #endif
 int
-sys_rename(td, uap)
-	struct thread *td;
-	register struct rename_args /* {
-		char *from;
-		char *to;
-	} */ *uap;
+sys_rename(struct thread *td, struct rename_args *uap)
 {
 
 	return (kern_renameat(td, AT_FDCWD, uap->from, AT_FDCWD,
@@ -3579,12 +3339,7 @@ struct mkdir_args {
 };
 #endif
 int
-sys_mkdir(td, uap)
-	struct thread *td;
-	register struct mkdir_args /* {
-		char *path;
-		int mode;
-	} */ *uap;
+sys_mkdir(struct thread *td, struct mkdir_args *uap)
 {
 
 	return (kern_mkdirat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
@@ -3677,11 +3432,7 @@ struct rmdir_args {
 };
 #endif
 int
-sys_rmdir(td, uap)
-	struct thread *td;
-	struct rmdir_args /* {
-		char *path;
-	} */ *uap;
+sys_rmdir(struct thread *td, struct rmdir_args *uap)
 {
 
 	return (kern_rmdirat(td, AT_FDCWD, uap->path, UIO_USERSPACE));
@@ -3918,14 +3669,7 @@ struct getdirentries_args {
 };
 #endif
 int
-sys_getdirentries(td, uap)
-	struct thread *td;
-	register struct getdirentries_args /* {
-		int fd;
-		char *buf;
-		u_int count;
-		long *basep;
-	} */ *uap;
+sys_getdirentries(struct thread *td, struct getdirentries_args *uap)
 {
 	long base;
 	int error;
@@ -4023,13 +3767,7 @@ struct getdents_args {
 };
 #endif
 int
-sys_getdents(td, uap)
-	struct thread *td;
-	register struct getdents_args /* {
-		int fd;
-		char *buf;
-		u_int count;
-	} */ *uap;
+sys_getdents(struct thread *td, struct getdents_args *uap)
 {
 	struct getdirentries_args ap;
 
@@ -4049,11 +3787,7 @@ struct umask_args {
 };
 #endif
 int
-sys_umask(td, uap)
-	struct thread *td;
-	struct umask_args /* {
-		int newmask;
-	} */ *uap;
+sys_umask(struct thread *td, struct umask_args *uap)
 {
 	struct filedesc *fdp;
 
@@ -4075,11 +3809,7 @@ struct revoke_args {
 };
 #endif
 int
-sys_revoke(td, uap)
-	struct thread *td;
-	register struct revoke_args /* {
-		char *path;
-	} */ *uap;
+sys_revoke(struct thread *td, struct revoke_args *uap)
 {
 	struct vnode *vp;
 	struct vattr vattr;
@@ -4162,13 +3892,11 @@ struct lgetfh_args {
 };
 #endif
 int
-sys_lgetfh(td, uap)
-	struct thread *td;
-	register struct lgetfh_args *uap;
+sys_lgetfh(struct thread *td, struct lgetfh_args *uap)
 {
 	struct nameidata nd;
 	fhandle_t fh;
-	register struct vnode *vp;
+	struct vnode *vp;
 	int error;
 
 	error = priv_check(td, PRIV_VFS_GETFH);
@@ -4197,13 +3925,11 @@ struct getfh_args {
 };
 #endif
 int
-sys_getfh(td, uap)
-	struct thread *td;
-	register struct getfh_args *uap;
+sys_getfh(struct thread *td, struct getfh_args *uap)
 {
 	struct nameidata nd;
 	fhandle_t fh;
-	register struct vnode *vp;
+	struct vnode *vp;
 	int error;
 
 	error = priv_check(td, PRIV_VFS_GETFH);
@@ -4239,12 +3965,7 @@ struct fhopen_args {
 };
 #endif
 int
-sys_fhopen(td, uap)
-	struct thread *td;
-	struct fhopen_args /* {
-		const struct fhandle *u_fhp;
-		int flags;
-	} */ *uap;
+sys_fhopen(struct thread *td, struct fhopen_args *uap)
 {
 	struct mount *mp;
 	struct vnode *vp;
@@ -4328,12 +4049,7 @@ struct fhstat_args {
 };
 #endif
 int
-sys_fhstat(td, uap)
-	struct thread *td;
-	register struct fhstat_args /* {
-		struct fhandle *u_fhp;
-		struct stat *sb;
-	} */ *uap;
+sys_fhstat(struct thread *td, struct fhstat_args *uap)
 {
 	struct stat sb;
 	struct fhandle fh;
@@ -4379,12 +4095,7 @@ struct fhstatfs_args {
 };
 #endif
 int
-sys_fhstatfs(td, uap)
-	struct thread *td;
-	struct fhstatfs_args /* {
-		struct fhandle *u_fhp;
-		struct statfs *buf;
-	} */ *uap;
+sys_fhstatfs(struct thread *td, struct fhstatfs_args *uap)
 {
 	struct statfs *sfp;
 	fhandle_t fh;
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 1329dc32fa04..d041c07920de 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -411,8 +411,7 @@ vn_open_vnode(struct vnode *vp, int fmode, struct ucred *cred,
  * Prototype text segments cannot be written.
  */
 int
-vn_writechk(vp)
-	register struct vnode *vp;
+vn_writechk(struct vnode *vp)
 {
 
 	ASSERT_VOP_LOCKED(vp, "vn_writechk");
@@ -1368,15 +1367,11 @@ vn_statfile(fp, sb, active_cred, td)
  * Stat a vnode; implementation for the stat syscall
  */
 int
-vn_stat(vp, sb, active_cred, file_cred, td)
-	struct vnode *vp;
-	register struct stat *sb;
-	struct ucred *active_cred;
-	struct ucred *file_cred;
-	struct thread *td;
+vn_stat(struct vnode *vp, struct stat *sb, struct ucred *active_cred,
+    struct ucred *file_cred, struct thread *td)
 {
 	struct vattr vattr;
-	register struct vattr *vap;
+	struct vattr *vap;
 	int error;
 	u_short mode;
 
@@ -1479,12 +1474,8 @@ vn_stat(vp, sb, active_cred, file_cred, td)
  * File table vnode ioctl routine.
  */
 static int
-vn_ioctl(fp, com, data, active_cred, td)
-	struct file *fp;
-	u_long com;
-	void *data;
-	struct ucred *active_cred;
-	struct thread *td;
+vn_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
+    struct thread *td)
 {
 	struct vattr vattr;
 	struct vnode *vp;
@@ -1522,11 +1513,8 @@ vn_ioctl(fp, com, data, active_cred, td)
  * File table vnode poll routine.
  */
 static int
-vn_poll(fp, events, active_cred, td)
-	struct file *fp;
-	int events;
-	struct ucred *active_cred;
-	struct thread *td;
+vn_poll(struct file *fp, int events, struct ucred *active_cred,
+    struct thread *td)
 {
 	struct vnode *vp;
 	int error;
@@ -1771,8 +1759,7 @@ vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
  * now in effect.
  */
 void
-vn_finished_write(mp)
-	struct mount *mp;
+vn_finished_write(struct mount *mp)
 {
 	if (mp == NULL || !vn_suspendable(mp))
 		return;
@@ -1794,8 +1781,7 @@ vn_finished_write(mp)
  * that the suspension is now in effect.
  */
 void
-vn_finished_secondary_write(mp)
-	struct mount *mp;
+vn_finished_secondary_write(struct mount *mp)
 {
 	if (mp == NULL || !vn_suspendable(mp))
 		return;
diff --git a/sys/libkern/zlib.c b/sys/libkern/zlib.c
index f5c3854800e3..7b1fe99387c3 100644
--- a/sys/libkern/zlib.c
+++ b/sys/libkern/zlib.c
@@ -1282,9 +1282,9 @@ local uInt longest_match(s, cur_match)
     IPos cur_match;                             /* current match */
 {
     unsigned chain_length = s->max_chain_length;/* max hash chain length */
-    register Bytef *scan = s->window + s->strstart; /* current string */
-    register Bytef *match;                       /* matched string */
-    register int len;                           /* length of current match */
+    Bytef *scan = s->window + s->strstart;      /* current string */
+    Bytef *match;                               /* matched string */
+    int len;                                    /* length of current match */
     int best_len = s->prev_length;              /* best match length so far */
     int nice_match = s->nice_match;             /* stop if match long enough */
     IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
@@ -1299,13 +1299,13 @@ local uInt longest_match(s, cur_match)
     /* Compare two bytes at a time. Note: this is not always beneficial.
      * Try with and without -DUNALIGNED_OK to check.
      */
-    register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1;
-    register ush scan_start = *(ushf*)scan;
-    register ush scan_end   = *(ushf*)(scan+best_len-1);
+    Bytef *strend = s->window + s->strstart + MAX_MATCH - 1;
+    ush scan_start = *(ushf*)scan;
+    ush scan_end   = *(ushf*)(scan+best_len-1);
 #else
-    register Bytef *strend = s->window + s->strstart + MAX_MATCH;
-    register Byte scan_end1  = scan[best_len-1];
-    register Byte scan_end   = scan[best_len];
+    Bytef *strend = s->window + s->strstart + MAX_MATCH;
+    Byte scan_end1  = scan[best_len-1];
+    Byte scan_end   = scan[best_len];
 #endif
 
     /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16.
@@ -1457,8 +1457,8 @@ local void check_match(s, start, match, length)
 local void fill_window(s)
     deflate_state *s;
 {
-    register unsigned n, m;
-    register Posf *p;
+    unsigned n, m;
+    Posf *p;
     unsigned more;    /* Amount of free space at the end of the window. */
     uInt wsize = s->w_size;
 
@@ -2923,7 +2923,7 @@ local unsigned bi_reverse(code, len)
     unsigned code; /* the value to invert */
     int len;       /* its bit length */
 {
-    register unsigned res = 0;
+    unsigned res = 0;
     do {
         res |= code & 1;
         code >>= 1, res <<= 1;
@@ -4222,16 +4222,16 @@ z_streamp zs;           /* for zalloc function */
   uInt f;                       /* i repeats in table every f entries */
   int g;                        /* maximum code length */
   int h;                        /* table level */
-  register uInt i;              /* counter, current code */
-  register uInt j;              /* counter */
-  register int k;               /* number of bits in current code */
+  uInt i;                       /* counter, current code */
+  uInt j;                       /* counter */
+  int k;                        /* number of bits in current code */
   int l;                        /* bits per table (returned in m) */
-  register uIntf *p;            /* pointer into c[], b[], or v[] */
+  uIntf *p;                     /* pointer into c[], b[], or v[] */
   inflate_huft *q;              /* points to current table */
   struct inflate_huft_s r;      /* table entry for structure assignment */
   inflate_huft *u[BMAX];        /* table stack */
   uInt v[N_MAX];                /* values in order of bit length */
-  register int w;               /* bits before this table == (l * h) */
+  int w;                        /* bits before this table == (l * h) */
   uInt x[BMAX+1];               /* bit offsets, then code stack */
   uIntf *xp;                    /* pointer into x */
   int y;                        /* number of dummy codes added */
@@ -4561,7 +4561,7 @@ z_streamp z;            /* for zfree function */
    list of the tables it made, with the links in a dummy first entry of
    each table. */
 {
-  register inflate_huft *p, *q, *r;
+  inflate_huft *p, *q, *r;
 
   /* Reverse linked list */
   p = Z_NULL;
diff --git a/sys/mips/cavium/cryptocteon/cavium_crypto.c b/sys/mips/cavium/cryptocteon/cavium_crypto.c
index ea01b8dc95aa..0ebab4fb3de1 100644
--- a/sys/mips/cavium/cryptocteon/cavium_crypto.c
+++ b/sys/mips/cavium/cryptocteon/cavium_crypto.c
@@ -587,7 +587,7 @@ octo_null_md5_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     uint64_t *data;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
@@ -689,7 +689,7 @@ octo_null_sha1_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     uint64_t *data;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
@@ -794,7 +794,7 @@ octo_des_cbc_md5_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -942,7 +942,7 @@ octo_des_cbc_md5_decrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1093,7 +1093,7 @@ octo_des_cbc_sha1_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1244,7 +1244,7 @@ octo_des_cbc_sha1_decrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1397,7 +1397,7 @@ octo_aes_cbc_md5_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1574,7 +1574,7 @@ octo_aes_cbc_md5_decrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1750,7 +1750,7 @@ octo_aes_cbc_sha1_encrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
@@ -1946,7 +1946,7 @@ octo_aes_cbc_sha1_decrypt(
     int crypt_off, int crypt_len,
     int icv_off, uint8_t *ivp)
 {
-    register int next = 0;
+    int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
diff --git a/sys/mips/mips/vm_machdep.c b/sys/mips/mips/vm_machdep.c
index 80b7c5989bd6..2e1c06e32bea 100644
--- a/sys/mips/mips/vm_machdep.c
+++ b/sys/mips/mips/vm_machdep.c
@@ -98,10 +98,9 @@ __FBSDID("$FreeBSD$");
  * ready to run and return to user mode.
  */
 void
-cpu_fork(register struct thread *td1,register struct proc *p2,
-    struct thread *td2,int flags)
+cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2,int flags)
 {
-	register struct proc *p1;
+	struct proc *p1;
 	struct pcb *pcb2;
 
 	p1 = td1->td_proc;
diff --git a/sys/net/altq/altq_rio.c b/sys/net/altq/altq_rio.c
index 7687701c2f94..d229140daddf 100644
--- a/sys/net/altq/altq_rio.c
+++ b/sys/net/altq/altq_rio.c
@@ -150,7 +150,7 @@
 #define	RIO_STATS		/* collect statistics */
 
 #define	TV_DELTA(a, b, delta) {					\
-	register int	xxs;					\
+	int	xxs;						\
 								\
 	delta = (a)->tv_usec - (b)->tv_usec; 			\
 	if ((xxs = (a)->tv_sec - (b)->tv_sec) != 0) { 		\
diff --git a/sys/net/altq/altq_rmclass.h b/sys/net/altq/altq_rmclass.h
index 6130c4ff3eee..268ed63f433e 100644
--- a/sys/net/altq/altq_rmclass.h
+++ b/sys/net/altq/altq_rmclass.h
@@ -77,7 +77,7 @@ struct red;
 	(((a)->tv_usec < (b)->tv_usec) && ((a)->tv_sec <= (b)->tv_sec)))
 
 #define	TV_DELTA(a, b, delta) { \
-	register int	xxs;	\
+	int	xxs;	\
 							\
 	delta = (a)->tv_usec - (b)->tv_usec; \
 	if ((xxs = (a)->tv_sec - (b)->tv_sec)) { \
@@ -98,7 +98,7 @@ struct red;
 }
 
 #define	TV_ADD_DELTA(a, delta, res) { \
-	register int xxus = (a)->tv_usec + (delta); \
+	int xxus = (a)->tv_usec + (delta); \
 	\
 	(res)->tv_sec = (a)->tv_sec; \
 	while (xxus >= 1000000) { \
diff --git a/sys/net/bpf_filter.c b/sys/net/bpf_filter.c
index 4948fb2d386f..8549428eecdc 100644
--- a/sys/net/bpf_filter.c
+++ b/sys/net/bpf_filter.c
@@ -74,7 +74,7 @@ __FBSDID("$FreeBSD$");
 #ifdef _KERNEL
 #define MINDEX(m, k) \
 { \
-	register int len = m->m_len; \
+	int len = m->m_len; \
  \
 	while (k >= len) { \
 		k -= len; \
@@ -341,7 +341,7 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
 			k = pc->k;
 			if (k >= buflen) {
 #ifdef _KERNEL
-				register struct mbuf *m;
+				struct mbuf *m;
 
 				if (buflen != 0)
 					return (0);
@@ -549,8 +549,8 @@ static const u_short	bpf_code_map[] = {
 int
 bpf_validate(const struct bpf_insn *f, int len)
 {
-	register int i;
-	register const struct bpf_insn *p;
+	int i;
+	const struct bpf_insn *p;
 
 	/* Do not accept negative length filter. */
 	if (len < 0)
@@ -572,7 +572,7 @@ bpf_validate(const struct bpf_insn *f, int len)
 		 * the code block.
 		 */
 		if (BPF_CLASS(p->code) == BPF_JMP) {
-			register u_int offset;
+			u_int offset;
 
 			if (p->code == (BPF_JMP|BPF_JA))
 				offset = p->k;
diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c
index a9b8c6d33d8c..feffe6c9547f 100644
--- a/sys/net/if_llatbl.c
+++ b/sys/net/if_llatbl.c
@@ -534,7 +534,7 @@ lltable_drain(int af)
 {
 	struct lltable	*llt;
 	struct llentry	*lle;
-	register int i;
+	int i;
 
 	LLTABLE_LIST_RLOCK();
 	SLIST_FOREACH(llt, &V_lltables, llt_link) {
diff --git a/sys/net/if_media.c b/sys/net/if_media.c
index 9834d525befa..ab77987218f8 100644
--- a/sys/net/if_media.c
+++ b/sys/net/if_media.c
@@ -121,7 +121,7 @@ ifmedia_add(ifm, mword, data, aux)
 	int data;
 	void *aux;
 {
-	register struct ifmedia_entry *entry;
+	struct ifmedia_entry *entry;
 
 #ifdef IFMEDIA_DEBUG
 	if (ifmedia_debug) {
diff --git a/sys/net/slcompress.c b/sys/net/slcompress.c
index bfaefdc104ec..63204b86c96a 100644
--- a/sys/net/slcompress.c
+++ b/sys/net/slcompress.c
@@ -60,12 +60,10 @@
 #define BCOPY(p1, p2, n) bcopy((void *)(p1), (void *)(p2), (int)(n))
 
 void
-sl_compress_init(comp, max_state)
-	struct slcompress *comp;
-	int max_state;
+sl_compress_init(struct slcompress *comp, int max_state)
 {
-	register u_int i;
-	register struct cstate *tstate = comp->tstate;
+	u_int i;
+	struct cstate *tstate = comp->tstate;
 
 	if (max_state == -1) {
 		max_state = MAX_STATES - 1;
@@ -152,20 +150,17 @@ sl_compress_init(comp, max_state)
  * if m is an M_PKTHDR mbuf.
  */
 u_int
-sl_compress_tcp(m, ip, comp, compress_cid)
-	struct mbuf *m;
-	register struct ip *ip;
-	struct slcompress *comp;
-	int compress_cid;
+sl_compress_tcp(struct mbuf *m, struct ip *ip, struct slcompress *comp,
+    int compress_cid)
 {
-	register struct cstate *cs = comp->last_cs->cs_next;
-	register u_int hlen = ip->ip_hl;
-	register struct tcphdr *oth;
-	register struct tcphdr *th;
-	register u_int deltaS, deltaA;
-	register u_int changes = 0;
+	struct cstate *cs = comp->last_cs->cs_next;
+	u_int hlen = ip->ip_hl;
+	struct tcphdr *oth;
+	struct tcphdr *th;
+	u_int deltaS, deltaA;
+	u_int changes = 0;
 	u_char new_seq[16];
-	register u_char *cp = new_seq;
+	u_char *cp = new_seq;
 
 	/*
 	 * Bail if this is an IP fragment or if the TCP packet isn't
@@ -202,8 +197,8 @@ sl_compress_tcp(m, ip, comp, compress_cid)
 		 * states via linear search.  If we don't find a state
 		 * for the datagram, the oldest state is (re-)used.
 		 */
-		register struct cstate *lcs;
-		register struct cstate *lastcs = comp->last_cs;
+		struct cstate *lcs;
+		struct cstate *lastcs = comp->last_cs;
 
 		do {
 			lcs = cs; cs = cs->cs_next;
@@ -412,11 +407,7 @@ sl_compress_tcp(m, ip, comp, compress_cid)
 
 
 int
-sl_uncompress_tcp(bufp, len, type, comp)
-	u_char **bufp;
-	int len;
-	u_int type;
-	struct slcompress *comp;
+sl_uncompress_tcp(u_char **bufp, int len, u_int type, struct slcompress *comp)
 {
 	u_char *hdr, *cp;
 	int hlen, vjlen;
@@ -460,21 +451,16 @@ sl_uncompress_tcp(bufp, len, type, comp)
  * in *hdrp and its length in *hlenp.
  */
 int
-sl_uncompress_tcp_core(buf, buflen, total_len, type, comp, hdrp, hlenp)
-	u_char *buf;
-	int buflen, total_len;
-	u_int type;
-	struct slcompress *comp;
-	u_char **hdrp;
-	u_int *hlenp;
+sl_uncompress_tcp_core(u_char *buf, int buflen, int total_len, u_int type,
+    struct slcompress *comp, u_char **hdrp, u_int *hlenp)
 {
-	register u_char *cp;
-	register u_int hlen, changes;
-	register struct tcphdr *th;
-	register struct cstate *cs;
-	register struct ip *ip;
-	register u_int16_t *bp;
-	register u_int vjlen;
+	u_char *cp;
+	u_int hlen, changes;
+	struct tcphdr *th;
+	struct cstate *cs;
+	struct ip *ip;
+	u_int16_t *bp;
+	u_int vjlen;
 
 	switch (type) {
 
@@ -542,7 +528,7 @@ sl_uncompress_tcp_core(buf, buflen, total_len, type, comp, hdrp, hlenp)
 	switch (changes & SPECIALS_MASK) {
 	case SPECIAL_I:
 		{
-		register u_int i = ntohs(cs->cs_ip.ip_len) - cs->cs_hlen;
+		u_int i = ntohs(cs->cs_ip.ip_len) - cs->cs_hlen;
 		th->th_ack = htonl(ntohl(th->th_ack) + i);
 		th->th_seq = htonl(ntohl(th->th_seq) + i);
 		}
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index e1611d50b4f0..00ae3a57327c 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -96,8 +96,8 @@ int
 in_localaddr(struct in_addr in)
 {
 	struct rm_priotracker in_ifa_tracker;
-	register u_long i = ntohl(in.s_addr);
-	register struct in_ifaddr *ia;
+	u_long i = ntohl(in.s_addr);
+	struct in_ifaddr *ia;
 
 	IN_IFADDR_RLOCK(&in_ifa_tracker);
 	TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
@@ -187,8 +187,8 @@ in_localip_more(struct in_ifaddr *ia)
 int
 in_canforward(struct in_addr in)
 {
-	register u_long i = ntohl(in.s_addr);
-	register u_long net;
+	u_long i = ntohl(in.s_addr);
+	u_long net;
 
 	if (IN_EXPERIMENTAL(i) || IN_MULTICAST(i) || IN_LINKLOCAL(i))
 		return (0);
@@ -206,8 +206,8 @@ in_canforward(struct in_addr in)
 static void
 in_socktrim(struct sockaddr_in *ap)
 {
-    register char *cplim = (char *) &ap->sin_addr;
-    register char *cp = (char *) (&ap->sin_addr + 1);
+    char *cplim = (char *) &ap->sin_addr;
+    char *cp = (char *) (&ap->sin_addr + 1);
 
     ap->sin_len = 0;
     while (--cp >= cplim)
@@ -962,7 +962,7 @@ in_ifaddr_broadcast(struct in_addr in, struct in_ifaddr *ia)
 int
 in_broadcast(struct in_addr in, struct ifnet *ifp)
 {
-	register struct ifaddr *ifa;
+	struct ifaddr *ifa;
 	int found;
 
 	if (in.s_addr == INADDR_BROADCAST ||
diff --git a/sys/netinet/in_cksum.c b/sys/netinet/in_cksum.c
index b4da4c8f7161..70061544b4ad 100644
--- a/sys/netinet/in_cksum.c
+++ b/sys/netinet/in_cksum.c
@@ -48,9 +48,9 @@ __FBSDID("$FreeBSD$");
 int
 in_cksum(struct mbuf *m, int len)
 {
-	register u_short *w;
-	register int sum = 0;
-	register int mlen = 0;
+	u_short *w;
+	int sum = 0;
+	int mlen = 0;
 	int byte_swapped = 0;
 
 	union {
diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c
index 46254ad548cb..9db8c2aa7f8d 100644
--- a/sys/netinet/ip_icmp.c
+++ b/sys/netinet/ip_icmp.c
@@ -185,10 +185,10 @@ kmod_icmpstat_inc(int statnum)
 void
 icmp_error(struct mbuf *n, int type, int code, uint32_t dest, int mtu)
 {
-	register struct ip *oip = mtod(n, struct ip *), *nip;
-	register unsigned oiphlen = oip->ip_hl << 2;
-	register struct icmp *icp;
-	register struct mbuf *m;
+	struct ip *oip = mtod(n, struct ip *), *nip;
+	unsigned oiphlen = oip->ip_hl << 2;
+	struct icmp *icp;
+	struct mbuf *m;
 	unsigned icmplen, icmpelen, nlen;
 
 	KASSERT((u_int)type <= ICMP_MAXTYPE, ("%s: illegal ICMP type", __func__));
@@ -814,7 +814,7 @@ icmp_reflect(struct mbuf *m)
 	ip->ip_ttl = V_ip_defttl;
 
 	if (optlen > 0) {
-		register u_char *cp;
+		u_char *cp;
 		int opt, cnt;
 		u_int len;
 
@@ -889,9 +889,9 @@ icmp_reflect(struct mbuf *m)
 static void
 icmp_send(struct mbuf *m, struct mbuf *opts)
 {
-	register struct ip *ip = mtod(m, struct ip *);
-	register int hlen;
-	register struct icmp *icp;
+	struct ip *ip = mtod(m, struct ip *);
+	int hlen;
+	struct icmp *icp;
 
 	hlen = ip->ip_hl << 2;
 	m->m_data += hlen;
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 9ba7946db468..13ea6aa9106c 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -113,7 +113,7 @@ static struct inpcb *in6_pcblookup_hash_locked(struct inpcbinfo *,
     struct in6_addr *, u_int, struct in6_addr *, u_int, int, struct ifnet *);
 
 int
-in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
+in6_pcbbind(struct inpcb *inp, struct sockaddr *nam,
     struct ucred *cred)
 {
 	struct socket *so = inp->inp_socket;
@@ -324,10 +324,10 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
  *   have forced minor changes in every protocol).
  */
 static int
-in6_pcbladdr(register struct inpcb *inp, struct sockaddr *nam,
+in6_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
     struct in6_addr *plocal_addr6)
 {
-	register struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
+	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
 	int error = 0;
 	int scope_ambiguous = 0;
 	struct in6_addr in6a;
@@ -389,11 +389,11 @@ in6_pcbladdr(register struct inpcb *inp, struct sockaddr *nam,
  * then pick one.
  */
 int
-in6_pcbconnect_mbuf(register struct inpcb *inp, struct sockaddr *nam,
+in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
     struct ucred *cred, struct mbuf *m)
 {
 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
-	register struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
+	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
 	struct in6_addr addr6;
 	int error;
 
@@ -494,7 +494,7 @@ in6_v4mapsin6_sockaddr(in_port_t port, struct in_addr *addr_p)
 int
 in6_getsockaddr(struct socket *so, struct sockaddr **nam)
 {
-	register struct inpcb *inp;
+	struct inpcb *inp;
 	struct in6_addr addr;
 	in_port_t port;
 
@@ -687,7 +687,7 @@ struct inpcb *
 in6_pcblookup_local(struct inpcbinfo *pcbinfo, struct in6_addr *laddr,
     u_short lport, int lookupflags, struct ucred *cred)
 {
-	register struct inpcb *inp;
+	struct inpcb *inp;
 	int matchwild = 3, wildcard;
 
 	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index d4ca8ab30a7b..62c55bc41efc 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -158,8 +158,8 @@ rip6_input(struct mbuf **mp, int *offp, int proto)
 {
 	struct ifnet *ifp;
 	struct mbuf *m = *mp;
-	register struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
-	register struct inpcb *in6p;
+	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
+	struct inpcb *in6p;
 	struct inpcb *last = NULL;
 	struct mbuf *opts = NULL;
 	struct sockaddr_in6 fromsa;
diff --git a/sys/netipsec/ipsec_mbuf.c b/sys/netipsec/ipsec_mbuf.c
index 6617d48af016..bf215acfbc78 100644
--- a/sys/netipsec/ipsec_mbuf.c
+++ b/sys/netipsec/ipsec_mbuf.c
@@ -168,8 +168,8 @@ m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
 caddr_t
 m_pad(struct mbuf *m, int n)
 {
-	register struct mbuf *m0, *m1;
-	register int len, pad;
+	struct mbuf *m0, *m1;
+	int len, pad;
 	caddr_t retval;
 
 	if (n <= 0) {  /* No stupid arguments. */
diff --git a/sys/rpc/clnt.h b/sys/rpc/clnt.h
index b9a5c6b213d9..8f593b6135ba 100644
--- a/sys/rpc/clnt.h
+++ b/sys/rpc/clnt.h
@@ -529,7 +529,7 @@ extern CLIENT *clnt_tli_create(const int, const struct netconfig *,
 			       struct netbuf *, const rpcprog_t,
 			       const rpcvers_t, const u_int, const u_int);
 /*
- *	const register int fd;		-- fd
+ *	const int fd;			-- fd
  *	const struct netconfig *nconf;	-- netconfig structure
  *	struct netbuf *svcaddr;		-- servers address
  *	const u_long prog;			-- program number
diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h
index c54c45b2443f..bf3d4afbdb8f 100644
--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@@ -71,8 +71,8 @@ extern void *dpcpu0;
 struct pcb;
 struct pcpu;
 
-register struct pcb *curpcb __asm__(__XSTRING(PCB_REG));
-register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
+struct pcb *curpcb __asm__(__XSTRING(PCB_REG));
+struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
 
 #define	get_pcpu()		(pcpup)
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)

From 43af5860113f52d7d6c49cd5f7b9a5dd9b321d28 Mon Sep 17 00:00:00 2001
From: Edward Tomasz Napierala <trasz@FreeBSD.org>
Date: Wed, 17 May 2017 08:38:41 +0000
Subject: [PATCH 09/73] Bump default MAXTSIZ (kern.maxtsiz) from 128MB to 32GB.
 The old limit prevents one from running eg clang built with debug; the new
 one is arbitrary (equal to MAXDSIZ) and... well, should be quite
 future-proof.

Same fix might be applicable to other 64 bit architectures; I'll ask
their respective maintainers to make sure it won't break anything.

Reviewed by:	kib
MFC after:	2 weeks
Sponsored by:	DARPA, AFRL
Differential Revision:	https://reviews.freebsd.org/D10758
---
 sys/amd64/include/vmparam.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h
index 07158e88d8cc..381097adf8b1 100644
--- a/sys/amd64/include/vmparam.h
+++ b/sys/amd64/include/vmparam.h
@@ -52,7 +52,7 @@
 /*
  * Virtual memory related constants, all in bytes
  */
-#define	MAXTSIZ		(128UL*1024*1024)	/* max text size */
+#define	MAXTSIZ		(32768UL*1024*1024)	/* max text size */
 #ifndef DFLDSIZ
 #define	DFLDSIZ		(32768UL*1024*1024)	/* initial data size limit */
 #endif

From 38cc96a8878d2c69a959fc1285380702f7c6354c Mon Sep 17 00:00:00 2001
From: "Andrey V. Elsukov" <ae@FreeBSD.org>
Date: Wed, 17 May 2017 09:04:09 +0000
Subject: [PATCH 10/73] Set M_BCAST and M_MCAST flags on mbuf sent via divert
 socket.

r290383 has changed how mbufs sent by divert socket are handled.
Previously they are always handled by slow path processing in ip_input().
Now ip_tryforward() is invoked from ip_input() before in_broadcast() check.
Since diverted packet lost all mbuf flags, it passes the broadcast check
in ip_tryforward() due to missing M_BCAST flag. In the result the broadcast
packet is forwarded to the wire instead of be consumed by network stack.

Add in_broadcast() check to the div_output() function. And restore the
M_BCAST flag if destination address is broadcast for the given network
interface.

PR:		209491
MFC after:	1 week
---
 sys/netinet/ip_divert.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index 8c1960d14eef..a3bd37e268de 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -481,6 +481,14 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin,
 		/* Send packet to input processing via netisr */
 		switch (ip->ip_v) {
 		case IPVERSION:
+			/*
+			 * Restore M_BCAST flag when destination address is
+			 * broadcast. It is expected by ip_tryforward().
+			 */
+			if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)))
+				m->m_flags |= M_MCAST;
+			else if (in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif))
+				m->m_flags |= M_BCAST;
 			netisr_queue_src(NETISR_IP, (uintptr_t)so, m);
 			break;
 #ifdef INET6

From 52772a85838656f847cb03aaa3b96bccd9f736cc Mon Sep 17 00:00:00 2001
From: "Andrey V. Elsukov" <ae@FreeBSD.org>
Date: Wed, 17 May 2017 10:56:22 +0000
Subject: [PATCH 11/73] Allow zero port specification in table entries with
 type flow.

PR:		217620
MFC after:	1 week
---
 sbin/ipfw/tables.c | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/sbin/ipfw/tables.c b/sbin/ipfw/tables.c
index bc0c61b55768..9aad4d959c7a 100644
--- a/sbin/ipfw/tables.c
+++ b/sbin/ipfw/tables.c
@@ -1260,16 +1260,14 @@ tentry_fill_key_type(char *arg, ipfw_obj_tentry *tentry, uint8_t type,
 			if ((p = strchr(arg, ',')) != NULL)
 				*p++ = '\0';
 
-			if ((port = htons(strtol(arg, NULL, 10))) == 0) {
+			port = htons(strtol(arg, &pp, 10));
+			if (*pp != '\0') {
 				if ((sent = getservbyname(arg, NULL)) == NULL)
 					errx(EX_DATAERR, "Unknown service: %s",
 					    arg);
-				else
-					key = sent->s_port;
+				port = sent->s_port;
 			}
-			
 			tfe->sport = port;
-
 			arg = p;
 		}
 
@@ -1304,16 +1302,14 @@ tentry_fill_key_type(char *arg, ipfw_obj_tentry *tentry, uint8_t type,
 			if ((p = strchr(arg, ',')) != NULL)
 				*p++ = '\0';
 
-			if ((port = htons(strtol(arg, NULL, 10))) == 0) {
+			port = htons(strtol(arg, &pp, 10));
+			if (*pp != '\0') {
 				if ((sent = getservbyname(arg, NULL)) == NULL)
 					errx(EX_DATAERR, "Unknown service: %s",
 					    arg);
-				else
-					key = sent->s_port;
+				port = sent->s_port;
 			}
-			
 			tfe->dport = port;
-
 			arg = p;
 		}
 

From 5bcd39c054262eab4d7f593eb7b6bec4967713aa Mon Sep 17 00:00:00 2001
From: Josh Paetzel <jpaetzel@FreeBSD.org>
Date: Wed, 17 May 2017 13:22:13 +0000
Subject: [PATCH 12/73] Increase the number of LUNs this hardware can support.

Experimentally we know this value works, but the hardware
may support an even higher value.

PR:	213876
Reported by:	J.Catrysse@proximedia.be
MFC after:	1 week
---
 sys/dev/tws/tws.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/dev/tws/tws.h b/sys/dev/tws/tws.h
index 8cb9791c0ad8..0af50c4f425d 100644
--- a/sys/dev/tws/tws.h
+++ b/sys/dev/tws/tws.h
@@ -67,7 +67,7 @@ extern int tws_queue_depth;
 
 #define TWS_DRIVER_VERSION_STRING "10.80.00.005"
 #define TWS_MAX_NUM_UNITS             65 
-#define TWS_MAX_NUM_LUNS              16
+#define TWS_MAX_NUM_LUNS              32
 #define TWS_MAX_IRQS                  2
 #define TWS_SCSI_INITIATOR_ID         66
 #define TWS_MAX_IO_SIZE               0x20000 /* 128kB */

From a564364892432e433bd06c0c45a4b11b07719ede Mon Sep 17 00:00:00 2001
From: Luiz Otavio O Souza <loos@FreeBSD.org>
Date: Wed, 17 May 2017 15:13:01 +0000
Subject: [PATCH 13/73] Move the IO Window Control Register defines out of the
 ARMADA38X ifdef.

Fixes the build of Marvell kernels (other than ARMADA38X) after r318336.

Reported by:	mmel
---
 sys/arm/mv/mvwin.h | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/sys/arm/mv/mvwin.h b/sys/arm/mv/mvwin.h
index 64e3914f47ed..dd65ee4a58db 100644
--- a/sys/arm/mv/mvwin.h
+++ b/sys/arm/mv/mvwin.h
@@ -305,18 +305,6 @@
 #define	MV_BOOTROM_WIN_SIZE	0xF
 #define	MV_CPU_SUBSYS_REGS_LEN	0x100
 
-/* IO Window Control Register fields */
-#define	IO_WIN_SIZE_SHIFT	16
-#define	IO_WIN_SIZE_MASK	0xFFFF
-#define	IO_WIN_ATTR_SHIFT	8
-#define	IO_WIN_ATTR_MASK	0xFF
-#define	IO_WIN_TGT_SHIFT	4
-#define	IO_WIN_TGT_MASK		0xF
-#define	IO_WIN_SYNC_SHIFT	1
-#define	IO_WIN_SYNC_MASK	0x1
-#define	IO_WIN_ENA_SHIFT	0
-#define	IO_WIN_ENA_MASK		0x1
-
 #define	IO_WIN_9_CTRL_OFFSET	0x98
 #define	IO_WIN_9_BASE_OFFSET	0x9C
 
@@ -329,6 +317,18 @@
 #define	MV_SYNC_BARRIER_CTRL_ALL	0xFFFF
 #endif
 
+/* IO Window Control Register fields */
+#define	IO_WIN_SIZE_SHIFT	16
+#define	IO_WIN_SIZE_MASK	0xFFFF
+#define	IO_WIN_ATTR_SHIFT	8
+#define	IO_WIN_ATTR_MASK	0xFF
+#define	IO_WIN_TGT_SHIFT	4
+#define	IO_WIN_TGT_MASK		0xF
+#define	IO_WIN_SYNC_SHIFT	1
+#define	IO_WIN_SYNC_MASK	0x1
+#define	IO_WIN_ENA_SHIFT	0
+#define	IO_WIN_ENA_MASK		0x1
+
 #define WIN_REG_IDX_RD(pre,reg,off,base)					\
 	static __inline uint32_t						\
 	pre ## _ ## reg ## _read(int i)						\

From abafc55b96a235656e44986eb1a4cd4740ae2396 Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:52:04 +0000
Subject: [PATCH 14/73] Fix USB3.0 decoding windows on Armada38x

Set correct offset for MBUS windows configuration in
USB3.0 interface.

Submitted by: Marcin Wojtas <mw@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10721
---
 sys/arm/mv/mvwin.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/arm/mv/mvwin.h b/sys/arm/mv/mvwin.h
index dd65ee4a58db..4c1062229eba 100644
--- a/sys/arm/mv/mvwin.h
+++ b/sys/arm/mv/mvwin.h
@@ -220,8 +220,8 @@
 #define MV_WIN_USB_BASE(n)		(0x10 * (n) + 0x324)
 #define MV_WIN_USB_MAX			4
 
-#define	MV_WIN_USB3_CTRL(n)		(0x8 * (n))
-#define	MV_WIN_USB3_BASE(n)		(0x8 * (n) + 0x4)
+#define	MV_WIN_USB3_CTRL(n)		(0x8 * (n) + 0x4000)
+#define	MV_WIN_USB3_BASE(n)		(0x8 * (n) + 0x4004)
 #define	MV_WIN_USB3_MAX			8
 
 #define MV_WIN_ETH_BASE(n)		(0x8 * (n) + 0x200)

From dbd1638a4896813b567b8c3a3339531ffbf04530 Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:53:13 +0000
Subject: [PATCH 15/73] Parse EHCI windows on Marvell platforms

Add missing compatibility string to allow proper
window configuration for EHCI devices.

Submitted by: Wojciech Macek <wma@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10722
---
 sys/arm/mv/mv_common.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/arm/mv/mv_common.c b/sys/arm/mv/mv_common.c
index 773272cb80fa..19f3e0a77025 100644
--- a/sys/arm/mv/mv_common.c
+++ b/sys/arm/mv/mv_common.c
@@ -139,6 +139,7 @@ struct soc_node_spec {
 static struct soc_node_spec soc_nodes[] = {
 	{ "mrvl,ge", &decode_win_eth_setup, &decode_win_eth_dump },
 	{ "mrvl,usb-ehci", &decode_win_usb_setup, &decode_win_usb_dump },
+	{ "marvell,orion-ehci", &decode_win_usb_setup, &decode_win_usb_dump },
 	{ "marvell,armada-380-xhci", &decode_win_usb3_setup, &decode_win_usb3_dump },
 	{ "marvell,armada-380-ahci", &decode_win_ahci_setup, &decode_win_ahci_dump },
 	{ "marvell,armada-380-sdhci", &decode_win_sdhci_setup, &decode_win_sdhci_dump },

From 49b5f559022dce0b0afe127c277926052e9e7ced Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:54:33 +0000
Subject: [PATCH 16/73] Enable proper parsing of nested simlpe-buses on Marvell
 platforms

OF_finddevice doesn't find the "simple-bus" node, which is problematic
for Marvell platforms, using nested buses in Device Tree, like
Armada 38x SoC.

Submitted by: Arnaud Ysmal <arnaud.ysmal@stormshield.eu>
Obtained from: Stormshield
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10719
---
 sys/arm/mv/mv_common.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sys/arm/mv/mv_common.c b/sys/arm/mv/mv_common.c
index 19f3e0a77025..c711998fcdab 100644
--- a/sys/arm/mv/mv_common.c
+++ b/sys/arm/mv/mv_common.c
@@ -2280,11 +2280,12 @@ win_cpu_from_dt(void)
 static int
 fdt_win_setup(void)
 {
-	phandle_t node, child;
+	phandle_t node, child, sb;
 	struct soc_node_spec *soc_node;
 	u_long size, base;
 	int err, i;
 
+	sb = 0;
 	node = OF_finddevice("/");
 	if (node == -1)
 		panic("fdt_win_setup: no root node");
@@ -2326,7 +2327,7 @@ fdt_win_setup(void)
 		 */
 		child = OF_peer(child);
 		if ((child == 0) && (node == OF_finddevice("/"))) {
-			node = fdt_find_compatible(node, "simple-bus", 0);
+			sb = node = fdt_find_compatible(node, "simple-bus", 0);
 			if (node == 0)
 				return (ENXIO);
 			child = OF_child(node);
@@ -2336,7 +2337,7 @@ fdt_win_setup(void)
 		 * it is present) and its children. This node also have
 		 * "simple-bus" compatible.
 		 */
-		if ((child == 0) && (node == OF_finddevice("simple-bus"))) {
+		if ((child == 0) && (node == sb)) {
 			node = fdt_find_compatible(node, "simple-bus", 0);
 			if (node == 0)
 				return (0);

From da081cb51d8ae03cf1fb38abd66225927a88e2c5 Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:56:09 +0000
Subject: [PATCH 17/73] Correct MPIC order of attachment

If MPIC happens to be a slave interrupt controller (as on Armada38x),
it should be attached after primary interrupt controller.
Thus BUS_PASS_ORDER_LATE was added to default BUS_PASS_INTERRUPT.

This change doesn't affect the cases when MPIC is standalone IC.

Submitted by:  Bartosz Szczepanek <bsz@semihalf.com>
Obtained from: Semihalf
Sponsored by:  Stormshield, Netgate
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10715
---
 sys/arm/mv/mpic.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/arm/mv/mpic.c b/sys/arm/mv/mpic.c
index 3fc06d73e32d..ce927ec675cf 100644
--- a/sys/arm/mv/mpic.c
+++ b/sys/arm/mv/mpic.c
@@ -398,7 +398,7 @@ static driver_t mv_mpic_driver = {
 static devclass_t mv_mpic_devclass;
 
 EARLY_DRIVER_MODULE(mpic, simplebus, mv_mpic_driver, mv_mpic_devclass, 0, 0,
-    BUS_PASS_INTERRUPT);
+    BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE);
 
 #ifndef INTRNG
 int

From b488f7aaa5601aba1e83860847025593d06c810a Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:57:14 +0000
Subject: [PATCH 18/73] Fix registration of MPIC driver

Submitted by:  Michal Mazur <mkm@semihalf.com>
Obtained from: Semihalf
Sponsored by:  Netgate
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10717
---
 sys/arm/mv/mpic.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/arm/mv/mpic.c b/sys/arm/mv/mpic.c
index ce927ec675cf..254eaa34df5c 100644
--- a/sys/arm/mv/mpic.c
+++ b/sys/arm/mv/mpic.c
@@ -273,6 +273,9 @@ mv_mpic_attach(device_t dev)
 		bus_release_resources(dev, mv_mpic_spec, sc->mpic_res);
 		return (ENXIO);
 	}
+
+	OF_device_register_xref(OF_xref_from_node(ofw_bus_get_node(dev)), dev);
+
 	if (intr_pic_register(dev, OF_xref_from_device(dev)) == NULL) {
 		device_printf(dev, "could not register PIC\n");
 		bus_release_resources(dev, mv_mpic_spec, sc->mpic_res);

From 7118192a7270706a111b04f87b48a096c1a8e1bb Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:58:39 +0000
Subject: [PATCH 19/73] Fix broken malloc in e6000sw

Malloc should always return something when M_WAITOK flag is used,
but keep this code and change flag to M_NOWAIT as it is under a lock
(allows for possible future change). Free ifnet structure to avoid
memory leak on failure.

Submitted by:  Zbigniew Bodek <zbb@semihalf.com>
Obtained from: Semihalf
Sponsored by:  Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10711
---
 sys/dev/etherswitch/e6000sw/e6000sw.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sys/dev/etherswitch/e6000sw/e6000sw.c b/sys/dev/etherswitch/e6000sw/e6000sw.c
index 3c86426d056b..1e31346a3437 100644
--- a/sys/dev/etherswitch/e6000sw/e6000sw.c
+++ b/sys/dev/etherswitch/e6000sw/e6000sw.c
@@ -321,9 +321,11 @@ e6000sw_init_interface(e6000sw_softc_t *sc, int port)
 	sc->ifp[port]->if_softc = sc;
 	sc->ifp[port]->if_flags |= IFF_UP | IFF_BROADCAST |
 	    IFF_DRV_RUNNING | IFF_SIMPLEX;
-	sc->ifname[port] = malloc(strlen(name) + 1, M_E6000SW, M_WAITOK);
-	if (sc->ifname[port] == NULL)
+	sc->ifname[port] = malloc(strlen(name) + 1, M_E6000SW, M_NOWAIT);
+	if (sc->ifname[port] == NULL) {
+		if_free(sc->ifp[port]);
 		return (ENOMEM);
+	}
 	memcpy(sc->ifname[port], name, strlen(name) + 1);
 	if_initname(sc->ifp[port], sc->ifname[port], port);
 

From 9c06c1c42a4ad110e2556c5c1d3e7a3a3bf68e90 Mon Sep 17 00:00:00 2001
From: Zbigniew Bodek <zbb@FreeBSD.org>
Date: Wed, 17 May 2017 15:59:45 +0000
Subject: [PATCH 20/73] Add missing unlock in e6000sw driver

This patch adds missing unlock on attach failure.

Submitted by:  Zbigniew Bodek <zbb@semihalf.com>
Obtained from: Semihalf
Sponsored by:  Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10712
---
 sys/dev/etherswitch/e6000sw/e6000sw.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/dev/etherswitch/e6000sw/e6000sw.c b/sys/dev/etherswitch/e6000sw/e6000sw.c
index 1e31346a3437..cfc20dbbf5f2 100644
--- a/sys/dev/etherswitch/e6000sw/e6000sw.c
+++ b/sys/dev/etherswitch/e6000sw/e6000sw.c
@@ -425,6 +425,7 @@ e6000sw_attach(device_t dev)
 	return (0);
 
 out_fail:
+	E6000SW_UNLOCK(sc);
 	e6000sw_detach(dev);
 
 	return (err);

From 366e9e4538645d24e8cf57831d41ab67fc349977 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 17 May 2017 16:32:24 +0000
Subject: [PATCH 21/73] fix sparc64 build by restoring 'register' in pcpu.h

Reported by:	jhb
Sponsored by:	The FreeBSD Foundation
---
 sys/sparc64/include/pcpu.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h
index bf3d4afbdb8f..c54c45b2443f 100644
--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@@ -71,8 +71,8 @@ extern void *dpcpu0;
 struct pcb;
 struct pcpu;
 
-struct pcb *curpcb __asm__(__XSTRING(PCB_REG));
-struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
+register struct pcb *curpcb __asm__(__XSTRING(PCB_REG));
+register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
 
 #define	get_pcpu()		(pcpup)
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)

From f56f89c7a43b67b9d5a077e53be8e30db09691f7 Mon Sep 17 00:00:00 2001
From: Adrian Chadd <adrian@FreeBSD.org>
Date: Wed, 17 May 2017 19:34:36 +0000
Subject: [PATCH 22/73] [net80211] initial VHT radiotap implementation defines
 from upstream radiotap.

---
 sys/net80211/ieee80211_radiotap.h | 116 ++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)

diff --git a/sys/net80211/ieee80211_radiotap.h b/sys/net80211/ieee80211_radiotap.h
index 388d70ed6c14..e42c66645091 100644
--- a/sys/net80211/ieee80211_radiotap.h
+++ b/sys/net80211/ieee80211_radiotap.h
@@ -178,6 +178,30 @@ struct ieee80211_radiotap_header {
  *      finally the maximum regulatory transmit power cap in .5 dBm
  *      units.  This property supersedes IEEE80211_RADIOTAP_CHANNEL
  *      and only one of the two should be present.
+ * IEEE80211_RADIOTAP_RX_FLAGS          guint16       bitmap
+ *
+ *     Properties of received frames. See flags defined below.
+ *
+ * IEEE80211_RADIOTAP_TX_FLAGS          guint16       bitmap
+ *
+ *     Properties of transmitted frames. See flags defined below.
+ *
+ * IEEE80211_RADIOTAP_RTS_RETRIES       u8           data
+ *
+ *     Number of rts retries a transmitted frame used.
+ *
+ * IEEE80211_RADIOTAP_DATA_RETRIES      u8           data
+ *
+ *     Number of unicast retries a transmitted frame used.
+ *
+ * IEEE80211_RADIOTAP_MCS       u8, u8, u8              unitless
+ *
+ *     Contains a bitmap of known fields/flags, the flags, and
+ *     the MCS index.
+ *
+ * IEEE80211_RADIOTAP_AMPDU_STATUS      u32, u16, u8, u8        unitlesss
+ *
+ *      Contains the AMPDU information for the subframe.
  */
 enum ieee80211_radiotap_type {
 	IEEE80211_RADIOTAP_TSFT = 0,
@@ -206,6 +230,7 @@ enum ieee80211_radiotap_type {
 	IEEE80211_RADIOTAP_XCHANNEL = 18,
 	IEEE80211_RADIOTAP_MCS = 19,
 	IEEE80211_RADIOTAP_AMPDU_STATUS = 20,
+	IEEE80211_RADIOTAP_VHT = 21,
 
         IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE = 29,
 	IEEE80211_RADIOTAP_VENDOREXT = 30,
@@ -250,4 +275,95 @@ enum ieee80211_radiotap_type {
 #define	IEEE80211_RADIOTAP_F_BADFCS	0x40	/* does not pass FCS check */
 #define	IEEE80211_RADIOTAP_F_SHORTGI	0x80	/* HT short GI */
 
+/* For IEEE80211_RADIOTAP_RX_FLAGS */
+#define	IEEE80211_RADIOTAP_F_RX_BADPLCP	0x0002	/* bad PLCP */
+
+/* For IEEE80211_RADIOTAP_TX_FLAGS */
+#define	IEEE80211_RADIOTAP_F_TX_FAIL	0x0001	/* failed due to excessive
+						 * retries */
+#define	IEEE80211_RADIOTAP_F_TX_CTS	0x0002	/* used cts 'protection' */
+#define	IEEE80211_RADIOTAP_F_TX_RTS	0x0004	/* used rts/cts handshake */
+
+
+/* For IEEE80211_RADIOTAP_MCS */
+#define	IEEE80211_RADIOTAP_MCS_HAVE_BW		0x01
+#define	IEEE80211_RADIOTAP_MCS_HAVE_MCS		0x02
+#define	IEEE80211_RADIOTAP_MCS_HAVE_GI		0x04
+#define	IEEE80211_RADIOTAP_MCS_HAVE_FMT		0x08
+#define	IEEE80211_RADIOTAP_MCS_HAVE_FEC		0x10
+#define	IEEE80211_RADIOTAP_MCS_HAVE_STBC	0x20
+#define	IEEE80211_RADIOTAP_MCS_HAVE_NESS	0x40
+#define	IEEE80211_RADIOTAP_MCS_NESS_BIT1	0x80
+
+#define	IEEE80211_RADIOTAP_MCS_BW_MASK		0x03
+#define	    IEEE80211_RADIOTAP_MCS_BW_20	0
+#define	    IEEE80211_RADIOTAP_MCS_BW_40	1
+#define	 IEEE80211_RADIOTAP_MCS_BW_20L		2
+#define	    IEEE80211_RADIOTAP_MCS_BW_20U	3
+#define	IEEE80211_RADIOTAP_MCS_SGI		0x04
+#define	IEEE80211_RADIOTAP_MCS_FMT_GF		0x08
+#define	IEEE80211_RADIOTAP_MCS_FEC_LDPC		0x10
+#define	IEEE80211_RADIOTAP_MCS_STBC_MASK	0x60
+#define	IEEE80211_RADIOTAP_MCS_STBC_SHIFT	5
+#define	    IEEE80211_RADIOTAP_MCS_STBC_1	1
+#define	    IEEE80211_RADIOTAP_MCS_STBC_2	2
+#define	    IEEE80211_RADIOTAP_MCS_STBC_3	3
+#define	IEEE80211_RADIOTAP_MCS_NESS_BIT0	0x80
+
+/* For IEEE80211_RADIOTAP_AMPDU_STATUS */
+#define	IEEE80211_RADIOTAP_AMPDU_REPORT_ZEROLEN		0x0001
+#define	IEEE80211_RADIOTAP_AMPDU_IS_ZEROLEN		0x0002
+#define	IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN		0x0004
+#define	IEEE80211_RADIOTAP_AMPDU_IS_LAST		0x0008
+#define	IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_ERR		0x0010
+#define	IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_KNOWN	0x0020
+
+/* For IEEE80211_RADIOTAP_VHT */
+#define	IEEE80211_RADIOTAP_VHT_HAVE_STBC	0x0001
+#define	IEEE80211_RADIOTAP_VHT_HAVE_TXOP_PS	0x0002
+#define	IEEE80211_RADIOTAP_VHT_HAVE_GI		0x0004
+#define	IEEE80211_RADIOTAP_VHT_HAVE_SGI_NSYM_DA	0x0008
+#define	IEEE80211_RADIOTAP_VHT_HAVE_LDPC_EXTRA	0x0010
+#define	IEEE80211_RADIOTAP_VHT_HAVE_BF		0x0020
+#define	IEEE80211_RADIOTAP_VHT_HAVE_BW		0x0040
+#define	IEEE80211_RADIOTAP_VHT_HAVE_GID		0x0080
+#define	IEEE80211_RADIOTAP_VHT_HAVE_PAID	0x0100
+#define	IEEE80211_RADIOTAP_VHT_STBC		0x01
+#define	IEEE80211_RADIOTAP_VHT_TXOP_PS		0x02
+#define	IEEE80211_RADIOTAP_VHT_SGI		0x04
+#define	IEEE80211_RADIOTAP_VHT_SGI_NSYM_DA	0x08
+#define	IEEE80211_RADIOTAP_VHT_LDPC_EXTRA	0x10
+#define	IEEE80211_RADIOTAP_VHT_BF		0x20
+#define	IEEE80211_RADIOTAP_VHT_NSS		0x0f
+#define	IEEE80211_RADIOTAP_VHT_MCS		0xf0
+#define	IEEE80211_RADIOTAP_VHT_CODING_LDPC	0x01
+
+#define	IEEE80211_RADIOTAP_VHT_BW_MASK		0x1f
+#define	IEEE80211_RADIOTAP_VHT_BW_20		IEEE80211_RADIOTAP_MCS_BW_20
+#define	IEEE80211_RADIOTAP_VHT_BW_40		IEEE80211_RADIOTAP_MCS_BW_40
+#define	IEEE80211_RADIOTAP_VHT_BW_20L		IEEE80211_RADIOTAP_MCS_BW_20L
+#define	IEEE80211_RADIOTAP_VHT_BW_20U		IEEE80211_RADIOTAP_MCS_BW_20U
+#define	IEEE80211_RADIOTAP_VHT_BW_80		4
+#define	IEEE80211_RADIOTAP_VHT_BW_40L		5
+#define	IEEE80211_RADIOTAP_VHT_BW_40U		6
+#define	IEEE80211_RADIOTAP_VHT_BW_20LL		7
+#define	IEEE80211_RADIOTAP_VHT_BW_20LU		8
+#define	IEEE80211_RADIOTAP_VHT_BW_20UL		9
+#define	IEEE80211_RADIOTAP_VHT_BW_20UU		10
+#define	IEEE80211_RADIOTAP_VHT_BW_160		11
+#define	IEEE80211_RADIOTAP_VHT_BW_80L		12
+#define	IEEE80211_RADIOTAP_VHT_BW_80U		13
+#define	IEEE80211_RADIOTAP_VHT_BW_40LL		14
+#define	IEEE80211_RADIOTAP_VHT_BW_40LU		15
+#define	IEEE80211_RADIOTAP_VHT_BW_40UL		16
+#define	IEEE80211_RADIOTAP_VHT_BW_40UU		17
+#define	IEEE80211_RADIOTAP_VHT_BW_20LLL		18
+#define	IEEE80211_RADIOTAP_VHT_BW_20LLU		19
+#define	IEEE80211_RADIOTAP_VHT_BW_20LUL		20
+#define	IEEE80211_RADIOTAP_VHT_BW_20LUU		21
+#define	IEEE80211_RADIOTAP_VHT_BW_20ULL		22
+#define	IEEE80211_RADIOTAP_VHT_BW_20ULU		23
+#define	IEEE80211_RADIOTAP_VHT_BW_20UUL		24
+#define	IEEE80211_RADIOTAP_VHT_BW_20UUU		25
+
 #endif /* !_NET80211_IEEE80211_RADIOTAP_H_ */

From 0044ecde83c378ac4ee4744a98d9b5931b9bb62b Mon Sep 17 00:00:00 2001
From: Luiz Otavio O Souza <loos@FreeBSD.org>
Date: Wed, 17 May 2017 21:14:27 +0000
Subject: [PATCH 23/73] Use the MACROS to access the Global mpic registers. 
 Makes the code consistent and easier to read.

While here, remove two unused static functions and fix a unused function
warning when building !INTRNG.

No functional changes.

Sponsored by:	Rubicon Communications, LLC (Netgate)
---
 sys/arm/mv/mpic.c | 39 ++++++++-------------------------------
 1 file changed, 8 insertions(+), 31 deletions(-)

diff --git a/sys/arm/mv/mpic.c b/sys/arm/mv/mpic.c
index 254eaa34df5c..70c17ae3faa2 100644
--- a/sys/arm/mv/mpic.c
+++ b/sys/arm/mv/mpic.c
@@ -148,12 +148,10 @@ static void	mpic_unmask_irq(uintptr_t nb);
 static void	mpic_mask_irq(uintptr_t nb);
 static void	mpic_mask_irq_err(uintptr_t nb);
 static void	mpic_unmask_irq_err(uintptr_t nb);
+#ifdef INTRNG
 static int	mpic_intr(void *arg);
-static void	mpic_unmask_msi(void);
-#ifndef INTRNG
-static void	arm_mask_irq_err(uintptr_t);
-static void	arm_unmask_irq_err(uintptr_t);
 #endif
+static void	mpic_unmask_msi(void);
 
 #define	MPIC_WRITE(softc, reg, val) \
     bus_space_write_4((softc)->mpic_bst, (softc)->mpic_bsh, (reg), (val))
@@ -260,8 +258,7 @@ mv_mpic_attach(device_t dev)
 		sc->drbl_bsh = rman_get_bushandle(sc->mpic_res[2]);
 	}
 
-	bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh,
-	    MPIC_CTRL, 1);
+	MPIC_WRITE(mv_mpic_sc, MPIC_CTRL, 1);
 	MPIC_CPU_WRITE(mv_mpic_sc, MPIC_CTP, 0);
 
 	val = MPIC_READ(mv_mpic_sc, MPIC_CTRL);
@@ -435,27 +432,12 @@ arm_mask_irq(uintptr_t nb)
 	mpic_mask_irq(nb);
 }
 
-
-static void
-arm_mask_irq_err(uintptr_t nb)
-{
-
-	mpic_mask_irq_err(nb);
-}
-
 void
 arm_unmask_irq(uintptr_t nb)
 {
 
 	mpic_unmask_irq(nb);
 }
-
-void
-arm_unmask_irq_err(uintptr_t nb)
-{
-
-	mpic_unmask_irq_err(nb);
-}
 #endif
 
 static void
@@ -471,8 +453,7 @@ mpic_unmask_irq_err(uintptr_t nb)
 	uint32_t mask;
 	uint8_t bit_off;
 
-	bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh,
-	    MPIC_ISE, MPIC_INT_ERR);
+	MPIC_WRITE(mv_mpic_sc, MPIC_ISE, MPIC_INT_ERR);
 	MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, MPIC_INT_ERR);
 
 	bit_off = nb - ERR_IRQ;
@@ -498,8 +479,7 @@ mpic_unmask_irq(uintptr_t nb)
 {
 
 	if (nb < ERR_IRQ) {
-		bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh,
-		    MPIC_ISE, nb);
+		MPIC_WRITE(mv_mpic_sc, MPIC_ISE, nb);
 		MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, nb);
 	} else if (nb < MSI_IRQ)
 		mpic_unmask_irq_err(nb);
@@ -513,8 +493,7 @@ mpic_mask_irq(uintptr_t nb)
 {
 
 	if (nb < ERR_IRQ) {
-		bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh,
-		    MPIC_ICE, nb);
+		MPIC_WRITE(mv_mpic_sc, MPIC_ICE, nb);
 		MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ISM, nb);
 	} else if (nb < MSI_IRQ)
 		mpic_mask_irq_err(nb);
@@ -533,8 +512,7 @@ mv_mpic_get_cause_err(void)
 	uint32_t err_cause;
 	uint8_t bit_off;
 
-	err_cause = bus_space_read_4(mv_mpic_sc->mpic_bst,
-	    mv_mpic_sc->mpic_bsh, MPIC_ERR_CAUSE);
+	err_cause = MPIC_READ(mv_mpic_sc, MPIC_ERR_CAUSE);
 
 	if (err_cause)
 		bit_off = ffs(err_cause) - 1;
@@ -615,8 +593,7 @@ pic_ipi_send(cpuset_t cpus, u_int ipi)
 		if (CPU_ISSET(i, &cpus))
 			val |= (1 << (8 + i));
 	val |= ipi;
-	bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh,
-	    MPIC_SOFT_INT, val);
+	MPIC_WRITE(mv_mpic_sc, MPIC_SOFT_INT, val);
 }
 
 int

From 67feec504576aacb04baaab0db77c475500c399e Mon Sep 17 00:00:00 2001
From: Stephen McConnell <slm@FreeBSD.org>
Date: Wed, 17 May 2017 21:33:37 +0000
Subject: [PATCH 24/73] Add tri-mode support (SAS/SATA/PCIe).

This includes NVMe device support and adds support for the following adapters:
    SAS 3408
    SAS 3416
    SAS 3508
    SAS 3516
    SAS 3616
    SAS 3708
    SAS 3716

Reviewed by:    ken, scottl, asomers, mav
Approved by:	ken, scottl, mav
MFC after:      2 weeks
Relnotes:	yes
Differential Revision: https://reviews.freebsd.org/D10095
---
 share/man/man4/mpr.4             |  91 +++-
 sys/dev/mpr/mpi/mpi2.h           |  74 +++-
 sys/dev/mpr/mpi/mpi2_cnfg.h      | 545 +++++++++++++++++++++++-
 sys/dev/mpr/mpi/mpi2_hbd.h       |   4 +-
 sys/dev/mpr/mpi/mpi2_history.txt | 124 ++++--
 sys/dev/mpr/mpi/mpi2_init.h      |  17 +-
 sys/dev/mpr/mpi/mpi2_ioc.h       | 246 ++++++++++-
 sys/dev/mpr/mpi/mpi2_pci.h       | 151 +++++++
 sys/dev/mpr/mpi/mpi2_tool.h      |  14 +-
 sys/dev/mpr/mpr.c                | 690 ++++++++++++++++++++++++++++++-
 sys/dev/mpr/mpr_config.c         | 301 +++++++++++++-
 sys/dev/mpr/mpr_mapping.c        | 549 +++++++++++++++++++++---
 sys/dev/mpr/mpr_mapping.h        |  51 ++-
 sys/dev/mpr/mpr_pci.c            |  56 ++-
 sys/dev/mpr/mpr_sas.c            | 467 +++++++++++++++++++--
 sys/dev/mpr/mpr_sas.h            |   2 +
 sys/dev/mpr/mpr_sas_lsi.c        | 216 +++++++++-
 sys/dev/mpr/mpr_table.c          |  17 +
 sys/dev/mpr/mpr_table.h          |   1 +
 sys/dev/mpr/mpr_user.c           |  82 +++-
 sys/dev/mpr/mprvar.h             | 127 +++++-
 21 files changed, 3610 insertions(+), 215 deletions(-)
 create mode 100755 sys/dev/mpr/mpi/mpi2_pci.h

diff --git a/share/man/man4/mpr.4 b/share/man/man4/mpr.4
index 8a2794d7b105..224907af50f0 100644
--- a/share/man/man4/mpr.4
+++ b/share/man/man4/mpr.4
@@ -1,8 +1,8 @@
 .\"
 .\" Copyright (c) 2010 Spectra Logic Corporation
 .\" Copyright (c) 2014 LSI Corp
-.\" Copyright (c) 2016 Avago Technologies
-.\" Copyright (c) 2016 Broadcom Ltd.
+.\" Copyright (c) 2017 Avago Technologies
+.\" Copyright (c) 2017 Broadcom Ltd.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -38,12 +38,12 @@
 .\" $Id$
 .\" $FreeBSD$
 .\"
-.Dd July 6, 2016
+.Dd May 17, 2017
 .Dt MPR 4
 .Os
 .Sh NAME
 .Nm mpr
-.Nd "LSI Fusion-MPT 3 IT/IR 12Gb/s Serial Attached SCSI/SATA driver"
+.Nd "LSI Fusion-MPT 3/3.5 IT/IR 12Gb/s Serial Attached SCSI/SATA/PCIe driver"
 .Sh SYNOPSIS
 To compile this driver into the kernel, place these lines in the kernel
 configuration file:
@@ -62,8 +62,8 @@ mpr_load="YES"
 The
 .Nm
 driver provides support for Broadcom Ltd./Avago Tech (LSI)
-Fusion-MPT 3 IT/IR
-.Tn SAS
+Fusion-MPT 3/3.5 IT/IR
+.Tn SAS/PCIe
 controllers.
 .Sh HARDWARE
 These controllers are supported by the
@@ -81,6 +81,24 @@ Broadcom Ltd./Avago Tech (LSI) SAS 3108 (8 Port SAS)
 Broadcom Ltd./Avago Tech (LSI) SAS 3216 (16 Port SAS)
 .It
 Broadcom Ltd./Avago Tech (LSI) SAS 3224 (24 Port SAS)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3316 (16 Port SAS)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3324 (24 Port SAS)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3408 (8 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3416 (16 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3508 (8 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3516 (16 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3616 (16 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3708 (8 Port SAS/PCIe)
+.It
+Broadcom Ltd./Avago Tech (LSI) SAS 3716 (16 Port SAS/PCIe)
 .El
 .Sh CONFIGURATION
 In all tunable descriptions below, X represents the adapter number.
@@ -156,6 +174,24 @@ dev.mpr.X.io_cmds_active
 .Xr sysctl 8
 variable.
 .Pp
+The current number of free PRP pages is stored in the
+dev.mpr.X.prp_pages_free
+.Xr sysctl 8
+variable.
+PRP pages are used by NVMe devices for I/O transfers, much like Scatter/Gather
+lists.
+.Pp
+The lowest number of free PRP pages seen since boot is stored in the
+dev.mpr.X.prp_pages_free_lowwater
+.Xr sysctl 8
+variable.
+.Pp
+The number of times that PRP page allocations have failed since boot is
+stored in the
+dev.mpr.X.prp_page_alloc_fail
+.Xr sysctl 8
+variable.
+.Pp
 To set the maximum number of pages that will be used per I/O for all adapters,
 set this tunable in
 .Xr loader.conf 5 :
@@ -229,13 +265,13 @@ Send SSU to HDDs, but not to SSDs.
 Send SSU to both HDDs and SSDs.
 .El
 .Pp
-To control the feature for a specific adapter, set this tunable value in
+To control this feature for a specific adapter, set this tunable value in
 .Xr loader.conf 5 :
 .Bd -literal -offset indent
 dev.mpr.X.enable_ssu
 .Ed
 .Pp
-The same set of values are valid when setting this tunable for all adapters.
+The same set of values are valid as when setting this tunable for all adapters.
 .Pp
 SATA disks that take several seconds to spin up and fail the SATA Identify
 command might not be discovered by the driver.
@@ -261,6 +297,45 @@ dev.mpr.X.spinup_wait_time=NNNN
 tunable.
 NNNN is the number of seconds to wait for SATA devices to spin up when they fail
 the initial SATA Identify command.
+.Pp
+The driver can map devices discovered by the adapter so that target IDs
+corresponding to a specific device persist across resets and reboots.
+In some cases it is possible for devices to lose their mapped IDs due to
+unexpected behavior from certain hardware, such as some types of enclosures.
+To overcome this problem, a tunable is provided that will force the driver to
+map devices using the Phy number associated with the device.
+This feature is not recommended if the topology includes multiple
+enclosures/expanders.
+If multiple enclosures/expanders are present in the topology, Phy numbers are
+repeated, causing all devices at these Phy numbers except the first device to
+fail enumeration.
+To control this feature for all adapters, set the
+.Bd -literal -offset indent
+hw.mpr.use_phy_num
+.Ed
+.Pp
+tunable in
+.Xr loader.conf 5
+to one of these values:
+.Bl -tag -width 6n -offset indent
+.It -1
+Only use Phy numbers to map devices and bypass the driver's mapping logic.
+.It 0
+Never use Phy numbers to map devices.
+.It 1
+Use Phy numbers to map devices, but only if the driver's mapping logic fails
+to map the device that is being enumerated.
+This is the default value.
+.El
+.Pp
+To control this feature for a specific adapter, set this tunable value in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+dev.mpr.X.use_phy_num
+.Ed
+.Pp
+The same set of values are valid as when setting this tunable for all adapters.
+.Pp
 .Sh DEBUGGING
 To enable debugging prints from the
 .Nm
diff --git a/sys/dev/mpr/mpi/mpi2.h b/sys/dev/mpr/mpi/mpi2.h
index 7084a8522000..2689b5ccdf1e 100644
--- a/sys/dev/mpr/mpi/mpi2.h
+++ b/sys/dev/mpr/mpi/mpi2.h
@@ -44,7 +44,7 @@
  *                  scatter/gather formats.
  *  Creation Date:  June 21, 2006
  *
- *  mpi2.h Version:  02.00.42
+ *  mpi2.h Version:  02.00.46
  *
  *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *        prefix are for use only on MPI v2.5 products, and must not be used
@@ -132,7 +132,8 @@
  *                      Bumped MPI2_HEADER_VERSION_UNIT.
  *  03-16-15  02.00.37  Updated for MPI v2.6.
  *                      Bumped MPI2_HEADER_VERSION_UNIT.
- *                      Added Scratchpad registers to
+ *                      Added Scratchpad registers and
+ *                      AtomicRequestDescriptorPost register to
  *                      MPI2_SYSTEM_INTERFACE_REGS.
  *                      Added MPI2_DIAG_SBR_RELOAD.
  *                      Added MPI2_IOCSTATUS_INSUFFICIENT_POWER.
@@ -142,6 +143,14 @@
  *                      Added V7 HostDiagnostic register defines
  *  12-15-15  02.00.41  Bumped MPI_HEADER_VERSION_UNIT
  *  01-01-16  02.00.42  Bumped MPI_HEADER_VERSION_UNIT
+ *  04-05-16  02.00.43  Modified  MPI26_DIAG_BOOT_DEVICE_SELECT defines
+ *                      to be unique within first 32 characters.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ *                      Bumped MPI2_HEADER_VERSION_UNIT.
+ *  04-10-16  02.00.44  Bumped MPI2_HEADER_VERSION_UNIT.
+ *  07-06-16  02.00.45  Bumped MPI2_HEADER_VERSION_UNIT.
+ *  09-02-16  02.00.46  Bumped MPI2_HEADER_VERSION_UNIT.
  *  --------------------------------------------------------------------------
  */
 
@@ -185,7 +194,7 @@
 
 
 /* Unit and Dev versioning for this MPI header set */
-#define MPI2_HEADER_VERSION_UNIT            (0x2A)
+#define MPI2_HEADER_VERSION_UNIT            (0x2E)
 #define MPI2_HEADER_VERSION_DEV             (0x00)
 #define MPI2_HEADER_VERSION_UNIT_MASK       (0xFF00)
 #define MPI2_HEADER_VERSION_UNIT_SHIFT      (8)
@@ -245,7 +254,8 @@ typedef volatile struct _MPI2_SYSTEM_INTERFACE_REGS
     U32         Scratchpad[4];              /* 0xB0 */
     U32         RequestDescriptorPostLow;   /* 0xC0 */
     U32         RequestDescriptorPostHigh;  /* 0xC4 */
-    U32         Reserved7[14];              /* 0xC8 */
+    U32         AtomicRequestDescriptorPost;/* 0xC8 */ /* MPI v2.6 and later; reserved in earlier versions */
+    U32         Reserved7[13];              /* 0xCC */
 } MPI2_SYSTEM_INTERFACE_REGS, MPI2_POINTER PTR_MPI2_SYSTEM_INTERFACE_REGS,
   Mpi2SystemInterfaceRegs_t, MPI2_POINTER pMpi2SystemInterfaceRegs_t;
 
@@ -293,10 +303,11 @@ typedef volatile struct _MPI2_SYSTEM_INTERFACE_REGS
 #define MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW       (0x00000800)
 
 /* Defines for V7A/V7R HostDiagnostic Register */
-#define MPI26_DIAG_BOOT_DEVICE_SELECT_FLASH64    (0x00000000)
-#define MPI26_DIAG_BOOT_DEVICE_SELECT_HCDW64     (0x00000800)
-#define MPI26_DIAG_BOOT_DEVICE_SELECT_FLASH32    (0x00001000)
-#define MPI26_DIAG_BOOT_DEVICE_SELECT_HCDW32     (0x00001800)
+#define MPI26_DIAG_BOOT_DEVICE_SEL_64FLASH      (0x00000000)
+#define MPI26_DIAG_BOOT_DEVICE_SEL_64HCDW       (0x00000800)
+#define MPI26_DIAG_BOOT_DEVICE_SEL_32FLASH      (0x00001000)
+#define MPI26_DIAG_BOOT_DEVICE_SEL_32HCDW       (0x00001800)
+
 #define MPI2_DIAG_CLEAR_FLASH_BAD_SIG           (0x00000400)
 #define MPI2_DIAG_FORCE_HCB_ON_RESET            (0x00000200)
 #define MPI2_DIAG_HCB_MODE                      (0x00000100)
@@ -379,6 +390,7 @@ typedef volatile struct _MPI2_SYSTEM_INTERFACE_REGS
  */
 #define MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET     (0x000000C0)
 #define MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET    (0x000000C4)
+#define MPI26_ATOMIC_REQUEST_DESCRIPTOR_POST_OFFSET (0x000000C8)
 
 
 /* Hard Reset delay timings */
@@ -415,6 +427,7 @@ typedef struct _MPI2_DEFAULT_REQUEST_DESCRIPTOR
 #define MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE            (0x08)
 #define MPI2_REQ_DESCRIPT_FLAGS_RAID_ACCELERATOR        (0x0A)
 #define MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO      (0x0C)
+#define MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED      (0x10)
 
 #define MPI2_REQ_DESCRIPT_FLAGS_IOC_FIFO_MARKER (0x01)
 
@@ -482,6 +495,14 @@ typedef MPI2_SCSI_IO_REQUEST_DESCRIPTOR
     MPI2_POINTER pMpi25FastPathSCSIIORequestDescriptor_t;
 
 
+/* PCIe Encapsulated Request Descriptor */
+typedef MPI2_SCSI_IO_REQUEST_DESCRIPTOR
+    MPI26_PCIE_ENCAPSULATED_REQUEST_DESCRIPTOR,
+    MPI2_POINTER PTR_MPI26_PCIE_ENCAPSULATED_REQUEST_DESCRIPTOR,
+    Mpi26PCIeEncapsulatedRequestDescriptor_t,
+    MPI2_POINTER pMpi26PCIeEncapsulatedRequestDescriptor_t;
+
+
 /* union of Request Descriptors */
 typedef union _MPI2_REQUEST_DESCRIPTOR_UNION
 {
@@ -491,11 +512,35 @@ typedef union _MPI2_REQUEST_DESCRIPTOR_UNION
     MPI2_SCSI_TARGET_REQUEST_DESCRIPTOR         SCSITarget;
     MPI2_RAID_ACCEL_REQUEST_DESCRIPTOR          RAIDAccelerator;
     MPI25_FP_SCSI_IO_REQUEST_DESCRIPTOR         FastPathSCSIIO;
+    MPI26_PCIE_ENCAPSULATED_REQUEST_DESCRIPTOR  PCIeEncapsulated;
     U64                                         Words;
 } MPI2_REQUEST_DESCRIPTOR_UNION, MPI2_POINTER PTR_MPI2_REQUEST_DESCRIPTOR_UNION,
   Mpi2RequestDescriptorUnion_t, MPI2_POINTER pMpi2RequestDescriptorUnion_t;
 
 
+/* Atomic Request Descriptors */
+
+/*
+ * All Atomic Request Descriptors have the same format, so the following
+ * structure is used for all Atomic Request Descriptors:
+ *      Atomic Default Request Descriptor
+ *      Atomic High Priority Request Descriptor
+ *      Atomic SCSI IO Request Descriptor
+ *      Atomic SCSI Target Request Descriptor
+ *      Atomic RAID Accelerator Request Descriptor
+ *      Atomic Fast Path SCSI IO Request Descriptor
+ *      Atomic PCIe Encapsulated Request Descriptor
+ */
+
+/* Atomic Request Descriptor */
+typedef struct _MPI26_ATOMIC_REQUEST_DESCRIPTOR
+{
+    U8              RequestFlags;               /* 0x00 */
+    U8              MSIxIndex;                  /* 0x01 */
+    U16             SMID;                       /* 0x02 */
+} MPI26_ATOMIC_REQUEST_DESCRIPTOR,
+  MPI2_POINTER PTR_MPI26_ATOMIC_REQUEST_DESCRIPTOR,
+  Mpi26AtomicRequestDescriptor_t, MPI2_POINTER pMpi26AtomicRequestDescriptor_t;
 
 /* for the RequestFlags field, use the same defines as MPI2_DEFAULT_REQUEST_DESCRIPTOR */
 
@@ -520,6 +565,7 @@ typedef struct _MPI2_DEFAULT_REPLY_DESCRIPTOR
 #define MPI2_RPY_DESCRIPT_FLAGS_TARGET_COMMAND_BUFFER       (0x03)
 #define MPI2_RPY_DESCRIPT_FLAGS_RAID_ACCELERATOR_SUCCESS    (0x05)
 #define MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS  (0x06)
+#define MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS  (0x08)
 #define MPI2_RPY_DESCRIPT_FLAGS_UNUSED                      (0x0F)
 
 /* values for marking a reply descriptor as unused */
@@ -607,6 +653,14 @@ typedef MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR
     MPI2_POINTER pMpi25FastPathSCSIIOSuccessReplyDescriptor_t;
 
 
+/* PCIe Encapsulated Success Reply Descriptor */
+typedef MPI2_RAID_ACCELERATOR_SUCCESS_REPLY_DESCRIPTOR
+    MPI26_PCIE_ENCAPSULATED_SUCCESS_REPLY_DESCRIPTOR,
+    MPI2_POINTER PTR_MPI26_PCIE_ENCAPSULATED_SUCCESS_REPLY_DESCRIPTOR,
+    Mpi26PCIeEncapsulatedSuccessReplyDescriptor_t,
+    MPI2_POINTER pMpi26PCIeEncapsulatedSuccessReplyDescriptor_t;
+
+
 /* union of Reply Descriptors */
 typedef union _MPI2_REPLY_DESCRIPTORS_UNION
 {
@@ -617,6 +671,7 @@ typedef union _MPI2_REPLY_DESCRIPTORS_UNION
     MPI2_TARGET_COMMAND_BUFFER_REPLY_DESCRIPTOR     TargetCommandBuffer;
     MPI2_RAID_ACCELERATOR_SUCCESS_REPLY_DESCRIPTOR  RAIDAcceleratorSuccess;
     MPI25_FP_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR       FastPathSCSIIOSuccess;
+    MPI26_PCIE_ENCAPSULATED_SUCCESS_REPLY_DESCRIPTOR    PCIeEncapsulatedSuccess;
     U64                                             Words;
 } MPI2_REPLY_DESCRIPTORS_UNION, MPI2_POINTER PTR_MPI2_REPLY_DESCRIPTORS_UNION,
   Mpi2ReplyDescriptorsUnion_t, MPI2_POINTER pMpi2ReplyDescriptorsUnion_t;
@@ -659,6 +714,7 @@ typedef union _MPI2_REPLY_DESCRIPTORS_UNION
 #define MPI2_FUNCTION_HOST_BASED_DISCOVERY_ACTION   (0x2F) /* Host Based Discovery Action */
 #define MPI2_FUNCTION_PWR_MGMT_CONTROL              (0x30) /* Power Management Control */
 #define MPI2_FUNCTION_SEND_HOST_MESSAGE             (0x31) /* Send Host Message */
+#define MPI2_FUNCTION_NVME_ENCAPSULATED             (0x33) /* NVMe Encapsulated (MPI v2.6) */
 #define MPI2_FUNCTION_MIN_PRODUCT_SPECIFIC          (0xF0) /* beginning of product-specific range */
 #define MPI2_FUNCTION_MAX_PRODUCT_SPECIFIC          (0xFF) /* end of product-specific range */
 
@@ -1232,6 +1288,8 @@ typedef union _MPI25_SGE_IO_UNION
 
 #define MPI26_IEEE_SGE_FLAGS_NSF_MASK           (0x1C)
 #define MPI26_IEEE_SGE_FLAGS_NSF_MPI_IEEE       (0x00)
+#define MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP       (0x08)
+#define MPI26_IEEE_SGE_FLAGS_NSF_NVME_SGL       (0x10)
 
 /* Data Location Address Space */
 
diff --git a/sys/dev/mpr/mpi/mpi2_cnfg.h b/sys/dev/mpr/mpi/mpi2_cnfg.h
index fb968b6a68d0..0595920e2f9d 100644
--- a/sys/dev/mpr/mpi/mpi2_cnfg.h
+++ b/sys/dev/mpr/mpi/mpi2_cnfg.h
@@ -42,7 +42,7 @@
  *          Title:  MPI Configuration messages and pages
  *  Creation Date:  November 10, 2006
  *
- *    mpi2_cnfg.h Version:  02.00.35
+ *    mpi2_cnfg.h Version:  02.00.39
  *
  *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *        prefix are for use only on MPI v2.5 products, and must not be used
@@ -223,9 +223,38 @@
  *                      Flags field to IO Unit Page 7.
  *                      Added IO Unit Page 11.
  *                      Added new SAS Phy Event codes
+ *                      Added PCIe configuration pages.
+ *  03-19-15  02.00.32  Fixed PCIe Link Config page structure names to be
+ *                      unique in first 32 characters.
  *  05-25-15  02.00.33  Added more defines for the BiosOptions field of
  *                      MPI2_CONFIG_PAGE_BIOS_1.
+ *  08-25-15  02.00.34  Added PCIe Device Page 2 SGL format capability.
  *  12-18-15  02.00.35  Added SATADeviceWaitTime to SAS IO Unit Page 4.
+ *  01-21-16  02.00.36  Added/modified MPI2_MFGPAGE_DEVID_SAS defines.
+ *                      Added Link field to PCIe Link Pages
+ *                      Added EnclosureLevel and ConnectorName to PCIe
+ *                      Device Page 0.
+ *                      Added define for PCIE IoUnit page 1 max rate shift.
+ *                      Added comment for reserved ExtPageTypes.
+ *                      Added SAS 4 22.5 gbs speed support.
+ *                      Added PCIe 4 16.0 GT/sec speec support.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ *                      Added NegotiatedLinkRate and NegotiatedPortWidth to
+ *                      PCIe device page 0.
+ *  04-10-16  02.00.37  Fixed MPI2_MFGPAGE_DEVID_SAS3616/3708 defines
+ *  07-01-16  02.00.38  Added Manufacturing page 7 Connector types.
+ *                      Changed declaration of ConnectorName in PCIe DevicePage0
+ *                      to match SAS DevicePage 0.
+ *                      Added SATADeviceWaitTime to IO Unit Page 11.
+ *                      Added MPI26_MFGPAGE_DEVID_SAS4008
+ *                      Added x16 PCIe width to IO Unit Page 7
+ *                      Added LINKFLAGS to control SRIS in PCIe IO Unit page 1
+ *                      phy data.
+ *                      Added InitStatus to PCIe IO Unit Page 1 header.
+ *  09-01-16  02.00.39  Added MPI26_CONFIG_PAGE_ENCLOSURE_0 and related defines.
+ *                      Added MPI26_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE and
+ *                      MPI26_ENCLOS_PGAD_FORM_HANDLE page address formats.
  *  --------------------------------------------------------------------------
  */
 
@@ -310,6 +339,12 @@ typedef union _MPI2_CONFIG_EXT_PAGE_HEADER_UNION
 #define MPI2_CONFIG_EXTPAGETYPE_SAS_PORT            (0x18)
 #define MPI2_CONFIG_EXTPAGETYPE_ETHERNET            (0x19)
 #define MPI2_CONFIG_EXTPAGETYPE_EXT_MANUFACTURING   (0x1A)
+#define MPI2_CONFIG_EXTPAGETYPE_PCIE_IO_UNIT        (0x1B) /* MPI v2.6 and later */
+#define MPI2_CONFIG_EXTPAGETYPE_PCIE_SWITCH         (0x1C) /* MPI v2.6 and later */
+#define MPI2_CONFIG_EXTPAGETYPE_PCIE_DEVICE         (0x1D) /* MPI v2.6 and later */
+#define MPI2_CONFIG_EXTPAGETYPE_PCIE_LINK           (0x1E) /* MPI v2.6 and later */
+/*  Product specific reserved values  0xE0 - 0xEF */
+/*  Vendor specific reserved values   0xF0 - 0xFF */
 
 
 /*****************************************************************************
@@ -377,6 +412,12 @@ typedef union _MPI2_CONFIG_EXT_PAGE_HEADER_UNION
 
 #define MPI2_SAS_ENCLOS_PGAD_HANDLE_MASK            (0x0000FFFF)
 
+/* Enclosure PageAddress format */
+#define MPI26_ENCLOS_PGAD_FORM_MASK                 (0xF0000000)
+#define MPI26_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE      (0x00000000)
+#define MPI26_ENCLOS_PGAD_FORM_HANDLE               (0x10000000)
+
+#define MPI26_ENCLOS_PGAD_HANDLE_MASK               (0x0000FFFF)
 
 /* RAID Configuration PageAddress format */
 #define MPI2_RAID_PGAD_FORM_MASK                    (0xF0000000)
@@ -403,6 +444,33 @@ typedef union _MPI2_CONFIG_EXT_PAGE_HEADER_UNION
 #define MPI2_ETHERNET_PGAD_IF_NUMBER_MASK           (0x000000FF)
 
 
+/* PCIe Switch PageAddress format */
+#define MPI26_PCIE_SWITCH_PGAD_FORM_MASK            (0xF0000000)
+#define MPI26_PCIE_SWITCH_PGAD_FORM_GET_NEXT_HNDL   (0x00000000)
+#define MPI26_PCIE_SWITCH_PGAD_FORM_HNDL_PORTNUM    (0x10000000)
+#define MPI26_PCIE_SWITCH_EXPAND_PGAD_FORM_HNDL     (0x20000000)
+
+#define MPI26_PCIE_SWITCH_PGAD_HANDLE_MASK          (0x0000FFFF)
+#define MPI26_PCIE_SWITCH_PGAD_PORTNUM_MASK         (0x00FF0000)
+#define MPI26_PCIE_SWITCH_PGAD_PORTNUM_SHIFT        (16)
+
+
+/* PCIe Device PageAddress format */
+#define MPI26_PCIE_DEVICE_PGAD_FORM_MASK            (0xF0000000)
+#define MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE (0x00000000)
+#define MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE          (0x20000000)
+
+#define MPI26_PCIE_DEVICE_PGAD_HANDLE_MASK          (0x0000FFFF)
+
+/* PCIe Link PageAddress format */
+#define MPI26_PCIE_LINK_PGAD_FORM_MASK            (0xF0000000)
+#define MPI26_PCIE_LINK_PGAD_FORM_GET_NEXT_LINK   (0x00000000)
+#define MPI26_PCIE_LINK_PGAD_FORM_LINK_NUM        (0x10000000)
+
+#define MPI26_PCIE_DEVICE_PGAD_LINKNUM_MASK       (0x000000FF)
+
+
+
 /****************************************************************************
 *   Configuration messages
 ****************************************************************************/
@@ -518,6 +586,20 @@ typedef struct _MPI2_CONFIG_REPLY
 #define MPI26_MFGPAGE_DEVID_SAS3324_3               (0x00C2)
 #define MPI26_MFGPAGE_DEVID_SAS3324_4               (0x00C3)
 
+#define MPI26_MFGPAGE_DEVID_SAS3516                 (0x00AA)
+#define MPI26_MFGPAGE_DEVID_SAS3516_1               (0x00AB)
+#define MPI26_MFGPAGE_DEVID_SAS3416                 (0x00AC)
+#define MPI26_MFGPAGE_DEVID_SAS3508                 (0x00AD)
+#define MPI26_MFGPAGE_DEVID_SAS3508_1               (0x00AE)
+#define MPI26_MFGPAGE_DEVID_SAS3408                 (0x00AF)
+
+#define MPI26_MFGPAGE_DEVID_SAS3716                 (0x00D0)
+#define MPI26_MFGPAGE_DEVID_SAS3616                 (0x00D1)
+#define MPI26_MFGPAGE_DEVID_SAS3708                 (0x00D2)
+
+#define MPI26_MFGPAGE_DEVID_SAS4008                 (0x00A1)
+
+
 /* Manufacturing Page 0 */
 
 typedef struct _MPI2_CONFIG_PAGE_MAN_0
@@ -755,6 +837,12 @@ typedef struct _MPI2_MANPAGE7_CONNECTOR_INFO
 #define MPI2_MANPAGE7_PINOUT_SFF_8644_8X                (0x0B)
 #define MPI2_MANPAGE7_PINOUT_SFF_8644_16X               (0x0C)
 #define MPI2_MANPAGE7_PINOUT_SFF_8436                   (0x0D)
+#define MPI2_MANPAGE7_PINOUT_SFF_8088_A                 (0x0E)
+#define MPI2_MANPAGE7_PINOUT_SFF_8643_16i               (0x0F)
+#define MPI2_MANPAGE7_PINOUT_SFF_8654_4i                (0x10)
+#define MPI2_MANPAGE7_PINOUT_SFF_8654_8i                (0x11)
+#define MPI2_MANPAGE7_PINOUT_SFF_8611_4i                (0x12)
+#define MPI2_MANPAGE7_PINOUT_SFF_8611_8i                (0x13)
 
 /* defines for the Location field */
 #define MPI2_MANPAGE7_LOCATION_UNKNOWN                  (0x01)
@@ -1017,11 +1105,13 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_7
 #define MPI2_IOUNITPAGE7_PCIE_WIDTH_X2              (0x02)
 #define MPI2_IOUNITPAGE7_PCIE_WIDTH_X4              (0x04)
 #define MPI2_IOUNITPAGE7_PCIE_WIDTH_X8              (0x08)
+#define MPI2_IOUNITPAGE7_PCIE_WIDTH_X16             (0x10)
 
 /* defines for IO Unit Page 7 PCIeSpeed field */
 #define MPI2_IOUNITPAGE7_PCIE_SPEED_2_5_GBPS        (0x00)
 #define MPI2_IOUNITPAGE7_PCIE_SPEED_5_0_GBPS        (0x01)
 #define MPI2_IOUNITPAGE7_PCIE_SPEED_8_0_GBPS        (0x02)
+#define MPI2_IOUNITPAGE7_PCIE_SPEED_16_0_GBPS       (0x03)
 
 /* defines for IO Unit Page 7 ProcessorState field */
 #define MPI2_IOUNITPAGE7_PSTATE_MASK_SECOND         (0x0000000F)
@@ -1079,6 +1169,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_7
 /* defines for IO Unit Page 7 Flags field */
 #define MPI2_IOUNITPAGE7_FLAG_CABLE_POWER_EXC       (0x01)
 
+
 /* IO Unit Page 8 */
 
 #define MPI2_IOUNIT8_NUM_THRESHOLDS     (4)
@@ -1228,7 +1319,7 @@ typedef struct _MPI26_CONFIG_PAGE_IO_UNIT_11
     U32                             Reserved3;                      /* 0x1C */
     U32                             Reserved4;                      /* 0x20 */
     U8                              BootDeviceWaitTime;             /* 0x24 */
-    U8                              Reserved5;                      /* 0x25 */
+    U8                              SATADeviceWaitTime;             /* 0x25 */
     U16                             Reserved6;                      /* 0x26 */
     U8                              NumPhys;                        /* 0x28 */
     U8                              PEInitialSpinupDelay;           /* 0x29 */
@@ -1249,9 +1340,6 @@ typedef struct _MPI26_CONFIG_PAGE_IO_UNIT_11
 
 
 
-
-
-
 /****************************************************************************
 *   IOC Config Pages
 ****************************************************************************/
@@ -1968,6 +2056,7 @@ typedef struct _MPI2_CONFIG_PAGE_RD_PDISK_1
 #define MPI2_SAS_NEG_LINK_RATE_3_0                      (0x09)
 #define MPI2_SAS_NEG_LINK_RATE_6_0                      (0x0A)
 #define MPI25_SAS_NEG_LINK_RATE_12_0                    (0x0B)
+#define MPI26_SAS_NEG_LINK_RATE_22_5                    (0x0C)
 
 
 /* values for AttachedPhyInfo fields */
@@ -2035,12 +2124,14 @@ typedef struct _MPI2_CONFIG_PAGE_RD_PDISK_1
 #define MPI2_SAS_PRATE_MAX_RATE_3_0                     (0x90)
 #define MPI2_SAS_PRATE_MAX_RATE_6_0                     (0xA0)
 #define MPI25_SAS_PRATE_MAX_RATE_12_0                   (0xB0)
+#define MPI26_SAS_PRATE_MAX_RATE_22_5                   (0xC0)
 #define MPI2_SAS_PRATE_MIN_RATE_MASK                    (0x0F)
 #define MPI2_SAS_PRATE_MIN_RATE_NOT_PROGRAMMABLE        (0x00)
 #define MPI2_SAS_PRATE_MIN_RATE_1_5                     (0x08)
 #define MPI2_SAS_PRATE_MIN_RATE_3_0                     (0x09)
 #define MPI2_SAS_PRATE_MIN_RATE_6_0                     (0x0A)
 #define MPI25_SAS_PRATE_MIN_RATE_12_0                   (0x0B)
+#define MPI26_SAS_PRATE_MIN_RATE_22_5                   (0x0C)
 
 
 /* values for SAS HwLinkRate fields */
@@ -2049,11 +2140,13 @@ typedef struct _MPI2_CONFIG_PAGE_RD_PDISK_1
 #define MPI2_SAS_HWRATE_MAX_RATE_3_0                    (0x90)
 #define MPI2_SAS_HWRATE_MAX_RATE_6_0                    (0xA0)
 #define MPI25_SAS_HWRATE_MAX_RATE_12_0                  (0xB0)
+#define MPI26_SAS_HWRATE_MAX_RATE_22_5                  (0xC0)
 #define MPI2_SAS_HWRATE_MIN_RATE_MASK                   (0x0F)
 #define MPI2_SAS_HWRATE_MIN_RATE_1_5                    (0x08)
 #define MPI2_SAS_HWRATE_MIN_RATE_3_0                    (0x09)
 #define MPI2_SAS_HWRATE_MIN_RATE_6_0                    (0x0A)
 #define MPI25_SAS_HWRATE_MIN_RATE_12_0                  (0x0B)
+#define MPI26_SAS_HWRATE_MIN_RATE_22_5                  (0x0C)
 
 
 
@@ -2227,11 +2320,13 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_1
 #define MPI2_SASIOUNIT1_MAX_RATE_3_0                                (0x90)
 #define MPI2_SASIOUNIT1_MAX_RATE_6_0                                (0xA0)
 #define MPI25_SASIOUNIT1_MAX_RATE_12_0                              (0xB0)
+#define MPI26_SASIOUNIT1_MAX_RATE_22_5                              (0xC0)
 #define MPI2_SASIOUNIT1_MIN_RATE_MASK                               (0x0F)
 #define MPI2_SASIOUNIT1_MIN_RATE_1_5                                (0x08)
 #define MPI2_SASIOUNIT1_MIN_RATE_3_0                                (0x09)
 #define MPI2_SASIOUNIT1_MIN_RATE_6_0                                (0x0A)
 #define MPI25_SASIOUNIT1_MIN_RATE_12_0                              (0x0B)
+#define MPI26_SASIOUNIT1_MIN_RATE_22_5                              (0x0C)
 
 /* see mpi2_sas.h for values for SAS IO Unit Page 1 ControllerPhyDeviceInfo values */
 
@@ -2718,7 +2813,6 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_DEV_0
 #define MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID             (0x0002)
 #define MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT               (0x0001)
 
-
 /* SAS Device Page 1 */
 
 typedef struct _MPI2_CONFIG_PAGE_SAS_DEV_1
@@ -2885,7 +2979,6 @@ typedef struct _MPI2_SASPHY3_PHY_EVENT_CONFIG
 #define MPI2_SASPHY3_EVENT_CODE_HOTPLUG_TIMEOUT             (0xD0)
 #define MPI2_SASPHY3_EVENT_CODE_MISALIGNED_MUX_PRIMITIVE    (0xD1)
 #define MPI2_SASPHY3_EVENT_CODE_RX_AIP                      (0xD2)
-
 /* Following codes are product specific and in MPI v2.6 and later */
 #define MPI2_SASPHY3_EVENT_CODE_LCARB_WAIT_TIME             (0xD3)
 #define MPI2_SASPHY3_EVENT_CODE_RCVD_CONN_RESP_WAIT_TIME    (0xD4)
@@ -2898,7 +2991,6 @@ typedef struct _MPI2_SASPHY3_PHY_EVENT_CONFIG
 #define MPI2_SASPHY3_EVENT_CODE_SATA_RX_START_RECEIVE       (0xDB)
 #define MPI2_SASPHY3_EVENT_CODE_SMP_RX_START_RECEIVE        (0xDC)
 
-
 /* values for the CounterType field */
 #define MPI2_SASPHY3_COUNTER_TYPE_WRAPPING                  (0x00)
 #define MPI2_SASPHY3_COUNTER_TYPE_SATURATING                (0x01)
@@ -2989,7 +3081,7 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_PORT_0
 *   SAS Enclosure Config Pages
 ****************************************************************************/
 
-/* SAS Enclosure Page 0 */
+/* SAS Enclosure Page 0, Enclosure Page 0 */
 
 typedef struct _MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0
 {
@@ -3007,7 +3099,10 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0
     U32                                 Reserved4;                  /* 0x24 */
 } MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0,
   MPI2_POINTER PTR_MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0,
-  Mpi2SasEnclosurePage0_t, MPI2_POINTER pMpi2SasEnclosurePage0_t;
+  Mpi2SasEnclosurePage0_t, MPI2_POINTER pMpi2SasEnclosurePage0_t,
+  MPI26_CONFIG_PAGE_ENCLOSURE_0,
+  MPI2_POINTER PTR_MPI26_CONFIG_PAGE_ENCLOSURE_0,
+  Mpi26EnclosurePage0_t, MPI2_POINTER pMpi26EnclosurePage0_t;
 
 #define MPI2_SASENCLOSURE0_PAGEVERSION      (0x04)
 
@@ -3021,6 +3116,17 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0
 #define MPI2_SAS_ENCLS0_FLAGS_MNG_SES_ENCLOSURE     (0x0004)
 #define MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_GPIO          (0x0005)
 
+#define MPI26_ENCLOSURE0_PAGEVERSION        (0x04)
+
+/* Values for Enclosure Page 0 Flags field */
+#define MPI26_ENCLS0_FLAGS_ENCL_LEVEL_VALID         (0x0010)
+#define MPI26_ENCLS0_FLAGS_MNG_MASK                 (0x000F)
+#define MPI26_ENCLS0_FLAGS_MNG_UNKNOWN              (0x0000)
+#define MPI26_ENCLS0_FLAGS_MNG_IOC_SES              (0x0001)
+#define MPI26_ENCLS0_FLAGS_MNG_IOC_SGPIO            (0x0002)
+#define MPI26_ENCLS0_FLAGS_MNG_EXP_SGPIO            (0x0003)
+#define MPI26_ENCLS0_FLAGS_MNG_SES_ENCLOSURE        (0x0004)
+#define MPI26_ENCLS0_FLAGS_MNG_IOC_GPIO             (0x0005)
 
 /****************************************************************************
 *   Log Config Page
@@ -3300,5 +3406,424 @@ typedef struct _MPI2_CONFIG_PAGE_EXT_MAN_PS
 
 /* PageVersion should be provided by product-specific code */
 
+
+/****************************************************************************
+*   values for fields used by several types of PCIe Config Pages
+****************************************************************************/
+
+/* values for NegotiatedLinkRates fields */
+#define MPI26_PCIE_NEG_LINK_RATE_MASK_PHYSICAL          (0x0F)
+/* link rates used for Negotiated Physical Link Rate */
+#define MPI26_PCIE_NEG_LINK_RATE_UNKNOWN                (0x00)
+#define MPI26_PCIE_NEG_LINK_RATE_PHY_DISABLED           (0x01)
+#define MPI26_PCIE_NEG_LINK_RATE_2_5                    (0x02)
+#define MPI26_PCIE_NEG_LINK_RATE_5_0                    (0x03)
+#define MPI26_PCIE_NEG_LINK_RATE_8_0                    (0x04)
+#define MPI26_PCIE_NEG_LINK_RATE_16_0                   (0x05)
+
+
+/****************************************************************************
+*   PCIe IO Unit Config Pages (MPI v2.6 and later)
+****************************************************************************/
+
+/* PCIe IO Unit Page 0 */
+
+typedef struct _MPI26_PCIE_IO_UNIT0_PHY_DATA
+{
+    U8          Link;                   /* 0x00 */
+    U8          LinkFlags;              /* 0x01 */
+    U8          PhyFlags;               /* 0x02 */
+    U8          NegotiatedLinkRate;     /* 0x03 */
+    U32         ControllerPhyDeviceInfo;/* 0x04 */
+    U16         AttachedDevHandle;      /* 0x08 */
+    U16         ControllerDevHandle;    /* 0x0A */
+    U32         EnumerationStatus;      /* 0x0C */
+    U32         Reserved1;              /* 0x10 */
+} MPI26_PCIE_IO_UNIT0_PHY_DATA, MPI2_POINTER PTR_MPI26_PCIE_IO_UNIT0_PHY_DATA,
+  Mpi26PCIeIOUnit0PhyData_t, MPI2_POINTER pMpi26PCIeIOUnit0PhyData_t;
+
+/*
+ * Host code (drivers, BIOS, utilities, etc.) should leave this define set to
+ * one and check the value returned for NumPhys at runtime.
+ */
+#ifndef MPI26_PCIE_IOUNIT0_PHY_MAX
+#define MPI26_PCIE_IOUNIT0_PHY_MAX      (1)
+#endif
+
+typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_0
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                                 /* 0x00 */
+    U32                                 Reserved1;                              /* 0x08 */
+    U8                                  NumPhys;                                /* 0x0C */
+    U8                                  InitStatus;                             /* 0x0D */
+    U16                                 Reserved3;                              /* 0x0E */
+    MPI26_PCIE_IO_UNIT0_PHY_DATA        PhyData[MPI26_PCIE_IOUNIT0_PHY_MAX];    /* 0x10 */
+} MPI26_CONFIG_PAGE_PIOUNIT_0,
+  MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PIOUNIT_0,
+  Mpi26PCIeIOUnitPage0_t, MPI2_POINTER pMpi26PCIeIOUnitPage0_t;
+
+#define MPI26_PCIEIOUNITPAGE0_PAGEVERSION                   (0x00)
+
+/* values for PCIe IO Unit Page 0 LinkFlags */
+#define MPI26_PCIEIOUNIT0_LINKFLAGS_ENUMERATION_IN_PROGRESS (0x08)
+
+/* values for PCIe IO Unit Page 0 PhyFlags */
+#define MPI26_PCIEIOUNIT0_PHYFLAGS_PHY_DISABLED             (0x08)
+
+/* use MPI26_PCIE_NEG_LINK_RATE_ defines for the NegotiatedLinkRate field */
+
+/* see mpi2_pci.h for values for PCIe IO Unit Page 0 ControllerPhyDeviceInfo values */
+
+/* values for PCIe IO Unit Page 0 EnumerationStatus */
+#define MPI26_PCIEIOUNIT0_ES_MAX_SWITCHES_EXCEEDED          (0x40000000)
+#define MPI26_PCIEIOUNIT0_ES_MAX_DEVICES_EXCEEDED           (0x20000000)
+
+
+/* PCIe IO Unit Page 1 */
+
+typedef struct _MPI26_PCIE_IO_UNIT1_PHY_DATA
+{
+    U8          Link;                       /* 0x00 */
+    U8          LinkFlags;                  /* 0x01 */
+    U8          PhyFlags;                   /* 0x02 */
+    U8          MaxMinLinkRate;             /* 0x03 */
+    U32         ControllerPhyDeviceInfo;    /* 0x04 */
+    U32         Reserved1;                  /* 0x08 */
+} MPI26_PCIE_IO_UNIT1_PHY_DATA, MPI2_POINTER PTR_MPI26_PCIE_IO_UNIT1_PHY_DATA,
+  Mpi26PCIeIOUnit1PhyData_t, MPI2_POINTER pMpi26PCIeIOUnit1PhyData_t;
+
+/* values for LinkFlags */
+#define MPI26_PCIEIOUNIT1_LINKFLAGS_DIS_SRIS    (0x00)
+#define MPI26_PCIEIOUNIT1_LINKFLAGS_EN_SRIS     (0x01)
+
+/*
+ * Host code (drivers, BIOS, utilities, etc.) should leave this define set to
+ * one and check the value returned for NumPhys at runtime.
+ */
+#ifndef MPI26_PCIE_IOUNIT1_PHY_MAX
+#define MPI26_PCIE_IOUNIT1_PHY_MAX      (1)
+#endif
+
+typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_1
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                             /* 0x00 */
+    U16                                 ControlFlags;                       /* 0x08 */
+    U16                                 Reserved;                           /* 0x0A */
+    U16                                 AdditionalControlFlags;             /* 0x0C */
+    U16                                 NVMeMaxQueueDepth;                  /* 0x0E */
+    U8                                  NumPhys;                            /* 0x10 */
+    U8                                  Reserved1;                          /* 0x11 */
+    U16                                 Reserved2;                          /* 0x12 */
+    MPI26_PCIE_IO_UNIT1_PHY_DATA        PhyData[MPI26_PCIE_IOUNIT1_PHY_MAX];/* 0x14 */
+} MPI26_CONFIG_PAGE_PIOUNIT_1,
+  MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PIOUNIT_1,
+  Mpi26PCIeIOUnitPage1_t, MPI2_POINTER pMpi26PCIeIOUnitPage1_t;
+
+#define MPI26_PCIEIOUNITPAGE1_PAGEVERSION   (0x00)
+
+/* values for PCIe IO Unit Page 1 PhyFlags */
+#define MPI26_PCIEIOUNIT1_PHYFLAGS_PHY_DISABLE                      (0x08)
+#define MPI26_PCIEIOUNIT1_PHYFLAGS_ENDPOINT_ONLY                    (0x01)
+
+/* values for PCIe IO Unit Page 1 MaxMinLinkRate */
+#define MPI26_PCIEIOUNIT1_MAX_RATE_MASK                             (0xF0)
+#define MPI26_PCIEIOUNIT1_MAX_RATE_SHIFT                            (4)
+#define MPI26_PCIEIOUNIT1_MAX_RATE_2_5                              (0x20)
+#define MPI26_PCIEIOUNIT1_MAX_RATE_5_0                              (0x30)
+#define MPI26_PCIEIOUNIT1_MAX_RATE_8_0                              (0x40)
+#define MPI26_PCIEIOUNIT1_MAX_RATE_16_0                             (0x50)
+
+/* see mpi2_pci.h for values for PCIe IO Unit Page 0 ControllerPhyDeviceInfo values */
+
+
+/****************************************************************************
+*   PCIe Switch Config Pages (MPI v2.6 and later)
+****************************************************************************/
+
+/* PCIe Switch Page 0 */
+
+typedef struct _MPI26_CONFIG_PAGE_PSWITCH_0
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                     /* 0x00 */
+    U8                                  PhysicalPort;               /* 0x08 */
+    U8                                  Reserved1;                  /* 0x09 */
+    U16                                 Reserved2;                  /* 0x0A */
+    U16                                 DevHandle;                  /* 0x0C */
+    U16                                 ParentDevHandle;            /* 0x0E */
+    U8                                  NumPorts;                   /* 0x10 */
+    U8                                  PCIeLevel;                  /* 0x11 */
+    U16                                 Reserved3;                  /* 0x12 */
+    U32                                 Reserved4;                  /* 0x14 */
+    U32                                 Reserved5;                  /* 0x18 */
+    U32                                 Reserved6;                  /* 0x1C */
+} MPI26_CONFIG_PAGE_PSWITCH_0, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PSWITCH_0,
+  Mpi26PCIeSwitchPage0_t, MPI2_POINTER pMpi26PCIeSwitchPage0_t;
+
+#define MPI26_PCIESWITCH0_PAGEVERSION       (0x00)
+
+
+/* PCIe Switch Page 1 */
+
+typedef struct _MPI26_CONFIG_PAGE_PSWITCH_1
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                     /* 0x00 */
+    U8                                  PhysicalPort;               /* 0x08 */
+    U8                                  Reserved1;                  /* 0x09 */
+    U16                                 Reserved2;                  /* 0x0A */
+    U8                                  NumPorts;                   /* 0x0C */
+    U8                                  PortNum;                    /* 0x0D */
+    U16                                 AttachedDevHandle;          /* 0x0E */
+    U16                                 SwitchDevHandle;            /* 0x10 */
+    U8                                  NegotiatedPortWidth;        /* 0x12 */
+    U8                                  NegotiatedLinkRate;         /* 0x13 */
+    U32                                 Reserved4;                  /* 0x14 */
+    U32                                 Reserved5;                  /* 0x18 */
+} MPI26_CONFIG_PAGE_PSWITCH_1, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PSWITCH_1,
+  Mpi26PCIeSwitchPage1_t, MPI2_POINTER pMpi26PCIeSwitchPage1_t;
+
+#define MPI26_PCIESWITCH1_PAGEVERSION       (0x00)
+
+/* use MPI26_PCIE_NEG_LINK_RATE_ defines for the NegotiatedLinkRate field */
+
+
+/****************************************************************************
+*   PCIe Device Config Pages (MPI v2.6 and later)
+****************************************************************************/
+
+/* PCIe Device Page 0 */
+
+typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_0
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                 /* 0x00 */
+    U16                                 Slot;                   /* 0x08 */
+    U16                                 EnclosureHandle;        /* 0x0A */
+    U64                                 WWID;                   /* 0x0C */
+    U16                                 ParentDevHandle;        /* 0x14 */
+    U8                                  PortNum;                /* 0x16 */
+    U8                                  AccessStatus;           /* 0x17 */
+    U16                                 DevHandle;              /* 0x18 */
+    U8                                  PhysicalPort;           /* 0x1A */
+    U8                                  Reserved1;              /* 0x1B */
+    U32                                 DeviceInfo;             /* 0x1C */
+    U32                                 Flags;                  /* 0x20 */
+    U8                                  SupportedLinkRates;     /* 0x24 */
+    U8                                  MaxPortWidth;           /* 0x25 */
+    U8                                  NegotiatedPortWidth;    /* 0x26 */
+    U8                                  NegotiatedLinkRate;     /* 0x27 */
+    U8                                  EnclosureLevel;         /* 0x28 */
+    U8                                  Reserved2;              /* 0x29 */
+    U16                                 Reserved3;              /* 0x2A */
+    U8                                  ConnectorName[4];       /* 0x2C */
+    U32                                 Reserved4;              /* 0x30 */
+    U32                                 Reserved5;              /* 0x34 */
+} MPI26_CONFIG_PAGE_PCIEDEV_0, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PCIEDEV_0,
+  Mpi26PCIeDevicePage0_t, MPI2_POINTER pMpi26PCIeDevicePage0_t;
+
+#define MPI26_PCIEDEVICE0_PAGEVERSION       (0x01)
+
+/* values for PCIe Device Page 0 AccessStatus field */
+#define MPI26_PCIEDEV0_ASTATUS_NO_ERRORS                    (0x00)
+#define MPI26_PCIEDEV0_ASTATUS_NEEDS_INITIALIZATION         (0x04)
+#define MPI26_PCIEDEV0_ASTATUS_CAPABILITY_FAILED            (0x02)
+#define MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED               (0x07)
+#define MPI26_PCIEDEV0_ASTATUS_MEMORY_SPACE_ACCESS_FAILED   (0x08)
+#define MPI26_PCIEDEV0_ASTATUS_UNSUPPORTED_DEVICE           (0x09)
+#define MPI26_PCIEDEV0_ASTATUS_MSIX_REQUIRED                (0x0A)
+#define MPI26_PCIEDEV0_ASTATUS_UNKNOWN                      (0x10)
+
+#define MPI26_PCIEDEV0_ASTATUS_NVME_READY_TIMEOUT           (0x30)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_DEVCFG_UNSUPPORTED      (0x31)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_IDENTIFY_FAILED         (0x32)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_QCONFIG_FAILED          (0x33)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_QCREATION_FAILED        (0x34)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_EVENTCFG_FAILED         (0x35)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_GET_FEATURE_STAT_FAILED (0x36)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_IDLE_TIMEOUT            (0x37)
+#define MPI26_PCIEDEV0_ASTATUS_NVME_FAILURE_STATUS          (0x38)
+
+#define MPI26_PCIEDEV0_ASTATUS_INIT_FAIL_MAX                (0x3F)
+
+/* see mpi2_pci.h for the MPI26_PCIE_DEVINFO_ defines used for the DeviceInfo field */
+
+/* values for PCIe Device Page 0 Flags field */
+#define MPI26_PCIEDEV0_FLAGS_UNAUTHORIZED_DEVICE            (0x8000)
+#define MPI26_PCIEDEV0_FLAGS_ENABLED_FAST_PATH              (0x4000)
+#define MPI26_PCIEDEV0_FLAGS_FAST_PATH_CAPABLE              (0x2000)
+#define MPI26_PCIEDEV0_FLAGS_ASYNCHRONOUS_NOTIFICATION      (0x0400)
+#define MPI26_PCIEDEV0_FLAGS_ATA_SW_PRESERVATION            (0x0200)
+#define MPI26_PCIEDEV0_FLAGS_UNSUPPORTED_DEVICE             (0x0100)
+#define MPI26_PCIEDEV0_FLAGS_ATA_48BIT_LBA_SUPPORTED        (0x0080)
+#define MPI26_PCIEDEV0_FLAGS_ATA_SMART_SUPPORTED            (0x0040)
+#define MPI26_PCIEDEV0_FLAGS_ATA_NCQ_SUPPORTED              (0x0020)
+#define MPI26_PCIEDEV0_FLAGS_ATA_FUA_SUPPORTED              (0x0010)
+#define MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID               (0x0002)
+#define MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT                 (0x0001)
+
+/* values for PCIe Device Page 0 SupportedLinkRates field */
+#define MPI26_PCIEDEV0_LINK_RATE_16_0_SUPPORTED             (0x08)
+#define MPI26_PCIEDEV0_LINK_RATE_8_0_SUPPORTED              (0x04)
+#define MPI26_PCIEDEV0_LINK_RATE_5_0_SUPPORTED              (0x02)
+#define MPI26_PCIEDEV0_LINK_RATE_2_5_SUPPORTED              (0x01)
+
+/* use MPI26_PCIE_NEG_LINK_RATE_ defines for the NegotiatedLinkRate field */
+
+
+/* PCIe Device Page 2 */
+
+typedef struct _MPI26_CONFIG_PAGE_PCIEDEV_2
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                 /* 0x00 */
+    U16                                 DevHandle;              /* 0x08 */
+    U16                                 Reserved1;              /* 0x0A */
+    U32                                 MaximumDataTransferSize;/* 0x0C */
+    U32                                 Capabilities;           /* 0x10 */
+    U32                                 Reserved2;              /* 0x14 */
+} MPI26_CONFIG_PAGE_PCIEDEV_2, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PCIEDEV_2,
+  Mpi26PCIeDevicePage2_t, MPI2_POINTER pMpi26PCIeDevicePage2_t;
+
+#define MPI26_PCIEDEVICE2_PAGEVERSION       (0x00)
+
+/* defines for PCIe Device Page 2 Capabilities field */
+#define MPI26_PCIEDEV2_CAP_SGL_FORMAT                  (0x00000004)
+#define MPI26_PCIEDEV2_CAP_BIT_BUCKET_SUPPORT          (0x00000002)
+#define MPI26_PCIEDEV2_CAP_SGL_SUPPORT                 (0x00000001)
+
+
+/****************************************************************************
+*   PCIe Link Config Pages (MPI v2.6 and later)
+****************************************************************************/
+
+/* PCIe Link Page 1 */
+
+typedef struct _MPI26_CONFIG_PAGE_PCIELINK_1
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                     /* 0x00 */
+    U8                                  Link;                       /* 0x08 */
+    U8                                  Reserved1;                  /* 0x09 */
+    U16                                 Reserved2;                  /* 0x0A */
+    U32                                 CorrectableErrorCount;      /* 0x0C */
+    U16                                 NonFatalErrorCount;         /* 0x10 */
+    U16                                 Reserved3;                  /* 0x12 */
+    U16                                 FatalErrorCount;            /* 0x14 */
+    U16                                 Reserved4;                  /* 0x16 */
+} MPI26_CONFIG_PAGE_PCIELINK_1, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PCIELINK_1,
+  Mpi26PcieLinkPage1_t, MPI2_POINTER pMpi26PcieLinkPage1_t;
+
+#define MPI26_PCIELINK1_PAGEVERSION            (0x00)
+
+/* PCIe Link Page 2 */
+
+typedef struct _MPI26_PCIELINK2_LINK_EVENT
+{
+    U8          LinkEventCode;      /* 0x00 */
+    U8          Reserved1;          /* 0x01 */
+    U16         Reserved2;          /* 0x02 */
+    U32         LinkEventInfo;      /* 0x04 */
+} MPI26_PCIELINK2_LINK_EVENT, MPI2_POINTER PTR_MPI26_PCIELINK2_LINK_EVENT,
+  Mpi26PcieLink2LinkEvent_t, MPI2_POINTER pMpi26PcieLink2LinkEvent_t;
+
+/* use MPI26_PCIELINK3_EVTCODE_ for the LinkEventCode field */
+
+
+/*
+ * Host code (drivers, BIOS, utilities, etc.) should leave this define set to
+ * one and check the value returned for NumLinkEvents at runtime.
+ */
+#ifndef MPI26_PCIELINK2_LINK_EVENT_MAX
+#define MPI26_PCIELINK2_LINK_EVENT_MAX      (1)
+#endif
+
+typedef struct _MPI26_CONFIG_PAGE_PCIELINK_2
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                     /* 0x00 */
+    U8                                  Link;                       /* 0x08 */
+    U8                                  Reserved1;                  /* 0x09 */
+    U16                                 Reserved2;                  /* 0x0A */
+    U8                                  NumLinkEvents;              /* 0x0C */
+    U8                                  Reserved3;                  /* 0x0D */
+    U16                                 Reserved4;                  /* 0x0E */
+    MPI26_PCIELINK2_LINK_EVENT          LinkEvent[MPI26_PCIELINK2_LINK_EVENT_MAX]; /* 0x10 */
+} MPI26_CONFIG_PAGE_PCIELINK_2, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PCIELINK_2,
+  Mpi26PcieLinkPage2_t, MPI2_POINTER pMpi26PcieLinkPage2_t;
+
+#define MPI26_PCIELINK2_PAGEVERSION            (0x00)
+
+
+/* PCIe Link Page 3 */
+
+typedef struct _MPI26_PCIELINK3_LINK_EVENT_CONFIG
+{
+    U8          LinkEventCode;      /* 0x00 */
+    U8          Reserved1;          /* 0x01 */
+    U16         Reserved2;          /* 0x02 */
+    U8          CounterType;        /* 0x04 */
+    U8          ThresholdWindow;    /* 0x05 */
+    U8          TimeUnits;          /* 0x06 */
+    U8          Reserved3;          /* 0x07 */
+    U32         EventThreshold;     /* 0x08 */
+    U16         ThresholdFlags;     /* 0x0C */
+    U16         Reserved4;          /* 0x0E */
+} MPI26_PCIELINK3_LINK_EVENT_CONFIG, MPI2_POINTER PTR_MPI26_PCIELINK3_LINK_EVENT_CONFIG,
+  Mpi26PcieLink3LinkEventConfig_t, MPI2_POINTER pMpi26PcieLink3LinkEventConfig_t;
+
+/* values for LinkEventCode field */
+#define MPI26_PCIELINK3_EVTCODE_NO_EVENT                              (0x00)
+#define MPI26_PCIELINK3_EVTCODE_CORRECTABLE_ERROR_RECEIVED            (0x01)
+#define MPI26_PCIELINK3_EVTCODE_NON_FATAL_ERROR_RECEIVED              (0x02)
+#define MPI26_PCIELINK3_EVTCODE_FATAL_ERROR_RECEIVED                  (0x03)
+#define MPI26_PCIELINK3_EVTCODE_DATA_LINK_ERROR_DETECTED              (0x04)
+#define MPI26_PCIELINK3_EVTCODE_TRANSACTION_LAYER_ERROR_DETECTED      (0x05)
+#define MPI26_PCIELINK3_EVTCODE_TLP_ECRC_ERROR_DETECTED               (0x06)
+#define MPI26_PCIELINK3_EVTCODE_POISONED_TLP                          (0x07)
+#define MPI26_PCIELINK3_EVTCODE_RECEIVED_NAK_DLLP                     (0x08)
+#define MPI26_PCIELINK3_EVTCODE_SENT_NAK_DLLP                         (0x09)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_RECOVERY_STATE                  (0x0A)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_RXL0S_STATE                     (0x0B)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_TXL0S_STATE                     (0x0C)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_L1_STATE                        (0x0D)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_DISABLED_STATE                  (0x0E)
+#define MPI26_PCIELINK3_EVTCODE_LTSSM_HOT_RESET_STATE                 (0x0F)
+#define MPI26_PCIELINK3_EVTCODE_SYSTEM_ERROR                          (0x10)
+#define MPI26_PCIELINK3_EVTCODE_DECODE_ERROR                          (0x11)
+#define MPI26_PCIELINK3_EVTCODE_DISPARITY_ERROR                       (0x12)
+
+/* values for the CounterType field */
+#define MPI26_PCIELINK3_COUNTER_TYPE_WRAPPING               (0x00)
+#define MPI26_PCIELINK3_COUNTER_TYPE_SATURATING             (0x01)
+#define MPI26_PCIELINK3_COUNTER_TYPE_PEAK_VALUE             (0x02)
+
+/* values for the TimeUnits field */
+#define MPI26_PCIELINK3_TM_UNITS_10_MICROSECONDS            (0x00)
+#define MPI26_PCIELINK3_TM_UNITS_100_MICROSECONDS           (0x01)
+#define MPI26_PCIELINK3_TM_UNITS_1_MILLISECOND              (0x02)
+#define MPI26_PCIELINK3_TM_UNITS_10_MILLISECONDS            (0x03)
+
+/* values for the ThresholdFlags field */
+#define MPI26_PCIELINK3_TFLAGS_EVENT_NOTIFY                 (0x0001)
+
+/*
+ * Host code (drivers, BIOS, utilities, etc.) should leave this define set to
+ * one and check the value returned for NumLinkEvents at runtime.
+ */
+#ifndef MPI26_PCIELINK3_LINK_EVENT_MAX
+#define MPI26_PCIELINK3_LINK_EVENT_MAX      (1)
+#endif
+
+typedef struct _MPI26_CONFIG_PAGE_PCIELINK_3
+{
+    MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                     /* 0x00 */
+    U8                                  Link;                       /* 0x08 */
+    U8                                  Reserved1;                  /* 0x09 */
+    U16                                 Reserved2;                  /* 0x0A */
+    U8                                  NumLinkEvents;              /* 0x0C */
+    U8                                  Reserved3;                  /* 0x0D */
+    U16                                 Reserved4;                  /* 0x0E */
+    MPI26_PCIELINK3_LINK_EVENT_CONFIG   LinkEventConfig[MPI26_PCIELINK3_LINK_EVENT_MAX]; /* 0x10 */
+} MPI26_CONFIG_PAGE_PCIELINK_3, MPI2_POINTER PTR_MPI26_CONFIG_PAGE_PCIELINK_3,
+  Mpi26PcieLinkPage3_t, MPI2_POINTER pMpi26PcieLinkPage3_t;
+
+#define MPI26_PCIELINK3_PAGEVERSION            (0x00)
+
+
 #endif
 
diff --git a/sys/dev/mpr/mpi/mpi2_hbd.h b/sys/dev/mpr/mpi/mpi2_hbd.h
index 48700717f379..37e3c0498adc 100644
--- a/sys/dev/mpr/mpi/mpi2_hbd.h
+++ b/sys/dev/mpr/mpi/mpi2_hbd.h
@@ -42,7 +42,7 @@
  *          Title:  MPI Host Based Discovery messages and structures
  *  Creation Date:  October 21, 2009
  *
- *  mpi2_hbd.h Version:  02.00.03
+ *  mpi2_hbd.h Version:  02.00.04
  *
  *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *        prefix are for use only on MPI v2.5 products, and must not be used
@@ -59,6 +59,7 @@
  *                      HBD Action request, replaced by AdditionalInfo field.
  *  11-18-11  02.00.02  Incorporating additions for MPI v2.5.
  *  11-18-14  02.00.03  Updated copyright information.
+ *  02-17-16  02.00.04  Added SAS 4 22.5 gbs speed support.
  *  --------------------------------------------------------------------------
  */
 
@@ -129,6 +130,7 @@ typedef struct _MPI2_HBD_ACTION_REQUEST
 #define MPI2_HBD_MAX_RATE_3_0                   (0x09)
 #define MPI2_HBD_MAX_RATE_6_0                   (0x0A)
 #define MPI25_HBD_MAX_RATE_12_0                 (0x0B)
+#define MPI26_HBD_MAX_RATE_22_5                 (0x0C)
 
 
 /* Host Based Discovery Action Reply Message */
diff --git a/sys/dev/mpr/mpi/mpi2_history.txt b/sys/dev/mpr/mpi/mpi2_history.txt
index 64498a2f3943..482c7fa35abe 100644
--- a/sys/dev/mpr/mpi/mpi2_history.txt
+++ b/sys/dev/mpr/mpi/mpi2_history.txt
@@ -41,24 +41,25 @@
  All rights reserved.
 
  ---------------------------------------
- Header Set Release Version:    02.00.42
- Header Set Release Date:       01-04-16
+ Header Set Release Version:    02.00.46
+ Header Set Release Date:       09-07-16
  ---------------------------------------
 
  Filename               Current version     Prior version
  ----------             ---------------     -------------
- mpi2.h                 02.00.42            02.00.41
- mpi2_cnfg.h            02.00.35            02.00.34
- mpi2_init.h            02.00.20            02.00.19
- mpi2_ioc.h             02.00.27            02.00.27
+ mpi2.h                 02.00.46            02.00.45
+ mpi2_cnfg.h            02.00.39            02.00.38
+ mpi2_init.h            02.00.21            02.00.21
+ mpi2_ioc.h             02.00.30            02.00.29
  mpi2_raid.h            02.00.11            02.00.11
  mpi2_sas.h             02.00.10            02.00.10
  mpi2_targ.h            02.00.09            02.00.09
- mpi2_tool.h            02.00.13            02.00.13
+ mpi2_tool.h            02.00.14            02.00.13
  mpi2_type.h            02.00.01            02.00.01
  mpi2_ra.h              02.00.01            02.00.01
- mpi2_hbd.h             02.00.03            02.00.03
- mpi2_history.txt       02.00.41            02.00.40
+ mpi2_hbd.h             02.00.04            02.00.04
+ mpi2_pci.h             02.00.02            02.00.02
+ mpi2_history.txt       02.00.43            02.00.43
 
 
  *  Date      Version   Description
@@ -141,7 +142,8 @@ mpi2.h
  *                      Bumped MPI2_HEADER_VERSION_UNIT.
  *  03-16-15  02.00.37  Updated for MPI v2.6.
  *                      Bumped MPI2_HEADER_VERSION_UNIT.
- *                      Added Scratchpad registers to
+ *                      Added Scratchpad registers and
+ *                      AtomicRequestDescriptorPost register to
  *                      MPI2_SYSTEM_INTERFACE_REGS.
  *                      Added MPI2_DIAG_SBR_RELOAD.
  *                      Added MPI2_IOCSTATUS_INSUFFICIENT_POWER.
@@ -151,6 +153,14 @@ mpi2.h
  *                      Added V7 HostDiagnostic register defines
  *  12-15-15  02.00.41  Bumped MPI_HEADER_VERSION_UNIT
  *  01-04-16  02.00.42  Bumped MPI_HEADER_VERSION_UNIT
+ *  04-05-16  02.00.43  Modified  MPI26_DIAG_BOOT_DEVICE_SELECT defines
+ *                      to be unique within first 32 characters.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ *                      Bumped MPI2_HEADER_VERSION_UNIT.
+ * 04-10-16  02.00.44   Bumped MPI2_HEADER_VERSION_UNIT.
+ * 07-06-16  02.00.45   Bumped MPI2_HEADER_VERSION_UNIT.
+ * 09-02-16  02.00.46   Bumped MPI2_HEADER_VERSION_UNIT.
  *  --------------------------------------------------------------------------
 
 mpi2_cnfg.h
@@ -323,9 +333,38 @@ mpi2_cnfg.h
  *                      Flags field to IO Unit Page 7.
  *                      Added IO Unit Page 11.
  *                      Added new SAS Phy Event codes
+ *                      Added PCIe configuration pages.
+ *  03-19-15  02.00.32  Fixed PCIe Link Config page structure names to be
+ *                      unique in first 32 characters.
  *  05-25-15  02.00.33  Added more defines for the BiosOptions field of
  *                      MPI2_CONFIG_PAGE_BIOS_1.
+ *  08-25-15  02.00.34  Added PCIe Device Page 2 SGL format capability.
  *  12-18-15  02.00.35  Added SATADeviceWaitTime to SAS IO Unit Page 4.
+ *  01-21-16  02.00.36  Added/modified MPI2_MFGPAGE_DEVID_SAS defines.
+ *                      Added Link field to PCIe Link Pages
+ *                      Added EnclosureLevel and ConnectorName to PCIe
+ *                      Device Page 0.
+ *                      Added define for PCIE IoUnit page 1 max rate shift.
+ *                      Added comment for reserved ExtPageTypes.
+ *                      Added SAS 4 22.5 gbs speed support.
+ *                      Added PCIe 4 16.0 GT/sec speec support.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ *                      Added NegotiatedLinkRate and NegotiatedPortWidth to
+ *                      PCIe device page 0.
+ *  04-10-16  02.00.37  Fixed MPI2_MFGPAGE_DEVID_SAS3616/3708 defines
+ *  07-01-16  02.00.38  Added Manufacturing page 7 Connector types.
+ *                      Changed declaration of ConnectorName in PCIe DevicePage0
+ *                      to match SAS DevicePage 0.
+ *                      Added SATADeviceWaitTime to IO Unit Page 11.
+ *                      Added MPI26_MFGPAGE_DEVID_SAS4008
+ *                      Added x16 PCIe width to IO Unit Page 7
+ *                      Added LINKFLAGS to control SRIS in PCIe IO Unit page 1
+ *                      phy data.
+ *                      Added InitStatus to PCIe IO Unit Page 1 header.
+ *  09-01-16  02.00.39  Added MPI26_CONFIG_PAGE_ENCLOSURE_0 and related defines.
+ *                      Added MPI26_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE and 
+ *                      MPI26_ENCLOS_PGAD_FORM_HANDLE page address formats. 
  *  --------------------------------------------------------------------------
 
 mpi2_init.h
@@ -365,6 +404,8 @@ mpi2_init.h
  *  08-26-15  02.00.18  Added SCSITASKMGMT_MSGFLAGS for Target Reset.
  *  12-18-15  02.00.19  Added EEDPObservedValue added to SCSI IO Reply message.
  *  01-04-16  02.00.20  Modified EEDP reported values in SCSI IO Reply message.
+ *  01-21-16  02.00.21  Modified MPI26_SCSITASKMGMT_MSGFLAGS_PCIE* defines to
+ *                      be unique within first 32 characters.
  *  --------------------------------------------------------------------------
 
 mpi2_ioc.h
@@ -491,9 +532,30 @@ mpi2_ioc.h
  *                      MPI26_EVENT_DATA_PCIE_LINK_COUNTER.
  *                      Added MPI26_CTRL_OP_SHUTDOWN.
  *                      Added MPI26_CTRL_OP_LINK_CLEAR_ERROR_LOG
- *                      Added MPI26_FW_HEADER_PID_FAMILY_3324_SAS
+ *                      Added MPI26_FW_HEADER_PID_FAMILY_3324_SAS and
+ *                      MPI26_FW_HEADER_PID_FAMILY_3516_SAS.
  *  08-25-15  02.00.27  Added IC ARCH Class based signature defines.
- *  --------------------------------------------------------------------------
+ *                      Added MPI26_EVENT_PCIE_ENUM_ES_RESOURCES_EXHAUSTED event.
+ *                      Added ConigurationFlags field to IOCInit message to
+ *                      support NVMe SGL format control.
+ *                      Added PCIe SRIOV support.
+ * 02-17-16   02.00.28  Added SAS 4 22.5 gbs speed support.
+ *                      Added PCIe 4 16.0 GT/sec speec support.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ * 07-01-16   02.00.29  Added Archclass for 4008 product.
+ *                      Added IOCException MPI2_IOCFACTS_EXCEPT_PCIE_DISABLED.
+ * 08-23-16   02.00.30  Added new defines for the ImageType field of FWDownload
+ *                      Request Message.
+ *                      Added new defines for the ImageType field of FWUpload
+ *                      Request Message.
+ *                      Added new values for the RegionType field in the Layout
+ *                      Data sections of the FLASH Layout Extended Image Data.
+ *                      Added new defines for the ReasonCode field of 
+ *                      Active Cable Exception Event.
+ *                      Added MPI2_EVENT_ENCL_DEVICE_STATUS_CHANGE and 
+ *                      MPI26_EVENT_DATA_ENCL_DEV_STATUS_CHANGE. 
+ * --------------------------------------------------------------------------
 
 mpi2_raid.h
  *  04-30-07  02.00.00  Corresponds to Fusion-MPT MPI Specification Rev A.
@@ -581,6 +643,8 @@ mpi2_tool.h
  *  08-19-13  02.00.11  Added MPI2_TOOLBOX_TEXT_DISPLAY_TOOL and related info.
  *  01-08-14  02.00.12  Added MPI2_TOOLBOX_CLEAN_BIT26_PRODUCT_SPECIFIC.
  *  11-18-14  02.00.13  Updated copyright information.
+ *  08-25-16  02.00.14  Added new values for the Flags field of Toolbox Clean
+ *                      Tool Request Message.
  *  --------------------------------------------------------------------------
 
 mpi2_type.h
@@ -599,24 +663,33 @@ mpi2_hbd.h
  *                      HBD Action request, replaced by AdditionalInfo field.
  *  11-18-11  02.00.02  Incorporating additions for MPI v2.5.
  *  11-18-14  02.00.03  Updated copyright information.
+ *  02-17-16  02.00.04  Added SAS 4 22.5 gbs speed support.
  *  --------------------------------------------------------------------------
 
+mpi2_pci.h
+ *  03-16-15  02.00.00  Initial version.
+ *  02-17-16  02.00.01  Removed AHCI support.
+ *                      Removed SOP support.
+ *  07-01-16  02.00.02  Added MPI26_NVME_FLAGS_FORCE_ADMIN_ERR_RESP to
+ *                      NVME Encapsulated Request.
+ *  --------------------------------------------------------------------------
 
 mpi2_history.txt         Parts list history
 
-Filename     02.00.42
-----------   --------
-mpi2.h       02.00.42
-mpi2_cnfg.h  02.00.35
-mpi2_init.h  02.00.20
-mpi2_ioc.h   02.00.27
-mpi2_raid.h  02.00.11
-mpi2_sas.h   02.00.10
-mpi2_targ.h  02.00.09
-mpi2_tool.h  02.00.13
-mpi2_type.h  02.00.01
-mpi2_ra.h    02.00.01
-mpi2_hbd.h   02.00.03
+Filename     02.00.46  02.00.45  02.00.44  02.00.43  02.00.42
+----------   --------  --------  --------  --------  --------
+mpi2.h       02.00.46  02.00.45  02.00.44  02.00.43  02.00.42
+mpi2_cnfg.h  02.00.39  02.00.38  02.00.37  02.00.36  02.00.35
+mpi2_init.h  02.00.21  02.00.21  02.00.21  02.00.21  02.00.20
+mpi2_ioc.h   02.00.30  02.00.29  02.00.28  02.00.28  02.00.27
+mpi2_raid.h  02.00.11  02.00.11  02.00.11  02.00.11  02.00.11
+mpi2_sas.h   02.00.10  02.00.10  02.00.10  02.00.10  02.00.10
+mpi2_targ.h  02.00.09  02.00.09  02.00.09  02.00.09  02.00.09
+mpi2_tool.h  02.00.14  02.00.13  02.00.13  02.00.13  02.00.13
+mpi2_type.h  02.00.01  02.00.01  02.00.01  02.00.01  02.00.01
+mpi2_ra.h    02.00.01  02.00.01  02.00.01  02.00.01  02.00.01
+mpi2_hbd.h   02.00.04  02.00.04  02.00.04  02.00.04  02.00.03
+mpi2_pci.h   02.00.02  02.00.02  02.00.01  02.00.01  02.00.00
 
 Filename     02.00.41  02.00.40  02.00.39  02.00.38  02.00.37  02.00.36
 ----------   --------  --------  --------  --------  --------  --------
@@ -631,6 +704,7 @@ mpi2_tool.h  02.00.13  02.00.13  02.00.13  02.00.13  02.00.13  02.00.13
 mpi2_type.h  02.00.01  02.00.01  02.00.01  02.00.01  02.00.01  02.00.01
 mpi2_ra.h    02.00.01  02.00.01  02.00.01  02.00.01  02.00.01  02.00.01
 mpi2_hbd.h   02.00.03  02.00.03  02.00.03  02.00.03  02.00.03  02.00.03
+mpi2_pci.h   02.00.00  02.00.00  02.00.00  02.00.00  02.00.00
 
 Filename     02.00.35  02.00.34  02.00.33  02.00.32  02.00.31  02.00.30
 ----------   --------  --------  --------  --------  --------  --------
diff --git a/sys/dev/mpr/mpi/mpi2_init.h b/sys/dev/mpr/mpi/mpi2_init.h
index 12b2689d685b..17bf383374ed 100644
--- a/sys/dev/mpr/mpi/mpi2_init.h
+++ b/sys/dev/mpr/mpi/mpi2_init.h
@@ -42,7 +42,7 @@
  *          Title:  MPI SCSI initiator mode messages and structures
  *  Creation Date:  June 23, 2006
  *
- *  mpi2_init.h Version:  02.00.20
+ *  mpi2_init.h Version:  02.00.21
  *
  *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *        prefix are for use only on MPI v2.5 products, and must not be used
@@ -62,7 +62,7 @@
  *  05-21-08  02.00.05  Fixed typo in name of Mpi2SepRequest_t.
  *  10-02-08  02.00.06  Removed Untagged and No Disconnect values from SCSI IO
  *                      Control field Task Attribute flags.
- *                      Moved LUN field defines to mpi2.h becasue they are
+ *                      Moved LUN field defines to mpi2.h because they are
  *                      common to many structures.
  *  05-06-09  02.00.07  Changed task management type of Query Unit Attention to
  *                      Query Asynchronous Event.
@@ -90,6 +90,8 @@
  *  08-26-15  02.00.18  Added SCSITASKMGMT_MSGFLAGS for Target Reset.
  *  12-18-15  02.00.19  Added EEDPObservedValue added to SCSI IO Reply message.
  *  01-04-16  02.00.20  Modified EEDP reported values in SCSI IO Reply message.
+ *  01-21-16  02.00.21  Modified MPI26_SCSITASKMGMT_MSGFLAGS_PCIE* defines to
+ *                      be unique within first 32 characters.
  *  --------------------------------------------------------------------------
  */
 
@@ -491,12 +493,13 @@ typedef struct _MPI2_SCSI_TASK_MANAGE_REQUEST
 #define MPI2_SCSITASKMGMT_TASKTYPE_QRY_UNIT_ATTENTION   (MPI2_SCSITASKMGMT_TASKTYPE_QRY_ASYNC_EVENT)
 
 /* MsgFlags bits */
-
-#define MPI2_SCSITASKMGMT_MSGFLAGS_MASK_TARGET_RESET    (0x18)
-#define MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET           (0x00)
+#define MPI2_SCSITASKMGMT_MSGFLAGS_MASK_TARGET_RESET      (0x18)
+#define MPI26_SCSITASKMGMT_MSGFLAGS_HOT_RESET_PCIE        (0x00)
+#define MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET             (0x00)
 #define MPI2_SCSITASKMGMT_MSGFLAGS_DO_NOT_SEND_TASK_IU    (0x01)
-#define MPI2_SCSITASKMGMT_MSGFLAGS_NEXUS_RESET_SRST     (0x08)
-#define MPI2_SCSITASKMGMT_MSGFLAGS_SAS_HARD_LINK_RESET  (0x10)
+#define MPI2_SCSITASKMGMT_MSGFLAGS_NEXUS_RESET_SRST       (0x08)
+#define MPI2_SCSITASKMGMT_MSGFLAGS_SAS_HARD_LINK_RESET    (0x10)
+#define MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE (0x18)
 
 
 /* SCSI Task Management Reply Message */
diff --git a/sys/dev/mpr/mpi/mpi2_ioc.h b/sys/dev/mpr/mpi/mpi2_ioc.h
index 003f7cd62284..5e3b7aeb540e 100644
--- a/sys/dev/mpr/mpi/mpi2_ioc.h
+++ b/sys/dev/mpr/mpi/mpi2_ioc.h
@@ -42,7 +42,7 @@
  *          Title:  MPI IOC, Port, Event, FW Download, and FW Upload messages
  *  Creation Date:  October 11, 2006
  *
- *  mpi2_ioc.h Version:  02.00.27
+ *  mpi2_ioc.h Version:  02.00.30
  *
  *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
  *        prefix are for use only on MPI v2.5 products, and must not be used
@@ -177,9 +177,29 @@
  *                      MPI26_EVENT_DATA_PCIE_LINK_COUNTER.
  *                      Added MPI26_CTRL_OP_SHUTDOWN.
  *                      Added MPI26_CTRL_OP_LINK_CLEAR_ERROR_LOG
- *                      Added MPI26_FW_HEADER_PID_FAMILY_3324_SAS
- *  08-25-15  02.00.27  Added IC ARCH Class based signature defines
- *                      
+ *                      Added MPI26_FW_HEADER_PID_FAMILY_3324_SAS and
+ *                      MPI26_FW_HEADER_PID_FAMILY_3516_SAS.
+ *  08-25-15  02.00.27  Added IC ARCH Class based signature defines.
+ *                      Added MPI26_EVENT_PCIE_ENUM_ES_RESOURCES_EXHAUSTED event.
+ *                      Added ConigurationFlags field to IOCInit message to
+ *                      support NVMe SGL format control.
+ *                      Added PCIe SRIOV support.
+ * 02-17-16   02.00.28  Added SAS 4 22.5 gbs speed support.
+ *                      Added PCIe 4 16.0 GT/sec speec support.
+ *                      Removed AHCI support.
+ *                      Removed SOP support.
+ * 07-01-16   02.00.29  Added Archclass for 4008 product.
+ *                      Added IOCException MPI2_IOCFACTS_EXCEPT_PCIE_DISABLED
+ * 08-23-16   02.00.30  Added new defines for the ImageType field of FWDownload
+ *                      Request Message.
+ *                      Added new defines for the ImageType field of FWUpload
+ *                      Request Message.
+ *                      Added new values for the RegionType field in the Layout
+ *                      Data sections of the FLASH Layout Extended Image Data.
+ *                      Added new defines for the ReasonCode field of
+ *                      Active Cable Exception Event.
+ *                      Added MPI2_EVENT_ENCL_DEVICE_STATUS_CHANGE and
+ *                      MPI26_EVENT_DATA_ENCL_DEV_STATUS_CHANGE.
  *  --------------------------------------------------------------------------
  */
 
@@ -251,6 +271,9 @@ typedef struct _MPI2_IOC_INIT_REQUEST
 #define MPI2_IOCINIT_HDRVERSION_DEV_MASK        (0x00FF)
 #define MPI2_IOCINIT_HDRVERSION_DEV_SHIFT       (0)
 
+/* ConfigurationFlags */
+#define MPI26_IOCINIT_CFGFLAGS_NVME_SGL_FORMAT  (0x0001)
+
 /* minimum depth for a Reply Descriptor Post Queue */
 #define MPI2_RDPQ_DEPTH_MIN                     (16)
 
@@ -363,6 +386,7 @@ typedef struct _MPI2_IOC_FACTS_REPLY
 #define MPI2_IOCFACTS_HDRVERSION_DEV_SHIFT              (0)
 
 /* IOCExceptions */
+#define MPI2_IOCFACTS_EXCEPT_PCIE_DISABLED              (0x0400)
 #define MPI2_IOCFACTS_EXCEPT_PARTIAL_MEMORY_FAILURE     (0x0200)
 #define MPI2_IOCFACTS_EXCEPT_IR_FOREIGN_CONFIG_MAX      (0x0100)
 
@@ -383,6 +407,8 @@ typedef struct _MPI2_IOC_FACTS_REPLY
 /* ProductID field uses MPI2_FW_HEADER_PID_ */
 
 /* IOCCapabilities */
+#define MPI26_IOCFACTS_CAPABILITY_PCIE_SRIOV            (0x00100000)
+#define MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ            (0x00080000)
 #define MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE     (0x00040000)
 #define MPI25_IOCFACTS_CAPABILITY_FAST_PATH_CAPABLE     (0x00020000)
 #define MPI2_IOCFACTS_CAPABILITY_HOST_BASED_DISCOVERY   (0x00010000)
@@ -400,6 +426,7 @@ typedef struct _MPI2_IOC_FACTS_REPLY
 #define MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING (0x00000004)
 
 /* ProtocolFlags */
+#define MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES             (0x0008) /* MPI v2.6 and later */
 #define MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR           (0x0002)
 #define MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET              (0x0001)
 
@@ -452,6 +479,7 @@ typedef struct _MPI2_PORT_FACTS_REPLY
 #define MPI2_PORTFACTS_PORTTYPE_ISCSI               (0x20)
 #define MPI2_PORTFACTS_PORTTYPE_SAS_PHYSICAL        (0x30)
 #define MPI2_PORTFACTS_PORTTYPE_SAS_VIRTUAL         (0x31)
+#define MPI2_PORTFACTS_PORTTYPE_TRI_MODE            (0x40) /* MPI v2.6 and later */
 
 
 /****************************************************************************
@@ -564,6 +592,7 @@ typedef struct _MPI2_EVENT_NOTIFICATION_REPLY
 #define MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW          (0x0019)
 #define MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST         (0x001C)
 #define MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE    (0x001D)
+#define MPI2_EVENT_ENCL_DEVICE_STATUS_CHANGE        (0x001D) /* MPI v2.6 and later */
 #define MPI2_EVENT_IR_VOLUME                        (0x001E)
 #define MPI2_EVENT_IR_PHYSICAL_DISK                 (0x001F)
 #define MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST     (0x0020)
@@ -576,6 +605,10 @@ typedef struct _MPI2_EVENT_NOTIFICATION_REPLY
 #define MPI2_EVENT_TEMP_THRESHOLD                   (0x0027)
 #define MPI2_EVENT_HOST_MESSAGE                     (0x0028)
 #define MPI2_EVENT_POWER_PERFORMANCE_CHANGE         (0x0029)
+#define MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE        (0x0030) /* MPI v2.6 and later */
+#define MPI2_EVENT_PCIE_ENUMERATION                 (0x0031) /* MPI v2.6 and later */
+#define MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST        (0x0032) /* MPI v2.6 and later */
+#define MPI2_EVENT_PCIE_LINK_COUNTER                (0x0033) /* MPI v2.6 and later */
 #define MPI2_EVENT_ACTIVE_CABLE_EXCEPTION           (0x0034) /* MPI v2.6 and later */
 #define MPI2_EVENT_MIN_PRODUCT_SPECIFIC             (0x006E)
 #define MPI2_EVENT_MAX_PRODUCT_SPECIFIC             (0x007F)
@@ -688,11 +721,9 @@ typedef struct _MPI26_EVENT_DATA_ACTIVE_CABLE_EXCEPT
   MPI2_POINTER pMpi26EventDataActiveCableExcept_t;
 
 /* defines for ReasonCode field */
-#define MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER	(0x00)
-#define MPI26_EVENT_ACTIVE_CABLE_PRESENT		(0x01) 
-#define MPI26_EVENT_ACTIVE_CABLE_DEGRADED		(0x02) 
-
-
+#define MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER     (0x00)
+#define MPI26_EVENT_ACTIVE_CABLE_PRESENT                (0x01)
+#define MPI26_EVENT_ACTIVE_CABLE_DEGRADED               (0x02)
 
 /* Hard Reset Received Event data */
 
@@ -1048,6 +1079,7 @@ typedef struct _MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST
 #define MPI2_EVENT_SAS_TOPO_LR_RATE_3_0                     (0x09)
 #define MPI2_EVENT_SAS_TOPO_LR_RATE_6_0                     (0x0A)
 #define MPI25_EVENT_SAS_TOPO_LR_RATE_12_0                   (0x0B)
+#define MPI26_EVENT_SAS_TOPO_LR_RATE_22_5                   (0x0C)
 
 /* values for the PhyStatus field */
 #define MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT                (0x80)
@@ -1075,12 +1107,19 @@ typedef struct _MPI2_EVENT_DATA_SAS_ENCL_DEV_STATUS_CHANGE
 } MPI2_EVENT_DATA_SAS_ENCL_DEV_STATUS_CHANGE,
   MPI2_POINTER PTR_MPI2_EVENT_DATA_SAS_ENCL_DEV_STATUS_CHANGE,
   Mpi2EventDataSasEnclDevStatusChange_t,
-  MPI2_POINTER pMpi2EventDataSasEnclDevStatusChange_t;
+  MPI2_POINTER pMpi2EventDataSasEnclDevStatusChange_t,
+  MPI26_EVENT_DATA_ENCL_DEV_STATUS_CHANGE,
+  MPI2_POINTER PTR_MPI26_EVENT_DATA_ENCL_DEV_STATUS_CHANGE,
+  Mpi26EventDataEnclDevStatusChange_t,
+  MPI2_POINTER pMpi26EventDataEnclDevStatusChange_t;
 
 /* SAS Enclosure Device Status Change event ReasonCode values */
 #define MPI2_EVENT_SAS_ENCL_RC_ADDED                (0x01)
 #define MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING       (0x02)
 
+/* Enclosure Device Status Change event ReasonCode values */
+#define MPI26_EVENT_ENCL_RC_ADDED                   (0x01)
+#define MPI26_EVENT_ENCL_RC_NOT_RESPONDING          (0x02)
 
 /* SAS PHY Counter Event data */
 
@@ -1168,6 +1207,167 @@ typedef struct _MPI2_EVENT_DATA_HBD_PHY
 #define MPI2_EVENT_HBD_DT_SAS               (0x01)
 
 
+/* PCIe Device Status Change Event data (MPI v2.6 and later) */
+
+typedef struct _MPI26_EVENT_DATA_PCIE_DEVICE_STATUS_CHANGE
+{
+    U16                     TaskTag;                        /* 0x00 */
+    U8                      ReasonCode;                     /* 0x02 */
+    U8                      PhysicalPort;                   /* 0x03 */
+    U8                      ASC;                            /* 0x04 */
+    U8                      ASCQ;                           /* 0x05 */
+    U16                     DevHandle;                      /* 0x06 */
+    U32                     Reserved2;                      /* 0x08 */
+    U64                     WWID;                           /* 0x0C */
+    U8                      LUN[8];                         /* 0x14 */
+} MPI26_EVENT_DATA_PCIE_DEVICE_STATUS_CHANGE,
+  MPI2_POINTER PTR_MPI26_EVENT_DATA_PCIE_DEVICE_STATUS_CHANGE,
+  Mpi26EventDataPCIeDeviceStatusChange_t,
+  MPI2_POINTER pMpi26EventDataPCIeDeviceStatusChange_t;
+
+/* PCIe Device Status Change Event data ReasonCode values */
+#define MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA                           (0x05)
+#define MPI26_EVENT_PCIDEV_STAT_RC_UNSUPPORTED                          (0x07)
+#define MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET                (0x08)
+#define MPI26_EVENT_PCIDEV_STAT_RC_TASK_ABORT_INTERNAL                  (0x09)
+#define MPI26_EVENT_PCIDEV_STAT_RC_ABORT_TASK_SET_INTERNAL              (0x0A)
+#define MPI26_EVENT_PCIDEV_STAT_RC_CLEAR_TASK_SET_INTERNAL              (0x0B)
+#define MPI26_EVENT_PCIDEV_STAT_RC_QUERY_TASK_INTERNAL                  (0x0C)
+#define MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION                   (0x0D)
+#define MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET               (0x0E)
+#define MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL              (0x0F)
+#define MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE                     (0x10)
+
+
+/* PCIe Enumeration Event data (MPI v2.6 and later) */
+
+typedef struct _MPI26_EVENT_DATA_PCIE_ENUMERATION
+{
+    U8                      Flags;                      /* 0x00 */
+    U8                      ReasonCode;                 /* 0x01 */
+    U8                      PhysicalPort;               /* 0x02 */
+    U8                      Reserved1;                  /* 0x03 */
+    U32                     EnumerationStatus;          /* 0x04 */
+} MPI26_EVENT_DATA_PCIE_ENUMERATION,
+  MPI2_POINTER PTR_MPI26_EVENT_DATA_PCIE_ENUMERATION,
+  Mpi26EventDataPCIeEnumeration_t,
+  MPI2_POINTER pMpi26EventDataPCIeEnumeration_t;
+
+/* PCIe Enumeration Event data Flags values */
+#define MPI26_EVENT_PCIE_ENUM_DEVICE_CHANGE                 (0x02)
+#define MPI26_EVENT_PCIE_ENUM_IN_PROGRESS                   (0x01)
+
+/* PCIe Enumeration Event data ReasonCode values */
+#define MPI26_EVENT_PCIE_ENUM_RC_STARTED                    (0x01)
+#define MPI26_EVENT_PCIE_ENUM_RC_COMPLETED                  (0x02)
+
+/* PCIe Enumeration Event data EnumerationStatus values */
+#define MPI26_EVENT_PCIE_ENUM_ES_MAX_SWITCHES_EXCEED            (0x40000000)
+#define MPI26_EVENT_PCIE_ENUM_ES_MAX_DEVICES_EXCEED             (0x20000000)
+#define MPI26_EVENT_PCIE_ENUM_ES_RESOURCES_EXHAUSTED            (0x10000000)
+
+
+/* PCIe Topology Change List Event data (MPI v2.6 and later) */
+
+/*
+ * Host code (drivers, BIOS, utilities, etc.) should leave this define set to
+ * one and check NumEntries at runtime.
+ */
+#ifndef MPI26_EVENT_PCIE_TOPO_PORT_COUNT
+#define MPI26_EVENT_PCIE_TOPO_PORT_COUNT        (1)
+#endif
+
+typedef struct _MPI26_EVENT_PCIE_TOPO_PORT_ENTRY
+{
+    U16         AttachedDevHandle;      /* 0x00 */
+    U8          PortStatus;             /* 0x02 */
+    U8          Reserved1;              /* 0x03 */
+    U8          CurrentPortInfo;        /* 0x04 */
+    U8          Reserved2;              /* 0x05 */
+    U8          PreviousPortInfo;       /* 0x06 */
+    U8          Reserved3;              /* 0x07 */
+} MPI26_EVENT_PCIE_TOPO_PORT_ENTRY,
+  MPI2_POINTER PTR_MPI26_EVENT_PCIE_TOPO_PORT_ENTRY,
+  Mpi26EventPCIeTopoPortEntry_t,
+  MPI2_POINTER pMpi26EventPCIeTopoPortEntry_t;
+
+/* PCIe Topology Change List Event data PortStatus values */
+#define MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED                  (0x01)
+#define MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING             (0x02)
+#define MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED               (0x03)
+#define MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE                  (0x04)
+#define MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING       (0x05)
+
+/* PCIe Topology Change List Event data defines for CurrentPortInfo and PreviousPortInfo */
+#define MPI26_EVENT_PCIE_TOPO_PI_LANE_MASK                  (0xF0)
+#define MPI26_EVENT_PCIE_TOPO_PI_LANES_UNKNOWN              (0x00)
+#define MPI26_EVENT_PCIE_TOPO_PI_1_LANE                     (0x10)
+#define MPI26_EVENT_PCIE_TOPO_PI_2_LANES                    (0x20)
+#define MPI26_EVENT_PCIE_TOPO_PI_4_LANES                    (0x30)
+#define MPI26_EVENT_PCIE_TOPO_PI_8_LANES                    (0x40)
+
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK                  (0x0F)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_UNKNOWN               (0x00)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_DISABLED              (0x01)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5                   (0x02)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_5_0                   (0x03)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_8_0                   (0x04)
+#define MPI26_EVENT_PCIE_TOPO_PI_RATE_16_0                  (0x05)
+
+typedef struct _MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST
+{
+    U16                                 EnclosureHandle;        /* 0x00 */
+    U16                                 SwitchDevHandle;        /* 0x02 */
+    U8                                  NumPorts;               /* 0x04 */
+    U8                                  Reserved1;              /* 0x05 */
+    U16                                 Reserved2;              /* 0x06 */
+    U8                                  NumEntries;             /* 0x08 */
+    U8                                  StartPortNum;           /* 0x09 */
+    U8                                  SwitchStatus;           /* 0x0A */
+    U8                                  PhysicalPort;           /* 0x0B */
+    MPI26_EVENT_PCIE_TOPO_PORT_ENTRY    PortEntry[MPI26_EVENT_PCIE_TOPO_PORT_COUNT]; /* 0x0C */
+} MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST,
+  MPI2_POINTER PTR_MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST,
+  Mpi26EventDataPCIeTopologyChangeList_t,
+  MPI2_POINTER pMpi26EventDataPCIeTopologyChangeList_t;
+
+/* PCIe Topology Change List Event data SwitchStatus values */
+#define MPI26_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH             (0x00)
+#define MPI26_EVENT_PCIE_TOPO_SS_ADDED                      (0x01)
+#define MPI26_EVENT_PCIE_TOPO_SS_NOT_RESPONDING             (0x02)
+#define MPI26_EVENT_PCIE_TOPO_SS_RESPONDING                 (0x03)
+#define MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING       (0x04)
+
+/* PCIe Link Counter Event data (MPI v2.6 and later) */
+
+typedef struct _MPI26_EVENT_DATA_PCIE_LINK_COUNTER
+{
+    U64         TimeStamp;          /* 0x00 */
+    U32         Reserved1;          /* 0x08 */
+    U8          LinkEventCode;      /* 0x0C */
+    U8          LinkNum;            /* 0x0D */
+    U16         Reserved2;          /* 0x0E */
+    U32         LinkEventInfo;      /* 0x10 */
+    U8          CounterType;        /* 0x14 */
+    U8          ThresholdWindow;    /* 0x15 */
+    U8          TimeUnits;          /* 0x16 */
+    U8          Reserved3;          /* 0x17 */
+    U32         EventThreshold;     /* 0x18 */
+    U16         ThresholdFlags;     /* 0x1C */
+    U16         Reserved4;          /* 0x1E */
+} MPI26_EVENT_DATA_PCIE_LINK_COUNTER,
+  MPI2_POINTER PTR_MPI26_EVENT_DATA_PCIE_LINK_COUNTER,
+  Mpi26EventDataPcieLinkCounter_t, MPI2_POINTER pMpi26EventDataPcieLinkCounter_t;
+
+
+/* use MPI26_PCIELINK3_EVTCODE_ values from mpi2_cnfg.h for the LinkEventCode field */
+
+/* use MPI26_PCIELINK3_COUNTER_TYPE_ values from mpi2_cnfg.h for the CounterType field */
+
+/* use MPI26_PCIELINK3_TIME_UNITS_ values from mpi2_cnfg.h for the TimeUnits field */
+
+/* use MPI26_PCIELINK3_TFLAGS_ values from mpi2_cnfg.h for the ThresholdFlags field */
+
 /****************************************************************************
 *  EventAck message
 ****************************************************************************/
@@ -1293,6 +1493,13 @@ typedef struct _MPI2_FW_DOWNLOAD_REQUEST
 #define MPI2_FW_DOWNLOAD_ITYPE_COMPLETE             (0x0A)
 #define MPI2_FW_DOWNLOAD_ITYPE_COMMON_BOOT_BLOCK    (0x0B)
 #define MPI2_FW_DOWNLOAD_ITYPE_PUBLIC_KEY           (0x0C) /* MPI v2.5 and newer */
+#define MPI2_FW_DOWNLOAD_ITYPE_SBR                  (0x0E)
+#define MPI2_FW_DOWNLOAD_ITYPE_SBR_BACKUP           (0x0F)
+#define MPI2_FW_DOWNLOAD_ITYPE_HIIM                 (0x10)
+#define MPI2_FW_DOWNLOAD_ITYPE_HIIA                 (0x11)
+#define MPI2_FW_DOWNLOAD_ITYPE_CTLR                 (0x12)
+#define MPI2_FW_DOWNLOAD_ITYPE_IMR_FIRMWARE         (0x13)
+#define MPI2_FW_DOWNLOAD_ITYPE_MR_NVDATA            (0x14)
 #define MPI2_FW_DOWNLOAD_ITYPE_MIN_PRODUCT_SPECIFIC (0xF0)
 
 /* MPI v2.0 FWDownload TransactionContext Element */
@@ -1386,6 +1593,13 @@ typedef struct _MPI2_FW_UPLOAD_REQUEST
 #define MPI2_FW_UPLOAD_ITYPE_COMPLETE           (0x0A)
 #define MPI2_FW_UPLOAD_ITYPE_COMMON_BOOT_BLOCK  (0x0B)
 #define MPI2_FW_UPLOAD_ITYPE_CBB_BACKUP         (0x0D)
+#define MPI2_FW_UPLOAD_ITYPE_SBR                (0x0E)
+#define MPI2_FW_UPLOAD_ITYPE_SBR_BACKUP         (0x0F)
+#define MPI2_FW_UPLOAD_ITYPE_HIIM               (0x10)
+#define MPI2_FW_UPLOAD_ITYPE_HIIA               (0x11)
+#define MPI2_FW_UPLOAD_ITYPE_CTLR               (0x12)
+#define MPI2_FW_UPLOAD_ITYPE_IMR_FIRMWARE       (0x13)
+#define MPI2_FW_UPLOAD_ITYPE_MR_NVDATA          (0x14)
 
 /* MPI v2.0 FWUpload TransactionContext Element */
 typedef struct _MPI2_FW_UPLOAD_TCSGE
@@ -1509,8 +1723,10 @@ typedef struct _MPI2_FW_IMAGE_HEADER
 #define MPI26_FW_HEADER_SIGNATURE0_ARC_0        (0x5A)
 #define MPI26_FW_HEADER_SIGNATURE0_ARC_1        (0x00)
 #define MPI26_FW_HEADER_SIGNATURE0_ARC_2        (0x01)
+#define MPI26_FW_HEADER_SIGNATURE0_ARC_3        (0x02)
 #define MPI26_FW_HEADER_SIGNATURE0              (MPI26_FW_HEADER_SIGNATURE0_BASE+MPI26_FW_HEADER_SIGNATURE0_ARC_0) // legacy (0x5AEAA55A)
 #define MPI26_FW_HEADER_SIGNATURE0_3516         (MPI26_FW_HEADER_SIGNATURE0_BASE+MPI26_FW_HEADER_SIGNATURE0_ARC_1)
+#define MPI26_FW_HEADER_SIGNATURE0_4008         (MPI26_FW_HEADER_SIGNATURE0_BASE+MPI26_FW_HEADER_SIGNATURE0_ARC_3)
 
 /* Signature1 field */
 #define MPI2_FW_HEADER_SIGNATURE1_OFFSET        (0x08)
@@ -1665,7 +1881,13 @@ typedef struct _MPI2_FLASH_LAYOUT_DATA
 #define MPI2_FLASH_REGION_COMMON_BOOT_BLOCK     (0x0A)
 #define MPI2_FLASH_REGION_INIT                  (MPI2_FLASH_REGION_COMMON_BOOT_BLOCK) /* older name */
 #define MPI2_FLASH_REGION_CBB_BACKUP            (0x0D)
-
+#define MPI2_FLASH_REGION_SBR                   (0x0E)
+#define MPI2_FLASH_REGION_SBR_BACKUP            (0x0F)
+#define MPI2_FLASH_REGION_HIIM                  (0x10)
+#define MPI2_FLASH_REGION_HIIA                  (0x11)
+#define MPI2_FLASH_REGION_CTLR                  (0x12)
+#define MPI2_FLASH_REGION_IMR_FIRMWARE          (0x13)
+#define MPI2_FLASH_REGION_MR_NVDATA             (0x14)
 
 /* ImageRevision */
 #define MPI2_FLASH_LAYOUT_IMAGE_REVISION        (0x00)
@@ -1960,6 +2182,8 @@ typedef struct _MPI26_IOUNIT_CONTROL_REQUEST
 #define MPI26_CTRL_OP_DEV_ENABLE_PERSIST_CONNECTION     (0x17)
 #define MPI26_CTRL_OP_DEV_DISABLE_PERSIST_CONNECTION    (0x18)
 #define MPI26_CTRL_OP_DEV_CLOSE_PERSIST_CONNECTION      (0x19)
+#define MPI26_CTRL_OP_ENABLE_NVME_SGL_FORMAT            (0x1A)
+#define MPI26_CTRL_OP_DISABLE_NVME_SGL_FORMAT           (0x1B)
 #define MPI26_CTRL_OP_PRODUCT_SPECIFIC_MIN              (0x80)
 
 /* values for the PrimFlags field */
diff --git a/sys/dev/mpr/mpi/mpi2_pci.h b/sys/dev/mpr/mpi/mpi2_pci.h
new file mode 100755
index 000000000000..a48433fc50db
--- /dev/null
+++ b/sys/dev/mpr/mpi/mpi2_pci.h
@@ -0,0 +1,151 @@
+/*-
+ * Copyright (c) 2012-2015 LSI Corp.
+ * Copyright (c) 2013-2016 Avago Technologies
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
+ *
+ * $FreeBSD$
+ */
+
+/*
+ *  Copyright (c) 2000-2015 LSI Corporation.
+ *  Copyright (c) 2013-2016 Avago Technologies
+ *  All rights reserved.
+ *
+ *
+ *           Name:  mpi2_pci.h
+ *          Title:  MPI PCIe Attached Devices structures and definitions.
+ *  Creation Date:  October 9, 2012
+ *
+ *  mpi2_pci.h Version:  02.00.02
+ *
+ *  NOTE: Names (typedefs, defines, etc.) beginning with an MPI25 or Mpi25
+ *        prefix are for use only on MPI v2.5 products, and must not be used
+ *        with MPI v2.0 products. Unless otherwise noted, names beginning with
+ *        MPI2 or Mpi2 are for use with both MPI v2.0 and MPI v2.5 products.
+ *
+ *  Version History
+ *  ---------------
+ *
+ *  Date      Version   Description
+ *  --------  --------  ------------------------------------------------------
+ *  03-16-15  02.00.00  Initial version.
+ *  02-17-16  02.00.01  Removed AHCI support.
+ *                      Removed SOP support.
+ *  07-01-16  02.00.02  Added MPI26_NVME_FLAGS_FORCE_ADMIN_ERR_RESP to
+ *                      NVME Encapsulated Request.
+ *  --------------------------------------------------------------------------
+ */
+
+#ifndef MPI2_PCI_H
+#define MPI2_PCI_H
+
+
+/*
+ * Values for the PCIe DeviceInfo field used in PCIe Device Status Change Event
+ * data and PCIe Configuration pages.
+ */
+#define MPI26_PCIE_DEVINFO_DIRECT_ATTACH        (0x00000010)
+
+#define MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE     (0x0000000F)
+#define MPI26_PCIE_DEVINFO_NO_DEVICE            (0x00000000)
+#define MPI26_PCIE_DEVINFO_PCI_SWITCH           (0x00000001)
+#define MPI26_PCIE_DEVINFO_NVME                 (0x00000003)
+
+
+/****************************************************************************
+*  NVMe Encapsulated message
+****************************************************************************/
+
+/* NVME Encapsulated Request Message */
+typedef struct _MPI26_NVME_ENCAPSULATED_REQUEST
+{
+    U16                     DevHandle;                      /* 0x00 */
+    U8                      ChainOffset;                    /* 0x02 */
+    U8                      Function;                       /* 0x03 */
+    U16                     EncapsulatedCommandLength;      /* 0x04 */
+    U8                      Reserved1;                      /* 0x06 */
+    U8                      MsgFlags;                       /* 0x07 */
+    U8                      VP_ID;                          /* 0x08 */
+    U8                      VF_ID;                          /* 0x09 */
+    U16                     Reserved2;                      /* 0x0A */
+    U32                     Reserved3;                      /* 0x0C */
+    U64                     ErrorResponseBaseAddress;       /* 0x10 */
+    U16                     ErrorResponseAllocationLength;  /* 0x18 */
+    U16                     Flags;                          /* 0x1A */
+    U32                     DataLength;                     /* 0x1C */
+    U8                      NVMe_Command[4];                /* 0x20 */ /* variable length */
+
+} MPI26_NVME_ENCAPSULATED_REQUEST, MPI2_POINTER PTR_MPI26_NVME_ENCAPSULATED_REQUEST,
+  Mpi26NVMeEncapsulatedRequest_t, MPI2_POINTER pMpi26NVMeEncapsulatedRequest_t;
+
+/* defines for the Flags field */
+#define MPI26_NVME_FLAGS_FORCE_ADMIN_ERR_RESP       (0x0020)
+/* Submission Queue Type*/
+#define MPI26_NVME_FLAGS_SUBMISSIONQ_MASK           (0x0010)
+#define MPI26_NVME_FLAGS_SUBMISSIONQ_IO             (0x0000)
+#define MPI26_NVME_FLAGS_SUBMISSIONQ_ADMIN          (0x0010)
+/* Error Response Address Space */
+#define MPI26_NVME_FLAGS_MASK_ERROR_RSP_ADDR        (0x000C)
+#define MPI26_NVME_FLAGS_SYSTEM_RSP_ADDR            (0x0000)
+#define MPI26_NVME_FLAGS_IOCPLB_RSP_ADDR            (0x0008)
+#define MPI26_NVME_FLAGS_IOCPLBNTA_RSP_ADDR         (0x000C)
+/* Data Direction*/
+#define MPI26_NVME_FLAGS_DATADIRECTION_MASK         (0x0003)
+#define MPI26_NVME_FLAGS_NODATATRANSFER             (0x0000)
+#define MPI26_NVME_FLAGS_WRITE                      (0x0001)
+#define MPI26_NVME_FLAGS_READ                       (0x0002)
+#define MPI26_NVME_FLAGS_BIDIRECTIONAL              (0x0003)
+
+
+/* NVMe Encapuslated Reply Message */
+typedef struct _MPI26_NVME_ENCAPSULATED_ERROR_REPLY
+{
+    U16                     DevHandle;                      /* 0x00 */
+    U8                      MsgLength;                      /* 0x02 */
+    U8                      Function;                       /* 0x03 */
+    U16                     EncapsulatedCommandLength;      /* 0x04 */
+    U8                      Reserved1;                      /* 0x06 */
+    U8                      MsgFlags;                       /* 0x07 */
+    U8                      VP_ID;                          /* 0x08 */
+    U8                      VF_ID;                          /* 0x09 */
+    U16                     Reserved2;                      /* 0x0A */
+    U16                     Reserved3;                      /* 0x0C */
+    U16                     IOCStatus;                      /* 0x0E */
+    U32                     IOCLogInfo;                     /* 0x10 */
+    U16                     ErrorResponseCount;             /* 0x14 */
+    U16                     Reserved4;                      /* 0x16 */
+} MPI26_NVME_ENCAPSULATED_ERROR_REPLY,
+  MPI2_POINTER PTR_MPI26_NVME_ENCAPSULATED_ERROR_REPLY,
+  Mpi26NVMeEncapsulatedErrorReply_t,
+  MPI2_POINTER pMpi26NVMeEncapsulatedErrorReply_t;
+
+
+#endif
+
+
diff --git a/sys/dev/mpr/mpi/mpi2_tool.h b/sys/dev/mpr/mpi/mpi2_tool.h
index fce98ae638b0..beebc93595aa 100644
--- a/sys/dev/mpr/mpi/mpi2_tool.h
+++ b/sys/dev/mpr/mpi/mpi2_tool.h
@@ -42,7 +42,7 @@
  *          Title:  MPI diagnostic tool structures and definitions
  *  Creation Date:  March 26, 2007
  *
- *    mpi2_tool.h Version:  02.00.13
+ *    mpi2_tool.h Version:  02.00.14
  *
  *  Version History
  *  ---------------
@@ -71,6 +71,8 @@
  *  08-19-13  02.00.11  Added MPI2_TOOLBOX_TEXT_DISPLAY_TOOL and related info.
  *  01-08-14  02.00.12  Added MPI2_TOOLBOX_CLEAN_BIT26_PRODUCT_SPECIFIC.
  *  11-18-14  02.00.13  Updated copyright information.
+ *  08-25-16  02.00.14  Added new values for the Flags field of Toolbox Clean
+ *                      Tool Request Message.
  *  --------------------------------------------------------------------------
  */
 
@@ -145,6 +147,16 @@ typedef struct _MPI2_TOOLBOX_CLEAN_REQUEST
 #define MPI2_TOOLBOX_CLEAN_BIT26_PRODUCT_SPECIFIC   (0x04000000)
 #define MPI2_TOOLBOX_CLEAN_MEGARAID                 (0x02000000)
 #define MPI2_TOOLBOX_CLEAN_INITIALIZATION           (0x01000000)
+#define MPI2_TOOLBOX_CLEAN_SBR                      (0x00800000)
+#define MPI2_TOOLBOX_CLEAN_SBR_BACKUP               (0x00400000)
+#define MPI2_TOOLBOX_CLEAN_HIIM                     (0x00200000)
+#define MPI2_TOOLBOX_CLEAN_HIIA                     (0x00100000)
+#define MPI2_TOOLBOX_CLEAN_CTLR                     (0x00080000)
+#define MPI2_TOOLBOX_CLEAN_IMR_FIRMWARE             (0x00040000)
+#define MPI2_TOOLBOX_CLEAN_MR_NVDATA                (0x00020000)
+#define MPI2_TOOLBOX_CLEAN_RESERVED_5_16            (0x0001FFE0)
+#define MPI2_TOOLBOX_CLEAN_ALL_BUT_MPB              (0x00000010)
+#define MPI2_TOOLBOX_CLEAN_ENTIRE_FLASH             (0x00000008)
 #define MPI2_TOOLBOX_CLEAN_FLASH                    (0x00000004)
 #define MPI2_TOOLBOX_CLEAN_SEEPROM                  (0x00000002)
 #define MPI2_TOOLBOX_CLEAN_NVSRAM                   (0x00000001)
diff --git a/sys/dev/mpr/mpr.c b/sys/dev/mpr/mpr.c
index b6124ba03915..adc2a886058c 100644
--- a/sys/dev/mpr/mpr.c
+++ b/sys/dev/mpr/mpr.c
@@ -63,18 +63,21 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcivar.h>
 
 #include <cam/cam.h>
+#include <cam/cam_ccb.h>
 #include <cam/scsi/scsi_all.h>
 
 #include <dev/mpr/mpi/mpi2_type.h>
 #include <dev/mpr/mpi/mpi2.h>
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_sas.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
 #include <dev/mpr/mpr_ioctl.h>
 #include <dev/mpr/mprvar.h>
 #include <dev/mpr/mpr_table.h>
+#include <dev/mpr/mpr_sas.h>
 
 static int mpr_diag_reset(struct mpr_softc *sc, int sleep_flag);
 static int mpr_init_queues(struct mpr_softc *sc);
@@ -87,6 +90,7 @@ static int mpr_send_iocinit(struct mpr_softc *sc);
 static int mpr_alloc_queues(struct mpr_softc *sc);
 static int mpr_alloc_replies(struct mpr_softc *sc);
 static int mpr_alloc_requests(struct mpr_softc *sc);
+static int mpr_alloc_nvme_prp_pages(struct mpr_softc *sc);
 static int mpr_attach_log(struct mpr_softc *sc);
 static __inline void mpr_complete_command(struct mpr_softc *sc,
     struct mpr_command *cm);
@@ -110,7 +114,7 @@ static char mpt2_reset_magic[] = { 0x00, 0x0f, 0x04, 0x0b, 0x02, 0x07, 0x0d };
 
 /* 
  * Added this union to smoothly convert le64toh cm->cm_desc.Words.
- * Compiler only supports unint64_t to be passed as an argument.
+ * Compiler only supports uint64_t to be passed as an argument.
  * Otherwise it will through this error:
  * "aggregate value used where an integer was expected"
  */
@@ -120,7 +124,7 @@ typedef union _reply_descriptor {
                 u32 low;
                 u32 high;
         } u;
-}reply_descriptor,address_descriptor;
+} reply_descriptor, request_descriptor;
 
 /* Rate limit chain-fail messages to 1 per minute */
 static struct timeval mpr_chainfail_interval = { 60, 0 };
@@ -311,7 +315,6 @@ mpr_transition_ready(struct mpr_softc *sc)
 
 	if (error)
 		device_printf(sc->mpr_dev, "Cannot transition IOC to ready\n");
-
 	return (error);
 }
 
@@ -392,7 +395,8 @@ mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching)
 	mpr_printf(sc, "IOCCapabilities: %b\n", sc->facts->IOCCapabilities,
 	    "\20" "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf"
 	    "\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR"
-	    "\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc");
+	    "\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc"
+	    "\22FastPath" "\23RDPQArray" "\24AtomicReqDesc" "\25PCIeSRIOV");
 
 	/*
 	 * If the chip doesn't support event replay then a hard reset will be
@@ -480,12 +484,15 @@ mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching)
 			    enabled = TRUE;
 
 		/*
-		 * Set flag if EEDP is supported and if TLR is supported.
+		 * Set flags for some supported items.
 		 */
 		if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP)
 			sc->eedp_enabled = TRUE;
 		if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR)
 			sc->control_TLR = TRUE;
+		if (sc->facts->IOCCapabilities &
+		    MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ)
+			sc->atomic_desc_capable = TRUE;
 
 		/*
 		 * Size the queues. Since the reply queues always need one free
@@ -501,6 +508,7 @@ mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching)
 		TAILQ_INIT(&sc->req_list);
 		TAILQ_INIT(&sc->high_priority_req_list);
 		TAILQ_INIT(&sc->chain_list);
+		TAILQ_INIT(&sc->prp_page_list);
 		TAILQ_INIT(&sc->tm_list);
 	}
 
@@ -634,6 +642,14 @@ mpr_iocfacts_free(struct mpr_softc *sc)
 	if (sc->sense_dmat != NULL)
 		bus_dma_tag_destroy(sc->sense_dmat);
 
+	if (sc->prp_page_busaddr != 0)
+		bus_dmamap_unload(sc->prp_page_dmat, sc->prp_page_map);
+	if (sc->prp_pages != NULL)
+		bus_dmamem_free(sc->prp_page_dmat, sc->prp_pages,
+		    sc->prp_page_map);
+	if (sc->prp_page_dmat != NULL)
+		bus_dma_tag_destroy(sc->prp_page_dmat);
+
 	if (sc->reply_busaddr != 0)
 		bus_dmamap_unload(sc->reply_dmat, sc->reply_map);
 	if (sc->reply_frames != NULL)
@@ -651,6 +667,8 @@ mpr_iocfacts_free(struct mpr_softc *sc)
 
 	if (sc->chains != NULL)
 		free(sc->chains, M_MPR);
+	if (sc->prps != NULL)
+		free(sc->prps, M_MPR);
 	if (sc->commands != NULL) {
 		for (i = 1; i < sc->num_reqs; i++) {
 			cm = &sc->commands[i];
@@ -804,7 +822,7 @@ mpr_wait_db_ack(struct mpr_softc *sc, int timeout, int sleep_flag)
 		count++;
 	} while (--cntdn);
 
-	out:
+out:
 	mpr_dprint(sc, MPR_FAULT, "%s: failed due to timeout count(%d), "
 		"int_status(%x)!\n", __func__, count, int_status);
 	return (ETIMEDOUT);
@@ -959,7 +977,7 @@ mpr_request_sync(struct mpr_softc *sc, void *req, MPI2_DEFAULT_REPLY *reply,
 static void
 mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm)
 {
-	reply_descriptor rd;
+	request_descriptor rd;
 
 	MPR_FUNCTRACE(sc);
 	mpr_dprint(sc, MPR_TRACE, "SMID %u cm %p ccb %p\n",
@@ -972,14 +990,19 @@ mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm)
 	if (++sc->io_cmds_active > sc->io_cmds_highwater)
 		sc->io_cmds_highwater++;
 
-	rd.u.low = cm->cm_desc.Words.Low;
-	rd.u.high = cm->cm_desc.Words.High;
-	rd.word = htole64(rd.word);
-	/* TODO-We may need to make below regwrite atomic */
-	mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
-	    rd.u.low);
-	mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET,
-	    rd.u.high);
+	if (sc->atomic_desc_capable) {
+		rd.u.low = cm->cm_desc.Words.Low;
+		mpr_regwrite(sc, MPI26_ATOMIC_REQUEST_DESCRIPTOR_POST_OFFSET,
+		    rd.u.low);
+	} else {
+		rd.u.low = cm->cm_desc.Words.Low;
+		rd.u.high = cm->cm_desc.Words.High;
+		rd.word = htole64(rd.word);
+		mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
+		    rd.u.low);
+		mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET,
+		    rd.u.high);
+	}
 }
 
 /*
@@ -1047,6 +1070,7 @@ mpr_send_iocinit(struct mpr_softc *sc)
 	time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000);
 	init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF);
 	init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF);
+	init.HostPageSize = HOST_PAGE_SIZE_4K;
 
 	error = mpr_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
 	if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
@@ -1276,6 +1300,16 @@ mpr_alloc_requests(struct mpr_softc *sc)
 		sc->chain_free_lowwater++;
 	}
 
+	/*
+	 * Allocate NVMe PRP Pages for NVMe SGL support only if the FW supports
+	 * these devices.
+	 */
+	if ((sc->facts->MsgVersion >= MPI2_VERSION_02_06) &&
+	    (sc->facts->ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES)) {
+		if (mpr_alloc_nvme_prp_pages(sc) == ENOMEM)
+			return (ENOMEM);
+	}
+
 	/* XXX Need to pick a more precise value */
 	nsegs = (MAXPHYS / PAGE_SIZE) + 1;
         if (bus_dma_tag_create( sc->mpr_parent_dmat,    /* parent */
@@ -1316,15 +1350,17 @@ mpr_alloc_requests(struct mpr_softc *sc)
 		cm->cm_desc.Default.SMID = i;
 		cm->cm_sc = sc;
 		TAILQ_INIT(&cm->cm_chain_list);
+		TAILQ_INIT(&cm->cm_prp_page_list);
 		callout_init_mtx(&cm->cm_callout, &sc->mpr_mtx, 0);
 
 		/* XXX Is a failure here a critical problem? */
-		if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) == 0)
+		if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap)
+		    == 0) {
 			if (i <= sc->facts->HighPriorityCredit)
 				mpr_free_high_priority_command(sc, cm);
 			else
 				mpr_free_command(sc, cm);
-		else {
+		} else {
 			panic("failed to allocate command %d\n", i);
 			sc->num_reqs = i;
 			break;
@@ -1334,6 +1370,86 @@ mpr_alloc_requests(struct mpr_softc *sc)
 	return (0);
 }
 
+/*
+ * Allocate contiguous buffers for PCIe NVMe devices for building native PRPs,
+ * which are scatter/gather lists for NVMe devices. 
+ *
+ * This buffer must be contiguous due to the nature of how NVMe PRPs are built
+ * and translated by FW.
+ *
+ * returns ENOMEM if memory could not be allocated, otherwise returns 0.
+ */
+static int
+mpr_alloc_nvme_prp_pages(struct mpr_softc *sc)
+{
+	int PRPs_per_page, PRPs_required, pages_required;
+	int rsize, i;
+	struct mpr_prp_page *prp_page;
+
+	/*
+	 * Assuming a MAX_IO_SIZE of 1MB and a PAGE_SIZE of 4k, the max number
+	 * of PRPs (NVMe's Scatter/Gather Element) needed per I/O is:
+	 * MAX_IO_SIZE / PAGE_SIZE = 256
+	 * 
+	 * 1 PRP entry in main frame for PRP list pointer still leaves 255 PRPs
+	 * required for the remainder of the 1MB I/O. 512 PRPs can fit into one
+	 * page (4096 / 8 = 512), so only one page is required for each I/O.
+	 *
+	 * Each of these buffers will need to be contiguous. For simplicity,
+	 * only one buffer is allocated here, which has all of the space
+	 * required for the NVMe Queue Depth. If there are problems allocating
+	 * this one buffer, this function will need to change to allocate
+	 * individual, contiguous NVME_QDEPTH buffers.
+	 *
+	 * The real calculation will use the real max io size. Above is just an
+	 * example.
+	 *
+	 */
+	PRPs_required = sc->maxio / PAGE_SIZE;
+	PRPs_per_page = (PAGE_SIZE / PRP_ENTRY_SIZE) - 1;
+	pages_required = (PRPs_required / PRPs_per_page) + 1;
+
+	sc->prp_buffer_size = PAGE_SIZE * pages_required; 
+	rsize = sc->prp_buffer_size * NVME_QDEPTH; 
+	if (bus_dma_tag_create( sc->mpr_parent_dmat,	/* parent */
+				4, 0,			/* algnmnt, boundary */
+				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+				BUS_SPACE_MAXADDR,	/* highaddr */
+				NULL, NULL,		/* filter, filterarg */
+				rsize,			/* maxsize */
+				1,			/* nsegments */
+				rsize,			/* maxsegsize */
+				0,			/* flags */
+				NULL, NULL,		/* lockfunc, lockarg */
+				&sc->prp_page_dmat)) {
+		device_printf(sc->mpr_dev, "Cannot allocate NVMe PRP DMA "
+		    "tag\n");
+		return (ENOMEM);
+	}
+	if (bus_dmamem_alloc(sc->prp_page_dmat, (void **)&sc->prp_pages,
+	    BUS_DMA_NOWAIT, &sc->prp_page_map)) {
+		device_printf(sc->mpr_dev, "Cannot allocate NVMe PRP memory\n");
+		return (ENOMEM);
+	}
+	bzero(sc->prp_pages, rsize);
+	bus_dmamap_load(sc->prp_page_dmat, sc->prp_page_map, sc->prp_pages,
+	    rsize, mpr_memaddr_cb, &sc->prp_page_busaddr, 0);
+
+	sc->prps = malloc(sizeof(struct mpr_prp_page) * NVME_QDEPTH, M_MPR,
+	    M_WAITOK | M_ZERO);
+	for (i = 0; i < NVME_QDEPTH; i++) {
+		prp_page = &sc->prps[i];
+		prp_page->prp_page = (uint64_t *)(sc->prp_pages +
+		    i * sc->prp_buffer_size);
+		prp_page->prp_page_busaddr = (uint64_t)(sc->prp_page_busaddr +
+		    i * sc->prp_buffer_size);
+		mpr_free_prp_page(sc, prp_page);
+		sc->prp_pages_free_lowwater++;
+	}
+
+	return (0);
+}
+
 static int
 mpr_init_queues(struct mpr_softc *sc)
 {
@@ -1352,8 +1468,10 @@ mpr_init_queues(struct mpr_softc *sc)
 	/*
 	 * Initialize all of the free queue entries.
 	 */
-	for (i = 0; i < sc->fqdepth; i++)
-		sc->free_queue[i] = sc->reply_busaddr + (i * sc->facts->ReplyFrameSize * 4);
+	for (i = 0; i < sc->fqdepth; i++) {
+		sc->free_queue[i] = sc->reply_busaddr +
+		    (i * sc->facts->ReplyFrameSize * 4);
+	}
 	sc->replyfreeindex = sc->num_replies;
 
 	return (0);
@@ -1520,6 +1638,18 @@ mpr_setup_sysctl(struct mpr_softc *sc)
 	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 	    OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0,
 	    "Use the phy number for enumeration");
+
+	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
+	    OID_AUTO, "prp_pages_free", CTLFLAG_RD,
+	    &sc->prp_pages_free, 0, "number of free PRP pages");
+
+	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
+	    OID_AUTO, "prp_pages_free_lowwater", CTLFLAG_RD,
+	    &sc->prp_pages_free_lowwater, 0,"lowest number of free PRP pages");
+
+	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
+	    OID_AUTO, "prp_page_alloc_fail", CTLFLAG_RD,
+	    &sc->prp_page_alloc_fail, "PRP page allocation failures");
 }
 
 int
@@ -1912,6 +2042,7 @@ mpr_intr_locked(void *data)
 		switch (flags) {
 		case MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS:
 		case MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS:
+		case MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS:
 			cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];
 			cm->cm_reply = NULL;
 			break;
@@ -2200,6 +2331,519 @@ mpr_deregister_events(struct mpr_softc *sc, struct mpr_event_handle *handle)
 	return (mpr_update_events(sc, NULL, NULL));
 }
 
+/**
+* mpr_build_nvme_prp - This function is called for NVMe end devices to build a
+* native SGL (NVMe PRP). The native SGL is built starting in the first PRP entry
+* of the NVMe message (PRP1). If the data buffer is small enough to be described
+* entirely using PRP1, then PRP2 is not used. If needed, PRP2 is used to
+* describe a larger data buffer. If the data buffer is too large to describe
+* using the two PRP entriess inside the NVMe message, then PRP1 describes the
+* first data memory segment, and PRP2 contains a pointer to a PRP list located
+* elsewhere in memory to describe the remaining data memory segments. The PRP
+* list will be contiguous.
+
+* The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP
+* consists of a list of PRP entries to describe a number of noncontigous
+* physical memory segments as a single memory buffer, just as a SGL does. Note
+* however, that this function is only used by the IOCTL call, so the memory
+* given will be guaranteed to be contiguous. There is no need to translate
+* non-contiguous SGL into a PRP in this case. All PRPs will describe contiguous
+* space that is one page size each.
+*
+* Each NVMe message contains two PRP entries. The first (PRP1) either contains
+* a PRP list pointer or a PRP element, depending upon the command. PRP2 contains
+* the second PRP element if the memory being described fits within 2 PRP
+* entries, or a PRP list pointer if the PRP spans more than two entries.
+*
+* A PRP list pointer contains the address of a PRP list, structured as a linear
+* array of PRP entries. Each PRP entry in this list describes a segment of
+* physical memory.
+*
+* Each 64-bit PRP entry comprises an address and an offset field. The address
+* always points to the beginning of a PAGE_SIZE physical memory page, and the
+* offset describes where within that page the memory segment begins. Only the
+* first element in a PRP list may contain a non-zero offest, implying that all
+* memory segments following the first begin at the start of a PAGE_SIZE page.
+*
+* Each PRP element normally describes a chunck of PAGE_SIZE physical memory,
+* with exceptions for the first and last elements in the list. If the memory
+* being described by the list begins at a non-zero offset within the first page,
+* then the first PRP element will contain a non-zero offset indicating where the
+* region begins within the page. The last memory segment may end before the end
+* of the PAGE_SIZE segment, depending upon the overall size of the memory being
+* described by the PRP list. 
+*
+* Since PRP entries lack any indication of size, the overall data buffer length
+* is used to determine where the end of the data memory buffer is located, and
+* how many PRP entries are required to describe it.
+*
+* Returns nothing.
+*/
+void 
+mpr_build_nvme_prp(struct mpr_softc *sc, struct mpr_command *cm,
+    Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, void *data,
+    uint32_t data_in_sz, uint32_t data_out_sz)
+{
+	int			prp_size = PRP_ENTRY_SIZE;
+	uint64_t		*prp_entry, *prp1_entry, *prp2_entry;
+	uint64_t		*prp_entry_phys, *prp_page, *prp_page_phys;
+	uint32_t		offset, entry_len, page_mask_result, page_mask;
+	bus_addr_t		paddr;
+	size_t			length;
+	struct mpr_prp_page	*prp_page_info = NULL;
+
+	/*
+	 * Not all commands require a data transfer. If no data, just return
+	 * without constructing any PRP.
+	 */
+	if (!data_in_sz && !data_out_sz)
+		return;
+
+	/*
+	 * Set pointers to PRP1 and PRP2, which are in the NVMe command. PRP1 is
+	 * located at a 24 byte offset from the start of the NVMe command. Then
+	 * set the current PRP entry pointer to PRP1.
+	 */
+	prp1_entry = (uint64_t *)(nvme_encap_request->NVMe_Command +
+	    NVME_CMD_PRP1_OFFSET);
+	prp2_entry = (uint64_t *)(nvme_encap_request->NVMe_Command +
+	    NVME_CMD_PRP2_OFFSET);
+	prp_entry = prp1_entry;
+
+	/*
+	 * For the PRP entries, use the specially allocated buffer of
+	 * contiguous memory. PRP Page allocation failures should not happen
+	 * because there should be enough PRP page buffers to account for the
+	 * possible NVMe QDepth.
+	 */
+	prp_page_info = mpr_alloc_prp_page(sc);
+	KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be "
+	    "used for building a native NVMe SGL.\n", __func__));
+	prp_page = (uint64_t *)prp_page_info->prp_page;
+	prp_page_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr;
+
+	/*
+	 * Insert the allocated PRP page into the command's PRP page list. This
+	 * will be freed when the command is freed.
+	 */
+	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);
+
+	/*
+	 * Check if we are within 1 entry of a page boundary we don't want our
+	 * first entry to be a PRP List entry.
+	 */
+	page_mask = PAGE_SIZE - 1;
+	page_mask_result = (uintptr_t)((uint8_t *)prp_page + prp_size) &
+	    page_mask;
+	if (!page_mask_result)
+	{
+		/* Bump up to next page boundary. */
+		prp_page = (uint64_t *)((uint8_t *)prp_page + prp_size);
+		prp_page_phys = (uint64_t *)((uint8_t *)prp_page_phys +
+		    prp_size);
+	}
+
+	/*
+	 * Set PRP physical pointer, which initially points to the current PRP
+	 * DMA memory page.
+	 */
+	prp_entry_phys = prp_page_phys;
+
+	/* Get physical address and length of the data buffer. */
+	paddr = (bus_addr_t)data;
+	if (data_in_sz)
+		length = data_in_sz;
+	else
+		length = data_out_sz;
+
+	/* Loop while the length is not zero. */
+	while (length)
+	{
+		/*
+		 * Check if we need to put a list pointer here if we are at page
+		 * boundary - prp_size (8 bytes).
+		 */
+		page_mask_result = (uintptr_t)((uint8_t *)prp_entry_phys +
+		    prp_size) & page_mask;
+		if (!page_mask_result)
+		{
+			/*
+			 * This is the last entry in a PRP List, so we need to
+			 * put a PRP list pointer here. What this does is:
+			 *   - bump the current memory pointer to the next
+			 *     address, which will be the next full page.
+			 *   - set the PRP Entry to point to that page. This is
+			 *     now the PRP List pointer.
+			 *   - bump the PRP Entry pointer the start of the next
+			 *     page. Since all of this PRP memory is contiguous,
+			 *     no need to get a new page - it's just the next
+			 *     address.
+			 */
+			prp_entry_phys++;
+			*prp_entry =
+			    htole64((uint64_t)(uintptr_t)prp_entry_phys);
+			prp_entry++;
+		}
+
+		/* Need to handle if entry will be part of a page. */
+		offset = (uint32_t)paddr & page_mask;
+		entry_len = PAGE_SIZE - offset;
+
+		if (prp_entry == prp1_entry)
+		{
+			/*
+			 * Must fill in the first PRP pointer (PRP1) before
+			 * moving on.
+			 */
+			*prp1_entry = htole64((uint64_t)paddr);
+
+			/*
+			 * Now point to the second PRP entry within the
+			 * command (PRP2).
+			 */
+			prp_entry = prp2_entry;
+		}
+		else if (prp_entry == prp2_entry)
+		{
+			/*
+			 * Should the PRP2 entry be a PRP List pointer or just a
+			 * regular PRP pointer? If there is more than one more
+			 * page of data, must use a PRP List pointer.
+			 */
+			if (length > PAGE_SIZE)
+			{
+				/*
+				 * PRP2 will contain a PRP List pointer because
+				 * more PRP's are needed with this command. The
+				 * list will start at the beginning of the
+				 * contiguous buffer.
+				 */
+				*prp2_entry =
+				    htole64(
+				    (uint64_t)(uintptr_t)prp_entry_phys);
+
+				/*
+				 * The next PRP Entry will be the start of the
+				 * first PRP List.
+				 */
+				prp_entry = prp_page;
+			}
+			else
+			{
+				/*
+				 * After this, the PRP Entries are complete.
+				 * This command uses 2 PRP's and no PRP list.
+				 */
+				*prp2_entry = htole64((uint64_t)paddr);
+			}
+		}
+		else
+		{
+			/*
+			 * Put entry in list and bump the addresses.
+			 *
+			 * After PRP1 and PRP2 are filled in, this will fill in
+			 * all remaining PRP entries in a PRP List, one per each
+			 * time through the loop.
+			 */
+			*prp_entry = htole64((uint64_t)paddr);
+			prp_entry++;
+			prp_entry_phys++;
+		}
+
+		/*
+		 * Bump the phys address of the command's data buffer by the
+		 * entry_len.
+		 */
+		paddr += entry_len;
+
+		/* Decrement length accounting for last partial page. */
+		if (entry_len > length)
+			length = 0;
+		else
+			length -= entry_len;
+	}
+}
+
+/*
+ * mpr_check_pcie_native_sgl - This function is called for PCIe end devices to
+ * determine if the driver needs to build a native SGL. If so, that native SGL
+ * is built in the contiguous buffers allocated especially for PCIe SGL
+ * creation. If the driver will not build a native SGL, return TRUE and a
+ * normal IEEE SGL will be built. Currently this routine supports NVMe devices
+ * only.
+ *
+ * Returns FALSE (0) if native SGL was built, TRUE (1) if no SGL was built.
+ */
+static int
+mpr_check_pcie_native_sgl(struct mpr_softc *sc, struct mpr_command *cm,
+    bus_dma_segment_t *segs, int segs_left)
+{
+	uint32_t		i, sge_dwords, length, offset, entry_len;
+	uint32_t		num_entries, buff_len = 0, sges_in_segment;
+	uint32_t		page_mask, page_mask_result, *curr_buff;
+	uint32_t		*ptr_sgl, *ptr_first_sgl, first_page_offset;
+	uint32_t		first_page_data_size, end_residual;
+	uint64_t		*msg_phys;
+	bus_addr_t		paddr;
+	int			build_native_sgl = 0, first_prp_entry;
+	int			prp_size = PRP_ENTRY_SIZE;
+	Mpi25IeeeSgeChain64_t	*main_chain_element = NULL;
+	struct mpr_prp_page	*prp_page_info = NULL;
+
+	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);
+
+	/*
+	 * Add up the sizes of each segment length to get the total transfer
+	 * size, which will be checked against the Maximum Data Transfer Size.
+	 * If the data transfer length exceeds the MDTS for this device, just
+	 * return 1 so a normal IEEE SGL will be built. F/W will break the I/O
+	 * up into multiple I/O's. [nvme_mdts = 0 means unlimited]
+	 */
+	for (i = 0; i < segs_left; i++)
+		buff_len += htole32(segs[i].ds_len);
+	if ((cm->cm_targ->MDTS > 0) && (buff_len > cm->cm_targ->MDTS))
+		return 1;
+
+	/* Create page_mask (to get offset within page) */
+	page_mask = PAGE_SIZE - 1;
+
+	/*
+	 * Check if the number of elements exceeds the max number that can be
+	 * put in the main message frame (H/W can only translate an SGL that
+	 * is contained entirely in the main message frame).
+	 */
+	sges_in_segment = (sc->facts->IOCRequestFrameSize -
+	    offsetof(Mpi25SCSIIORequest_t, SGL)) / sizeof(MPI25_SGE_IO_UNION);
+	if (segs_left > sges_in_segment)
+		build_native_sgl = 1;
+	else
+	{
+		/*
+		 * NVMe uses one PRP for each physical page (or part of physical
+		 * page).
+		 *    if 4 pages or less then IEEE is OK
+		 *    if > 5 pages then we need to build a native SGL
+		 *    if > 4 and <= 5 pages, then check the physical address of
+		 *      the first SG entry, then if this first size in the page
+		 *      is >= the residual beyond 4 pages then use IEEE,
+		 *      otherwise use native SGL
+		 */
+		if (buff_len > (PAGE_SIZE * 5))
+			build_native_sgl = 1;
+		else if ((buff_len > (PAGE_SIZE * 4)) &&
+		    (buff_len <= (PAGE_SIZE * 5)) )
+		{
+			msg_phys = (uint64_t *)segs[0].ds_addr;
+			first_page_offset =
+			    ((uint32_t)(uint64_t)(uintptr_t)msg_phys &
+			    page_mask);
+			first_page_data_size = PAGE_SIZE - first_page_offset;
+			end_residual = buff_len % PAGE_SIZE;
+
+			/*
+			 * If offset into first page pushes the end of the data
+			 * beyond end of the 5th page, we need the extra PRP
+			 * list.
+			 */
+			if (first_page_data_size < end_residual)
+				build_native_sgl = 1;
+
+			/*
+			 * Check if first SG entry size is < residual beyond 4
+			 * pages.
+			 */
+			if (htole32(segs[0].ds_len) <
+			    (buff_len - (PAGE_SIZE * 4)))
+				build_native_sgl = 1;
+		}
+	}
+
+	/* check if native SGL is needed */
+	if (!build_native_sgl)
+		return 1;
+
+	/*
+	 * Native SGL is needed.
+	 * Put a chain element in main message frame that points to the first
+	 * chain buffer.
+	 *
+	 * NOTE:  The ChainOffset field must be 0 when using a chain pointer to
+	 *        a native SGL.
+	 */
+
+	/* Set main message chain element pointer */
+	main_chain_element = (pMpi25IeeeSgeChain64_t)cm->cm_sge;
+
+	/*
+	 * For NVMe the chain element needs to be the 2nd SGL entry in the main
+	 * message.
+	 */
+	main_chain_element = (Mpi25IeeeSgeChain64_t *)
+	    ((uint8_t *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
+
+	/*
+	 * For the PRP entries, use the specially allocated buffer of
+	 * contiguous memory. PRP Page allocation failures should not happen
+	 * because there should be enough PRP page buffers to account for the
+	 * possible NVMe QDepth.
+	 */
+	prp_page_info = mpr_alloc_prp_page(sc);
+	KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be "
+	    "used for building a native NVMe SGL.\n", __func__));
+	curr_buff = (uint32_t *)prp_page_info->prp_page;
+	msg_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr;
+
+	/*
+	 * Insert the allocated PRP page into the command's PRP page list. This
+	 * will be freed when the command is freed.
+	 */
+	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);
+
+	/*
+	 * Check if we are within 1 entry of a page boundary we don't want our
+	 * first entry to be a PRP List entry.
+	 */
+	page_mask_result = (uintptr_t)((uint8_t *)curr_buff + prp_size) &
+	    page_mask;
+	if (!page_mask_result) {
+		/* Bump up to next page boundary. */
+		curr_buff = (uint32_t *)((uint8_t *)curr_buff + prp_size);
+		msg_phys = (uint64_t *)((uint8_t *)msg_phys + prp_size);
+	}
+
+	/* Fill in the chain element and make it an NVMe segment type. */
+	main_chain_element->Address.High =
+	    htole32((uint32_t)((uint64_t)(uintptr_t)msg_phys >> 32));
+	main_chain_element->Address.Low =
+	    htole32((uint32_t)(uintptr_t)msg_phys);
+	main_chain_element->NextChainOffset = 0;
+	main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
+	    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
+	    MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
+
+	/* Set SGL pointer to start of contiguous PCIe buffer. */
+	ptr_sgl = curr_buff;
+	sge_dwords = 2;
+	num_entries = 0;
+
+	/*
+	 * NVMe has a very convoluted PRP format. One PRP is required for each
+	 * page or partial page. We need to split up OS SG entries if they are
+	 * longer than one page or cross a page boundary. We also have to insert
+	 * a PRP list pointer entry as the last entry in each physical page of
+	 * the PRP list.
+	 *
+	 * NOTE: The first PRP "entry" is actually placed in the first SGL entry
+	 * in the main message in IEEE 64 format. The 2nd entry in the main
+	 * message is the chain element, and the rest of the PRP entries are
+	 * built in the contiguous PCIe buffer.
+	 */
+	first_prp_entry = 1;
+	ptr_first_sgl = (uint32_t *)cm->cm_sge;
+
+	for (i = 0; i < segs_left; i++) {
+		/* Get physical address and length of this SG entry. */
+		paddr = segs[i].ds_addr;
+		length = segs[i].ds_len;
+
+		/*
+		 * Check whether a given SGE buffer lies on a non-PAGED
+		 * boundary if this is not the first page. If so, this is not
+		 * expected so have FW build the SGL.
+		 */
+		if (i) {
+			if ((uint32_t)paddr & page_mask) {
+				mpr_dprint(sc, MPR_ERROR, "Unaligned SGE while "
+				    "building NVMe PRPs, low address is 0x%x\n",
+				    (uint32_t)paddr);
+				return 1;
+			}
+		}
+
+		/* Apart from last SGE, if any other SGE boundary is not page
+		 * aligned then it means that hole exists. Existence of hole
+		 * leads to data corruption. So fallback to IEEE SGEs.
+		 */
+		if (i != (segs_left - 1)) {
+			if (((uint32_t)paddr + length) & page_mask) {
+				mpr_dprint(sc, MPR_ERROR, "Unaligned SGE "
+				    "boundary while building NVMe PRPs, low "
+				    "address: 0x%x and length: %u\n",
+				    (uint32_t)paddr, length);
+				return 1;
+			}
+		}
+
+		/* Loop while the length is not zero. */
+		while (length) {
+			/*
+			 * Check if we need to put a list pointer here if we are
+			 * at page boundary - prp_size.
+			 */
+			page_mask_result = (uintptr_t)((uint8_t *)ptr_sgl +
+			    prp_size) & page_mask;
+			if (!page_mask_result) {
+				/*
+				 * Need to put a PRP list pointer here.
+				 */
+				msg_phys = (uint64_t *)((uint8_t *)msg_phys +
+				    prp_size);
+				*ptr_sgl = htole32((uintptr_t)msg_phys);
+				*(ptr_sgl+1) = htole32((uint64_t)(uintptr_t)
+				    msg_phys >> 32);
+				ptr_sgl += sge_dwords;
+				num_entries++;
+			}
+
+			/* Need to handle if entry will be part of a page. */
+			offset = (uint32_t)paddr & page_mask;
+			entry_len = PAGE_SIZE - offset;
+			if (first_prp_entry) {
+				/*
+				 * Put IEEE entry in first SGE in main message.
+				 * (Simple element, System addr, not end of
+				 * list.)
+				 */
+				*ptr_first_sgl = htole32((uint32_t)paddr);
+				*(ptr_first_sgl + 1) =
+				    htole32((uint32_t)((uint64_t)paddr >> 32));
+				*(ptr_first_sgl + 2) = htole32(entry_len);
+				*(ptr_first_sgl + 3) = 0;
+
+				/* No longer the first PRP entry. */
+				first_prp_entry = 0;
+			} else {
+				/* Put entry in list. */
+				*ptr_sgl = htole32((uint32_t)paddr);
+				*(ptr_sgl + 1) =
+				    htole32((uint32_t)((uint64_t)paddr >> 32));
+
+				/* Bump ptr_sgl, msg_phys, and num_entries. */
+				ptr_sgl += sge_dwords;
+				msg_phys = (uint64_t *)((uint8_t *)msg_phys +
+				    prp_size);
+				num_entries++;
+			}
+
+			/* Bump the phys address by the entry_len. */
+			paddr += entry_len;
+
+			/* Decrement length accounting for last partial page. */
+			if (entry_len > length)
+				length = 0;
+			else
+				length -= entry_len;
+		}
+	}
+
+	/* Set chain element Length. */
+	main_chain_element->Length = htole32(num_entries * prp_size);
+
+	/* Return 0, indicating we built a native SGL. */
+	return 0;
+}
+
 /*
  * Add a chain element as the next SGE for the specified command.
  * Reset cm_sge and cm_sgesize to indicate all the available space. Chains are
@@ -2540,6 +3184,13 @@ mpr_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 	} else
 		dir = BUS_DMASYNC_PREREAD;
 
+	/* Check if a native SG list is needed for an NVMe PCIe device. */
+	if (cm->cm_targ && cm->cm_targ->is_nvme &&
+	    mpr_check_pcie_native_sgl(sc, cm, segs, nsegs) == 0) {
+		/* A native SG list was built, skip to end. */
+		goto out;
+	}
+
 	for (i = 0; i < nsegs; i++) {
 		if ((cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) && (i != 0)) {
 			sflags &= ~MPI2_SGE_FLAGS_DIRECTION;
@@ -2557,6 +3208,7 @@ mpr_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 		}
 	}
 
+out:
 	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
 	mpr_enqueue_request(sc, cm);
 
diff --git a/sys/dev/mpr/mpr_config.c b/sys/dev/mpr/mpr_config.c
index 0ba44ea7914e..1ae070056159 100644
--- a/sys/dev/mpr/mpr_config.c
+++ b/sys/dev/mpr/mpr_config.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2.h>
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_sas.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
@@ -91,7 +92,7 @@ mpr_config_get_ioc_pg8(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_IOC;
 	request->Header.PageNumber = 8;
-	request->Header.PageVersion = MPI2_IOCPAGE8_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
@@ -137,7 +138,7 @@ mpr_config_get_ioc_pg8(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_IOC;
 	request->Header.PageNumber = 8;
-	request->Header.PageVersion = MPI2_IOCPAGE8_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->Header.PageLength = mpi_reply->Header.PageLength;
 	cm->cm_length =  le16toh(mpi_reply->Header.PageLength) * 4;
 	cm->cm_sge = &request->PageBufferSGE;
@@ -221,7 +222,7 @@ mpr_config_get_iounit_pg8(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_IO_UNIT;
 	request->Header.PageNumber = 8;
-	request->Header.PageVersion = MPI2_IOUNITPAGE8_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
@@ -267,7 +268,7 @@ mpr_config_get_iounit_pg8(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_IO_UNIT;
 	request->Header.PageNumber = 8;
-	request->Header.PageVersion = MPI2_IOUNITPAGE8_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->Header.PageLength = mpi_reply->Header.PageLength;
 	cm->cm_length =  le16toh(mpi_reply->Header.PageLength) * 4;
 	cm->cm_sge = &request->PageBufferSGE;
@@ -387,7 +388,7 @@ mpr_config_get_dpm_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_DRIVER_MAPPING;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_DRIVERMAPPING0_PAGEVERSION;
+	request->ExtPageLength = request->Header.PageVersion = 0;
 	request->PageAddress = sc->max_dpm_entries <<
 	    MPI2_DPM_PGAD_ENTRY_COUNT_SHIFT;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
@@ -436,7 +437,7 @@ mpr_config_get_dpm_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_DRIVER_MAPPING;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_DRIVERMAPPING0_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->PageAddress = sc->max_dpm_entries <<
 	    MPI2_DPM_PGAD_ENTRY_COUNT_SHIFT;
 	request->ExtPageLength = mpi_reply->ExtPageLength;
@@ -522,7 +523,7 @@ int mpr_config_set_dpm_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_DRIVER_MAPPING;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_DRIVERMAPPING0_PAGEVERSION;
+	request->ExtPageLength = request->Header.PageVersion = 0;
 	/* We can remove below two lines ????*/
 	request->PageAddress = 1 << MPI2_DPM_PGAD_ENTRY_COUNT_SHIFT;
 	request->PageAddress |= htole16(entry_idx);
@@ -572,7 +573,7 @@ int mpr_config_set_dpm_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_DRIVER_MAPPING;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_DRIVERMAPPING0_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->ExtPageLength = mpi_reply->ExtPageLength;
 	request->PageAddress = 1 << MPI2_DPM_PGAD_ENTRY_COUNT_SHIFT;
 	request->PageAddress |= htole16(entry_idx);
@@ -660,7 +661,7 @@ mpr_config_get_sas_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_SAS_DEVICE;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_SASDEVICE0_PAGEVERSION;
+	request->ExtPageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
@@ -707,7 +708,7 @@ mpr_config_get_sas_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
 	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_SAS_DEVICE;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_SASDEVICE0_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->ExtPageLength = mpi_reply->ExtPageLength;
 	request->PageAddress = htole32(form | handle);
 	cm->cm_length = le16toh(mpi_reply->ExtPageLength) * 4;
@@ -758,6 +759,276 @@ mpr_config_get_sas_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
 	return (error);
 }
 
+/**
+ * mpr_config_get_pcie_device_pg0 - obtain PCIe device page 0
+ * @sc: per adapter object
+ * @mpi_reply: reply mf payload returned from firmware
+ * @config_page: contents of the config page
+ * @form: GET_NEXT_HANDLE or HANDLE
+ * @handle: device handle
+ * Context: sleep.
+ *
+ * Returns 0 for success, non-zero for failure.
+ */
+int
+mpr_config_get_pcie_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
+    *mpi_reply, Mpi26PCIeDevicePage0_t *config_page, u32 form, u16 handle)
+{
+	MPI2_CONFIG_REQUEST *request;
+	MPI2_CONFIG_REPLY *reply;
+	struct mpr_command *cm;
+	Mpi26PCIeDevicePage0_t *page = NULL;
+	int error = 0;
+	u16 ioc_status;
+
+	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);
+
+	if ((cm = mpr_alloc_command(sc)) == NULL) {
+		printf("%s: command alloc failed @ line %d\n", __func__,
+		    __LINE__);
+		error = EBUSY;
+		goto out;
+	}
+	request = (MPI2_CONFIG_REQUEST *)cm->cm_req;
+	bzero(request, sizeof(MPI2_CONFIG_REQUEST));
+	request->Function = MPI2_FUNCTION_CONFIG;
+	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
+	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
+	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_PCIE_DEVICE;
+	request->Header.PageNumber = 0;
+	request->ExtPageLength = request->Header.PageVersion = 0;
+	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
+	cm->cm_data = NULL;
+	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
+	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
+	if (error || (reply == NULL)) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: request for header completed with error %d",
+		    __func__, error);
+		error = ENXIO;
+		goto out;
+	}
+	ioc_status = le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
+	bcopy(reply, mpi_reply, sizeof(MPI2_CONFIG_REPLY));
+	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: header read with error; iocstatus = 0x%x\n",
+		    __func__, ioc_status);
+		error = ENXIO;
+		goto out;
+	}
+	/* We have to do free and alloc for the reply-free and reply-post
+	 * counters to match - Need to review the reply FIFO handling.
+	 */
+	mpr_free_command(sc, cm);
+
+	if ((cm = mpr_alloc_command(sc)) == NULL) {
+		printf("%s: command alloc failed @ line %d\n", __func__,
+		    __LINE__);
+		error = EBUSY;
+		goto out;
+	}
+	request = (MPI2_CONFIG_REQUEST *)cm->cm_req;
+	bzero(request, sizeof(MPI2_CONFIG_REQUEST));
+	request->Function = MPI2_FUNCTION_CONFIG;
+	request->Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
+	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
+	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_PCIE_DEVICE;
+	request->Header.PageNumber = 0;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
+	request->ExtPageLength = mpi_reply->ExtPageLength;
+	request->PageAddress = htole32(form | handle);
+	cm->cm_length = le16toh(mpi_reply->ExtPageLength) * 4;
+	cm->cm_sge = &request->PageBufferSGE;
+	cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
+	cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE | MPR_CM_FLAGS_DATAIN;
+	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
+	page = malloc(cm->cm_length, M_MPR, M_ZERO | M_NOWAIT);
+	if (!page) {
+		printf("%s: page alloc failed\n", __func__);
+		error = ENOMEM;
+		goto out;
+	}
+	cm->cm_data = page;
+
+	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
+	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
+	if (error || (reply == NULL)) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: request for page completed with error %d",
+		    __func__, error);
+		error = ENXIO;
+		goto out;
+	}
+	ioc_status = le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
+	bcopy(reply, mpi_reply, sizeof(MPI2_CONFIG_REPLY));
+	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: page read with error; iocstatus = 0x%x\n",
+		    __func__, ioc_status);
+		error = ENXIO;
+		goto out;
+	}
+	bcopy(page, config_page, MIN(cm->cm_length, 
+	    sizeof(Mpi26PCIeDevicePage0_t)));
+out:
+	free(page, M_MPR);
+	if (cm)
+		mpr_free_command(sc, cm);
+	return (error);
+}
+
+/**
+ * mpr_config_get_pcie_device_pg2 - obtain PCIe device page 2
+ * @sc: per adapter object
+ * @mpi_reply: reply mf payload returned from firmware
+ * @config_page: contents of the config page
+ * @form: GET_NEXT_HANDLE or HANDLE
+ * @handle: device handle
+ * Context: sleep.
+ *
+ * Returns 0 for success, non-zero for failure.
+ */
+int
+mpr_config_get_pcie_device_pg2(struct mpr_softc *sc, Mpi2ConfigReply_t
+    *mpi_reply, Mpi26PCIeDevicePage2_t *config_page, u32 form, u16 handle)
+{
+	MPI2_CONFIG_REQUEST *request;
+	MPI2_CONFIG_REPLY *reply;
+	struct mpr_command *cm;
+	Mpi26PCIeDevicePage2_t *page = NULL;
+	int error = 0;
+	u16 ioc_status;
+
+	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);
+
+	if ((cm = mpr_alloc_command(sc)) == NULL) {
+		printf("%s: command alloc failed @ line %d\n", __func__,
+		    __LINE__);
+		error = EBUSY;
+		goto out;
+	}
+	request = (MPI2_CONFIG_REQUEST *)cm->cm_req;
+	bzero(request, sizeof(MPI2_CONFIG_REQUEST));
+	request->Function = MPI2_FUNCTION_CONFIG;
+	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
+	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
+	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_PCIE_DEVICE;
+	request->Header.PageNumber = 2;
+	request->ExtPageLength = request->Header.PageVersion = 0;
+	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
+	cm->cm_data = NULL;
+	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
+	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
+	if (error || (reply == NULL)) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: request for header completed with error %d",
+		    __func__, error);
+		error = ENXIO;
+		goto out;
+	}
+	ioc_status = le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
+	bcopy(reply, mpi_reply, sizeof(MPI2_CONFIG_REPLY));
+	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: header read with error; iocstatus = 0x%x\n",
+		    __func__, ioc_status);
+		error = ENXIO;
+		goto out;
+	}
+	/* We have to do free and alloc for the reply-free and reply-post
+	 * counters to match - Need to review the reply FIFO handling.
+	 */
+	mpr_free_command(sc, cm);
+
+	if ((cm = mpr_alloc_command(sc)) == NULL) {
+		printf("%s: command alloc failed @ line %d\n", __func__,
+		    __LINE__);
+		error = EBUSY;
+		goto out;
+	}
+	request = (MPI2_CONFIG_REQUEST *)cm->cm_req;
+	bzero(request, sizeof(MPI2_CONFIG_REQUEST));
+	request->Function = MPI2_FUNCTION_CONFIG;
+	request->Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
+	request->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
+	request->ExtPageType = MPI2_CONFIG_EXTPAGETYPE_PCIE_DEVICE;
+	request->Header.PageNumber = 2;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
+	request->ExtPageLength = mpi_reply->ExtPageLength;
+	request->PageAddress = htole32(form | handle);
+	cm->cm_length = le16toh(mpi_reply->ExtPageLength) * 4;
+	cm->cm_sge = &request->PageBufferSGE;
+	cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
+	cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE | MPR_CM_FLAGS_DATAIN;
+	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
+	page = malloc(cm->cm_length, M_MPR, M_ZERO | M_NOWAIT);
+	if (!page) {
+		printf("%s: page alloc failed\n", __func__);
+		error = ENOMEM;
+		goto out;
+	}
+	cm->cm_data = page;
+
+	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
+	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
+	if (error || (reply == NULL)) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: request for page completed with error %d",
+		    __func__, error);
+		error = ENXIO;
+		goto out;
+	}
+	ioc_status = le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
+	bcopy(reply, mpi_reply, sizeof(MPI2_CONFIG_REPLY));
+	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
+		/* FIXME */
+		/*
+		 * If the request returns an error then we need to do a diag
+		 * reset
+		 */ 
+		printf("%s: page read with error; iocstatus = 0x%x\n",
+		    __func__, ioc_status);
+		error = ENXIO;
+		goto out;
+	}
+	bcopy(page, config_page, MIN(cm->cm_length, 
+	    sizeof(Mpi26PCIeDevicePage2_t)));
+out:
+	free(page, M_MPR);
+	if (cm)
+		mpr_free_command(sc, cm);
+	return (error);
+}
+
 /**
  * mpr_config_get_bios_pg3 - obtain BIOS page 3
  * @sc: per adapter object
@@ -792,7 +1063,7 @@ mpr_config_get_bios_pg3(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_BIOS;
 	request->Header.PageNumber = 3;
-	request->Header.PageVersion = MPI2_BIOSPAGE3_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
@@ -838,7 +1109,7 @@ mpr_config_get_bios_pg3(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_BIOS;
 	request->Header.PageNumber = 3;
-	request->Header.PageVersion = MPI2_BIOSPAGE3_PAGEVERSION;
+	request->Header.PageVersion = mpi_reply->Header.PageVersion;
 	request->Header.PageLength = mpi_reply->Header.PageLength;
 	cm->cm_length = le16toh(mpi_reply->Header.PageLength) * 4;
 	cm->cm_sge = &request->PageBufferSGE;
@@ -922,7 +1193,7 @@ mpr_config_get_raid_volume_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_RAID_VOLUME;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_RAIDVOLPAGE0_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 
@@ -1051,7 +1322,7 @@ mpr_config_get_raid_volume_pg1(struct mpr_softc *sc, Mpi2ConfigReply_t
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_RAID_VOLUME;
 	request->Header.PageNumber = 1;
-	request->Header.PageVersion = MPI2_RAIDVOLPAGE1_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 	error = mpr_wait_command(sc, cm, 60, CAN_SLEEP);
@@ -1208,7 +1479,7 @@ mpr_config_get_raid_pd_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t *mpi_reply,
 	request->Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
 	request->Header.PageType = MPI2_CONFIG_PAGETYPE_RAID_PHYSDISK;
 	request->Header.PageNumber = 0;
-	request->Header.PageVersion = MPI2_RAIDPHYSDISKPAGE0_PAGEVERSION;
+	request->Header.PageLength = request->Header.PageVersion = 0;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 	cm->cm_data = NULL;
 
diff --git a/sys/dev/mpr/mpr_mapping.c b/sys/dev/mpr/mpr_mapping.c
index 0feb55570551..a97522c0e724 100644
--- a/sys/dev/mpr/mpr_mapping.c
+++ b/sys/dev/mpr/mpr_mapping.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2.h>
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_sas.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
@@ -675,6 +676,55 @@ _mapping_add_to_removal_table(struct mpr_softc *sc, u16 handle,
 
 }
 
+/**
+ * _mapping_inc_missing_count
+ * @sc: per adapter object
+ * @map_idx: index into the mapping table for the device that is missing 
+ *
+ * Increment the missing count in the mapping table for a SAS, SATA, or PCIe
+ * device that is not responding. If Persitent Mapping is used, increment the
+ * DPM entry as well. Also, add this device to the removal table for possible
+ * removal if a new device is added.
+ *
+ * Returns nothing.
+ */
+static void
+_mapping_inc_missing_count(struct mpr_softc *sc, u32 map_idx)
+{
+	u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags);
+	struct dev_mapping_table *mt_entry;
+	Mpi2DriverMap0Entry_t *dpm_entry;
+
+	if (map_idx == MPR_MAPTABLE_BAD_IDX) {
+		mpr_dprint(sc, MPR_INFO, "%s: device is already removed from "
+		    "mapping table\n", __func__);
+		return;
+	}
+	mt_entry = &sc->mapping_table[map_idx];
+	if (!mt_entry->init_complete) {
+		if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT)
+			mt_entry->missing_count++;
+		else
+			mt_entry->init_complete = 1;
+	}
+	if (!mt_entry->missing_count)
+		mt_entry->missing_count++;
+	_mapping_add_to_removal_table(sc, mt_entry->dev_handle, 0);
+	mt_entry->dev_handle = 0;
+
+	if (((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
+	    MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) &&
+	    sc->is_dpm_enable && !mt_entry->init_complete &&
+	    mt_entry->dpm_entry_num != MPR_DPM_BAD_IDX) {
+		dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 +
+		    sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER));
+		dpm_entry += mt_entry->dpm_entry_num;
+		dpm_entry->MappingInformation = mt_entry->missing_count;
+		sc->dpm_flush_entry[mt_entry->dpm_entry_num] = 1;
+	}
+	mt_entry->init_complete = 1;
+}
+
 /**
  * _mapping_update_missing_count - Update missing count for a device
  * @sc: per adapter object
@@ -689,12 +739,9 @@ static void
 _mapping_update_missing_count(struct mpr_softc *sc,
     struct _map_topology_change *topo_change)
 {
-	u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags);
 	u8 entry;
 	struct _map_phy_change *phy_change;
 	u32 map_idx;
-	struct dev_mapping_table *mt_entry;
-	Mpi2DriverMap0Entry_t *dpm_entry;
 
 	for (entry = 0; entry < topo_change->num_entries; entry++) {
 		phy_change = &topo_change->phy_details[entry];
@@ -704,35 +751,37 @@ _mapping_update_missing_count(struct mpr_softc *sc,
 		map_idx = _mapping_get_mt_idx_from_handle(sc, phy_change->
 		    dev_handle);
 		phy_change->is_processed = 1;
-		if (map_idx == MPR_MAPTABLE_BAD_IDX) {
-			printf("%s: device is already removed from mapping "
-			    "table\n", __func__);
-			continue;
-		}
-		mt_entry = &sc->mapping_table[map_idx];
-		if (!mt_entry->init_complete) {
-			if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT)
-				mt_entry->missing_count++;
-			else
-				mt_entry->init_complete = 1;
-		}
-		if (!mt_entry->missing_count)
-			mt_entry->missing_count++;
-		_mapping_add_to_removal_table(sc, mt_entry->dev_handle, 0);
-		mt_entry->dev_handle = 0;
+		_mapping_inc_missing_count(sc, map_idx);
+	}
+}
 
-		if (((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
-		    MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) &&
-		    sc->is_dpm_enable && !mt_entry->init_complete &&
-		    mt_entry->dpm_entry_num != MPR_DPM_BAD_IDX) {
-			dpm_entry =
-			    (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 +
-			    sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER));
-			dpm_entry += mt_entry->dpm_entry_num;
-			dpm_entry->MappingInformation = mt_entry->missing_count;
-			sc->dpm_flush_entry[mt_entry->dpm_entry_num] = 1;
-		}
-		mt_entry->init_complete = 1;
+/**
+ * _mapping_update_pcie_missing_count - Update missing count for a PCIe device
+ * @sc: per adapter object
+ * @topo_change: Topology change event entry
+ *
+ * Search through the PCIe topology change list and if any device is found not
+ * responding it's associated map table entry and DPM entry is updated
+ *
+ * Returns nothing.
+ */
+static void
+_mapping_update_pcie_missing_count(struct mpr_softc *sc,
+    struct _map_pcie_topology_change *topo_change)
+{
+	u8 entry;
+	struct _map_port_change *port_change;
+	u32 map_idx;
+
+	for (entry = 0; entry < topo_change->num_entries; entry++) {
+		port_change = &topo_change->port_details[entry];
+		if (!port_change->dev_handle || (port_change->reason !=
+		    MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING))
+			continue;
+		map_idx = _mapping_get_mt_idx_from_handle(sc, port_change->
+		    dev_handle);
+		port_change->is_processed = 1;
+		_mapping_inc_missing_count(sc, map_idx);
 	}
 }
 
@@ -940,7 +989,7 @@ _mapping_get_dev_info(struct mpr_softc *sc,
 
 		phy_change->physical_id = sas_address;
 		phy_change->slot = le16toh(sas_device_pg0.Slot);
-		phy_change->device_info = le32toh(sas_device_pg0.DeviceInfo);
+		phy_change->device_info = device_info;
 
 		if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
 		    MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) {
@@ -990,12 +1039,118 @@ _mapping_get_dev_info(struct mpr_softc *sc,
 					break;
 				}
 			}
+
+			/* Found space in enclosure for mapping entry */
 			mt_entry = &sc->mapping_table[map_idx];
 			for (index = map_idx; index < (et_entry->num_slots
 			    + map_idx); index++, mt_entry++) {
 				mt_entry->device_info = MPR_DEV_RESERVED;
 				mt_entry->physical_id = et_entry->enclosure_id;
 				mt_entry->phy_bits = et_entry->phy_bits;
+				mt_entry->missing_count = 0;
+			}
+		}
+	}
+}
+
+/**
+ * _mapping_get_pcie_dev_info -get information about newly added PCIe devices
+ * @sc: per adapter object
+ * @topo_change: Topology change event entry
+ *
+ * Searches through the PCIe topology change event list and issues PCIe device
+ * pg0 requests for the newly added PCIe device. If the device is in an
+ * enclosure, search for available space in the enclosure mapping table for the
+ * device and reserve that space.
+ *
+ * Returns nothing
+ */
+static void
+_mapping_get_pcie_dev_info(struct mpr_softc *sc,
+    struct _map_pcie_topology_change *topo_change)
+{
+	u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags);
+	Mpi2ConfigReply_t mpi_reply;
+	Mpi26PCIeDevicePage0_t pcie_device_pg0;
+	u8 entry, enc_idx, port_idx;
+	u32 map_idx, index;
+	struct _map_port_change *port_change, *tmp_port_change;
+	uint64_t pcie_wwid;
+	struct enc_mapping_table *et_entry;
+	struct dev_mapping_table *mt_entry;
+	u8 add_code = MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED;
+
+	for (entry = 0; entry < topo_change->num_entries; entry++) {
+		port_change = &topo_change->port_details[entry];
+		if (port_change->is_processed || !port_change->dev_handle ||
+		    port_change->reason != MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED)
+			continue;
+		if (mpr_config_get_pcie_device_pg0(sc, &mpi_reply,
+		    &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE,
+		    port_change->dev_handle)) {
+			port_change->is_processed = 1;
+			continue;
+		}
+
+		pcie_wwid = pcie_device_pg0.WWID.High;
+		pcie_wwid = (pcie_wwid << 32) | pcie_device_pg0.WWID.Low;
+		port_change->physical_id = pcie_wwid;
+		port_change->slot = le16toh(pcie_device_pg0.Slot);
+		port_change->device_info = le32toh(pcie_device_pg0.DeviceInfo);
+
+		if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
+		    MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) {
+			enc_idx = _mapping_get_enc_idx_from_handle(sc,
+			    topo_change->enc_handle);
+			if (enc_idx == MPR_ENCTABLE_BAD_IDX) {
+				port_change->is_processed = 1;
+				mpr_dprint(sc, MPR_MAPPING, "%s: failed to add "
+				    "the device with handle 0x%04x because the "
+				    "enclosure is not in the mapping table\n",
+				    __func__, port_change->dev_handle);
+				continue;
+			}
+			if (!(port_change->device_info &
+			    MPI26_PCIE_DEVINFO_NVME)) {
+				port_change->is_processed = 1;
+				continue;
+			}
+			et_entry = &sc->enclosure_table[enc_idx];
+			if (et_entry->start_index != MPR_MAPTABLE_BAD_IDX)
+				continue;
+			if (!topo_change->switch_dev_handle) {
+				map_idx	= sc->num_rsvd_entries;
+				et_entry->start_index = map_idx;
+			} else {
+				map_idx = _mapping_find_enc_map_space(sc,
+				    et_entry);
+				et_entry->start_index = map_idx;
+				if (et_entry->start_index ==
+				    MPR_MAPTABLE_BAD_IDX) {
+					port_change->is_processed = 1;
+					for (port_idx = 0; port_idx <
+					    topo_change->num_entries;
+					    port_idx++) {
+						tmp_port_change =
+						    &topo_change->port_details
+						    [port_idx];
+						if (tmp_port_change->reason ==
+						    add_code)
+							tmp_port_change->
+							    is_processed = 1;
+					}
+					break;
+				}
+			}
+
+			/* Found space in enclosure for mapping entry */
+			mt_entry = &sc->mapping_table[map_idx];
+			for (index = map_idx; index < (et_entry->num_slots
+			    + map_idx); index++, mt_entry++) {
+				mt_entry->device_info = MPR_DEV_RESERVED;
+				mt_entry->physical_id = et_entry->enclosure_id;
+				mt_entry->phy_bits = et_entry->phy_bits;
+				mt_entry->missing_count = 0;
 			}
 		}
 	}
@@ -1106,8 +1261,8 @@ _mapping_clear_removed_entries(struct mpr_softc *sc)
  * @sc: per adapter object
  * @topo_change: Topology change event entry
  *
- * Search through the topology change event list and updates map table,
- * enclosure table and DPM pages for for the newly added devices.
+ * Search through the topology change event list and update map table,
+ * enclosure table and DPM pages for the newly added devices.
  *
  * Returns nothing
  */
@@ -1144,10 +1299,10 @@ _mapping_add_new_device(struct mpr_softc *sc,
 			    (sc, topo_change->enc_handle);
 			if (enc_idx == MPR_ENCTABLE_BAD_IDX) {
 				phy_change->is_processed = 1;
-				printf("%s: failed to add the device with "
-				    "handle 0x%04x because the enclosure is "
-				    "not in the mapping table\n", __func__,
-				    phy_change->dev_handle);
+				mpr_dprint(sc, MPR_ERROR, "%s: failed to add "
+				    "the device with handle 0x%04x because the "
+				    "enclosure is not in the mapping table\n",
+				    __func__, phy_change->dev_handle);
 				continue;
 			}
 			et_entry = &sc->enclosure_table[enc_idx];
@@ -1157,10 +1312,11 @@ _mapping_add_new_device(struct mpr_softc *sc,
 					sc->mt_add_device_failed = 1;
 					continue;
 				}
-				printf("%s: failed to add the device with "
-				    "handle 0x%04x because there is no free "
-				    "space available in the mapping table\n",
-				    __func__, phy_change->dev_handle);
+				mpr_dprint(sc, MPR_INFO, "%s: failed to add "
+				    "the device with handle 0x%04x because "
+				    "there is no free space available in the "
+				    "mapping table\n", __func__,
+				    phy_change->dev_handle);
 				continue;
 			}
 			map_idx = et_entry->start_index + phy_change->slot -
@@ -1268,10 +1424,11 @@ _mapping_add_new_device(struct mpr_softc *sc,
 					sc->mt_add_device_failed = 1;
 					continue;
 				}
-				printf("%s: failed to add the device with "
-				    "handle 0x%04x because there is no free "
-				    "space available in the mapping table\n",
-				    __func__, phy_change->dev_handle);
+				mpr_dprint(sc, MPR_INFO, "%s: failed to add "
+				    "the device with handle 0x%04x because "
+				    "there is no free space available in the "
+				    "mapping table\n", __func__,
+				    phy_change->dev_handle);
 				continue;
 			}
 			if (sc->is_dpm_enable) {
@@ -1314,14 +1471,13 @@ _mapping_add_new_device(struct mpr_softc *sc,
 					sc->dpm_flush_entry[dpm_idx] = 1;
 					phy_change->is_processed = 1;
 				} else if (dpm_idx == MPR_DPM_BAD_IDX) {
-						phy_change->is_processed = 1;
-						mpr_dprint(sc, MPR_INFO, "%s: "
-						    "failed to add the device "
-						    "with handle 0x%04x to "
-						    "persistent table because "
-						    "there is no free space "
-						    "available\n", __func__,
-						    phy_change->dev_handle);
+					phy_change->is_processed = 1;
+					mpr_dprint(sc, MPR_INFO, "%s: failed "
+					    "to add the device with handle "
+					    "0x%04x to persistent table "
+					    "because there is no free space "
+					    "available\n", __func__,
+					    phy_change->dev_handle);
 				}
 			}
 			mt_entry->init_complete = 1;
@@ -1333,6 +1489,241 @@ _mapping_add_new_device(struct mpr_softc *sc,
 		_mapping_clear_removed_entries(sc);
 }
 
+/**
+ * _mapping_add_new_pcie_device -Add the new PCIe device into mapping table
+ * @sc: per adapter object
+ * @topo_change: Topology change event entry
+ *
+ * Search through the PCIe topology change event list and update map table,
+ * enclosure table and DPM pages for the newly added devices.
+ *
+ * Returns nothing
+ */
+static void
+_mapping_add_new_pcie_device(struct mpr_softc *sc,
+    struct _map_pcie_topology_change *topo_change)
+{
+	u8 enc_idx, missing_cnt, is_removed = 0;
+	u16 dpm_idx;
+	u32 search_idx, map_idx;
+	u32 entry;
+	struct dev_mapping_table *mt_entry;
+	struct enc_mapping_table *et_entry;
+	struct _map_port_change *port_change;
+	u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags);
+	Mpi2DriverMap0Entry_t *dpm_entry;
+	uint64_t temp64_var;
+	u8 map_shift = MPI2_DRVMAP0_MAPINFO_SLOT_SHIFT;
+	u8 hdr_sz = sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER);
+	u16 max_num_phy_ids = le16toh(sc->ioc_pg8.MaxNumPhysicalMappedIDs);
+
+	for (entry = 0; entry < topo_change->num_entries; entry++) {
+		port_change = &topo_change->port_details[entry];
+		if (port_change->is_processed)
+			continue;
+		if (port_change->reason != MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED ||
+		    !port_change->dev_handle) {
+			port_change->is_processed = 1;
+			continue;
+		}
+		if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
+		    MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) {
+			enc_idx = _mapping_get_enc_idx_from_handle
+			    (sc, topo_change->enc_handle);
+			if (enc_idx == MPR_ENCTABLE_BAD_IDX) {
+				port_change->is_processed = 1;
+				mpr_dprint(sc, MPR_ERROR, "%s: failed to add "
+				    "the device with handle 0x%04x because the "
+				    "enclosure is not in the mapping table\n",
+				    __func__, port_change->dev_handle);
+				continue;
+			}
+			et_entry = &sc->enclosure_table[enc_idx];
+			if (et_entry->start_index == MPR_MAPTABLE_BAD_IDX) {
+				port_change->is_processed = 1;
+				if (!sc->mt_full_retry) {
+					sc->mt_add_device_failed = 1;
+					continue;
+				}
+				mpr_dprint(sc, MPR_INFO, "%s: failed to add "
+				    "the device with handle 0x%04x because "
+				    "there is no free space available in the "
+				    "mapping table\n", __func__,
+				    port_change->dev_handle);
+				continue;
+			}
+			map_idx = et_entry->start_index + port_change->slot -
+			    et_entry->start_slot;
+			mt_entry = &sc->mapping_table[map_idx];
+			mt_entry->physical_id = port_change->physical_id;
+			mt_entry->channel = 0;
+			mt_entry->id = map_idx;
+			mt_entry->dev_handle = port_change->dev_handle;
+			mt_entry->missing_count = 0;
+			mt_entry->dpm_entry_num = et_entry->dpm_entry_num;
+			mt_entry->device_info = port_change->device_info |
+			    (MPR_DEV_RESERVED | MPR_MAP_IN_USE);
+			if (sc->is_dpm_enable) {
+				dpm_idx = et_entry->dpm_entry_num;
+				if (dpm_idx == MPR_DPM_BAD_IDX)
+					dpm_idx = _mapping_get_dpm_idx_from_id
+					    (sc, et_entry->enclosure_id,
+					     et_entry->phy_bits);
+				if (dpm_idx == MPR_DPM_BAD_IDX) {
+					dpm_idx = _mapping_get_free_dpm_idx(sc);
+					if (dpm_idx != MPR_DPM_BAD_IDX) {
+						dpm_entry =
+						    (Mpi2DriverMap0Entry_t *)
+						    ((u8 *) sc->dpm_pg0 +
+						     hdr_sz);
+						dpm_entry += dpm_idx;
+						dpm_entry->
+						    PhysicalIdentifier.Low =
+						    (0xFFFFFFFF &
+						    et_entry->enclosure_id);
+						dpm_entry->
+						    PhysicalIdentifier.High =
+						    ( et_entry->enclosure_id
+						     >> 32);
+						dpm_entry->DeviceIndex =
+						    (U16)et_entry->start_index;
+						dpm_entry->MappingInformation =
+							et_entry->num_slots;
+						dpm_entry->MappingInformation
+						    <<= map_shift;
+						dpm_entry->PhysicalBitsMapping
+						    = et_entry->phy_bits;
+						et_entry->dpm_entry_num =
+						    dpm_idx;
+		/* FIXME Do I need to set the dpm_idxin mt_entry too */
+						sc->dpm_entry_used[dpm_idx] = 1;
+						sc->dpm_flush_entry[dpm_idx] =
+						    1;
+						port_change->is_processed = 1;
+					} else {
+						port_change->is_processed = 1;
+						mpr_dprint(sc, MPR_INFO, "%s: "
+						    "failed to add the device "
+						    "with handle 0x%04x to "
+						    "persistent table because "
+						    "there is no free space "
+						    "available\n", __func__,
+						    port_change->dev_handle);
+					}
+				} else {
+					et_entry->dpm_entry_num = dpm_idx;
+					mt_entry->dpm_entry_num = dpm_idx;
+				}
+			}
+			/* FIXME Why not mt_entry too? */
+			et_entry->init_complete = 1;
+		} else if ((ioc_pg8_flags &
+		    MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) ==
+		    MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) {
+			map_idx = _mapping_get_mt_idx_from_id
+			    (sc, port_change->physical_id);
+			if (map_idx == MPR_MAPTABLE_BAD_IDX) {
+				search_idx = sc->num_rsvd_entries;
+				if (topo_change->switch_dev_handle)
+					search_idx += max_num_phy_ids;
+				map_idx = _mapping_get_free_mt_idx(sc,
+				    search_idx);
+			}
+			if (map_idx == MPR_MAPTABLE_BAD_IDX) {
+				map_idx = _mapping_get_high_missing_mt_idx(sc);
+				if (map_idx != MPR_MAPTABLE_BAD_IDX) {
+					mt_entry = &sc->mapping_table[map_idx];
+					if (mt_entry->dev_handle) {
+						_mapping_add_to_removal_table
+						    (sc, mt_entry->dev_handle,
+						     0);
+						is_removed = 1;
+					}
+					mt_entry->init_complete = 0;
+				}
+			}
+			if (map_idx != MPR_MAPTABLE_BAD_IDX) {
+				mt_entry = &sc->mapping_table[map_idx];
+				mt_entry->physical_id =
+				    port_change->physical_id;
+				mt_entry->channel = 0;
+				mt_entry->id = map_idx;
+				mt_entry->dev_handle = port_change->dev_handle;
+				mt_entry->missing_count = 0;
+				mt_entry->device_info =
+				    port_change->device_info |
+				    (MPR_DEV_RESERVED | MPR_MAP_IN_USE);
+			} else {
+				port_change->is_processed = 1;
+				if (!sc->mt_full_retry) {
+					sc->mt_add_device_failed = 1;
+					continue;
+				}
+				mpr_dprint(sc, MPR_INFO, "%s: failed to add "
+				    "the device with handle 0x%04x because "
+				    "there is no free space available in the "
+				    "mapping table\n", __func__,
+				    port_change->dev_handle);
+				continue;
+			}
+			if (sc->is_dpm_enable) {
+				if (mt_entry->dpm_entry_num !=
+				    MPR_DPM_BAD_IDX) {
+					dpm_idx = mt_entry->dpm_entry_num;
+					dpm_entry = (Mpi2DriverMap0Entry_t *)
+					    ((u8 *)sc->dpm_pg0 + hdr_sz);
+					dpm_entry += dpm_idx;
+					missing_cnt = dpm_entry->
+					    MappingInformation &
+					    MPI2_DRVMAP0_MAPINFO_MISSING_MASK;
+					temp64_var = dpm_entry->
+					    PhysicalIdentifier.High;
+					temp64_var = (temp64_var << 32) |
+					   dpm_entry->PhysicalIdentifier.Low;
+					if ((mt_entry->physical_id ==
+					    temp64_var) && !missing_cnt)
+						mt_entry->init_complete = 1;
+				} else {
+					dpm_idx = _mapping_get_free_dpm_idx(sc);
+					mt_entry->init_complete = 0;
+				}
+				if (dpm_idx != MPR_DPM_BAD_IDX &&
+				    !mt_entry->init_complete) {
+					mt_entry->init_complete = 1;
+					mt_entry->dpm_entry_num = dpm_idx;
+					dpm_entry = (Mpi2DriverMap0Entry_t *)
+					    ((u8 *)sc->dpm_pg0 + hdr_sz);
+					dpm_entry += dpm_idx;
+					dpm_entry->PhysicalIdentifier.Low =
+					    (0xFFFFFFFF &
+					    mt_entry->physical_id);
+					dpm_entry->PhysicalIdentifier.High =
+					    (mt_entry->physical_id >> 32);
+					dpm_entry->DeviceIndex = (U16) map_idx;
+					dpm_entry->MappingInformation = 0;
+					dpm_entry->PhysicalBitsMapping = 0;
+					sc->dpm_entry_used[dpm_idx] = 1;
+					sc->dpm_flush_entry[dpm_idx] = 1;
+					port_change->is_processed = 1;
+				} else if (dpm_idx == MPR_DPM_BAD_IDX) {
+					port_change->is_processed = 1;
+					mpr_dprint(sc, MPR_INFO, "%s: failed "
+					    "to add the device with handle "
+					    "0x%04x to persistent table "
+					    "because there is no free space "
+					    "available\n", __func__,
+					    port_change->dev_handle);
+				}
+			}
+			mt_entry->init_complete = 1;
+		}
+
+		port_change->is_processed = 1;
+	}
+	if (is_removed)
+		_mapping_clear_removed_entries(sc);
+}
+
 /**
  * _mapping_flush_dpm_pages -Flush the DPM pages to NVRAM
  * @sc: per adapter object
@@ -2072,6 +2463,56 @@ mpr_mapping_topology_change_event(struct mpr_softc *sc,
 		sc->pending_map_events--;
 }
 
+/**
+ * mpr_mapping_pcie_topology_change_event - handle PCIe topology change events
+ * @sc: per adapter object
+ * @event_data: event data payload
+ *
+ * Returns nothing.
+ */
+void
+mpr_mapping_pcie_topology_change_event(struct mpr_softc *sc,
+    Mpi26EventDataPCIeTopologyChangeList_t *event_data)
+{
+	struct _map_pcie_topology_change topo_change;
+	struct _map_port_change *port_change;
+	Mpi26EventPCIeTopoPortEntry_t *event_port_change;
+	u8 i, num_entries;
+
+	topo_change.switch_dev_handle = le16toh(event_data->SwitchDevHandle);
+	topo_change.enc_handle = le16toh(event_data->EnclosureHandle);
+	num_entries = event_data->NumEntries;
+	topo_change.num_entries = num_entries;
+	topo_change.start_port_num = event_data->StartPortNum;
+	topo_change.num_ports = event_data->NumPorts;
+	topo_change.switch_status = event_data->SwitchStatus;
+	event_port_change = event_data->PortEntry;
+	topo_change.port_details = NULL;
+
+	if (!num_entries)
+		goto out;
+	port_change = malloc(sizeof(struct _map_port_change) * num_entries,
+	    M_MPR, M_NOWAIT|M_ZERO);
+	topo_change.port_details = port_change;
+	if (!port_change)
+		goto out;
+	for (i = 0; i < num_entries; i++, event_port_change++, port_change++) {
+		port_change->dev_handle = le16toh(event_port_change->
+		    AttachedDevHandle);
+		port_change->reason = event_port_change->PortStatus;
+	}
+	_mapping_update_pcie_missing_count(sc, &topo_change);
+	_mapping_get_pcie_dev_info(sc, &topo_change);
+	_mapping_clear_removed_entries(sc);
+	_mapping_add_new_pcie_device(sc, &topo_change);
+
+out:
+	free(topo_change.port_details, M_MPR);
+	_mapping_flush_dpm_pages(sc);
+	if (sc->pending_map_events)
+		sc->pending_map_events--;
+}
+
 /**
  * _mapping_check_update_ir_mt_idx - Check and update IR map table index
  * @sc: per adapter object
diff --git a/sys/dev/mpr/mpr_mapping.h b/sys/dev/mpr/mpr_mapping.h
index 05571f2ed9e8..925eb03a9088 100644
--- a/sys/dev/mpr/mpr_mapping.h
+++ b/sys/dev/mpr/mpr_mapping.h
@@ -53,9 +53,36 @@ struct _map_phy_change {
 };
 
 /**
- * struct _map_topology_change - entries to be removed from mapping table
- * @dpm_entry_num: index of this device in device persistent map table
+ * struct _map_port_change - PCIe Port entries received in PCIe Topology change
+ * list event
+ * @physical_id: WWID of the device attached to the associated port
+ * @device_info: bitfield provides detailed info about the device
+ * @MDTS: Maximum Data Transfer Size for the device
  * @dev_handle: device handle for the device pointed by this entry
+ * @slot: slot ID
+ * @is_processed: Flag to indicate whether this entry is processed or not
+ */
+struct _map_port_change {
+	uint64_t	physical_id;
+	uint32_t	device_info;
+	uint32_t	MDTS;
+	uint16_t	dev_handle;
+	uint16_t	slot;
+	uint8_t		reason;
+	uint8_t		is_processed;
+	uint8_t		reserved[2];
+};
+
+/**
+ * struct _map_topology_change - SAS/SATA entries to be removed from mapping
+ * table
+ * @enc_handle: enclosure handle where this device is located
+ * @exp_handle: expander handle where this device is located
+ * @num_entries: number of entries in the SAS Topology Change List event
+ * @start_phy_num: PHY number of the first PHY in the event data
+ * @num_phys: number of PHYs in the expander where this device is located
+ * @exp_status: status for the expander where this device is located
+ * @phy_details: more details about each PHY in the event data
  */
 struct _map_topology_change {
 	uint16_t	enc_handle;
@@ -67,6 +94,26 @@ struct _map_topology_change {
 	struct _map_phy_change *phy_details;
 };
 
+/**
+ * struct _map_pcie_topology_change - PCIe entries to be removed from mapping
+ * table
+ * @enc_handle: enclosure handle where this device is located
+ * @switch_dev_handle:  PCIe switch device handle where this device is located
+ * @num_entries: number of entries in the PCIe Topology Change List event
+ * @start_port_num: port number of the first port in the event data
+ * @num_ports: number of ports in the PCIe switch device
+ * @switch_status: status for the PCIe switch where this device is located
+ * @port_details: more details about each Port in the event data
+ */
+struct _map_pcie_topology_change {
+	uint16_t	enc_handle;
+	uint16_t	switch_dev_handle;
+	uint8_t	num_entries;
+	uint8_t	start_port_num;
+	uint8_t	num_ports;
+	uint8_t switch_status;
+	struct _map_port_change *port_details;
+};
 
 extern int
 mprsas_get_sas_address_for_sata_disk(struct mpr_softc *ioc,
diff --git a/sys/dev/mpr/mpr_pci.c b/sys/dev/mpr/mpr_pci.c
index 17f3f3e7e34e..4b9f0aaa1513 100644
--- a/sys/dev/mpr/mpr_pci.c
+++ b/sys/dev/mpr/mpr_pci.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 
 #include <sys/queue.h>
 #include <sys/kthread.h>
@@ -110,6 +111,10 @@ struct mpr_ident {
 	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3108_5" },
 	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_6,
 	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3108_6" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3216,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3216" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3224,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3224" },
 	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_1,
 	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3316_1" },
 	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_2,
@@ -118,10 +123,24 @@ struct mpr_ident {
 	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3324_1" },
 	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_2,
 	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3324_2" },
-	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3216,
-	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3216" },
-	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3224,
-	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3224" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3408" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3416" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3508" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3508_1" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3516" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3516_1" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3616,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3616" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3708,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3708" },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3716,
+	    0xffff, 0xffff, 0, "Avago Technologies (LSI) SAS3716" },
 	{ 0, 0, 0, 0, 0, NULL }
 };
 
@@ -164,7 +183,7 @@ mpr_pci_attach(device_t dev)
 {
 	struct mpr_softc *sc;
 	struct mpr_ident *m;
-	int error;
+	int error, i;
 
 	sc = device_get_softc(dev);
 	bzero(sc, sizeof(*sc));
@@ -175,13 +194,32 @@ mpr_pci_attach(device_t dev)
 	/* Twiddle basic PCI config bits for a sanity check */
 	pci_enable_busmaster(dev);
 
-	/* Allocate the System Interface Register Set */
-	sc->mpr_regs_rid = PCIR_BAR(1);
-	if ((sc->mpr_regs_resource = bus_alloc_resource_any(dev,
-	    SYS_RES_MEMORY, &sc->mpr_regs_rid, RF_ACTIVE)) == NULL) {
+	/* Set flag if this is a Gen3.5 IOC */
+	if ((m->device == MPI26_MFGPAGE_DEVID_SAS3508) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3508_1) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3408) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3516) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3516_1) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3416) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3716) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3616) ||
+	    (m->device == MPI26_MFGPAGE_DEVID_SAS3708)) {
+		sc->mpr_flags |= MPR_FLAGS_GEN35_IOC;
+	}
+
+	for (i = 0; i < PCI_MAXMAPS_0; i++) {
+		sc->mpr_regs_rid = PCIR_BAR(i);
+
+		if ((sc->mpr_regs_resource = bus_alloc_resource_any(dev,
+		    SYS_RES_MEMORY, &sc->mpr_regs_rid, RF_ACTIVE)) != NULL)
+			break;
+	}
+
+	if (sc->mpr_regs_resource == NULL) {
 		mpr_printf(sc, "Cannot allocate PCI registers\n");
 		return (ENXIO);
 	}
+
 	sc->mpr_btag = rman_get_bustag(sc->mpr_regs_resource);
 	sc->mpr_bhandle = rman_get_bushandle(sc->mpr_regs_resource);
 
diff --git a/sys/dev/mpr/mpr_sas.c b/sys/dev/mpr/mpr_sas.c
index 2974e06a64a8..0138c2ca50c1 100644
--- a/sys/dev/mpr/mpr_sas.c
+++ b/sys/dev/mpr/mpr_sas.c
@@ -72,10 +72,13 @@ __FBSDID("$FreeBSD$");
 #include <cam/scsi/smp_all.h>
 #endif
 
+#include <dev/nvme/nvme.h>
+
 #include <dev/mpr/mpi/mpi2_type.h>
 #include <dev/mpr/mpi/mpi2.h>
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_sas.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
@@ -477,13 +480,13 @@ mprsas_prepare_volume_remove(struct mprsas_softc *sassc, uint16_t handle)
 }
 
 /*
- * The MPT3 firmware performs debounce on the link to avoid transient link
- * errors and false removals.  When it does decide that link has been lost
- * and a device needs to go away, it expects that the host will perform a
- * target reset and then an op remove.  The reset has the side-effect of
- * aborting any outstanding requests for the device, which is required for
- * the op-remove to succeed.  It's not clear if the host should check for
- * the device coming back alive after the reset.
+ * The firmware performs debounce on the link to avoid transient link errors
+ * and false removals.  When it does decide that link has been lost and a
+ * device needs to go away, it expects that the host will perform a target reset
+ * and then an op remove.  The reset has the side-effect of aborting any
+ * outstanding requests for the device, which is required for the op-remove to
+ * succeed.  It's not clear if the host should check for the device coming back
+ * alive after the reset.
  */
 void
 mprsas_prepare_remove(struct mprsas_softc *sassc, uint16_t handle)
@@ -705,7 +708,14 @@ mprsas_register_events(struct mpr_softc *sc)
 	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
 	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
 	setbit(events, MPI2_EVENT_TEMP_THRESHOLD);
-	setbit(events, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
+	if (sc->facts->MsgVersion >= MPI2_VERSION_02_06) {
+		setbit(events, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
+		if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
+			setbit(events, MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
+			setbit(events, MPI2_EVENT_PCIE_ENUMERATION);
+			setbit(events, MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
+		}
+	}
 
 	mpr_register_events(sc, events, mprsas_evt_handler, NULL,
 	    &sc->sassc->mprsas_eh);
@@ -1018,6 +1028,7 @@ mprsas_action(struct cam_sim *sim, union ccb *ccb)
 		if ((sc->max_io_pages > 0) && (sc->max_io_pages * PAGE_SIZE <
 		    cpi->maxio))
 			cpi->maxio = sc->max_io_pages * PAGE_SIZE;
+		sc->maxio = cpi->maxio;
 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
 		break;
 	}
@@ -1636,7 +1647,7 @@ mprsas_scsiio_timeout(void *data)
 	targ->timeouts++;
 
 	mprsas_log_command(cm, MPR_ERROR, "command timeout %d cm %p target "
-	    "%u, handle(0x%04x)\n", cm->cm_ccb->ccb_h.timeout, cm,  targ->tid,
+	    "%u, handle(0x%04x)\n", cm->cm_ccb->ccb_h.timeout, cm, targ->tid,
 	    targ->handle);
 	if (targ->encl_level_valid) {
 		mpr_dprint(sc, MPR_ERROR, "At enclosure level %d, slot %d, "
@@ -1680,6 +1691,160 @@ mprsas_scsiio_timeout(void *data)
 	}
 }
 
+/** 
+ * mprsas_build_nvme_unmap - Build Native NVMe DSM command equivalent
+ *			     to SCSI Unmap.
+ * Return 0 - for success,
+ *	  1 - to immediately return back the command with success status to CAM
+ *	  negative value - to fallback to firmware path i.e. issue scsi unmap
+ *			   to FW without any translation.
+ */
+static int
+mprsas_build_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm,
+    union ccb *ccb, struct mprsas_target *targ)
+{
+	Mpi26NVMeEncapsulatedRequest_t *req = NULL;
+	struct ccb_scsiio *csio;
+	struct unmap_parm_list *plist;
+	struct nvme_dsm_range *nvme_dsm_ranges = NULL;
+	struct nvme_command *c;
+	int i, res;
+	uint16_t ndesc, list_len, data_length;
+	struct mpr_prp_page *prp_page_info;
+	uint64_t nvme_dsm_ranges_dma_handle;
+
+	csio = &ccb->csio;
+#if __FreeBSD_version >= 1100103
+	list_len = (scsiio_cdb_ptr(csio)[7] << 8 | scsiio_cdb_ptr(csio)[8]);
+#else
+	if (csio->ccb_h.flags & CAM_CDB_POINTER) {
+		list_len = (ccb->csio.cdb_io.cdb_ptr[7] << 8 |
+		    ccb->csio.cdb_io.cdb_ptr[8]);
+	} else {
+		list_len = (ccb->csio.cdb_io.cdb_bytes[7] << 8 |
+		    ccb->csio.cdb_io.cdb_bytes[8]);
+	}
+#endif
+	if (!list_len) {
+		mpr_dprint(sc, MPR_ERROR, "Parameter list length is Zero\n");
+		return -EINVAL;
+	}
+
+	plist = malloc(csio->dxfer_len, M_MPR, M_ZERO|M_NOWAIT);
+	if (!plist) {
+		mpr_dprint(sc, MPR_ERROR, "Unable to allocate memory to "
+		    "save UNMAP data\n");
+		return -ENOMEM;
+	}
+
+	/* Copy SCSI unmap data to a local buffer */
+	bcopy(csio->data_ptr, plist, csio->dxfer_len);
+
+	/* return back the unmap command to CAM with success status,
+	 * if number of descripts is zero.
+	 */
+	ndesc = be16toh(plist->unmap_blk_desc_data_len) >> 4;
+	if (!ndesc) {
+		mpr_dprint(sc, MPR_XINFO, "Number of descriptors in "
+		    "UNMAP cmd is Zero\n");
+		res = 1;
+		goto out;
+	}
+
+	data_length = ndesc * sizeof(struct nvme_dsm_range);
+	if (data_length > targ->MDTS) {
+		mpr_dprint(sc, MPR_ERROR, "data length: %d is greater than "
+		    "Device's MDTS: %d\n", data_length, targ->MDTS);
+		res = -EINVAL;
+		goto out;
+	}
+
+	prp_page_info = mpr_alloc_prp_page(sc);
+	KASSERT(prp_page_info != NULL, ("%s: There is no PRP Page for "
+	    "UNMAP command.\n", __func__));
+
+	/*
+	 * Insert the allocated PRP page into the command's PRP page list. This
+	 * will be freed when the command is freed.
+	 */
+	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);
+
+	nvme_dsm_ranges = (struct nvme_dsm_range *)prp_page_info->prp_page;
+	nvme_dsm_ranges_dma_handle = prp_page_info->prp_page_busaddr;
+
+	bzero(nvme_dsm_ranges, data_length);
+
+	/* Convert SCSI unmap's descriptor data to NVMe DSM specific Range data
+	 * for each descriptors contained in SCSI UNMAP data.
+	 */
+	for (i = 0; i < ndesc; i++) {
+		nvme_dsm_ranges[i].length =
+		    htole32(be32toh(plist->desc[i].nlb));
+		nvme_dsm_ranges[i].starting_lba =
+		    htole64(be64toh(plist->desc[i].slba));
+		nvme_dsm_ranges[i].attributes = 0;
+	}
+
+	/* Build MPI2.6's NVMe Encapsulated Request Message */
+	req = (Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req;
+	bzero(req, sizeof(*req));
+	req->DevHandle = htole16(targ->handle);
+	req->Function = MPI2_FUNCTION_NVME_ENCAPSULATED;
+	req->Flags = MPI26_NVME_FLAGS_WRITE;
+	req->ErrorResponseBaseAddress.High =
+	    htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32));
+	req->ErrorResponseBaseAddress.Low =
+	    htole32(cm->cm_sense_busaddr);
+	req->ErrorResponseAllocationLength =
+	    htole16(sizeof(struct nvme_completion));
+	req->EncapsulatedCommandLength =
+	    htole16(sizeof(struct nvme_command));
+	req->DataLength = htole32(data_length);
+
+	/* Build NVMe DSM command */
+	c = (struct nvme_command *) req->NVMe_Command;
+	c->opc = NVME_OPC_DATASET_MANAGEMENT;
+	c->nsid = htole32(csio->ccb_h.target_lun + 1);
+	c->cdw10 = htole32(ndesc - 1);
+	c->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE);
+
+	cm->cm_length = data_length;
+	cm->cm_data = NULL;
+
+	cm->cm_complete = mprsas_scsiio_complete;
+	cm->cm_complete_data = ccb;
+	cm->cm_targ = targ;
+	cm->cm_lun = csio->ccb_h.target_lun;
+	cm->cm_ccb = ccb;
+
+	cm->cm_desc.Default.RequestFlags =
+	    MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
+
+#if __FreeBSD_version >= 1000029
+	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
+	    mprsas_scsiio_timeout, cm, 0);
+#else //__FreeBSD_version < 1000029
+	callout_reset(&cm->cm_callout, (ccb->ccb_h.timeout * hz) / 1000,
+	    mprsas_scsiio_timeout, cm);
+#endif //__FreeBSD_version >= 1000029
+
+	targ->issued++;
+	targ->outstanding++;
+	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
+	ccb->ccb_h.status |= CAM_SIM_QUEUED;
+
+	mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n",
+	    __func__, cm, ccb, targ->outstanding);
+
+	mpr_build_nvme_prp(sc, cm, req, (void *)nvme_dsm_ranges_dma_handle, 0,
+	    data_length);
+	mpr_map_command(sc, cm);
+
+out:
+	free(plist, M_MPR);
+	return 0;
+}
+
 static void
 mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 {
@@ -1689,9 +1854,10 @@ mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 	struct mprsas_target *targ;
 	struct mprsas_lun *lun;
 	struct mpr_command *cm;
-	uint8_t i, lba_byte, *ref_tag_addr;
+	uint8_t i, lba_byte, *ref_tag_addr, scsi_opcode;
 	uint16_t eedp_flags;
 	uint32_t mpi_control;
+	int rc;
 
 	sc = sassc->sc;
 	MPR_FUNCTRACE(sc);
@@ -1777,6 +1943,30 @@ mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 		return;
 	}
 
+	/* For NVME device's issue UNMAP command directly to NVME drives by
+	 * constructing equivalent native NVMe DataSetManagement command.
+	 */
+#if __FreeBSD_version >= 1100103
+	scsi_opcode = scsiio_cdb_ptr(csio)[0];
+#else
+	if (csio->ccb_h.flags & CAM_CDB_POINTER)
+		scsi_opcode = csio->cdb_io.cdb_ptr[0];
+	else
+		scsi_opcode = csio->cdb_io.cdb_bytes[0];
+#endif
+	if (scsi_opcode == UNMAP &&
+	    targ->is_nvme &&
+	    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
+		rc = mprsas_build_nvme_unmap(sc, cm, ccb, targ);
+		if (rc == 1) { /* return command to CAM with success status */
+			mpr_free_command(sc, cm);
+			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
+			xpt_done(ccb);
+			return;
+		} else if (!rc) /* Issued NVMe Encapsulated Request Message */
+			return;
+	}
+
 	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
 	bzero(req, sizeof(*req));
 	req->DevHandle = htole16(targ->handle);
@@ -1849,8 +2039,8 @@ mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
 	else {
 		KASSERT(csio->cdb_len <= IOCDBLEN,
-		    ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER is not set",
-		     csio->cdb_len));
+		    ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER "
+		    "is not set", csio->cdb_len));
 		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
 	}
 	req->IoFlags = htole16(csio->cdb_len);
@@ -1874,6 +2064,10 @@ mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
+			if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
+				eedp_flags |=
+				    MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
+			}
 			req->EEDPFlags = htole16(eedp_flags);
 
 			/*
@@ -1933,11 +2127,15 @@ mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
 		req->IoFlags |= MPI25_SCSIIO_IOFLAGS_FAST_PATH;
 		cm->cm_desc.FastPathSCSIIO.RequestFlags =
 		    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
-		cm->cm_desc.FastPathSCSIIO.DevHandle = htole16(targ->handle);
+		if (!sc->atomic_desc_capable) {
+			cm->cm_desc.FastPathSCSIIO.DevHandle =
+			    htole16(targ->handle);
+		}
 	} else {
 		cm->cm_desc.SCSIIO.RequestFlags =
 		    MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
-		cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
+		if (!sc->atomic_desc_capable)
+			cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
 	}
 
 #if __FreeBSD_version >= 1000029
@@ -2160,6 +2358,200 @@ mpr_sc_failed_io_info(struct mpr_softc *sc, struct ccb_scsiio *csio,
 	}
 }
 
+/** mprsas_nvme_trans_status_code
+ *
+ * Convert Native NVMe command error status to
+ * equivalent SCSI error status.
+ *
+ * Returns appropriate scsi_status
+ */
+static u8
+mprsas_nvme_trans_status_code(struct nvme_status nvme_status,
+    struct mpr_command *cm)
+{
+	u8 status = MPI2_SCSI_STATUS_GOOD;
+	int skey, asc, ascq;
+	union ccb *ccb = cm->cm_complete_data;
+	int returned_sense_len;
+
+	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+	skey = SSD_KEY_ILLEGAL_REQUEST;
+	asc = SCSI_ASC_NO_SENSE;
+	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+
+	switch (nvme_status.sct) {
+	case NVME_SCT_GENERIC:
+		switch (nvme_status.sc) {
+		case NVME_SC_SUCCESS:
+			status = MPI2_SCSI_STATUS_GOOD;
+			skey = SSD_KEY_NO_SENSE;
+			asc = SCSI_ASC_NO_SENSE;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_INVALID_OPCODE:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_ILLEGAL_COMMAND;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_INVALID_FIELD:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_INVALID_CDB;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_DATA_TRANSFER_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_NO_SENSE;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_ABORTED_POWER_LOSS:
+			status = MPI2_SCSI_STATUS_TASK_ABORTED;
+			skey = SSD_KEY_ABORTED_COMMAND;
+			asc = SCSI_ASC_WARNING;
+			ascq = SCSI_ASCQ_POWER_LOSS_EXPECTED;
+			break;
+		case NVME_SC_INTERNAL_DEVICE_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_HARDWARE_ERROR;
+			asc = SCSI_ASC_INTERNAL_TARGET_FAILURE;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_ABORTED_BY_REQUEST:
+		case NVME_SC_ABORTED_SQ_DELETION:
+		case NVME_SC_ABORTED_FAILED_FUSED:
+		case NVME_SC_ABORTED_MISSING_FUSED:
+			status = MPI2_SCSI_STATUS_TASK_ABORTED;
+			skey = SSD_KEY_ABORTED_COMMAND;
+			asc = SCSI_ASC_NO_SENSE;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
+			ascq = SCSI_ASCQ_INVALID_LUN_ID;
+			break;
+		case NVME_SC_LBA_OUT_OF_RANGE:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_ILLEGAL_BLOCK;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_CAPACITY_EXCEEDED:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_NO_SENSE;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_NAMESPACE_NOT_READY:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_NOT_READY; 
+			asc = SCSI_ASC_LUN_NOT_READY;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		}
+		break;
+	case NVME_SCT_COMMAND_SPECIFIC:
+		switch (nvme_status.sc) {
+		case NVME_SC_INVALID_FORMAT:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_FORMAT_COMMAND_FAILED;
+			ascq = SCSI_ASCQ_FORMAT_COMMAND_FAILED;
+			break;
+		case NVME_SC_CONFLICTING_ATTRIBUTES:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_INVALID_CDB;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		}
+		break;
+	case NVME_SCT_MEDIA_ERROR:
+		switch (nvme_status.sc) {
+		case NVME_SC_WRITE_FAULTS:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_UNRECOVERED_READ_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_UNRECOVERED_READ_ERROR;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_GUARD_CHECK_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED;
+			ascq = SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED;
+			break;
+		case NVME_SC_APPLICATION_TAG_CHECK_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED;
+			ascq = SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED;
+			break;
+		case NVME_SC_REFERENCE_TAG_CHECK_ERROR:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MEDIUM_ERROR;
+			asc = SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED;
+			ascq = SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED;
+			break;
+		case NVME_SC_COMPARE_FAILURE:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_MISCOMPARE;
+			asc = SCSI_ASC_MISCOMPARE_DURING_VERIFY;
+			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
+			break;
+		case NVME_SC_ACCESS_DENIED:
+			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
+			skey = SSD_KEY_ILLEGAL_REQUEST;
+			asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
+			ascq = SCSI_ASCQ_INVALID_LUN_ID;
+			break;
+		}
+		break;
+	}
+	
+	returned_sense_len = sizeof(struct scsi_sense_data);
+	if (returned_sense_len < ccb->csio.sense_len)
+		ccb->csio.sense_resid = ccb->csio.sense_len -
+		    returned_sense_len;
+	else
+		ccb->csio.sense_resid = 0;
+
+	scsi_set_sense_data(&ccb->csio.sense_data, SSD_TYPE_FIXED,
+	    1, skey, asc, ascq, SSD_ELEM_NONE);
+	ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
+
+	return status;
+}
+
+/** mprsas_complete_nvme_unmap 
+ *
+ * Complete native NVMe command issued using NVMe Encapsulated
+ * Request Message.
+ */
+static u8
+mprsas_complete_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm)
+{
+	Mpi26NVMeEncapsulatedErrorReply_t *mpi_reply;
+	struct nvme_completion *nvme_completion = NULL;
+	u8 scsi_status = MPI2_SCSI_STATUS_GOOD;
+
+	mpi_reply =(Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply;
+	if (le16toh(mpi_reply->ErrorResponseCount)){
+		nvme_completion = (struct nvme_completion *)cm->cm_sense;
+		scsi_status = mprsas_nvme_trans_status_code(
+		    nvme_completion->status, cm);
+	}
+	return scsi_status;
+}
+
 static void
 mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 {
@@ -2168,7 +2560,7 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 	struct ccb_scsiio *csio;
 	struct mprsas_softc *sassc;
 	struct scsi_vpd_supported_page_list *vpd_list = NULL;
-	u8 *TLR_bits, TLR_on;
+	u8 *TLR_bits, TLR_on, *scsi_cdb;
 	int dir = 0, i;
 	u16 alloc_len;
 	struct mprsas_target *target;
@@ -2266,6 +2658,20 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		}
 	}
 
+	/*
+	 * Point to the SCSI CDB, which is dependent on the CAM_CDB_POINTER
+	 * flag, and use it in a few places in the rest of this function for
+	 * convenience. Use the macro if available.
+	 */
+#if __FreeBSD_version >= 1100103
+	scsi_cdb = scsiio_cdb_ptr(csio);
+#else
+	if (csio->ccb_h.flags & CAM_CDB_POINTER)
+		scsi_cdb = csio->cdb_io.cdb_ptr;
+	else
+		scsi_cdb = csio->cdb_io.cdb_bytes;
+#endif
+
 	/*
 	 * If this is a Start Stop Unit command and it was issued by the driver
 	 * during shutdown, decrement the refcount to account for all of the
@@ -2273,7 +2679,7 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
 	 * is TRUE.
 	 */
-	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
+	if (sc->SSU_started && (scsi_cdb[0] == START_STOP_UNIT)) {
 		mpr_dprint(sc, MPR_INFO, "Decrementing SSU count.\n");
 		sc->SSU_refcount--;
 	}
@@ -2314,6 +2720,14 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		return;
 	}
 
+	target = &sassc->targets[target_id];
+	if (scsi_cdb[0] == UNMAP &&
+	    target->is_nvme &&
+	    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
+		rep->SCSIStatus = mprsas_complete_nvme_unmap(sc, cm);
+		csio->scsi_status = rep->SCSIStatus;
+	}
+
 	mprsas_log_command(cm, MPR_XINFO,
 	    "ioc %x scsi %x state %x xfer %u\n",
 	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
@@ -2325,7 +2739,6 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		/* FALLTHROUGH */
 	case MPI2_IOCSTATUS_SUCCESS:
 	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
-
 		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
 		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
 			mprsas_log_command(cm, MPR_XINFO, "recovered error\n");
@@ -2403,9 +2816,9 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		 * controller, turn the TLR_bits value ON if page 0x90 is
 		 * supported.
 		 */
-		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
-		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
-		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
+		if ((scsi_cdb[0] == INQUIRY) &&
+		    (scsi_cdb[1] & SI_EVPD) &&
+		    (scsi_cdb[2] == SVPD_SUPPORTED_PAGE_LIST) &&
 		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
 		    (csio->data_ptr != NULL) &&
 		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
@@ -2417,8 +2830,7 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
 			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
 			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
-			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
-			    csio->cdb_io.cdb_bytes[4];
+			alloc_len = ((u16)scsi_cdb[3] << 8) + scsi_cdb[4];
 			alloc_len -= csio->resid;
 			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
 				if (vpd_list->list[i] == 0x90) {
@@ -2433,7 +2845,7 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		 * a SCSI StartStopUnit command will be sent to it when the
 		 * driver is being shutdown.
 		 */
-		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
+		if ((scsi_cdb[0] == INQUIRY) &&
 		    (csio->data_ptr != NULL) &&
 		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
 		    (sc->mapping_table[target_id].device_info &
@@ -2524,7 +2936,14 @@ mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
 		    rep->SCSIStatus, rep->SCSIState,
 		    le32toh(rep->TransferCount));
 		csio->resid = cm->cm_length;
-		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
+
+		if (scsi_cdb[0] == UNMAP &&
+		    target->is_nvme &&
+		    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR)
+			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
+		else
+			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
+
 		break;
 	}
 	
diff --git a/sys/dev/mpr/mpr_sas.h b/sys/dev/mpr/mpr_sas.h
index 614b5fe347f4..c3e9bf6ddee3 100644
--- a/sys/dev/mpr/mpr_sas.h
+++ b/sys/dev/mpr/mpr_sas.h
@@ -83,6 +83,8 @@ struct mprsas_target {
 	uint8_t		scsi_req_desc_type;
 	uint8_t		stop_at_shutdown;
 	uint8_t		supports_SSU;
+	uint8_t		is_nvme;
+	uint32_t	MDTS;
 };
 
 struct mprsas_softc {
diff --git a/sys/dev/mpr/mpr_sas_lsi.c b/sys/dev/mpr/mpr_sas_lsi.c
index 9555a719c8ce..2f3b3602b4a9 100644
--- a/sys/dev/mpr/mpr_sas_lsi.c
+++ b/sys/dev/mpr/mpr_sas_lsi.c
@@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2.h>
 #include <dev/mpr/mpi/mpi2_ioc.h>
 #include <dev/mpr/mpi/mpi2_sas.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_raid.h>
@@ -116,6 +117,8 @@ static void mprsas_fw_work(struct mpr_softc *sc,
 static void mprsas_fw_event_free(struct mpr_softc *,
     struct mpr_fw_event_work *);
 static int mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate);
+static int mprsas_add_pcie_device(struct mpr_softc *sc, u16 handle,
+    u8 linkrate);
 static int mprsas_get_sata_identify(struct mpr_softc *sc, u16 handle,
     Mpi2SataPassthroughReply_t *mpi_reply, char *id_buffer, int sz,
     u32 devinfo);
@@ -156,6 +159,7 @@ mprsas_evt_handler(struct mpr_softc *sc, uintptr_t data,
 	bcopy(event->EventData, fw_event->event_data, sz);
 	fw_event->event = event->Event;
 	if ((event->Event == MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST ||
+	    event->Event == MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST ||
 	    event->Event == MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE ||
 	    event->Event == MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST) &&
 	    sc->track_mapping_events)
@@ -167,13 +171,13 @@ mprsas_evt_handler(struct mpr_softc *sc, uintptr_t data,
 	 * events are processed.
 	 */
 	if ((event->Event == MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST ||
+	    event->Event == MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST ||
 	    event->Event == MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST) &&
 	    sc->wait_for_port_enable)
 		mprsas_startup_increment(sc->sassc);
 
 	TAILQ_INSERT_TAIL(&sc->sassc->ev_queue, fw_event, ev_link);
 	taskqueue_enqueue(sc->sassc->ev_tq, &sc->sassc->ev_task);
-
 }
 
 static void
@@ -205,7 +209,7 @@ mprsas_fw_work(struct mpr_softc *sc, struct mpr_fw_event_work *fw_event)
 	{
 		MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST *data;
 		MPI2_EVENT_SAS_TOPO_PHY_ENTRY *phy;
-		int i;
+		uint8_t i;
 
 		data = (MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST *)
 		    fw_event->event_data;
@@ -674,6 +678,60 @@ mprsas_fw_work(struct mpr_softc *sc, struct mpr_fw_event_work *fw_event)
 		}
 		break;
 	}
+	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: 
+	{
+		MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST *data;
+		MPI26_EVENT_PCIE_TOPO_PORT_ENTRY *port_entry;
+		uint8_t i, link_rate;
+		uint16_t handle;
+
+		data = (MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST *)
+		    fw_event->event_data;
+
+		mpr_mapping_pcie_topology_change_event(sc,
+		    fw_event->event_data);
+
+		for (i = 0; i < data->NumEntries; i++) {
+			port_entry = &data->PortEntry[i];
+			handle = le16toh(port_entry->AttachedDevHandle);
+			link_rate = port_entry->CurrentPortInfo &
+			    MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
+			switch (port_entry->PortStatus) {
+			case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
+				if (link_rate <
+				    MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5) {
+					mpr_dprint(sc, MPR_ERROR, "%s: Cannot "
+					    "add PCIe device with handle 0x%x "
+					    "with unknown link rate.\n",
+					    __func__, handle);
+					break;
+				}
+				if (mprsas_add_pcie_device(sc, handle,
+				    link_rate)) {
+					mpr_dprint(sc, MPR_ERROR, "%s: failed "
+					    "to add PCIe device with handle "
+					    "0x%x\n", __func__, handle);
+					mprsas_prepare_remove(sassc, handle);
+				}
+				break;
+			case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
+				mprsas_prepare_remove(sassc, handle);
+				break;
+			case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
+			case MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE:
+			case MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
+			default:
+				break;
+			}
+		}
+		/*
+		 * refcount was incremented for this event in
+		 * mprsas_evt_handler.  Decrement it here because the event has
+		 * been processed.
+		 */
+		mprsas_startup_decrement(sassc);
+		break;
+	}
 	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
 	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
 	default:
@@ -703,7 +761,8 @@ mprsas_firmware_event_work(void *arg, int pending)
 }
 
 static int
-mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate){
+mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate)
+{
 	char devstring[80];
 	struct mprsas_softc *sassc;
 	struct mprsas_target *targ;
@@ -779,8 +838,8 @@ mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate){
 	if (sc->use_phynum != -1) 
 		id = mpr_mapping_get_sas_id(sc, sas_address, handle);
 	if (id == MPR_MAP_BAD_ID) {
-		if ((sc->use_phynum == 0)
-		 || ((id = config_page.PhyNum) > sassc->maxtargets)) {
+		if ((sc->use_phynum == 0) ||
+		    ((id = config_page.PhyNum) > sassc->maxtargets)) {
 			mpr_dprint(sc, MPR_INFO, "failure at %s:%d/%s()! "
 			    "Could not get ID for device with handle 0x%04x\n",
 			    __FILE__, __LINE__, __func__, handle);
@@ -827,8 +886,10 @@ mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate){
 	if (is_SATA_SSD) {
 		targ->flags = MPR_TARGET_IS_SATA_SSD;
 	}
-	if (le16toh(config_page.Flags) &
-	    MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) {
+	if ((le16toh(config_page.Flags) &
+	    MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
+	    (le16toh(config_page.Flags) &
+	    MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE)) {
 		targ->scsi_req_desc_type =
 		    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
 	}
@@ -908,7 +969,7 @@ mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate){
 	mprsas_startup_decrement(sassc);
 	return (error);
 }
-	
+
 int
 mprsas_get_sas_address_for_sata_disk(struct mpr_softc *sc,
     u64 *sas_address, u16 handle, u32 device_info, u8 *is_SATA_SSD)
@@ -1139,6 +1200,141 @@ mprsas_ata_id_timeout(void *data)
 	wakeup(cm);
 }
 
+static int
+mprsas_add_pcie_device(struct mpr_softc *sc, u16 handle, u8 linkrate)
+{
+	char devstring[80];
+	struct mprsas_softc *sassc;
+	struct mprsas_target *targ;
+	Mpi2ConfigReply_t mpi_reply;
+	Mpi26PCIeDevicePage0_t config_page;
+	Mpi26PCIeDevicePage2_t config_page2;
+	uint64_t pcie_wwid, parent_wwid = 0;
+	u32 device_info, parent_devinfo = 0;
+	unsigned int id;
+	int error = 0;
+	struct mprsas_lun *lun;
+
+	sassc = sc->sassc;
+	mprsas_startup_increment(sassc);
+	if ((mpr_config_get_pcie_device_pg0(sc, &mpi_reply, &config_page,
+	     MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle))) {
+		printf("%s: error reading PCIe device page0\n", __func__);
+		error = ENXIO;
+		goto out;
+	}
+
+	device_info = le32toh(config_page.DeviceInfo);
+
+	if (((device_info & MPI26_PCIE_DEVINFO_PCI_SWITCH) == 0)
+	    && (le16toh(config_page.ParentDevHandle) != 0)) {
+		Mpi2ConfigReply_t tmp_mpi_reply;
+		Mpi26PCIeDevicePage0_t parent_config_page;
+
+		if ((mpr_config_get_pcie_device_pg0(sc, &tmp_mpi_reply,
+		     &parent_config_page, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE,
+		     le16toh(config_page.ParentDevHandle)))) {
+			printf("%s: error reading PCIe device %#x page0\n",
+			    __func__, le16toh(config_page.ParentDevHandle));
+		} else {
+			parent_wwid = parent_config_page.WWID.High;
+			parent_wwid = (parent_wwid << 32) |
+			    parent_config_page.WWID.Low;
+			parent_devinfo = le32toh(parent_config_page.DeviceInfo);
+		}
+	}
+	/* TODO Check proper endianness */
+	pcie_wwid = config_page.WWID.High;
+	pcie_wwid = (pcie_wwid << 32) | config_page.WWID.Low;
+	mpr_dprint(sc, MPR_INFO, "PCIe WWID from PCIe device page0 = %jx\n",
+	    pcie_wwid);
+
+	if ((mpr_config_get_pcie_device_pg2(sc, &mpi_reply, &config_page2,
+	     MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle))) {
+		printf("%s: error reading PCIe device page2\n", __func__);
+		error = ENXIO;
+		goto out;
+	}
+
+	id = mpr_mapping_get_sas_id(sc, pcie_wwid, handle);
+	if (id == MPR_MAP_BAD_ID) {
+		printf("failure at %s:%d/%s()! Could not get ID for device "
+		    "with handle 0x%04x\n", __FILE__, __LINE__, __func__,
+		    handle);
+		error = ENXIO;
+		goto out;
+	}
+
+	if (mprsas_check_id(sassc, id) != 0) {
+		device_printf(sc->mpr_dev, "Excluding target id %d\n", id);
+		error = ENXIO;
+		goto out;
+	}
+
+	mpr_dprint(sc, MPR_MAPPING, "WWID from PCIe device page0 = %jx\n",
+	    pcie_wwid);
+	targ = &sassc->targets[id];
+	targ->devinfo = device_info;
+	targ->encl_handle = le16toh(config_page.EnclosureHandle);
+	targ->encl_slot = le16toh(config_page.Slot);
+	targ->encl_level = config_page.EnclosureLevel;
+	targ->connector_name[0] = ((char *)&config_page.ConnectorName)[0];
+	targ->connector_name[1] = ((char *)&config_page.ConnectorName)[1];
+	targ->connector_name[2] = ((char *)&config_page.ConnectorName)[2];
+	targ->connector_name[3] = ((char *)&config_page.ConnectorName)[3];
+	targ->is_nvme = device_info & MPI26_PCIE_DEVINFO_NVME;
+	targ->MDTS = config_page2.MaximumDataTransferSize;
+	/*
+	 * Assume always TRUE for encl_level_valid because there is no valid
+	 * flag for PCIe.
+	 */
+	targ->encl_level_valid = TRUE;
+	targ->handle = handle;
+	targ->parent_handle = le16toh(config_page.ParentDevHandle);
+	targ->sasaddr = mpr_to_u64(&config_page.WWID);
+	targ->parent_sasaddr = le64toh(parent_wwid);
+	targ->parent_devinfo = parent_devinfo;
+	targ->tid = id;
+	targ->linkrate = linkrate;
+	targ->flags = 0;
+	if ((le16toh(config_page.Flags) &
+	    MPI26_PCIEDEV0_FLAGS_ENABLED_FAST_PATH) && 
+	    (le16toh(config_page.Flags) &
+	    MPI26_PCIEDEV0_FLAGS_FAST_PATH_CAPABLE)) {
+		targ->scsi_req_desc_type =
+		    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
+	}
+	TAILQ_INIT(&targ->commands);
+	TAILQ_INIT(&targ->timedout_commands);
+	while (!SLIST_EMPTY(&targ->luns)) {
+		lun = SLIST_FIRST(&targ->luns);
+		SLIST_REMOVE_HEAD(&targ->luns, lun_link);
+		free(lun, M_MPR);
+	}
+	SLIST_INIT(&targ->luns);
+
+	mpr_describe_devinfo(targ->devinfo, devstring, 80);
+	mpr_dprint(sc, (MPR_INFO|MPR_MAPPING), "Found PCIe device <%s> <%s> "
+	    "handle<0x%04x> enclosureHandle<0x%04x> slot %d\n", devstring,
+	    mpr_describe_table(mpr_pcie_linkrate_names, targ->linkrate),
+	    targ->handle, targ->encl_handle, targ->encl_slot);
+	if (targ->encl_level_valid) {
+		mpr_dprint(sc, (MPR_INFO|MPR_MAPPING), "At enclosure level %d "
+		    "and connector name (%4s)\n", targ->encl_level,
+		    targ->connector_name);
+	}
+#if ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000039)) || \
+    (__FreeBSD_version < 902502)
+	if ((sassc->flags & MPRSAS_IN_STARTUP) == 0)
+#endif
+		mprsas_rescan_target(sc, targ);
+	mpr_dprint(sc, MPR_MAPPING, "Target id 0x%x added\n", targ->tid);
+
+out:
+	mprsas_startup_decrement(sassc);
+	return (error);
+}
+
 static int
 mprsas_volume_add(struct mpr_softc *sc, u16 handle)
 {
@@ -1235,8 +1431,8 @@ mprsas_SSU_to_SATA_devices(struct mpr_softc *sc)
 		if (target->stop_at_shutdown) {
 			ccb = xpt_alloc_ccb_nowait();
 			if (ccb == NULL) {
-				mpr_dprint(sc, MPR_FAULT, "Unable to alloc CCB to stop "
-				    "unit.\n");
+				mpr_dprint(sc, MPR_FAULT, "Unable to alloc CCB "
+				    "to stop unit.\n");
 				return;
 			}
 
diff --git a/sys/dev/mpr/mpr_table.c b/sys/dev/mpr/mpr_table.c
index 2dab6103483c..44420fdbf260 100644
--- a/sys/dev/mpr/mpr_table.c
+++ b/sys/dev/mpr/mpr_table.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpr_ioctl.h>
 #include <dev/mpr/mprvar.h>
 #include <dev/mpr/mpr_table.h>
@@ -74,6 +75,7 @@ mpr_describe_table(struct mpr_table_lookup *table, u_int code)
 	return(table[i+1].string);
 }
 
+//SLM-Add new PCIe info to all of these tables
 struct mpr_table_lookup mpr_event_names[] = {
 	{"LogData",			0x01},
 	{"StateChange",			0x02},
@@ -100,6 +102,10 @@ struct mpr_table_lookup mpr_event_names[] = {
 	{"TempThreshold",		0x27},
 	{"HostMessage",			0x28},
 	{"PowerPerformanceChange",	0x29},
+	{"PCIeDeviceStatusChange",	0x30},
+	{"PCIeEnumeration",		0x31},
+	{"PCIeTopologyChangeList",	0x32},
+	{"PCIeLinkCounter",		0x33},
 	{"CableEvent",			0x34},
 	{NULL, 0},
 	{"Unknown Event", 0}
@@ -192,6 +198,16 @@ struct mpr_table_lookup mpr_sasdev_reason[] = {
 	{"Unknown",			0x00}
 };
 
+struct mpr_table_lookup mpr_pcie_linkrate_names[] = {
+	{"Port disabled",		0x01},
+	{"2.5GT/sec",			0x02},
+	{"5.0GT/sec",			0x03},
+	{"8.0GT/sec",			0x04},
+	{"16.0GT/sec",			0x05},
+	{NULL, 0},
+	{"LinkRate Unknown",		0x00}
+};
+
 void
 mpr_describe_devinfo(uint32_t devinfo, char *string, int len)
 {
@@ -321,6 +337,7 @@ _mpr_print_evt_sas(struct mpr_softc *sc, MPI2_EVENT_NOTIFICATION_REPLY *event)
 		    "\40MaxEnclosures");
 		break;
 	}
+//SLM-add for PCIE EVENT too
 	case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 	{
 		MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST *data;
diff --git a/sys/dev/mpr/mpr_table.h b/sys/dev/mpr/mpr_table.h
index 45a8eed1837b..162689a2aaf8 100644
--- a/sys/dev/mpr/mpr_table.h
+++ b/sys/dev/mpr/mpr_table.h
@@ -40,6 +40,7 @@ void mpr_describe_devinfo(uint32_t devinfo, char *string, int len);
 extern struct mpr_table_lookup mpr_event_names[];
 extern struct mpr_table_lookup mpr_phystatus_names[];
 extern struct mpr_table_lookup mpr_linkrate_names[];
+extern struct mpr_table_lookup mpr_pcie_linkrate_names[];
 
 void _mpr_print_iocfacts(struct mpr_softc *, MPI2_IOC_FACTS_REPLY *);
 void _mpr_print_portfacts(struct mpr_softc *, MPI2_PORT_FACTS_REPLY *);
diff --git a/sys/dev/mpr/mpr_user.c b/sys/dev/mpr/mpr_user.c
index 0a847de189ed..b1c11093341d 100644
--- a/sys/dev/mpr/mpr_user.c
+++ b/sys/dev/mpr/mpr_user.c
@@ -99,6 +99,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/mpr/mpi/mpi2_cnfg.h>
 #include <dev/mpr/mpi/mpi2_init.h>
 #include <dev/mpr/mpi/mpi2_tool.h>
+#include <dev/mpr/mpi/mpi2_pci.h>
 #include <dev/mpr/mpr_ioctl.h>
 #include <dev/mpr/mprvar.h>
 #include <dev/mpr/mpr_table.h>
@@ -747,6 +748,8 @@ mpr_user_pass_thru(struct mpr_softc *sc, mpr_pass_thru_t *data)
 {
 	MPI2_REQUEST_HEADER	*hdr, tmphdr;	
 	MPI2_DEFAULT_REPLY	*rpl;
+	Mpi26NVMeEncapsulatedErrorReply_t *nvme_error_reply = NULL;
+	Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request = NULL;
 	struct mpr_command	*cm = NULL;
 	int			i, err = 0, dir = 0, sz;
 	uint8_t			tool, function = 0;
@@ -923,8 +926,8 @@ mpr_user_pass_thru(struct mpr_softc *sc, mpr_pass_thru_t *data)
 			    cm->cm_data, data->DataSize);
 		}
 		if (err != 0)
-			mpr_dprint(sc, MPR_FAULT, "%s: failed to copy "
-			    "IOCTL data from user space\n", __func__);
+			mpr_dprint(sc, MPR_FAULT, "%s: failed to copy IOCTL "
+			    "data from user space\n", __func__);
 	}
 	/*
 	 * Set this flag only if processing a command that does not need an
@@ -946,6 +949,35 @@ mpr_user_pass_thru(struct mpr_softc *sc, mpr_pass_thru_t *data)
 	}
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 
+	if (function == MPI2_FUNCTION_NVME_ENCAPSULATED) {
+		nvme_encap_request =
+		    (Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req;
+		cm->cm_desc.Default.RequestFlags =
+		    MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
+
+		/*
+		 * Get the Physical Address of the sense buffer.
+		 * Save the user's Error Response buffer address and use that
+		 *   field to hold the sense buffer address.
+		 * Clear the internal sense buffer, which will potentially hold
+		 *   the Completion Queue Entry on return, or 0 if no Entry.
+		 * Build the PRPs and set direction bits.
+		 * Send the request.
+		 */
+		cm->nvme_error_response =
+		    (uint64_t *)(uintptr_t)(((uint64_t)nvme_encap_request->
+		    ErrorResponseBaseAddress.High << 32) |
+		    (uint64_t)nvme_encap_request->
+		    ErrorResponseBaseAddress.Low);
+		nvme_encap_request->ErrorResponseBaseAddress.High =
+		    htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32));
+		nvme_encap_request->ErrorResponseBaseAddress.Low =
+		    htole32(cm->cm_sense_busaddr);
+		memset(cm->cm_sense, 0, NVME_ERROR_RESPONSE_SIZE);
+		mpr_build_nvme_prp(sc, cm, nvme_encap_request, cm->cm_data,
+		    data->DataSize, data->DataOutSize);
+	}
+
 	/*
 	 * Set up Sense buffer and SGL offset for IO passthru.  SCSI IO request
 	 * uses SCSI IO or Fast Path SCSI IO descriptor.
@@ -994,15 +1026,19 @@ mpr_user_pass_thru(struct mpr_softc *sc, mpr_pass_thru_t *data)
 			    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO) {
 				cm->cm_desc.FastPathSCSIIO.RequestFlags =
 				    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
-				cm->cm_desc.FastPathSCSIIO.DevHandle =
-				    scsi_io_req->DevHandle;
+				if (!sc->atomic_desc_capable) {
+					cm->cm_desc.FastPathSCSIIO.DevHandle =
+					    scsi_io_req->DevHandle;
+				}
 				scsi_io_req->IoFlags |=
 				    MPI25_SCSIIO_IOFLAGS_FAST_PATH;
 			} else {
 				cm->cm_desc.SCSIIO.RequestFlags =
 				    MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
-				cm->cm_desc.SCSIIO.DevHandle =
-				    scsi_io_req->DevHandle;
+				if (!sc->atomic_desc_capable) {
+					cm->cm_desc.SCSIIO.DevHandle =
+					    scsi_io_req->DevHandle;
+				}
 			}
 
 			/*
@@ -1079,6 +1115,38 @@ mpr_user_pass_thru(struct mpr_softc *sc, mpr_pass_thru_t *data)
 				mpr_lock(sc);
 			}
 		}
+
+		/*
+		 * Copy out the NVMe Error Reponse to user. The Error Response
+		 * buffer is given by the user, but a sense buffer is used to
+		 * get that data from the IOC. The user's
+		 * ErrorResponseBaseAddress is saved in the
+		 * 'nvme_error_response' field before the command because that
+		 * field is set to a sense buffer. When the command is
+		 * complete, the Error Response data from the IOC is copied to
+		 * that user address after it is checked for validity.
+		 * Also note that 'sense' buffers are not defined for
+		 * NVMe commands. Sense terminalogy is only used here so that
+		 * the same IOCTL structure and sense buffers can be used for
+		 * NVMe.
+		 */
+		if (function == MPI2_FUNCTION_NVME_ENCAPSULATED) {
+			if (cm->nvme_error_response == NULL) {
+				mpr_dprint(sc, MPR_INFO, "NVMe Error Response "
+				    "buffer is NULL. Response data will not be "
+				    "returned.\n");
+				mpr_unlock(sc);
+				goto RetFreeUnlocked;
+			}
+
+			nvme_error_reply =
+			    (Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply;
+			sz = MIN(le32toh(nvme_error_reply->ErrorResponseCount),
+			    NVME_ERROR_RESPONSE_SIZE);
+			mpr_unlock(sc);
+			copyout(cm->cm_sense, cm->nvme_error_response, sz);
+			mpr_lock(sc);
+		}
 	}
 	mpr_unlock(sc);
 
@@ -2068,7 +2136,7 @@ mpr_user_btdh(struct mpr_softc *sc, mpr_btdh_mapping_t *data)
 			return (EINVAL);
 
 		if (target > sc->max_devices) {
-			mpr_dprint(sc, MPR_FAULT, "Target ID is out of range "
+			mpr_dprint(sc, MPR_XINFO, "Target ID is out of range "
 			   "for Bus/Target to DevHandle mapping.");
 			return (EINVAL);
 		}
diff --git a/sys/dev/mpr/mprvar.h b/sys/dev/mpr/mprvar.h
index 981358bcf915..9ef0c76d9c33 100644
--- a/sys/dev/mpr/mprvar.h
+++ b/sys/dev/mpr/mprvar.h
@@ -33,7 +33,7 @@
 #ifndef _MPRVAR_H
 #define _MPRVAR_H
 
-#define MPR_DRIVER_VERSION	"15.01.00.00-fbsd"
+#define MPR_DRIVER_VERSION	"15.02.00.00-fbsd"
 
 #define MPR_DB_MAX_WAIT		2500
 
@@ -50,6 +50,17 @@
 #define MPR_DEFAULT_CHAIN_SEG_SIZE	8
 #define MPR_MAX_CHAIN_ELEMENT_SIZE	16
 
+/*
+ * PCIe NVMe Specific defines
+ */
+//SLM-for now just use the same value as a SAS disk
+#define NVME_QDEPTH			MPR_REQ_FRAMES
+#define PRP_ENTRY_SIZE			8
+#define NVME_CMD_PRP1_OFFSET		24	/* PRP1 offset in NVMe cmd */
+#define NVME_CMD_PRP2_OFFSET		32	/* PRP2 offset in NVMe cmd */
+#define NVME_ERROR_RESPONSE_SIZE	16	/* Max NVME Error Response */
+#define HOST_PAGE_SIZE_4K		12
+
 #define MPR_FUNCTRACE(sc)			\
 	mpr_dprint((sc), MPR_TRACE, "%s\n", __func__)
 
@@ -184,6 +195,12 @@ struct mpr_chain {
 	uint64_t			chain_busaddr;
 };
 
+struct mpr_prp_page {
+	TAILQ_ENTRY(mpr_prp_page)	prp_page_link;
+	uint64_t			*prp_page;
+	uint64_t			prp_page_busaddr;
+};
+
 /*
  * This needs to be at least 2 to support SMP passthrough.
  */
@@ -229,9 +246,11 @@ struct mpr_command {
 #define MPR_CM_STATE_TIMEDOUT		2
 	bus_dmamap_t			cm_dmamap;
 	struct scsi_sense_data		*cm_sense;
+	uint64_t			*nvme_error_response;
 	TAILQ_HEAD(, mpr_chain)		cm_chain_list;
+ 	TAILQ_HEAD(, mpr_prp_page)	cm_prp_page_list;
 	uint32_t			cm_req_busaddr;
-	uint32_t			cm_sense_busaddr;
+	bus_addr_t			cm_sense_busaddr;
 	struct callout			cm_callout;
 };
 
@@ -257,27 +276,35 @@ struct mpr_softc {
 #define MPR_FLAGS_SHUTDOWN	(1 << 3)
 #define MPR_FLAGS_DIAGRESET	(1 << 4)
 #define	MPR_FLAGS_ATTACH_DONE	(1 << 5)
+#define	MPR_FLAGS_GEN35_IOC	(1 << 6)
 	u_int				mpr_debug;
 	u_int				disable_msix;
 	u_int				disable_msi;
+	u_int				atomic_desc_capable;
 	int				tm_cmds_active;
 	int				io_cmds_active;
 	int				io_cmds_highwater;
 	int				chain_free;
 	int				max_chains;
 	int				max_io_pages;
+	u_int				maxio;
 	int				chain_free_lowwater;
 	uint32_t			chain_frame_size;
 	uint16_t			chain_seg_size;
+	int				prp_buffer_size;
+	int				prp_pages_free;
+	int				prp_pages_free_lowwater;
 	u_int				enable_ssu;
 	int				spinup_wait_time;
 	int				use_phynum;
 	uint64_t			chain_alloc_fail;
+	uint64_t			prp_page_alloc_fail;
 	struct sysctl_ctx_list		sysctl_ctx;
 	struct sysctl_oid		*sysctl_tree;
 	char                            fw_version[16];
 	struct mpr_command		*commands;
 	struct mpr_chain		*chains;
+	struct mpr_prp_page		*prps;
 	struct callout			periodic;
 
 	struct mprsas_softc		*sassc;
@@ -285,6 +312,7 @@ struct mpr_softc {
 	TAILQ_HEAD(, mpr_command)	req_list;
 	TAILQ_HEAD(, mpr_command)	high_priority_req_list;
 	TAILQ_HEAD(, mpr_chain)		chain_list;
+	TAILQ_HEAD(, mpr_prp_page)	prp_page_list;
 	TAILQ_HEAD(, mpr_command)	tm_list;
 	int				replypostindex;
 	int				replyfreeindex;
@@ -333,6 +361,11 @@ struct mpr_softc {
 	bus_dma_tag_t			chain_dmat;
 	bus_dmamap_t			chain_map;
 
+	uint8_t				*prp_pages;
+	bus_addr_t			prp_page_busaddr;
+	bus_dma_tag_t			prp_page_dmat;
+	bus_dmamap_t			prp_page_map;
+
 	MPI2_REPLY_DESCRIPTORS_UNION	*post_queue;
 	bus_addr_t			post_busaddr;
 	uint32_t			*free_queue;
@@ -471,10 +504,33 @@ mpr_free_chain(struct mpr_softc *sc, struct mpr_chain *chain)
 	TAILQ_INSERT_TAIL(&sc->chain_list, chain, chain_link);
 }
 
+static __inline struct mpr_prp_page *
+mpr_alloc_prp_page(struct mpr_softc *sc)
+{
+	struct mpr_prp_page *prp_page;
+
+	if ((prp_page = TAILQ_FIRST(&sc->prp_page_list)) != NULL) {
+		TAILQ_REMOVE(&sc->prp_page_list, prp_page, prp_page_link);
+		sc->prp_pages_free--;
+		if (sc->prp_pages_free < sc->prp_pages_free_lowwater)
+			sc->prp_pages_free_lowwater = sc->prp_pages_free;
+	} else
+		sc->prp_page_alloc_fail++;
+	return (prp_page);
+}
+
+static __inline void
+mpr_free_prp_page(struct mpr_softc *sc, struct mpr_prp_page *prp_page)
+{
+	sc->prp_pages_free++;
+	TAILQ_INSERT_TAIL(&sc->prp_page_list, prp_page, prp_page_link);
+}
+
 static __inline void
 mpr_free_command(struct mpr_softc *sc, struct mpr_command *cm)
 {
 	struct mpr_chain *chain, *chain_temp;
+	struct mpr_prp_page *prp_page, *prp_page_temp;
 
 	if (cm->cm_reply != NULL)
 		mpr_free_reply(sc, cm->cm_reply_data);
@@ -497,6 +553,11 @@ mpr_free_command(struct mpr_softc *sc, struct mpr_command *cm)
 		TAILQ_REMOVE(&cm->cm_chain_list, chain, chain_link);
 		mpr_free_chain(sc, chain);
 	}
+	TAILQ_FOREACH_SAFE(prp_page, &cm->cm_prp_page_list, prp_page_link,
+	    prp_page_temp) {
+		TAILQ_REMOVE(&cm->cm_prp_page_list, prp_page, prp_page_link);
+		mpr_free_prp_page(sc, prp_page);
+	}
 	TAILQ_INSERT_TAIL(&sc->req_list, cm, cm_link);
 }
 
@@ -510,7 +571,8 @@ mpr_alloc_command(struct mpr_softc *sc)
 		return (NULL);
 
 	TAILQ_REMOVE(&sc->req_list, cm, cm_link);
-	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy command\n"));
+	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy "
+	    "command\n"));
 	cm->cm_state = MPR_CM_STATE_BUSY;
 	return (cm);
 }
@@ -547,7 +609,8 @@ mpr_alloc_high_priority_command(struct mpr_softc *sc)
 		return (NULL);
 
 	TAILQ_REMOVE(&sc->high_priority_req_list, cm, cm_link);
-	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy command\n"));
+	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy "
+	    "command\n"));
 	cm->cm_state = MPR_CM_STATE_BUSY;
 	return (cm);
 }
@@ -653,6 +716,9 @@ int mpr_register_events(struct mpr_softc *, uint8_t *, mpr_evt_callback_t *,
 int mpr_restart(struct mpr_softc *);
 int mpr_update_events(struct mpr_softc *, struct mpr_event_handle *, uint8_t *);
 int mpr_deregister_events(struct mpr_softc *, struct mpr_event_handle *);
+void mpr_build_nvme_prp(struct mpr_softc *sc, struct mpr_command *cm,
+    Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, void *data,
+    uint32_t data_in_sz, uint32_t data_out_sz);
 int mpr_push_sge(struct mpr_command *, MPI2_SGE_SIMPLE64 *, size_t, int);
 int mpr_push_ieee_sge(struct mpr_command *, void *, int);
 int mpr_add_dmaseg(struct mpr_command *, vm_paddr_t, size_t, u_int, int);
@@ -682,6 +748,10 @@ int mpr_config_get_iounit_pg8(struct mpr_softc *sc,
     Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage8_t *config_page);
 int mpr_config_get_sas_device_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
     Mpi2SasDevicePage0_t *, u32 , u16 );
+int mpr_config_get_pcie_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
+    *mpi_reply, Mpi26PCIeDevicePage0_t *config_page, u32 form, u16 handle);
+int mpr_config_get_pcie_device_pg2(struct mpr_softc *sc, Mpi2ConfigReply_t
+    *mpi_reply, Mpi26PCIeDevicePage2_t *config_page, u32 form, u16 handle);
 int mpr_config_get_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
     Mpi2DriverMappingPage0_t *, u16 );
 int mpr_config_get_raid_volume_pg1(struct mpr_softc *sc,
@@ -702,6 +772,8 @@ void mpr_base_static_config_pages(struct mpr_softc *sc);
 int mpr_mapping_initialize(struct mpr_softc *);
 void mpr_mapping_topology_change_event(struct mpr_softc *,
     Mpi2EventDataSasTopologyChangeList_t *);
+void mpr_mapping_pcie_topology_change_event(struct mpr_softc *sc,
+    Mpi26EventDataPCIeTopologyChangeList_t *event_data);
 int mpr_mapping_is_reinit_required(struct mpr_softc *);
 void mpr_mapping_free_memory(struct mpr_softc *sc);
 int mpr_config_set_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
@@ -769,5 +841,52 @@ SYSCTL_DECL(_hw_mpr);
 #define CAM_PRIORITY_NORMAL CAM_PRIORITY_NONE
 #endif
 
+/* Definitions for SCSI unmap translation to NVMe DSM command */
+
+/* UNMAP block descriptor structure */
+struct unmap_blk_desc {
+	uint64_t slba;
+	uint32_t nlb;
+	uint32_t resv;
+};
+
+/* UNMAP command's data */
+struct unmap_parm_list {
+	uint16_t unmap_data_len;
+	uint16_t unmap_blk_desc_data_len;
+	uint32_t resv;
+	struct unmap_blk_desc desc[0];
+};
+
+/* SCSI ADDITIONAL SENSE Codes */
+#define FIXED_SENSE_DATA                                0x70
+#define SCSI_ASC_NO_SENSE                               0x00
+#define SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT             0x03
+#define SCSI_ASC_LUN_NOT_READY                          0x04
+#define SCSI_ASC_WARNING                                0x0B
+#define SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED           0x10
+#define SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED          0x10
+#define SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED          0x10
+#define SCSI_ASC_UNRECOVERED_READ_ERROR                 0x11
+#define SCSI_ASC_MISCOMPARE_DURING_VERIFY               0x1D
+#define SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID           0x20
+#define SCSI_ASC_ILLEGAL_COMMAND                        0x20
+#define SCSI_ASC_ILLEGAL_BLOCK                          0x21
+#define SCSI_ASC_INVALID_CDB                            0x24
+#define SCSI_ASC_INVALID_LUN                            0x25
+#define SCSI_ASC_INVALID_PARAMETER                      0x26
+#define SCSI_ASC_FORMAT_COMMAND_FAILED                  0x31
+#define SCSI_ASC_INTERNAL_TARGET_FAILURE                0x44
+                
+/* SCSI ADDITIONAL SENSE Code Qualifiers */ 
+#define SCSI_ASCQ_CAUSE_NOT_REPORTABLE                  0x00
+#define SCSI_ASCQ_FORMAT_COMMAND_FAILED                 0x01
+#define SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED          0x01
+#define SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED         0x02
+#define SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED         0x03
+#define SCSI_ASCQ_FORMAT_IN_PROGRESS                    0x04
+#define SCSI_ASCQ_POWER_LOSS_EXPECTED                   0x08
+#define SCSI_ASCQ_INVALID_LUN_ID                        0x09
+
 #endif
 

From c1212a7a74d33a6e844bc869bcb7e42cbb7c1a91 Mon Sep 17 00:00:00 2001
From: Luiz Otavio O Souza <loos@FreeBSD.org>
Date: Wed, 17 May 2017 22:05:07 +0000
Subject: [PATCH 25/73] Fix the offset for the CPU0 MPIC registers.

Please note that only a subset of CPU0 registers are exported.  CPU1
registers are not touched.

Obtained from:	ARMADA38X Functional Specifications
Sponsored by:	Rubicon Communications, LLC (Netgate)
---
 sys/boot/fdt/dts/arm/armada-38x.dtsi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/boot/fdt/dts/arm/armada-38x.dtsi b/sys/boot/fdt/dts/arm/armada-38x.dtsi
index c4673ecc4d09..e8d1b05c0d27 100644
--- a/sys/boot/fdt/dts/arm/armada-38x.dtsi
+++ b/sys/boot/fdt/dts/arm/armada-38x.dtsi
@@ -419,7 +419,7 @@
 
 			mpic: interrupt-controller@20a00 {
 				compatible = "marvell,mpic";
-				reg = <0x20a00 0x2d0>, <0x21070 0x58>;
+				reg = <0x20a00 0x2d0>, <0x21870 0x58>;
 				#interrupt-cells = <1>;
 				#size-cells = <1>;
 				interrupt-controller;

From 5033c43b7a5c86f71be4058a8209503be2d65c49 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Wed, 17 May 2017 22:13:07 +0000
Subject: [PATCH 26/73] Add a driver for the Chelsio T6 crypto accelerator
 engine.

The ccr(4) driver supports use of the crypto accelerator engine on
Chelsio T6 NICs in "lookaside" mode via the opencrypto framework.

Currently, the driver supports AES-CBC, AES-CTR, AES-GCM, and AES-XTS
cipher algorithms as well as the SHA1-HMAC, SHA2-256-HMAC, SHA2-384-HMAC,
and SHA2-512-HMAC authentication algorithms.  The driver also supports
chaining one of AES-CBC, AES-CTR, or AES-XTS with an authentication
algorithm for encrypt-then-authenticate operations.

Note that this driver is still under active development and testing and
may not yet be ready for production use.  It does pass the tests in
tests/sys/opencrypto with the exception that the AES-GCM implementation
in the driver does not yet support requests with a zero byte payload.

To use this driver currently, the "uwire" configuration must be used
along with explicitly enabling support for lookaside crypto capabilities
in the cxgbe(4) driver.  These can be done by setting the following
tunables before loading the cxgbe(4) driver:

    hw.cxgbe.config_file=uwire
    hw.cxgbe.cryptocaps_allowed=-1

MFC after:	1 month
Relnotes:	yes
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D10763
---
 share/man/man4/Makefile          |    1 +
 share/man/man4/ccr.4             |  110 ++
 share/man/man4/cxgbe.4           |    3 +-
 sys/conf/NOTES                   |    2 +
 sys/conf/files                   |    2 +
 sys/dev/cxgbe/adapter.h          |    1 +
 sys/dev/cxgbe/crypto/t4_crypto.c | 2102 ++++++++++++++++++++++++++++++
 sys/dev/cxgbe/crypto/t4_crypto.h |  186 +++
 sys/modules/cxgbe/Makefile       |    1 +
 sys/modules/cxgbe/ccr/Makefile   |   19 +
 sys/powerpc/conf/NOTES           |    1 +
 11 files changed, 2427 insertions(+), 1 deletion(-)
 create mode 100644 share/man/man4/ccr.4
 create mode 100644 sys/dev/cxgbe/crypto/t4_crypto.c
 create mode 100644 sys/dev/cxgbe/crypto/t4_crypto.h
 create mode 100644 sys/modules/cxgbe/ccr/Makefile

diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 2d4efa810f32..205093c9df63 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -100,6 +100,7 @@ MAN=	aac.4 \
 	cc_newreno.4 \
 	cc_vegas.4 \
 	${_ccd.4} \
+	ccr.4 \
 	cd.4 \
 	cdce.4 \
 	cfi.4 \
diff --git a/share/man/man4/ccr.4 b/share/man/man4/ccr.4
new file mode 100644
index 000000000000..c75eb905a24c
--- /dev/null
+++ b/share/man/man4/ccr.4
@@ -0,0 +1,110 @@
+.\" Copyright (c) 2017, Chelsio Inc
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd May 16, 2017
+.Dt CCR 4
+.Os
+.Sh NAME
+.Nm ccr
+.Nd "Chelsio T6 crypto accelerator driver"
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indeunt
+.Cd "device ccr"
+.Ed
+.Pp
+To load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+ccr_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver provides support for the crypto accelerator engine included on
+PCI Express Ethernet adapters based on the Chelsio Terminator 6 ASIC (T6).
+The driver accelerates AES-CBC, AES-CTR, AES-GCM, AES-XTS, SHA1-HMAC,
+SHA2-256-HMAC, SHA2-384-HMAC, and SHA2-512-HMAC operations for
+.Xr crypto 4
+and
+.Xr ipsec 4 .
+The driver also supports chaining one of AES-CBC, AES-CTR, or AES-XTS with
+SHA1-HMAC, SHA2-256-HMAC, SHA2-384-HMAC, or SHA2-512-HMAC for
+encrypt-then-authenticate operations.
+For further hardware information and questions related to hardware
+requirements, see
+.Pa http://www.chelsio.com/ .
+.Pp
+The
+.Nm
+driver attaches as a child of an existing Chelsio NIC device and thus
+requires that the
+.Xr cxgbe 4
+driver be active.
+.Sh HARDWARE
+The
+.Nm
+driver supports the crypto accelerator engine included on adapters
+based on the T6 ASIC:
+.Pp
+.Bl -bullet -compact
+.It
+Chelsio T6225-CR
+.It
+Chelsio T6225-SO-CR
+.It
+Chelsio T62100-LP-CR
+.It
+Chelsio T62100-SO-CR
+.It
+Chelsio T62100-CR
+.El
+.Sh SUPPORT
+For general information and support,
+go to the Chelsio support website at:
+.Pa http://www.chelsio.com/ .
+.Pp
+If an issue is identified with this driver with a supported adapter,
+email all the specific information related to the issue to
+.Aq Mt support@chelsio.com .
+.Sh SEE ALSO
+.Xr crypto 4 ,
+.Xr cxgbe 4 ,
+.Xr ipsec 4
+.Sh HISTORY
+The
+.Nm
+device driver first appeared in
+.Fx 12.0 .
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+driver was written by
+.An John Baldwin Aq Mt jhb@FreeBSD.org .
diff --git a/share/man/man4/cxgbe.4 b/share/man/man4/cxgbe.4
index 9ee9f84d1b92..8833ba8efdbc 100644
--- a/share/man/man4/cxgbe.4
+++ b/share/man/man4/cxgbe.4
@@ -31,7 +31,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 22, 2016
+.Dd May 16, 2017
 .Dt CXGBE 4
 .Os
 .Sh NAME
@@ -367,6 +367,7 @@ email all the specific information related to the issue to
 .Sh SEE ALSO
 .Xr altq 4 ,
 .Xr arp 4 ,
+.Xr ccr 4 ,
 .Xr cxgb 4 ,
 .Xr cxgbev 4 ,
 .Xr netintro 4 ,
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index 87257b6110dc..70d14ef9f4a9 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -2849,6 +2849,8 @@ device		cryptodev	# /dev/crypto for access to h/w
 
 device		rndtest		# FIPS 140-2 entropy tester
 
+device		ccr		# Chelsio T6
+
 device		hifn		# Hifn 7951, 7781, etc.
 options 	HIFN_DEBUG	# enable debugging support: hw.hifn.debug
 options 	HIFN_RNDTEST	# enable rndtest support
diff --git a/sys/conf/files b/sys/conf/files
index e0ed1c12c278..5d116ee6f288 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1428,6 +1428,8 @@ t6fw.fw			optional cxgbe					\
 	compile-with	"${NORMAL_FW}"					\
 	no-obj no-implicit-rule						\
 	clean		"t6fw.fw"
+dev/cxgbe/crypto/t4_crypto.c	optional ccr \
+	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cy/cy.c			optional cy
 dev/cy/cy_isa.c			optional cy isa
 dev/cy/cy_pci.c			optional cy pci
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 02453c299007..9866e0cf9028 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -796,6 +796,7 @@ struct adapter {
 	struct tom_tunables tt;
 	void *iwarp_softc;	/* (struct c4iw_dev *) */
 	void *iscsi_ulp_softc;	/* (struct cxgbei_data *) */
+	void *ccr_softc;	/* (struct ccr_softc *) */
 	struct l2t_data *l2t;	/* L2 table */
 	struct tid_info tids;
 
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
new file mode 100644
index 000000000000..7cbb17ea7324
--- /dev/null
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -0,0 +1,2102 @@
+/*-
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/bus.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/module.h>
+#include <sys/sglist.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform.h>
+
+#include "cryptodev_if.h"
+
+#include "common/common.h"
+#include "crypto/t4_crypto.h"
+
+/*
+ * Requests consist of:
+ *
+ * +-------------------------------+
+ * | struct fw_crypto_lookaside_wr |
+ * +-------------------------------+
+ * | struct ulp_txpkt              |
+ * +-------------------------------+
+ * | struct ulptx_idata            |
+ * +-------------------------------+
+ * | struct cpl_tx_sec_pdu         |
+ * +-------------------------------+
+ * | struct cpl_tls_tx_scmd_fmt    |
+ * +-------------------------------+
+ * | key context header            |
+ * +-------------------------------+
+ * | AES key                       |  ----- For requests with AES
+ * +-------------------------------+ -
+ * | IPAD (16-byte aligned)        |  \
+ * +-------------------------------+  +---- For requests with HMAC
+ * | OPAD (16-byte aligned)        |  /
+ * +-------------------------------+ -
+ * | GMAC H                        |  ----- For AES-GCM
+ * +-------------------------------+ -
+ * | struct cpl_rx_phys_dsgl       |  \
+ * +-------------------------------+  +---- Destination buffer for
+ * | PHYS_DSGL entries             |  /     non-hash-only requests
+ * +-------------------------------+ -
+ * | 16 dummy bytes                |  ----- Only for hash-only requests
+ * +-------------------------------+
+ * | IV                            |  ----- If immediate IV
+ * +-------------------------------+
+ * | Payload                       |  ----- If immediate Payload
+ * +-------------------------------+ -
+ * | struct ulptx_sgl              |  \
+ * +-------------------------------+  +---- If payload via SGL
+ * | SGL entries                   |  /
+ * +-------------------------------+ -
+ *
+ * Note that the key context must be padded to ensure 16-byte alignment.
+ * For HMAC requests, the key consists of the partial hash of the IPAD
+ * followed by the partial hash of the OPAD.
+ *
+ * Replies consist of:
+ *
+ * +-------------------------------+
+ * | struct cpl_fw6_pld            |
+ * +-------------------------------+
+ * | hash digest                   |  ----- For HMAC request with
+ * +-------------------------------+        'hash_size' set in work request
+ *
+ * A 32-bit big-endian error status word is supplied in the last 4
+ * bytes of data[0] in the CPL_FW6_PLD message.  bit 0 indicates a
+ * "MAC" error and bit 1 indicates a "PAD" error.
+ *
+ * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message
+ * in the request is returned in data[1] of the CPL_FW6_PLD message.
+ *
+ * For block cipher replies, the updated IV is supplied in data[2] and
+ * data[3] of the CPL_FW6_PLD message.
+ *
+ * For hash replies where the work request set 'hash_size' to request
+ * a copy of the hash in the reply, the hash digest is supplied
+ * immediately following the CPL_FW6_PLD message.
+ */
+
+/*
+ * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32
+ * SG entries.
+ */
+#define	MAX_RX_PHYS_DSGL_SGE	32
+#define	DSGL_SGE_MAXLEN		65535
+
+static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto");
+
+struct ccr_session_hmac {
+	struct auth_hash *auth_hash;
+	int hash_len;
+	unsigned int partial_digest_len;
+	unsigned int auth_mode;
+	unsigned int mk_size;
+	char ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+	char opad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+};
+
+struct ccr_session_gmac {
+	int hash_len;
+	char ghash_h[GMAC_BLOCK_LEN];
+};
+
+struct ccr_session_blkcipher {
+	unsigned int cipher_mode;
+	unsigned int key_len;
+	unsigned int iv_len;
+	__be32 key_ctx_hdr;
+	char enckey[CHCR_AES_MAX_KEY_LEN];
+	char deckey[CHCR_AES_MAX_KEY_LEN];
+};
+
+struct ccr_session {
+	bool active;
+	int pending;
+	enum { HMAC, BLKCIPHER, AUTHENC, GCM } mode;
+	union {
+		struct ccr_session_hmac hmac;
+		struct ccr_session_gmac gmac;
+	};
+	struct ccr_session_blkcipher blkcipher;
+};
+
+struct ccr_softc {
+	struct adapter *adapter;
+	device_t dev;
+	uint32_t cid;
+	int tx_channel_id;
+	struct ccr_session *sessions;
+	int nsessions;
+	struct mtx lock;
+	bool detaching;
+	struct sge_wrq *txq;
+	struct sge_rxq *rxq;
+
+	/*
+	 * Pre-allocate S/G lists used when preparing a work request.
+	 * 'sg_crp' contains an sglist describing the entire buffer
+	 * for a 'struct cryptop'.  'sg_ulptx' is used to describe
+	 * the data the engine should DMA as input via ULPTX_SGL.
+	 * 'sg_dsgl' is used to describe the destination that cipher
+	 * text and a tag should be written to.
+	 */
+	struct sglist *sg_crp;
+	struct sglist *sg_ulptx;
+	struct sglist *sg_dsgl;
+
+	/* Statistics. */
+	uint64_t stats_blkcipher_encrypt;
+	uint64_t stats_blkcipher_decrypt;
+	uint64_t stats_hmac;
+	uint64_t stats_authenc_encrypt;
+	uint64_t stats_authenc_decrypt;
+	uint64_t stats_gcm_encrypt;
+	uint64_t stats_gcm_decrypt;
+	uint64_t stats_wr_nomem;
+	uint64_t stats_inflight;
+	uint64_t stats_mac_error;
+	uint64_t stats_pad_error;
+	uint64_t stats_bad_session;
+	uint64_t stats_sglist_error;
+	uint64_t stats_process_error;
+};
+
+/*
+ * Crypto requests involve two kind of scatter/gather lists.
+ *
+ * Non-hash-only requests require a PHYS_DSGL that describes the
+ * location to store the results of the encryption or decryption
+ * operation.  This SGL uses a different format (PHYS_DSGL) and should
+ * exclude the crd_skip bytes at the start of the data as well as
+ * any AAD or IV.  For authenticated encryption requests it should
+ * cover include the destination of the hash or tag.
+ *
+ * The input payload may either be supplied inline as immediate data,
+ * or via a standard ULP_TX SGL.  This SGL should include AAD,
+ * ciphertext, and the hash or tag for authenticated decryption
+ * requests.
+ *
+ * These scatter/gather lists can describe different subsets of the
+ * buffer described by the crypto operation.  ccr_populate_sglist()
+ * generates a scatter/gather list that covers the entire crypto
+ * operation buffer that is then used to construct the other
+ * scatter/gather lists.
+ */
+static int
+ccr_populate_sglist(struct sglist *sg, struct cryptop *crp)
+{
+	int error;
+
+	sglist_reset(sg);
+	if (crp->crp_flags & CRYPTO_F_IMBUF)
+		error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
+	else if (crp->crp_flags & CRYPTO_F_IOV)
+		error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
+	else
+		error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+	return (error);
+}
+
+/*
+ * Segments in 'sg' larger than 'maxsegsize' are counted as multiple
+ * segments.
+ */
+static int
+ccr_count_sgl(struct sglist *sg, int maxsegsize)
+{
+	int i, nsegs;
+
+	nsegs = 0;
+	for (i = 0; i < sg->sg_nseg; i++)
+		nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize);
+	return (nsegs);
+}
+
+/* These functions deal with PHYS_DSGL for the reply buffer. */
+static inline int
+ccr_phys_dsgl_len(int nsegs)
+{
+	int len;
+
+	len = (nsegs / 8) * sizeof(struct phys_sge_pairs);
+	if ((nsegs % 8) != 0) {
+		len += sizeof(uint16_t) * 8;
+		len += roundup2(nsegs % 8, 2) * sizeof(uint64_t);
+	}
+	return (len);
+}
+
+static void
+ccr_write_phys_dsgl(struct ccr_softc *sc, void *dst, int nsegs)
+{
+	struct sglist *sg;
+	struct cpl_rx_phys_dsgl *cpl;
+	struct phys_sge_pairs *sgl;
+	vm_paddr_t paddr;
+	size_t seglen;
+	u_int i, j;
+
+	sg = sc->sg_dsgl;
+	cpl = dst;
+	cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) |
+	    V_CPL_RX_PHYS_DSGL_ISRDMA(0));
+	cpl->pcirlxorder_to_noofsgentr = htobe32(
+	    V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) |
+	    V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) |
+	    V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) |
+	    V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs));
+	cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
+	cpl->rss_hdr_int.qid = htobe16(sc->rxq->iq.abs_id);
+	cpl->rss_hdr_int.hash_val = 0;
+	sgl = (struct phys_sge_pairs *)(cpl + 1);
+	j = 0;
+	for (i = 0; i < sg->sg_nseg; i++) {
+		seglen = sg->sg_segs[i].ss_len;
+		paddr = sg->sg_segs[i].ss_paddr;
+		do {
+			sgl->addr[j] = htobe64(paddr);
+			if (seglen > DSGL_SGE_MAXLEN) {
+				sgl->len[j] = htobe16(DSGL_SGE_MAXLEN);
+				paddr += DSGL_SGE_MAXLEN;
+				seglen -= DSGL_SGE_MAXLEN;
+			} else {
+				sgl->len[j] = htobe16(seglen);
+				seglen = 0;
+			}
+			j++;
+			if (j == 8) {
+				sgl++;
+				j = 0;
+			}
+		} while (seglen != 0);
+	}
+	MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs);
+}
+
+/* These functions deal with the ULPTX_SGL for input payload. */
+static inline int
+ccr_ulptx_sgl_len(int nsegs)
+{
+	u_int n;
+
+	nsegs--; /* first segment is part of ulptx_sgl */
+	n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
+	return (roundup2(n, 16));
+}
+
+static void
+ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs)
+{
+	struct ulptx_sgl *usgl;
+	struct sglist *sg;
+	struct sglist_seg *ss;
+	int i;
+
+	sg = sc->sg_ulptx;
+	MPASS(nsegs == sg->sg_nseg);
+	ss = &sg->sg_segs[0];
+	usgl = dst;
+	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
+	    V_ULPTX_NSGE(nsegs));
+	usgl->len0 = htobe32(ss->ss_len);
+	usgl->addr0 = htobe64(ss->ss_paddr);
+	ss++;
+	for (i = 0; i < sg->sg_nseg - 1; i++) {
+		usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len);
+		usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr);
+		ss++;
+	}
+	
+}
+
+static bool
+ccr_use_imm_data(u_int transhdr_len, u_int input_len)
+{
+
+	if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN)
+		return (false);
+	if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) >
+	    SGE_MAX_WR_LEN)
+		return (false);
+	return (true);
+}
+
+static void
+ccr_populate_wreq(struct ccr_softc *sc, struct chcr_wr *crwr, u_int kctx_len,
+    u_int wr_len, uint32_t sid, u_int imm_len, u_int sgl_len, u_int hash_size,
+    u_int iv_loc, struct cryptop *crp)
+{
+	u_int cctx_size;
+
+	cctx_size = sizeof(struct _key_ctx) + kctx_len;
+	crwr->wreq.op_to_cctx_size = htobe32(
+	    V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4));
+	crwr->wreq.len16_pkd = htobe32(
+	    V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16));
+	crwr->wreq.session_id = htobe32(sid);
+	crwr->wreq.rx_chid_to_rx_q_id = htobe32(
+	    V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(sc->tx_channel_id) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_IV(iv_loc) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(sc->rxq->iq.abs_id));
+	crwr->wreq.key_addr = 0;
+	crwr->wreq.pld_size_hash_size = htobe32(
+	    V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) |
+	    V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size));
+	crwr->wreq.cookie = htobe64((uintptr_t)crp);
+
+	crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
+	    V_ULP_TXPKT_DATAMODIFY(0) |
+	    V_ULP_TXPKT_CHANNELID(sc->tx_channel_id) | V_ULP_TXPKT_DEST(0) |
+	    V_ULP_TXPKT_FID(0) | V_ULP_TXPKT_RO(1));
+	crwr->ulptx.len = htobe32(
+	    ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16));
+
+	crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
+	    V_ULP_TX_SC_MORE(imm_len != 0 ? 0 : 1));
+	crwr->sc_imm.len = htobe32(wr_len - offsetof(struct chcr_wr, sec_cpl) -
+	    sgl_len);
+}
+
+static int
+ccr_hmac(struct ccr_softc *sc, uint32_t sid, struct ccr_session *s,
+    struct cryptop *crp)
+{
+	struct chcr_wr *crwr;
+	struct wrqe *wr;
+	struct auth_hash *axf;
+	struct cryptodesc *crd;
+	char *dst;
+	u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len;
+	u_int imm_len, iopad_size;
+	int error, sgl_nsegs, sgl_len;
+
+	axf = s->hmac.auth_hash;
+
+	/* PADs must be 128-bit aligned. */
+	iopad_size = roundup2(s->hmac.partial_digest_len, 16);
+
+	/*
+	 * The 'key' part of the context includes the aligned IPAD and
+	 * OPAD.
+	 */
+	kctx_len = iopad_size * 2;
+	hash_size_in_response = axf->hashsize;
+	transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
+
+	crd = crp->crp_desc;
+	if (ccr_use_imm_data(transhdr_len, crd->crd_len)) {
+		imm_len = crd->crd_len;
+		sgl_nsegs = 0;
+		sgl_len = 0;
+	} else {
+		imm_len = 0;
+		sglist_reset(sc->sg_ulptx);
+		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+		    crd->crd_skip, crd->crd_len);
+		if (error)
+			return (error);
+		sgl_nsegs = sc->sg_ulptx->sg_nseg;
+		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
+	}
+
+	wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
+	wr = alloc_wrqe(wr_len, sc->txq);
+	if (wr == NULL) {
+		sc->stats_wr_nomem++;
+		return (ENOMEM);
+	}
+	crwr = wrtod(wr);
+	memset(crwr, 0, wr_len);
+
+	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, sid, imm_len, sgl_len,
+	    hash_size_in_response, IV_NOP, crp);
+
+	/* XXX: Hardcodes SGE loopback channel of 0. */
+	crwr->sec_cpl.op_ivinsrtofst = htobe32(
+	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
+	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
+	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
+	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
+	    V_CPL_TX_SEC_PDU_IVINSRTOFST(0));
+
+	crwr->sec_cpl.pldlen = htobe32(crd->crd_len);
+
+	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
+	    V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0));
+
+	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
+	crwr->sec_cpl.seqno_numivs = htobe32(
+	    V_SCMD_SEQ_NO_CTRL(0) |
+	    V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) |
+	    V_SCMD_CIPH_MODE(CHCR_SCMD_CIPHER_MODE_NOP) |
+	    V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
+	    V_SCMD_HMAC_CTRL(CHCR_SCMD_HMAC_CTRL_NO_TRUNC));
+	crwr->sec_cpl.ivgen_hdrlen = htobe32(
+	    V_SCMD_LAST_FRAG(0) | V_SCMD_MORE_FRAGS(0) | V_SCMD_MAC_ONLY(1));
+
+	memcpy(crwr->key_ctx.key, s->hmac.ipad, s->hmac.partial_digest_len);
+	memcpy(crwr->key_ctx.key + iopad_size, s->hmac.opad,
+	    s->hmac.partial_digest_len);
+
+	/* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */
+	kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
+	crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
+	    V_KEY_CONTEXT_OPAD_PRESENT(1) | V_KEY_CONTEXT_SALT_PRESENT(1) |
+	    V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) |
+	    V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1));
+
+	dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES;
+	if (imm_len != 0)
+		crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
+		    crd->crd_len, dst);
+	else
+		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
+
+	/* XXX: TODO backpressure */
+	t4_wrq_tx(sc->adapter, wr);
+
+	return (0);
+}
+
+static int
+ccr_hmac_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
+    const struct cpl_fw6_pld *cpl, int error)
+{
+	struct cryptodesc *crd;
+
+	crd = crp->crp_desc;
+	if (error == 0) {
+		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
+		    s->hmac.hash_len, (c_caddr_t)(cpl + 1));
+	}
+
+	return (error);
+}
+
+static int
+ccr_blkcipher(struct ccr_softc *sc, uint32_t sid, struct ccr_session *s,
+    struct cryptop *crp)
+{
+	char iv[CHCR_MAX_CRYPTO_IV_LEN];
+	struct chcr_wr *crwr;
+	struct wrqe *wr;
+	struct cryptodesc *crd;
+	char *dst;
+	u_int iv_loc, kctx_len, key_half, op_type, transhdr_len, wr_len;
+	u_int imm_len;
+	int dsgl_nsegs, dsgl_len;
+	int sgl_nsegs, sgl_len;
+	int error;
+
+	crd = crp->crp_desc;
+
+	if (s->blkcipher.key_len == 0)
+		return (EINVAL);
+	if (crd->crd_alg == CRYPTO_AES_CBC &&
+	    (crd->crd_len % AES_BLOCK_LEN) != 0)
+		return (EINVAL);
+
+	iv_loc = IV_NOP;
+	if (crd->crd_flags & CRD_F_ENCRYPT) {
+		op_type = CHCR_ENCRYPT_OP;
+		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
+		else
+			arc4rand(iv, s->blkcipher.iv_len, 0);
+		iv_loc = IV_IMMEDIATE;
+		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    crd->crd_inject, s->blkcipher.iv_len, iv);
+	} else {
+		op_type = CHCR_DECRYPT_OP;
+		if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
+			memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
+			iv_loc = IV_IMMEDIATE;
+		} else
+			iv_loc = IV_DSGL;
+	}
+
+	sglist_reset(sc->sg_dsgl);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip,
+	    crd->crd_len);
+	if (error)
+		return (error);
+	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
+	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
+		return (EFBIG);
+	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
+
+	/* The 'key' must be 128-bit aligned. */
+	kctx_len = roundup2(s->blkcipher.key_len, 16);
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
+
+	if (ccr_use_imm_data(transhdr_len, crd->crd_len +
+	    s->blkcipher.iv_len)) {
+		imm_len = crd->crd_len;
+		if (iv_loc == IV_DSGL) {
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crd->crd_inject, s->blkcipher.iv_len, iv);
+			iv_loc = IV_IMMEDIATE;
+		}
+		sgl_nsegs = 0;
+		sgl_len = 0;
+	} else {
+		imm_len = 0;
+		sglist_reset(sc->sg_ulptx);
+		if (iv_loc == IV_DSGL) {
+			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+			    crd->crd_inject, s->blkcipher.iv_len);
+			if (error)
+				return (error);
+		}
+		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+		    crd->crd_skip, crd->crd_len);
+		if (error)
+			return (error);
+		sgl_nsegs = sc->sg_ulptx->sg_nseg;
+		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
+	}
+
+	wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
+	if (iv_loc == IV_IMMEDIATE)
+		wr_len += s->blkcipher.iv_len;
+	wr = alloc_wrqe(wr_len, sc->txq);
+	if (wr == NULL) {
+		sc->stats_wr_nomem++;
+		return (ENOMEM);
+	}
+	crwr = wrtod(wr);
+	memset(crwr, 0, wr_len);
+
+	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, sid, imm_len, sgl_len, 0,
+	    iv_loc, crp);
+
+	/* XXX: Hardcodes SGE loopback channel of 0. */
+	crwr->sec_cpl.op_ivinsrtofst = htobe32(
+	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
+	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
+	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
+	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
+	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
+
+	crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len);
+
+	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
+	    V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
+	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0));
+
+	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
+	crwr->sec_cpl.seqno_numivs = htobe32(
+	    V_SCMD_SEQ_NO_CTRL(0) |
+	    V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) |
+	    V_SCMD_ENC_DEC_CTRL(op_type) |
+	    V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
+	    V_SCMD_AUTH_MODE(CHCR_SCMD_AUTH_MODE_NOP) |
+	    V_SCMD_HMAC_CTRL(CHCR_SCMD_HMAC_CTRL_NOP) |
+	    V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+	    V_SCMD_NUM_IVS(0));
+	crwr->sec_cpl.ivgen_hdrlen = htobe32(
+	    V_SCMD_IV_GEN_CTRL(0) |
+	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
+	    V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
+
+	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
+	switch (crd->crd_alg) {
+	case CRYPTO_AES_CBC:
+		if (crd->crd_flags & CRD_F_ENCRYPT)
+			memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
+			    s->blkcipher.key_len);
+		else
+			memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
+			    s->blkcipher.key_len);
+		break;
+	case CRYPTO_AES_ICM:
+		memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
+		    s->blkcipher.key_len);
+		break;
+	case CRYPTO_AES_XTS:
+		key_half = s->blkcipher.key_len / 2;
+		memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
+		    key_half);
+		if (crd->crd_flags & CRD_F_ENCRYPT)
+			memcpy(crwr->key_ctx.key + key_half,
+			    s->blkcipher.enckey, key_half);
+		else
+			memcpy(crwr->key_ctx.key + key_half,
+			    s->blkcipher.deckey, key_half);
+		break;
+	}
+
+	dst = (char *)(crwr + 1) + kctx_len;
+	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
+	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
+	if (iv_loc == IV_IMMEDIATE) {
+		memcpy(dst, iv, s->blkcipher.iv_len);
+		dst += s->blkcipher.iv_len;
+	}
+	if (imm_len != 0)
+		crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
+		    crd->crd_len, dst);
+	else
+		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
+
+	/* XXX: TODO backpressure */
+	t4_wrq_tx(sc->adapter, wr);
+
+	return (0);
+}
+
+static int
+ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s,
+    struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
+{
+
+	/*
+	 * The updated IV to permit chained requests is at
+	 * cpl->data[2], but OCF doesn't permit chained requests.
+	 */
+	return (error);
+}
+
+/*
+ * 'hashsize' is the length of a full digest.  'authsize' is the
+ * requested digest length for this operation which may be less
+ * than 'hashsize'.
+ */
+static int
+ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize)
+{
+
+	if (authsize == 10)
+		return (CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366);
+	if (authsize == 12)
+		return (CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT);
+	if (authsize == hashsize / 2)
+		return (CHCR_SCMD_HMAC_CTRL_DIV2);
+	return (CHCR_SCMD_HMAC_CTRL_NO_TRUNC);
+}
+
+static int
+ccr_authenc(struct ccr_softc *sc, uint32_t sid, struct ccr_session *s,
+    struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde)
+{
+	char iv[CHCR_MAX_CRYPTO_IV_LEN];
+	struct chcr_wr *crwr;
+	struct wrqe *wr;
+	struct auth_hash *axf;
+	char *dst;
+	u_int iv_loc, kctx_len, key_half, op_type, transhdr_len, wr_len;
+	u_int hash_size_in_response, imm_len, iopad_size;
+	u_int aad_start, aad_len, aad_stop;
+	u_int auth_start, auth_stop, auth_insert;
+	u_int cipher_start, cipher_stop;
+	u_int hmac_ctrl, input_len;
+	int dsgl_nsegs, dsgl_len;
+	int sgl_nsegs, sgl_len;
+	int error;
+
+	if (s->blkcipher.key_len == 0)
+		return (EINVAL);
+	if (crde->crd_alg == CRYPTO_AES_CBC &&
+	    (crde->crd_len % AES_BLOCK_LEN) != 0)
+		return (EINVAL);
+
+	/*
+	 * AAD is only permitted before the cipher/plain text, not
+	 * after.
+	 */
+	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+		return (EINVAL);
+
+	axf = s->hmac.auth_hash;
+	hash_size_in_response = s->hmac.hash_len;
+
+	/*
+	 * The IV is always stored at the start of the buffer even
+	 * though it may be duplicated in the payload.  The crypto
+	 * engine doesn't work properly if the IV offset points inside
+	 * of the AAD region, so a second copy is always required.
+	 */
+	iv_loc = IV_IMMEDIATE;
+	if (crde->crd_flags & CRD_F_ENCRYPT) {
+		op_type = CHCR_ENCRYPT_OP;
+		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
+		else
+			arc4rand(iv, s->blkcipher.iv_len, 0);
+		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    crde->crd_inject, s->blkcipher.iv_len, iv);
+	} else {
+		op_type = CHCR_DECRYPT_OP;
+		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
+		else
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crde->crd_inject, s->blkcipher.iv_len, iv);
+	}
+
+	/*
+	 * The output buffer consists of the cipher text followed by
+	 * the hash when encrypting.  For decryption it only contains
+	 * the plain text.
+	 */
+	sglist_reset(sc->sg_dsgl);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
+	    crde->crd_len);
+	if (error)
+		return (error);
+	if (op_type == CHCR_ENCRYPT_OP) {
+		error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+		    crda->crd_inject, hash_size_in_response);
+		if (error)
+			return (error);
+	}
+	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
+	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
+		return (EFBIG);
+	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
+
+	/* PADs must be 128-bit aligned. */
+	iopad_size = roundup2(s->hmac.partial_digest_len, 16);
+
+	/*
+	 * The 'key' part of the key context consists of the key followed
+	 * by the IPAD and OPAD.
+	 */
+	kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2;
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
+
+	/*
+	 * The input buffer consists of the IV, any AAD, and then the
+	 * cipher/plain text.  For decryption requests the hash is
+	 * appended after the cipher text.
+	 */
+	if (crda->crd_skip < crde->crd_skip) {
+		if (crda->crd_skip + crda->crd_len > crde->crd_skip)
+			aad_len = (crde->crd_skip - crda->crd_skip);
+		else
+			aad_len = crda->crd_len;
+	} else
+		aad_len = 0;
+	input_len = aad_len + crde->crd_len;
+	if (op_type == CHCR_DECRYPT_OP)
+		input_len += hash_size_in_response;
+	if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) {
+		imm_len = input_len;
+		sgl_nsegs = 0;
+		sgl_len = 0;
+	} else {
+		imm_len = 0;
+		sglist_reset(sc->sg_ulptx);
+		if (aad_len != 0) {
+			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+			    crda->crd_skip, aad_len);
+			if (error)
+				return (error);
+		}
+		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+		    crde->crd_skip, crde->crd_len);
+		if (error)
+			return (error);
+		if (op_type == CHCR_DECRYPT_OP) {
+			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+			    crda->crd_inject, hash_size_in_response);
+			if (error)
+				return (error);
+		}
+		sgl_nsegs = sc->sg_ulptx->sg_nseg;
+		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
+	}
+
+	/*
+	 * Any auth-only data before the cipher region is marked as AAD.
+	 * Auth-data that overlaps with the cipher region is placed in
+	 * the auth section.
+	 */
+	if (aad_len != 0) {
+		aad_start = s->blkcipher.iv_len + 1;
+		aad_stop = aad_start + aad_len - 1;
+	} else {
+		aad_start = 0;
+		aad_stop = 0;
+	}
+	cipher_start = s->blkcipher.iv_len + aad_len + 1;
+	if (op_type == CHCR_DECRYPT_OP)
+		cipher_stop = hash_size_in_response;
+	else
+		cipher_stop = 0;
+	if (aad_len == crda->crd_len) {
+		auth_start = 0;
+		auth_stop = 0;
+	} else {
+		if (aad_len != 0)
+			auth_start = cipher_start;
+		else
+			auth_start = s->blkcipher.iv_len + crda->crd_skip -
+			    crde->crd_skip + 1;
+		auth_stop = (crde->crd_skip + crde->crd_len) -
+		    (crda->crd_skip + crda->crd_len) + cipher_stop;
+	}
+	if (op_type == CHCR_DECRYPT_OP)
+		auth_insert = hash_size_in_response;
+	else
+		auth_insert = 0;
+
+	wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
+	if (iv_loc == IV_IMMEDIATE)
+		wr_len += s->blkcipher.iv_len;
+	wr = alloc_wrqe(wr_len, sc->txq);
+	if (wr == NULL) {
+		sc->stats_wr_nomem++;
+		return (ENOMEM);
+	}
+	crwr = wrtod(wr);
+	memset(crwr, 0, wr_len);
+
+	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, sid, imm_len, sgl_len,
+	    op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, iv_loc,
+	    crp);
+
+	/* XXX: Hardcodes SGE loopback channel of 0. */
+	crwr->sec_cpl.op_ivinsrtofst = htobe32(
+	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
+	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
+	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
+	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
+	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
+
+	crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len);
+
+	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
+	    V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
+	    V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
+	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
+	    V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
+	    V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
+	    V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
+
+	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
+	hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response);
+	crwr->sec_cpl.seqno_numivs = htobe32(
+	    V_SCMD_SEQ_NO_CTRL(0) |
+	    V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) |
+	    V_SCMD_ENC_DEC_CTRL(op_type) |
+	    V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
+	    V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
+	    V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
+	    V_SCMD_HMAC_CTRL(hmac_ctrl) |
+	    V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+	    V_SCMD_NUM_IVS(0));
+	crwr->sec_cpl.ivgen_hdrlen = htobe32(
+	    V_SCMD_IV_GEN_CTRL(0) |
+	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
+	    V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
+
+	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
+	switch (crde->crd_alg) {
+	case CRYPTO_AES_CBC:
+		if (crde->crd_flags & CRD_F_ENCRYPT)
+			memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
+			    s->blkcipher.key_len);
+		else
+			memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
+			    s->blkcipher.key_len);
+		break;
+	case CRYPTO_AES_ICM:
+		memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
+		    s->blkcipher.key_len);
+		break;
+	case CRYPTO_AES_XTS:
+		key_half = s->blkcipher.key_len / 2;
+		memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
+		    key_half);
+		if (crde->crd_flags & CRD_F_ENCRYPT)
+			memcpy(crwr->key_ctx.key + key_half,
+			    s->blkcipher.enckey, key_half);
+		else
+			memcpy(crwr->key_ctx.key + key_half,
+			    s->blkcipher.deckey, key_half);
+		break;
+	}
+
+	dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
+	memcpy(dst, s->hmac.ipad, s->hmac.partial_digest_len);
+	memcpy(dst + iopad_size, s->hmac.opad, s->hmac.partial_digest_len);
+
+	dst = (char *)(crwr + 1) + kctx_len;
+	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
+	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
+	if (iv_loc == IV_IMMEDIATE) {
+		memcpy(dst, iv, s->blkcipher.iv_len);
+		dst += s->blkcipher.iv_len;
+	}
+	if (imm_len != 0) {
+		if (aad_len != 0) {
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crda->crd_skip, aad_len, dst);
+			dst += aad_len;
+		}
+		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
+		    crde->crd_len, dst);
+		dst += crde->crd_len;
+		if (op_type == CHCR_DECRYPT_OP)
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crda->crd_inject, hash_size_in_response, dst);
+	} else
+		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
+
+	/* XXX: TODO backpressure */
+	t4_wrq_tx(sc->adapter, wr);
+
+	return (0);
+}
+
+static int
+ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s,
+    struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
+{
+	struct cryptodesc *crd;
+
+	/*
+	 * The updated IV to permit chained requests is at
+	 * cpl->data[2], but OCF doesn't permit chained requests.
+	 *
+	 * For a decryption request, the hardware may do a verification
+	 * of the HMAC which will fail if the existing HMAC isn't in the
+	 * buffer.  If that happens, clear the error and copy the HMAC
+	 * from the CPL reply into the buffer.
+	 *
+	 * For encryption requests, crd should be the cipher request
+	 * which will have CRD_F_ENCRYPT set.  For decryption
+	 * requests, crp_desc will be the HMAC request which should
+	 * not have this flag set.
+	 */
+	crd = crp->crp_desc;
+	if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) &&
+	    !(crd->crd_flags & CRD_F_ENCRYPT)) {
+		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
+		    s->hmac.hash_len, (c_caddr_t)(cpl + 1));
+		error = 0;
+	}
+	return (error);
+}
+
+static int
+ccr_gcm(struct ccr_softc *sc, uint32_t sid, struct ccr_session *s,
+    struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde)
+{
+	char iv[CHCR_MAX_CRYPTO_IV_LEN];
+	struct chcr_wr *crwr;
+	struct wrqe *wr;
+	char *dst;
+	u_int iv_len, iv_loc, kctx_len, op_type, transhdr_len, wr_len;
+	u_int hash_size_in_response, imm_len;
+	u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert;
+	u_int hmac_ctrl, input_len;
+	int dsgl_nsegs, dsgl_len;
+	int sgl_nsegs, sgl_len;
+	int error;
+
+	if (s->blkcipher.key_len == 0)
+		return (EINVAL);
+
+	/*
+	 * AAD is only permitted before the cipher/plain text, not
+	 * after.
+	 */
+	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+		return (EINVAL);
+
+	hash_size_in_response = s->gmac.hash_len;
+
+	/*
+	 * The IV is always stored at the start of the buffer even
+	 * though it may be duplicated in the payload.  The crypto
+	 * engine doesn't work properly if the IV offset points inside
+	 * of the AAD region, so a second copy is always required.
+	 *
+	 * The IV for GCM is further complicated in that IPSec
+	 * provides a full 16-byte IV (including the counter), whereas
+	 * the /dev/crypto interface sometimes provides a full 16-byte
+	 * IV (if no IV is provided in the ioctl) and sometimes a
+	 * 12-byte IV (if the IV was explicit).  For now the driver
+	 * always assumes a 12-byte IV and initializes the low 4 byte
+	 * counter to 1.
+	 */
+	iv_loc = IV_IMMEDIATE;
+	if (crde->crd_flags & CRD_F_ENCRYPT) {
+		op_type = CHCR_ENCRYPT_OP;
+		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
+		else
+			arc4rand(iv, s->blkcipher.iv_len, 0);
+		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    crde->crd_inject, s->blkcipher.iv_len, iv);
+	} else {
+		op_type = CHCR_DECRYPT_OP;
+		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
+			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
+		else
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crde->crd_inject, s->blkcipher.iv_len, iv);
+	}
+
+	/*
+	 * If the input IV is 12 bytes, append an explicit counter of
+	 * 1.
+	 */
+	if (s->blkcipher.iv_len == 12) {
+		*(uint32_t *)&iv[12] = htobe32(1);
+		iv_len = AES_BLOCK_LEN;
+	} else
+		iv_len = s->blkcipher.iv_len;
+
+	/*
+	 * The output buffer consists of the cipher text followed by
+	 * the tag when encrypting.  For decryption it only contains
+	 * the plain text.
+	 */
+	sglist_reset(sc->sg_dsgl);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
+	    crde->crd_len);
+	if (error)
+		return (error);
+	if (op_type == CHCR_ENCRYPT_OP) {
+		error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+		    crda->crd_inject, hash_size_in_response);
+		if (error)
+			return (error);
+	}
+	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
+	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
+		return (EFBIG);
+	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
+
+	/*
+	 * The 'key' part of the key context consists of the key followed
+	 * by the Galois hash key.
+	 */
+	kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN;
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
+
+	/*
+	 * The input buffer consists of the IV, any AAD, and then the
+	 * cipher/plain text.  For decryption requests the hash is
+	 * appended after the cipher text.
+	 */
+	input_len = crda->crd_len + crde->crd_len;
+	if (op_type == CHCR_DECRYPT_OP)
+		input_len += hash_size_in_response;
+	if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
+		imm_len = input_len;
+		sgl_nsegs = 0;
+		sgl_len = 0;
+	} else {
+		imm_len = 0;
+		sglist_reset(sc->sg_ulptx);
+		if (crda->crd_len != 0) {
+			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+			    crda->crd_skip, crda->crd_len);
+			if (error)
+				return (error);
+		}
+		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+		    crde->crd_skip, crde->crd_len);
+		if (error)
+			return (error);
+		if (op_type == CHCR_DECRYPT_OP) {
+			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
+			    crda->crd_inject, hash_size_in_response);
+			if (error)
+				return (error);
+		}
+		sgl_nsegs = sc->sg_ulptx->sg_nseg;
+		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
+	}
+
+	if (crda->crd_len != 0) {
+		aad_start = iv_len + 1;
+		aad_stop = aad_start + crda->crd_len - 1;
+	} else {
+		aad_start = 0;
+		aad_stop = 0;
+	}
+	cipher_start = iv_len + crda->crd_len + 1;
+	if (op_type == CHCR_DECRYPT_OP)
+		cipher_stop = hash_size_in_response;
+	else
+		cipher_stop = 0;
+	if (op_type == CHCR_DECRYPT_OP)
+		auth_insert = hash_size_in_response;
+	else
+		auth_insert = 0;
+
+	wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
+	if (iv_loc == IV_IMMEDIATE)
+		wr_len += iv_len;
+	wr = alloc_wrqe(wr_len, sc->txq);
+	if (wr == NULL) {
+		sc->stats_wr_nomem++;
+		return (ENOMEM);
+	}
+	crwr = wrtod(wr);
+	memset(crwr, 0, wr_len);
+
+	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, sid, imm_len, sgl_len,
+	    0, iv_loc, crp);
+
+	/* XXX: Hardcodes SGE loopback channel of 0. */
+	crwr->sec_cpl.op_ivinsrtofst = htobe32(
+	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
+	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
+	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
+	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
+	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
+
+	crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
+
+	/*
+	 * NB: cipherstop is explicitly set to 0.  On encrypt it
+	 * should normally be set to 0 anyway (as the encrypt crd ends
+	 * at the end of the input).  However, for decrypt the cipher
+	 * ends before the tag in the AUTHENC case (and authstop is
+	 * set to stop before the tag), but for GCM the cipher still
+	 * runs to the end of the buffer.  Not sure if this is
+	 * intentional or a firmware quirk, but it is required for
+	 * working tag validation with GCM decryption.
+	 */
+	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
+	    V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
+	    V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
+	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
+	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) |
+	    V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
+	    V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
+	    V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
+
+	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
+	hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response);
+	crwr->sec_cpl.seqno_numivs = htobe32(
+	    V_SCMD_SEQ_NO_CTRL(0) |
+	    V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) |
+	    V_SCMD_ENC_DEC_CTRL(op_type) |
+	    V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
+	    V_SCMD_CIPH_MODE(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
+	    V_SCMD_AUTH_MODE(CHCR_SCMD_AUTH_MODE_GHASH) |
+	    V_SCMD_HMAC_CTRL(hmac_ctrl) |
+	    V_SCMD_IV_SIZE(iv_len / 2) |
+	    V_SCMD_NUM_IVS(0));
+	crwr->sec_cpl.ivgen_hdrlen = htobe32(
+	    V_SCMD_IV_GEN_CTRL(0) |
+	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
+	    V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
+
+	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
+	memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len);
+	dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
+	memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);
+
+	dst = (char *)(crwr + 1) + kctx_len;
+	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
+	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
+	if (iv_loc == IV_IMMEDIATE) {
+		memcpy(dst, iv, iv_len);
+		dst += iv_len;
+	}
+	if (imm_len != 0) {
+		if (crda->crd_len != 0) {
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crda->crd_skip, crda->crd_len, dst);
+			dst += crda->crd_len;
+		}
+		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
+		    crde->crd_len, dst);
+		dst += crde->crd_len;
+		if (op_type == CHCR_DECRYPT_OP)
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    crda->crd_inject, hash_size_in_response, dst);
+	} else
+		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
+
+	/* XXX: TODO backpressure */
+	t4_wrq_tx(sc->adapter, wr);
+
+	return (0);
+}
+
+static int
+ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s,
+    struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
+{
+
+	/*
+	 * The updated IV to permit chained requests is at
+	 * cpl->data[2], but OCF doesn't permit chained requests.
+	 *
+	 * Note that the hardware should always verify the GMAC hash.
+	 */
+	return (error);
+}
+
+static void
+ccr_identify(driver_t *driver, device_t parent)
+{
+	struct adapter *sc;
+
+	sc = device_get_softc(parent);
+	if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE &&
+	    device_find_child(parent, "ccr", -1) == NULL)
+		device_add_child(parent, "ccr", -1);
+}
+
+static int
+ccr_probe(device_t dev)
+{
+
+	device_set_desc(dev, "Chelsio Crypto Accelerator");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static void
+ccr_sysctls(struct ccr_softc *sc)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *oid;
+	struct sysctl_oid_list *children;
+
+	ctx = device_get_sysctl_ctx(sc->dev);
+
+	/*
+	 * dev.ccr.X.
+	 */
+	oid = device_get_sysctl_tree(sc->dev);
+	children = SYSCTL_CHILDREN(oid);
+
+	/*
+	 * dev.ccr.X.stats.
+	 */
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
+	    NULL, "statistics");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
+	    &sc->stats_hmac, 0, "HMAC requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD,
+	    &sc->stats_blkcipher_encrypt, 0,
+	    "Cipher encryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD,
+	    &sc->stats_blkcipher_decrypt, 0,
+	    "Cipher decryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD,
+	    &sc->stats_authenc_encrypt, 0,
+	    "Combined AES+HMAC encryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD,
+	    &sc->stats_authenc_decrypt, 0,
+	    "Combined AES+HMAC decryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD,
+	    &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD,
+	    &sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
+	    &sc->stats_wr_nomem, 0, "Work request memory allocation failures");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
+	    &sc->stats_inflight, 0, "Requests currently pending");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
+	    &sc->stats_mac_error, 0, "MAC errors");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
+	    &sc->stats_pad_error, 0, "Padding errors");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD,
+	    &sc->stats_pad_error, 0, "Requests with invalid session ID");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD,
+	    &sc->stats_pad_error, 0, "Requests for which DMA mapping failed");
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD,
+	    &sc->stats_pad_error, 0, "Requests failed during queueing");
+}
+
+static int
+ccr_attach(device_t dev)
+{
+	struct ccr_softc *sc;
+	int32_t cid;
+
+	/*
+	 * TODO: Crypto requests will panic if the parent device isn't
+	 * initialized so that the queues are up and running.  Need to
+	 * figure out how to handle that correctly, maybe just reject
+	 * requests if the adapter isn't fully initialized?
+	 */
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+	sc->adapter = device_get_softc(device_get_parent(dev));
+	sc->txq = &sc->adapter->sge.ctrlq[0];
+	sc->rxq = &sc->adapter->sge.rxq[0];
+	cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
+	if (cid < 0) {
+		device_printf(dev, "could not get crypto driver id\n");
+		return (ENXIO);
+	}
+	sc->cid = cid;
+	sc->adapter->ccr_softc = sc;
+
+	/* XXX: TODO? */
+	sc->tx_channel_id = 0;
+
+	mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
+	sc->sg_crp = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
+	sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
+	sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK);
+	ccr_sysctls(sc);
+
+	crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0);
+	crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0);
+	crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0);
+	crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0);
+	crypto_register(cid, CRYPTO_AES_CBC, 0, 0);
+	crypto_register(cid, CRYPTO_AES_ICM, 0, 0);
+	crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
+	crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
+	crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
+	crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
+	crypto_register(cid, CRYPTO_AES_XTS, 0, 0);
+	return (0);
+}
+
+static int
+ccr_detach(device_t dev)
+{
+	struct ccr_softc *sc;
+	int i;
+
+	sc = device_get_softc(dev);
+
+	mtx_lock(&sc->lock);
+	for (i = 0; i < sc->nsessions; i++) {
+		if (sc->sessions[i].active || sc->sessions[i].pending != 0) {
+			mtx_unlock(&sc->lock);
+			return (EBUSY);
+		}
+	}
+	sc->detaching = true;
+	mtx_unlock(&sc->lock);
+
+	crypto_unregister_all(sc->cid);
+	free(sc->sessions, M_CCR);
+	mtx_destroy(&sc->lock);
+	sglist_free(sc->sg_dsgl);
+	sglist_free(sc->sg_ulptx);
+	sglist_free(sc->sg_crp);
+	sc->adapter->ccr_softc = NULL;
+	return (0);
+}
+
+static void
+ccr_copy_partial_hash(void *dst, int cri_alg, union authctx *auth_ctx)
+{
+	uint32_t *u32;
+	uint64_t *u64;
+	u_int i;
+
+	u32 = (uint32_t *)dst;
+	u64 = (uint64_t *)dst;
+	switch (cri_alg) {
+	case CRYPTO_SHA1_HMAC:
+		for (i = 0; i < SHA1_HASH_LEN / 4; i++)
+			u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]);
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		for (i = 0; i < SHA2_256_HASH_LEN / 4; i++)
+			u32[i] = htobe32(auth_ctx->sha256ctx.state[i]);
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		for (i = 0; i < SHA2_512_HASH_LEN / 8; i++)
+			u64[i] = htobe64(auth_ctx->sha384ctx.state[i]);
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		for (i = 0; i < SHA2_512_HASH_LEN / 8; i++)
+			u64[i] = htobe64(auth_ctx->sha512ctx.state[i]);
+		break;
+	}
+}
+
+static void
+ccr_init_hmac_digest(struct ccr_session *s, int cri_alg, char *key,
+    int klen)
+{
+	union authctx auth_ctx;
+	struct auth_hash *axf;
+	u_int i;
+
+	/*
+	 * If the key is larger than the block size, use the digest of
+	 * the key as the key instead.
+	 */
+	axf = s->hmac.auth_hash;
+	klen /= 8;
+	if (klen > axf->blocksize) {
+		axf->Init(&auth_ctx);
+		axf->Update(&auth_ctx, key, klen);
+		axf->Final(s->hmac.ipad, &auth_ctx);
+		klen = axf->hashsize;
+	} else
+		memcpy(s->hmac.ipad, key, klen);
+
+	memset(s->hmac.ipad + klen, 0, axf->blocksize);
+	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
+
+	for (i = 0; i < axf->blocksize; i++) {
+		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
+		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
+	}
+
+	/*
+	 * Hash the raw ipad and opad and store the partial result in
+	 * the same buffer.
+	 */
+	axf->Init(&auth_ctx);
+	axf->Update(&auth_ctx, s->hmac.ipad, axf->blocksize);
+	ccr_copy_partial_hash(s->hmac.ipad, cri_alg, &auth_ctx);
+
+	axf->Init(&auth_ctx);
+	axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize);
+	ccr_copy_partial_hash(s->hmac.opad, cri_alg, &auth_ctx);
+}
+
+/*
+ * Borrowed from AES_GMAC_Setkey().
+ */
+static void
+ccr_init_gmac_hash(struct ccr_session *s, char *key, int klen)
+{
+	static char zeroes[GMAC_BLOCK_LEN];
+	uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)];
+	int rounds;
+
+	rounds = rijndaelKeySetupEnc(keysched, key, klen);
+	rijndaelEncrypt(keysched, rounds, zeroes, s->gmac.ghash_h);
+}
+
+static int
+ccr_aes_check_keylen(int alg, int klen)
+{
+
+	switch (klen) {
+	case 128:
+	case 192:
+		if (alg == CRYPTO_AES_XTS)
+			return (EINVAL);
+		break;
+	case 256:
+		break;
+	case 512:
+		if (alg != CRYPTO_AES_XTS)
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (0);
+}
+
+/*
+ * Borrowed from cesa_prep_aes_key().  We should perhaps have a public
+ * function to generate this instead.
+ *
+ * NB: The crypto engine wants the words in the decryption key in reverse
+ * order.
+ */
+static void
+ccr_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits)
+{
+	uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
+	uint32_t *dkey;
+	int i;
+
+	rijndaelKeySetupEnc(ek, enc_key, kbits);
+	dkey = dec_key;
+	dkey += (kbits / 8) / 4;
+
+	switch (kbits) {
+	case 128:
+		for (i = 0; i < 4; i++)
+			*--dkey = htobe32(ek[4 * 10 + i]);
+		break;
+	case 192:
+		for (i = 0; i < 2; i++)
+			*--dkey = htobe32(ek[4 * 11 + 2 + i]);
+		for (i = 0; i < 4; i++)
+			*--dkey = htobe32(ek[4 * 12 + i]);
+		break;
+	case 256:
+		for (i = 0; i < 4; i++)
+			*--dkey = htobe32(ek[4 * 13 + i]);
+		for (i = 0; i < 4; i++)
+			*--dkey = htobe32(ek[4 * 14 + i]);
+		break;
+	}
+	MPASS(dkey == dec_key);
+}
+
+static void
+ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
+{
+	unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size;
+	unsigned int opad_present;
+
+	if (alg == CRYPTO_AES_XTS)
+		kbits = klen / 2;
+	else
+		kbits = klen;
+	switch (kbits) {
+	case 128:
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+		break;
+	case 192:
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+		break;
+	case 256:
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+		break;
+	default:
+		panic("should not get here");
+	}
+
+	s->blkcipher.key_len = klen / 8;
+	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
+	switch (alg) {
+	case CRYPTO_AES_CBC:
+	case CRYPTO_AES_XTS:
+		ccr_aes_getdeckey(s->blkcipher.deckey, key, kbits);
+		break;
+	}
+
+	kctx_len = roundup2(s->blkcipher.key_len, 16);
+	switch (s->mode) {
+	case AUTHENC:
+		mk_size = s->hmac.mk_size;
+		opad_present = 1;
+		iopad_size = roundup2(s->hmac.partial_digest_len, 16);
+		kctx_len += iopad_size * 2;
+		break;
+	case GCM:
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
+		opad_present = 0;
+		kctx_len += GMAC_BLOCK_LEN;
+		break;
+	default:
+		mk_size = CHCR_KEYCTX_NO_KEY;
+		opad_present = 0;
+		break;
+	}
+	kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
+	s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
+	    V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) |
+	    V_KEY_CONTEXT_OPAD_PRESENT(opad_present) |
+	    V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) |
+	    V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1));
+}
+
+static int
+ccr_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
+{
+	struct ccr_softc *sc;
+	struct ccr_session *s;
+	struct auth_hash *auth_hash;
+	struct cryptoini *c, *hash, *cipher;
+	unsigned int auth_mode, cipher_mode, iv_len, mk_size;
+	unsigned int partial_digest_len;
+	int error, i, sess;
+	bool gcm_hash;
+
+	if (sidp == NULL || cri == NULL)
+		return (EINVAL);
+
+	gcm_hash = false;
+	cipher = NULL;
+	hash = NULL;
+	auth_hash = NULL;
+	auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
+	cipher_mode = CHCR_SCMD_CIPHER_MODE_NOP;
+	iv_len = 0;
+	mk_size = 0;
+	partial_digest_len = 0;
+	for (c = cri; c != NULL; c = c->cri_next) {
+		switch (c->cri_alg) {
+		case CRYPTO_SHA1_HMAC:
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+		case CRYPTO_AES_128_NIST_GMAC:
+		case CRYPTO_AES_192_NIST_GMAC:
+		case CRYPTO_AES_256_NIST_GMAC:
+			if (hash)
+				return (EINVAL);
+			hash = c;
+			switch (c->cri_alg) {
+			case CRYPTO_SHA1_HMAC:
+				auth_hash = &auth_hash_hmac_sha1;
+				auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
+				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+				partial_digest_len = SHA1_HASH_LEN;
+				break;
+			case CRYPTO_SHA2_256_HMAC:
+				auth_hash = &auth_hash_hmac_sha2_256;
+				auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
+				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+				partial_digest_len = SHA2_256_HASH_LEN;
+				break;
+			case CRYPTO_SHA2_384_HMAC:
+				auth_hash = &auth_hash_hmac_sha2_384;
+				auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
+				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+				partial_digest_len = SHA2_512_HASH_LEN;
+				break;
+			case CRYPTO_SHA2_512_HMAC:
+				auth_hash = &auth_hash_hmac_sha2_512;
+				auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
+				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+				partial_digest_len = SHA2_512_HASH_LEN;
+				break;
+			case CRYPTO_AES_128_NIST_GMAC:
+			case CRYPTO_AES_192_NIST_GMAC:
+			case CRYPTO_AES_256_NIST_GMAC:
+				gcm_hash = true;
+				auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
+				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
+				break;
+			}
+			break;
+		case CRYPTO_AES_CBC:
+		case CRYPTO_AES_ICM:
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_XTS:
+			if (cipher)
+				return (EINVAL);
+			cipher = c;
+			switch (c->cri_alg) {
+			case CRYPTO_AES_CBC:
+				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
+				iv_len = AES_BLOCK_LEN;
+				break;
+			case CRYPTO_AES_ICM:
+				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
+				iv_len = AES_BLOCK_LEN;
+				break;
+			case CRYPTO_AES_NIST_GCM_16:
+				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_GCM;
+				iv_len = AES_GCM_IV_LEN;
+				break;
+			case CRYPTO_AES_XTS:
+				cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
+				iv_len = AES_BLOCK_LEN;
+				break;
+			}
+			if (c->cri_key != NULL) {
+				error = ccr_aes_check_keylen(c->cri_alg,
+				    c->cri_klen);
+				if (error)
+					return (error);
+			}
+			break;
+		default:
+			return (EINVAL);
+		}
+	}
+	if (gcm_hash != (cipher_mode == CHCR_SCMD_CIPHER_MODE_AES_GCM))
+		return (EINVAL);
+	if (hash == NULL && cipher == NULL)
+		return (EINVAL);
+	if (hash != NULL && hash->cri_key == NULL)
+		return (EINVAL);
+
+	sc = device_get_softc(dev);
+	mtx_lock(&sc->lock);
+	if (sc->detaching) {
+		mtx_unlock(&sc->lock);
+		return (ENXIO);
+	}
+	sess = -1;
+	for (i = 0; i < sc->nsessions; i++) {
+		if (!sc->sessions[i].active && sc->sessions[i].pending == 0) {
+			sess = i;
+			break;
+		}
+	}
+	if (sess == -1) {
+		s = malloc(sizeof(*s) * (sc->nsessions + 1), M_CCR,
+		    M_NOWAIT | M_ZERO);
+		if (s == NULL) {
+			mtx_unlock(&sc->lock);
+			return (ENOMEM);
+		}
+		if (sc->sessions != NULL)
+			memcpy(s, sc->sessions, sizeof(*s) * sc->nsessions);
+		sess = sc->nsessions;
+		free(sc->sessions, M_CCR);
+		sc->sessions = s;
+		sc->nsessions++;
+	}
+
+	s = &sc->sessions[sess];
+
+	if (gcm_hash)
+		s->mode = GCM;
+	else if (hash != NULL && cipher != NULL)
+		s->mode = AUTHENC;
+	else if (hash != NULL)
+		s->mode = HMAC;
+	else {
+		MPASS(cipher != NULL);
+		s->mode = BLKCIPHER;
+	}
+	if (gcm_hash) {
+		if (hash->cri_mlen == 0)
+			s->gmac.hash_len = AES_GMAC_HASH_LEN;
+		else
+			s->gmac.hash_len = hash->cri_mlen;
+		ccr_init_gmac_hash(s, hash->cri_key, hash->cri_klen);
+	} else if (hash != NULL) {
+		s->hmac.auth_hash = auth_hash;
+		s->hmac.auth_mode = auth_mode;
+		s->hmac.mk_size = mk_size;
+		s->hmac.partial_digest_len = partial_digest_len;
+		if (hash->cri_mlen == 0)
+			s->hmac.hash_len = auth_hash->hashsize;
+		else
+			s->hmac.hash_len = hash->cri_mlen;
+		ccr_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
+		    hash->cri_klen);
+	}
+	if (cipher != NULL) {
+		s->blkcipher.cipher_mode = cipher_mode;
+		s->blkcipher.iv_len = iv_len;
+		if (cipher->cri_key != NULL)
+			ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
+			    cipher->cri_klen);
+	}
+
+	s->active = true;
+	mtx_unlock(&sc->lock);
+
+	*sidp = sess;
+	return (0);
+}
+
+static int
+ccr_freesession(device_t dev, uint64_t tid)
+{
+	struct ccr_softc *sc;
+	uint32_t sid;
+	int error;
+
+	sc = device_get_softc(dev);
+	sid = CRYPTO_SESID2LID(tid);
+	mtx_lock(&sc->lock);
+	if (sid >= sc->nsessions || !sc->sessions[sid].active)
+		error = EINVAL;
+	else {
+		if (sc->sessions[sid].pending != 0)
+			device_printf(dev,
+			    "session %d freed with %d pending requests\n", sid,
+			    sc->sessions[sid].pending);
+		sc->sessions[sid].active = false;
+		error = 0;
+	}
+	mtx_unlock(&sc->lock);
+	return (error);
+}
+
+static int
+ccr_process(device_t dev, struct cryptop *crp, int hint)
+{
+	struct ccr_softc *sc;
+	struct ccr_session *s;
+	struct cryptodesc *crd, *crda, *crde;
+	uint32_t sid;
+	int error;
+
+	if (crp == NULL)
+		return (EINVAL);
+
+	crd = crp->crp_desc;
+	sid = CRYPTO_SESID2LID(crp->crp_sid);
+	sc = device_get_softc(dev);
+	mtx_lock(&sc->lock);
+	if (sid >= sc->nsessions || !sc->sessions[sid].active) {
+		sc->stats_bad_session++;
+		error = EINVAL;
+		goto out;
+	}
+
+	error = ccr_populate_sglist(sc->sg_crp, crp);
+	if (error) {
+		sc->stats_sglist_error++;
+		goto out;
+	}
+
+	s = &sc->sessions[sid];
+	switch (s->mode) {
+	case HMAC:
+		if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
+			ccr_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
+			    crd->crd_klen);
+		error = ccr_hmac(sc, sid, s, crp);
+		if (error == 0)
+			sc->stats_hmac++;
+		break;
+	case BLKCIPHER:
+		if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
+			error = ccr_aes_check_keylen(crd->crd_alg,
+			    crd->crd_klen);
+			if (error)
+				break;
+			ccr_aes_setkey(s, crd->crd_alg, crd->crd_key,
+			    crd->crd_klen);
+		}
+		error = ccr_blkcipher(sc, sid, s, crp);
+		if (error == 0) {
+			if (crd->crd_flags & CRD_F_ENCRYPT)
+				sc->stats_blkcipher_encrypt++;
+			else
+				sc->stats_blkcipher_decrypt++;
+		}
+		break;
+	case AUTHENC:
+		error = 0;
+		switch (crd->crd_alg) {
+		case CRYPTO_AES_CBC:
+		case CRYPTO_AES_ICM:
+		case CRYPTO_AES_XTS:
+			/* Only encrypt-then-authenticate supported. */
+			crde = crd;
+			crda = crd->crd_next;
+			if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
+				error = EINVAL;
+				break;
+			}
+			break;
+		default:
+			crda = crd;
+			crde = crd->crd_next;
+			if (crde->crd_flags & CRD_F_ENCRYPT) {
+				error = EINVAL;
+				break;
+			}
+			break;
+		}
+		if (error)
+			break;
+		if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
+			ccr_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
+			    crda->crd_klen);
+		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
+			error = ccr_aes_check_keylen(crde->crd_alg,
+			    crde->crd_klen);
+			if (error)
+				break;
+			ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
+			    crde->crd_klen);
+		}
+		error = ccr_authenc(sc, sid, s, crp, crda, crde);
+		if (error == 0) {
+			if (crde->crd_flags & CRD_F_ENCRYPT)
+				sc->stats_authenc_encrypt++;
+			else
+				sc->stats_authenc_decrypt++;
+		}
+		break;
+	case GCM:
+		error = 0;
+		if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
+			crde = crd;
+			crda = crd->crd_next;
+		} else {
+			crda = crd;
+			crde = crd->crd_next;
+		}
+		if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
+			ccr_init_gmac_hash(s, crda->crd_key, crda->crd_klen);
+		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
+			error = ccr_aes_check_keylen(crde->crd_alg,
+			    crde->crd_klen);
+			if (error)
+				break;
+			ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
+			    crde->crd_klen);
+		}
+		error = ccr_gcm(sc, sid, s, crp, crda, crde);
+		if (error == 0) {
+			if (crde->crd_flags & CRD_F_ENCRYPT)
+				sc->stats_gcm_encrypt++;
+			else
+				sc->stats_gcm_decrypt++;
+		}
+		break;
+	}
+
+	if (error == 0) {
+		s->pending++;
+		sc->stats_inflight++;
+	} else
+		sc->stats_process_error++;
+
+out:
+	mtx_unlock(&sc->lock);
+
+	if (error) {
+		crp->crp_etype = error;
+		crypto_done(crp);
+	}
+
+	return (0);
+}
+
+static int
+do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
+    struct mbuf *m)
+{
+	struct ccr_softc *sc = iq->adapter->ccr_softc;
+	struct ccr_session *s;
+	const struct cpl_fw6_pld *cpl;
+	struct cryptop *crp;
+	uint32_t sid, status;
+	int error;
+
+	if (m != NULL)
+		cpl = mtod(m, const void *);
+	else
+		cpl = (const void *)(rss + 1);
+
+	crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]);
+	sid = CRYPTO_SESID2LID(crp->crp_sid);
+	status = be64toh(cpl->data[0]);
+	if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status))
+		error = EBADMSG;
+	else
+		error = 0;
+
+	mtx_lock(&sc->lock);
+	MPASS(sid < sc->nsessions);
+	s = &sc->sessions[sid];
+	s->pending--;
+	sc->stats_inflight--;
+
+	switch (s->mode) {
+	case HMAC:
+		error = ccr_hmac_done(sc, s, crp, cpl, error);
+		break;
+	case BLKCIPHER:
+		error = ccr_blkcipher_done(sc, s, crp, cpl, error);
+		break;
+	case AUTHENC:
+		error = ccr_authenc_done(sc, s, crp, cpl, error);
+		break;
+	case GCM:
+		error = ccr_gcm_done(sc, s, crp, cpl, error);
+		break;
+	}
+
+	if (error == EBADMSG) {
+		if (CHK_MAC_ERR_BIT(status))
+			sc->stats_mac_error++;
+		if (CHK_PAD_ERR_BIT(status))
+			sc->stats_pad_error++;
+	}
+	mtx_unlock(&sc->lock);
+	crp->crp_etype = error;
+	crypto_done(crp);
+	m_freem(m);
+	return (0);
+}
+
+static int
+ccr_modevent(module_t mod, int cmd, void *arg)
+{
+
+	switch (cmd) {
+	case MOD_LOAD:
+		t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld);
+		return (0);
+	case MOD_UNLOAD:
+		t4_register_cpl_handler(CPL_FW6_PLD, NULL);
+		return (0);
+	default:
+		return (EOPNOTSUPP);
+	}
+}
+
+static device_method_t ccr_methods[] = {
+	DEVMETHOD(device_identify,	ccr_identify),
+	DEVMETHOD(device_probe,		ccr_probe),
+	DEVMETHOD(device_attach,	ccr_attach),
+	DEVMETHOD(device_detach,	ccr_detach),
+
+	DEVMETHOD(cryptodev_newsession,	ccr_newsession),
+	DEVMETHOD(cryptodev_freesession, ccr_freesession),
+	DEVMETHOD(cryptodev_process,	ccr_process),
+
+	DEVMETHOD_END
+};
+
+static driver_t ccr_driver = {
+	"ccr",
+	ccr_methods,
+	sizeof(struct ccr_softc)
+};
+
+static devclass_t ccr_devclass;
+
+DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL);
+MODULE_VERSION(ccr, 1);
+MODULE_DEPEND(ccr, crypto, 1, 1, 1);
+MODULE_DEPEND(ccr, t6nex, 1, 1, 1);
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.h b/sys/dev/cxgbe/crypto/t4_crypto.h
new file mode 100644
index 000000000000..811e0b7026b3
--- /dev/null
+++ b/sys/dev/cxgbe/crypto/t4_crypto.h
@@ -0,0 +1,186 @@
+/*-
+ * Copyright (c) 2017 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __T4_CRYPTO_H__
+#define	__T4_CRYPTO_H__
+
+/* From chr_core.h */
+#define PAD_ERROR_BIT		1
+#define CHK_PAD_ERR_BIT(x)	(((x) >> PAD_ERROR_BIT) & 1)
+
+#define MAC_ERROR_BIT		0
+#define CHK_MAC_ERR_BIT(x)	(((x) >> MAC_ERROR_BIT) & 1)
+#define MAX_SALT                4
+
+struct _key_ctx {
+	__be32 ctx_hdr;
+	u8 salt[MAX_SALT];
+	__be64 reserverd;
+	unsigned char key[0];
+};
+
+struct chcr_wr {
+	struct fw_crypto_lookaside_wr wreq;
+	struct ulp_txpkt ulptx;
+	struct ulptx_idata sc_imm;
+	struct cpl_tx_sec_pdu sec_cpl;
+	struct _key_ctx key_ctx;
+};
+
+/* From chr_algo.h */
+
+/* Crypto key context */
+#define S_KEY_CONTEXT_CTX_LEN           24
+#define M_KEY_CONTEXT_CTX_LEN           0xff
+#define V_KEY_CONTEXT_CTX_LEN(x)        ((x) << S_KEY_CONTEXT_CTX_LEN)
+#define G_KEY_CONTEXT_CTX_LEN(x) \
+	(((x) >> S_KEY_CONTEXT_CTX_LEN) & M_KEY_CONTEXT_CTX_LEN)
+
+#define S_KEY_CONTEXT_DUAL_CK      12
+#define M_KEY_CONTEXT_DUAL_CK      0x1
+#define V_KEY_CONTEXT_DUAL_CK(x)   ((x) << S_KEY_CONTEXT_DUAL_CK)
+#define G_KEY_CONTEXT_DUAL_CK(x)   \
+(((x) >> S_KEY_CONTEXT_DUAL_CK) & M_KEY_CONTEXT_DUAL_CK)
+#define F_KEY_CONTEXT_DUAL_CK      V_KEY_CONTEXT_DUAL_CK(1U)
+
+#define S_KEY_CONTEXT_OPAD_PRESENT      11
+#define M_KEY_CONTEXT_OPAD_PRESENT      0x1
+#define V_KEY_CONTEXT_OPAD_PRESENT(x)   ((x) << S_KEY_CONTEXT_OPAD_PRESENT)
+#define G_KEY_CONTEXT_OPAD_PRESENT(x)   \
+	(((x) >> S_KEY_CONTEXT_OPAD_PRESENT) & \
+	 M_KEY_CONTEXT_OPAD_PRESENT)
+#define F_KEY_CONTEXT_OPAD_PRESENT      V_KEY_CONTEXT_OPAD_PRESENT(1U)
+
+#define S_KEY_CONTEXT_SALT_PRESENT      10
+#define M_KEY_CONTEXT_SALT_PRESENT      0x1
+#define V_KEY_CONTEXT_SALT_PRESENT(x)   ((x) << S_KEY_CONTEXT_SALT_PRESENT)
+#define G_KEY_CONTEXT_SALT_PRESENT(x)   \
+	(((x) >> S_KEY_CONTEXT_SALT_PRESENT) & \
+	 M_KEY_CONTEXT_SALT_PRESENT)
+#define F_KEY_CONTEXT_SALT_PRESENT      V_KEY_CONTEXT_SALT_PRESENT(1U)
+
+#define S_KEY_CONTEXT_CK_SIZE           6
+#define M_KEY_CONTEXT_CK_SIZE           0xf
+#define V_KEY_CONTEXT_CK_SIZE(x)        ((x) << S_KEY_CONTEXT_CK_SIZE)
+#define G_KEY_CONTEXT_CK_SIZE(x)        \
+	(((x) >> S_KEY_CONTEXT_CK_SIZE) & M_KEY_CONTEXT_CK_SIZE)
+
+#define S_KEY_CONTEXT_MK_SIZE           2
+#define M_KEY_CONTEXT_MK_SIZE           0xf
+#define V_KEY_CONTEXT_MK_SIZE(x)        ((x) << S_KEY_CONTEXT_MK_SIZE)
+#define G_KEY_CONTEXT_MK_SIZE(x)        \
+	(((x) >> S_KEY_CONTEXT_MK_SIZE) & M_KEY_CONTEXT_MK_SIZE)
+
+#define S_KEY_CONTEXT_VALID     0
+#define M_KEY_CONTEXT_VALID     0x1
+#define V_KEY_CONTEXT_VALID(x)  ((x) << S_KEY_CONTEXT_VALID)
+#define G_KEY_CONTEXT_VALID(x)  \
+	(((x) >> S_KEY_CONTEXT_VALID) & \
+	 M_KEY_CONTEXT_VALID)
+#define F_KEY_CONTEXT_VALID     V_KEY_CONTEXT_VALID(1U)
+
+#define CHCR_HASH_MAX_DIGEST_SIZE 64
+
+#define DUMMY_BYTES 16
+
+#define TRANSHDR_SIZE(kctx_len)\
+	(sizeof(struct chcr_wr) +\
+	 kctx_len)
+#define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \
+	(TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\
+	 sizeof(struct cpl_rx_phys_dsgl))
+#define HASH_TRANSHDR_SIZE(kctx_len)\
+	(TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES)
+
+#define CRYPTO_MAX_IMM_TX_PKT_LEN 256
+
+struct phys_sge_pairs {
+	__be16 len[8];
+	__be64 addr[8];
+};
+
+/* From chr_crypto.h */
+#define CHCR_AES_MAX_KEY_LEN  (AES_XTS_MAX_KEY)
+#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */
+
+#define CHCR_ENCRYPT_OP 0
+#define CHCR_DECRYPT_OP 1
+
+#define CHCR_SCMD_PROTO_VERSION_GENERIC 4
+
+#define CHCR_SCMD_CIPHER_MODE_NOP               0
+#define CHCR_SCMD_CIPHER_MODE_AES_CBC           1
+#define CHCR_SCMD_CIPHER_MODE_AES_GCM           2
+#define CHCR_SCMD_CIPHER_MODE_AES_CTR           3
+#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES       4
+#define CHCR_SCMD_CIPHER_MODE_AES_XTS           6
+#define CHCR_SCMD_CIPHER_MODE_AES_CCM           7
+
+#define CHCR_SCMD_AUTH_MODE_NOP             0
+#define CHCR_SCMD_AUTH_MODE_SHA1            1
+#define CHCR_SCMD_AUTH_MODE_SHA224          2
+#define CHCR_SCMD_AUTH_MODE_SHA256          3
+#define CHCR_SCMD_AUTH_MODE_GHASH           4
+#define CHCR_SCMD_AUTH_MODE_SHA512_224      5
+#define CHCR_SCMD_AUTH_MODE_SHA512_256      6
+#define CHCR_SCMD_AUTH_MODE_SHA512_384      7
+#define CHCR_SCMD_AUTH_MODE_SHA512_512      8
+#define CHCR_SCMD_AUTH_MODE_CBCMAC          9
+#define CHCR_SCMD_AUTH_MODE_CMAC            10
+
+#define CHCR_SCMD_HMAC_CTRL_NOP             0
+#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC        1
+#define CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366   2
+#define CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT     3
+#define CHCR_SCMD_HMAC_CTRL_PL1		    4
+#define CHCR_SCMD_HMAC_CTRL_PL2		    5
+#define CHCR_SCMD_HMAC_CTRL_PL3		    6
+#define CHCR_SCMD_HMAC_CTRL_DIV2	    7
+
+/* This are not really mac key size. They are intermediate values
+ * of sha engine and its size
+ */
+#define CHCR_KEYCTX_MAC_KEY_SIZE_128        0
+#define CHCR_KEYCTX_MAC_KEY_SIZE_160        1
+#define CHCR_KEYCTX_MAC_KEY_SIZE_192        2
+#define CHCR_KEYCTX_MAC_KEY_SIZE_256        3
+#define CHCR_KEYCTX_MAC_KEY_SIZE_512        4
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128     0
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_192     1
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_256     2
+#define CHCR_KEYCTX_NO_KEY                  15
+
+#define IV_NOP                  0
+#define IV_IMMEDIATE            1
+#define IV_DSGL			2
+
+#define CHCR_HASH_MAX_BLOCK_SIZE_64  64
+#define CHCR_HASH_MAX_BLOCK_SIZE_128 128
+
+#endif /* !__T4_CRYPTO_H__ */
diff --git a/sys/modules/cxgbe/Makefile b/sys/modules/cxgbe/Makefile
index 434be3ba6a98..e3265d5ba5da 100644
--- a/sys/modules/cxgbe/Makefile
+++ b/sys/modules/cxgbe/Makefile
@@ -17,6 +17,7 @@ SUBDIR+=	t6_firmware
 SUBDIR+=	${_tom}
 SUBDIR+=	${_iw_cxgbe}
 SUBDIR+=	${_cxgbei}
+SUBDIR+=	ccr
 
 .if ${MACHINE_CPUARCH} == "amd64"
 _tom=	tom
diff --git a/sys/modules/cxgbe/ccr/Makefile b/sys/modules/cxgbe/ccr/Makefile
new file mode 100644
index 000000000000..73d34762719d
--- /dev/null
+++ b/sys/modules/cxgbe/ccr/Makefile
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+CXGBE=	${SRCTOP}/sys/dev/cxgbe
+.PATH: ${CXGBE}/crypto
+
+KMOD=	ccr
+
+SRCS=	t4_crypto.c
+SRCS+=	bus_if.h
+SRCS+=	device_if.h
+SRCS+=	cryptodev_if.h
+SRCS+=	pci_if.h
+
+CFLAGS+= -I${CXGBE}
+
+MFILES=	kern/bus_if.m kern/device_if.m opencrypto/cryptodev_if.m \
+	dev/pci/pci_if.m
+
+.include <bsd.kmod.mk>
diff --git a/sys/powerpc/conf/NOTES b/sys/powerpc/conf/NOTES
index 553dee5a6dac..86bc291d3668 100644
--- a/sys/powerpc/conf/NOTES
+++ b/sys/powerpc/conf/NOTES
@@ -76,6 +76,7 @@ device		adm1030		# Apple G4 MDD fan controller
 # Devices we don't want to deal with
 
 nodevice	bktr
+nodevice	ccr
 nodevice	cxgbe		# XXX: builds on powerpc64 only.
 nodevice	cxgbev
 nodevice	fdc

From d5a5e50d3bd2426a5b6f2fc11f6750234f2d7d48 Mon Sep 17 00:00:00 2001
From: Jonathan Anderson <jonathan@FreeBSD.org>
Date: Wed, 17 May 2017 22:51:28 +0000
Subject: [PATCH 27/73] Allow rtld direct-exec to take a file descriptor.

When executing rtld directly, allow a file descriptor to be explicitly
specified rather than opened from the given path. This, together with the
LD_LIBRARY_PATH_FDS environment variable, allows dynamically-linked
applications to be executed from within capability mode.

Also add some rudimentary argument parsing (without pulling in getopt or
the like) to accept this file descriptor, a help (-h) option and a basic
usage string.

Reviewed by:	kib
Sponsored by:	NSERC, RDC
Differential Revision:	https://reviews.freebsd.org/D10751
---
 libexec/rtld-elf/rtld.c | 126 ++++++++++++++++++++++++++++++++++------
 1 file changed, 109 insertions(+), 17 deletions(-)

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index e14a41a09689..7025acb96dfb 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -115,8 +115,10 @@ static void objlist_push_head(Objlist *, Obj_Entry *);
 static void objlist_push_tail(Objlist *, Obj_Entry *);
 static void objlist_put_after(Objlist *, Obj_Entry *, Obj_Entry *);
 static void objlist_remove(Objlist *, Obj_Entry *);
+static int parse_args(char* argv[], int argc, bool *use_pathp, int *fdp);
 static int parse_integer(const char *);
 static void *path_enumerate(const char *, path_enum_proc, void *);
+static void print_usage(const char *argv0);
 static void release_object(Obj_Entry *);
 static int relocate_object_dag(Obj_Entry *root, bool bind_now,
     Obj_Entry *rtldobj, int flags, RtldLockState *lockstate);
@@ -350,9 +352,9 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     char **argv, *argv0, **env, **envp, *kexecpath, *library_path_rpath;
     caddr_t imgentry;
     char buf[MAXPATHLEN];
-    int argc, fd, i, mib[2], phnum;
+    int argc, fd, i, mib[2], phnum, rtld_argc;
     size_t len;
-    bool dir_enable;
+    bool dir_enable, explicit_fd, search_in_path;
 
     /*
      * On entry, the dynamic linker itself has not been relocated yet.
@@ -428,15 +430,19 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	    }
 	    dbg("opening main program in direct exec mode");
 	    if (argc >= 2) {
-		argv0 = argv[1];
-		fd = open(argv0, O_RDONLY | O_CLOEXEC | O_VERIFY);
+		rtld_argc = parse_args(argv, argc, &search_in_path, &fd);
+		argv0 = argv[rtld_argc];
+		explicit_fd = (fd != -1);
+		if (!explicit_fd)
+		    fd = open(argv0, O_RDONLY | O_CLOEXEC | O_VERIFY);
 		if (fd == -1) {
 		    rtld_printf("Opening %s: %s\n", argv0,
 		      rtld_strerror(errno));
 		    rtld_die();
 		}
 		if (fstat(fd, &st) == -1) {
-		    rtld_printf("Stat %s: %s\n", argv0,
+		    _rtld_error("failed to fstat FD %d (%s): %s", fd,
+		      explicit_fd ? "user-provided descriptor" : argv0,
 		      rtld_strerror(errno));
 		    rtld_die();
 		}
@@ -469,26 +475,23 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 
 		/*
 		 * For direct exec mode, argv[0] is the interpreter
-		 * name, we must remove it and shift arguments left by
-		 * 1 before invoking binary main.  Since stack layout
+		 * name, we must remove it and shift arguments left
+		 * before invoking binary main.  Since stack layout
 		 * places environment pointers and aux vectors right
 		 * after the terminating NULL, we must shift
 		 * environment and aux as well.
-		 * XXX Shift will be > 1 when options are implemented.
 		 */
+		main_argc = argc - rtld_argc;
+		for (i = 0; i <= main_argc; i++)
+		    argv[i] = argv[i + rtld_argc];
+		*argcp -= rtld_argc;
+		environ = env = envp = argv + main_argc + 1;
 		do {
-		    *argv = *(argv + 1);
-		    argv++;
-		} while (*argv != NULL);
-		*argcp -= 1;
-		main_argc = argc - 1;
-		environ = env = envp = argv;
-		do {
-		    *envp = *(envp + 1);
+		    *envp = *(envp + rtld_argc);
 		    envp++;
 		} while (*envp != NULL);
 		aux = auxp = (Elf_Auxinfo *)envp;
-		auxpf = (Elf_Auxinfo *)(envp + 1);
+		auxpf = (Elf_Auxinfo *)(envp + rtld_argc);
 		for (;; auxp++, auxpf++) {
 		    *auxp = *auxpf;
 		    if (auxp->a_type == AT_NULL)
@@ -5273,6 +5276,81 @@ symlook_init_from_req(SymLook *dst, const SymLook *src)
 }
 
 
+/*
+ * Parse a set of command-line arguments.
+ */
+static int
+parse_args(char* argv[], int argc, bool *use_pathp, int *fdp)
+{
+	const char *arg;
+	int fd, i, j, arglen;
+	char opt;
+
+	dbg("Parsing command-line arguments");
+	*use_pathp = false;
+	*fdp = -1;
+
+	for (i = 1; i < argc; i++ ) {
+		arg = argv[i];
+		dbg("argv[%d]: '%s'", i, arg);
+
+		/*
+		 * rtld arguments end with an explicit "--" or with the first
+		 * non-prefixed argument.
+		 */
+		if (strcmp(arg, "--") == 0) {
+			i++;
+			break;
+		}
+		if (arg[0] != '-')
+			break;
+
+		/*
+		 * All other arguments are single-character options that can
+		 * be combined, so we need to search through `arg` for them.
+		 */
+		arglen = strlen(arg);
+		for (j = 1; j < arglen; j++) {
+			opt = arg[j];
+			if (opt == 'h') {
+				print_usage(argv[0]);
+				rtld_die();
+			} else if (opt == 'f') {
+			/*
+			 * -f XX can be used to specify a descriptor for the
+			 * binary named at the command line (i.e., the later
+			 * argument will specify the process name but the
+			 * descriptor is what will actually be executed)
+			 */
+			if (j != arglen - 1) {
+				/* -f must be the last option in, e.g., -abcf */
+				_rtld_error("invalid options: %s", arg);
+				rtld_die();
+			}
+			i++;
+			fd = parse_integer(argv[i]);
+			if (fd == -1) {
+				_rtld_error("invalid file descriptor: '%s'",
+				    argv[i]);
+				rtld_die();
+			}
+			*fdp = fd;
+			break;
+			/* TODO:
+			} else if (opt == 'p') {
+				*use_pathp = true;
+			*/
+			} else {
+				rtld_printf("invalid argument: '%s'\n", arg);
+				print_usage(argv[0]);
+				rtld_die();
+			}
+		}
+	}
+
+	return (i);
+}
+
 /*
  * Parse a file descriptor number without pulling in more of libc (e.g. atoi).
  */
@@ -5300,6 +5378,20 @@ parse_integer(const char *str)
 	return (n);
 }
 
+void print_usage(const char *argv0)
+{
+
+	rtld_printf("Usage: %s [-h] [-f <FD>] [--] <binary> [<args>]\n"
+		"\n"
+		"Options:\n"
+		"  -h        Display this help message\n"
+		/* TODO: "  -p        Search in PATH for named binary\n" */
+		"  -f <FD>   Execute <FD> instead of searching for <binary>\n"
+		"  --        End of RTLD options\n"
+		"  <binary>  Name of process to execute\n"
+		"  <args>    Arguments to the executed process\n", argv0);
+}
+
 /*
  * Overrides for libc_pic-provided functions.
  */

From dc902dbd60bcd4074ccbbec68c969e224d17e2cb Mon Sep 17 00:00:00 2001
From: Jonathan Anderson <jonathan@FreeBSD.org>
Date: Thu, 18 May 2017 00:32:05 +0000
Subject: [PATCH 28/73] Fix some nroff syntax in rtld.1.

When I originally documented the LD_LIBRARY_PATH_FDS environment variable,
I used `.Ev` rather than `.It Ev` to introduce it; this led to the
documentation being embedded in the previous paragraph (LD_LIBRARY_PATH).
---
 libexec/rtld-elf/rtld.1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libexec/rtld-elf/rtld.1 b/libexec/rtld-elf/rtld.1
index 79f1478bb90d..4347eb018171 100644
--- a/libexec/rtld-elf/rtld.1
+++ b/libexec/rtld-elf/rtld.1
@@ -192,7 +192,7 @@ the directories specified by
 will be searched first
 followed by the set of built-in standard directories.
 This variable is unset for set-user-ID and set-group-ID programs.
-.Ev LD_LIBRARY_PATH_FDS
+.It Ev LD_LIBRARY_PATH_FDS
 A colon separated list of file descriptor numbers for library directories.
 This is intended for use within
 .Xr capsicum 4

From ea642042132a319db70fb6f1f1912a345fff9e6a Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 01:35:07 +0000
Subject: [PATCH 29/73] Make the `.gperf.c` suffix rule depend on
 fake-gperf.awk

Parameterize out fake-gperf.awk to avoid duplicating the path

MFC after:	2 weeks
Sponsored by:	Dell EMC Isilon
---
 usr.bin/getconf/Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/usr.bin/getconf/Makefile b/usr.bin/getconf/Makefile
index eaaf628aad37..ca0f0c04530c 100644
--- a/usr.bin/getconf/Makefile
+++ b/usr.bin/getconf/Makefile
@@ -13,8 +13,9 @@ CLEANFILES+=	confstr.c limits.c pathconf.c progenv.c sysconf.c \
 
 all:	conflicts
 
-.gperf.c:
-	LC_ALL=C awk -f ${.CURDIR}/fake-gperf.awk ${.IMPSRC} >${.TARGET}
+FAKE_GPERF=	${.CURDIR}/fake-gperf.awk
+.gperf.c: ${FAKE_GPERF}
+	LC_ALL=C awk -f ${FAKE_GPERF} ${.IMPSRC} >${.TARGET}
 
 .gperf.names:
 	LC_ALL=C awk '/^[_A-Z]/ { print; }' ${.IMPSRC} | \

From 20d90b10b1510dd503348ec04268adcffce35591 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 01:43:30 +0000
Subject: [PATCH 30/73] usr.bin/getconf: add some initial tests

Items tested via this commit are:
- Some basic POSIX constants.
- Some valid programming environments with -v.
- Some invalid programming environments via -v.

NOTE: this test makes assumptions about ILP32/LP32 vs LP64 that are
currently not true on all architectures to avoid hardcoding some
architectures in the tests. I'm working on improving getconf(1) to be
more sane about handling ILP32/LP32 vs LP64. Future commits are coming
soon to address this.

MFC after:	2 weeks
Tested with:	amd64, i386
Sponsored by:	Dell EMC Isilon
---
 etc/mtree/BSD.tests.dist              |   2 +
 usr.bin/getconf/Makefile              |   6 ++
 usr.bin/getconf/getconf.h             |   9 +-
 usr.bin/getconf/tests/Makefile        |  11 +++
 usr.bin/getconf/tests/arch_type.c     |  59 +++++++++++++
 usr.bin/getconf/tests/getconf_test.sh | 120 ++++++++++++++++++++++++++
 6 files changed, 206 insertions(+), 1 deletion(-)
 create mode 100644 usr.bin/getconf/tests/Makefile
 create mode 100644 usr.bin/getconf/tests/arch_type.c
 create mode 100755 usr.bin/getconf/tests/getconf_test.sh

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index f3861c801b3c..1a38ea811d04 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -622,6 +622,8 @@
         ..
         file2c
         ..
+        getconf
+        ..
         grep
         ..
         gzip
diff --git a/usr.bin/getconf/Makefile b/usr.bin/getconf/Makefile
index ca0f0c04530c..1670465a0878 100644
--- a/usr.bin/getconf/Makefile
+++ b/usr.bin/getconf/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG=	getconf
 
 SRCS=	confstr.c getconf.c limits.c pathconf.c progenv.c sysconf.c
@@ -35,4 +37,8 @@ conflicting.names:	confstr.names limits.names sysconf.names
 unique.names:		conflicting.names
 	LC_ALL=C sort -u ${.ALLSRC} >${.TARGET}
 
+.if ${MK_TESTS} != "no"
+SUBDIR+=	tests
+.endif
+
 .include <bsd.prog.mk>
diff --git a/usr.bin/getconf/getconf.h b/usr.bin/getconf/getconf.h
index 266a0ff3b051..53cf783cc40d 100644
--- a/usr.bin/getconf/getconf.h
+++ b/usr.bin/getconf/getconf.h
@@ -36,8 +36,15 @@ typedef long long intmax_t;
 #include <inttypes.h>
 #endif
 
+typedef enum {
+	PROG_ENV_VALID_NO_ALT_PATH = -1,
+	PROG_ENV_INVALID = 0,
+	PROG_ENV_VALID_HAS_ALT_PATH = 1,
+	PROG_ENV_UNKNOWN = 2,
+} prog_env_validity;
+
 int	find_confstr(const char *name, int *key);
 int	find_limit(const char *name, intmax_t *value);
 int	find_pathconf(const char *name, int *key);
-int	find_progenv(const char *name, const char **alt_path);
+prog_env_validity	find_progenv(const char *name, const char **alt_path);
 int	find_sysconf(const char *name, int *key);
diff --git a/usr.bin/getconf/tests/Makefile b/usr.bin/getconf/tests/Makefile
new file mode 100644
index 000000000000..3b2fdafc7bd0
--- /dev/null
+++ b/usr.bin/getconf/tests/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+ATF_TESTS_SH+=	getconf_test
+
+PROGS+=		arch_type
+
+BINDIR=		${TESTSDIR}
+
+WARNS?=		6
+
+.include <bsd.test.mk>
diff --git a/usr.bin/getconf/tests/arch_type.c b/usr.bin/getconf/tests/arch_type.c
new file mode 100644
index 000000000000..f443f0a0413a
--- /dev/null
+++ b/usr.bin/getconf/tests/arch_type.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2017 Ngie Cooper <ngie@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$FreeBSD$");
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+int
+main(void)
+{
+	bool known_arch_type;
+
+	known_arch_type = false;
+#ifdef	__LP64__
+	printf("LP64\n");
+	known_arch_type = true;
+#endif
+#ifdef	__LP32__
+	printf("LP32\n");
+	known_arch_type = true;
+#endif
+#ifdef	__ILP32__
+	printf("ILP32\n");
+	known_arch_type = true;
+#endif
+
+	if (known_arch_type)
+		exit(0);
+
+	fprintf(stderr, "unknown architecture type detected\n");
+	assert(0);
+}
diff --git a/usr.bin/getconf/tests/getconf_test.sh b/usr.bin/getconf/tests/getconf_test.sh
new file mode 100755
index 000000000000..c5b7fddb1b02
--- /dev/null
+++ b/usr.bin/getconf/tests/getconf_test.sh
@@ -0,0 +1,120 @@
+#
+# Copyright (c) 2017 Dell EMC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+POSITIVE_EXP_EXPR_RE='match:[1-9][0-9]*'
+
+POSIX_CONSTANTS="ARG_MAX PAGESIZE"
+POSIX_PATH_CONSTANTS="NAME_MAX PATH_MAX"
+SUPPORTED_32BIT_PROGRAM_ENVS="POSIX_V6_ILP32_OFFBIG"
+SUPPORTED_64BIT_PROGRAM_ENVS="POSIX_V6_LP64_OFF64"
+UNAVAILABLE_PROGRAM_ENVS="I_AM_BOGUS"
+UNSUPPORTED_32BIT_PROGRAM_ENVS="POSIX_V6_LP64_OFF64"
+UNSUPPORTED_64BIT_PROGRAM_ENVS="POSIX_V6_ILP32_OFFBIG"
+
+XOPEN_CONSTANTS=
+
+# XXX: hardcoded sysexits
+EX_USAGE=64
+EX_UNAVAILABLE=69
+
+set_program_environments()
+{
+	atf_check -o save:arch_type.out $(atf_get_srcdir)/arch_type
+	arch_type=$(cat arch_type.out)
+	case "$arch_type" in
+	ILP32|LP32)
+		SUPPORTED_PROGRAM_ENVS="$SUPPORTED_PROGRAM_ENVS $SUPPORTED_32BIT_PROGRAM_ENVS"
+		UNSUPPORTED_PROGRAM_ENVS="$UNSUPPORTED_PROGRAM_ENVS $UNSUPPORTED_32BIT_PROGRAM_ENVS"
+		;;
+	LP64)
+		SUPPORTED_PROGRAM_ENVS="$SUPPORTED_PROGRAM_ENVS $SUPPORTED_64BIT_PROGRAM_ENVS"
+		UNSUPPORTED_PROGRAM_ENVS="$UNSUPPORTED_PROGRAM_ENVS $UNSUPPORTED_64BIT_PROGRAM_ENVS"
+		;;
+	*)
+		atf_fail "arch_type output unexpected: $arch_type"
+		;;
+	esac
+}
+
+atf_test_case no_programming_environment
+no_programming_environment_head()
+{
+	atf_set	"descr" "Test some POSIX constants as a positive functional test"
+}
+
+no_programming_environment_body()
+{
+	for var in $POSIX_CONSTANTS; do
+		atf_check -o "$POSITIVE_EXP_EXPR_RE" getconf $var
+	done
+	for var in $POSIX_PATH_CONSTANTS; do
+		atf_check -o "$POSITIVE_EXP_EXPR_RE" getconf $var .
+	done
+}
+
+atf_test_case programming_environment
+programming_environment_head()
+{
+	atf_set	"descr" "Test some constants with specific programming environments"
+}
+
+programming_environment_body()
+{
+	set_program_environments
+
+	for prog_env in ${SUPPORTED_PROGRAM_ENVS}; do
+		for var in $POSIX_CONSTANTS; do
+			atf_check -o "$POSITIVE_EXP_EXPR_RE" \
+			    getconf -v $prog_env $var
+		done
+	done
+}
+
+atf_test_case programming_environment_unsupported
+programming_environment_unsupported_head()
+{
+	atf_set	"descr" "Test for unsupported environments"
+}
+
+programming_environment_unsupported_body()
+{
+	set_program_environments
+
+	for prog_env in ${UNSUPPORTED_PROGRAM_ENVS}; do
+		for var in $POSIX_CONSTANTS; do
+			atf_check -e not-empty -s exit:$EX_UNAVAILABLE \
+			    getconf -v $prog_env $var
+		done
+	done
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case no_programming_environment
+	atf_add_test_case programming_environment
+	atf_add_test_case programming_environment_unsupported
+}

From 382cc5a49418becdfec0f857cc6b8c7c9450ec2b Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 01:46:30 +0000
Subject: [PATCH 31/73] Revert local changes to find_progenv accidentally
 committed in r318436

MFC after:	2 weeks
MFC with:	r318436
Sponsored by:	Dell EMC Isilon
---
 usr.bin/getconf/getconf.h | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/usr.bin/getconf/getconf.h b/usr.bin/getconf/getconf.h
index 53cf783cc40d..266a0ff3b051 100644
--- a/usr.bin/getconf/getconf.h
+++ b/usr.bin/getconf/getconf.h
@@ -36,15 +36,8 @@ typedef long long intmax_t;
 #include <inttypes.h>
 #endif
 
-typedef enum {
-	PROG_ENV_VALID_NO_ALT_PATH = -1,
-	PROG_ENV_INVALID = 0,
-	PROG_ENV_VALID_HAS_ALT_PATH = 1,
-	PROG_ENV_UNKNOWN = 2,
-} prog_env_validity;
-
 int	find_confstr(const char *name, int *key);
 int	find_limit(const char *name, intmax_t *value);
 int	find_pathconf(const char *name, int *key);
-prog_env_validity	find_progenv(const char *name, const char **alt_path);
+int	find_progenv(const char *name, const char **alt_path);
 int	find_sysconf(const char *name, int *key);

From 518613e4125d8fd663ddc6d75d02291f308c1e49 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 06:13:29 +0000
Subject: [PATCH 32/73] Normalize .PATH on SRCTOP

This will help Jenkins dedupe 9 warnings between the static build and
the module build of ipsec(4).

Missed in SRCTOP conversion in r314651.

MFC with:	r314651
Sponsored by:	Dell EMC Isilon
---
 sys/modules/ipsec/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/modules/ipsec/Makefile b/sys/modules/ipsec/Makefile
index d3f1811d1726..01b92c57643d 100644
--- a/sys/modules/ipsec/Makefile
+++ b/sys/modules/ipsec/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.PATH: ${SRCTOP}/sys/net ${.CURDIR}/../../netipsec
+.PATH: ${SRCTOP}/sys/net ${SRCTOP}/sys/netipsec
 
 KMOD=	ipsec
 SRCS=	if_ipsec.c ipsec.c ipsec_input.c ipsec_mbuf.c ipsec_mod.c \

From 5c63f26129665311272aac2b5ab42b3f10473b6a Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 06:15:42 +0000
Subject: [PATCH 33/73] Normalize SYSDIR on SRCTOP instead of .CURDIR

This is being done to simplify pathing for CFLAGS and source files.

MFC after:	2 weeks
Sponsored by:	Dell EMC Isilon
---
 sys/modules/dtrace/dtaudit/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/modules/dtrace/dtaudit/Makefile b/sys/modules/dtrace/dtaudit/Makefile
index d44ee4763f59..aea5bd590ac6 100644
--- a/sys/modules/dtrace/dtaudit/Makefile
+++ b/sys/modules/dtrace/dtaudit/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-SYSDIR?=	${.CURDIR}/../../..
+SYSDIR?=	${SRCTOP}/sys
 
 .PATH:		${SYSDIR}/security/audit
 

From 94af8db1e978eeea36ce6a0a5b16c325d40b8fc5 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 06:25:39 +0000
Subject: [PATCH 34/73] Handle the cron.d entry for MK_AT in cron conditionally

Install /etc/cron.d/at if MK_AT != no, always using it, which tries
to run a non-existent program via cron(8) every 5 minutes with the
default /etc/crontab, prior to this commit.

SHELL and PATH are duplicated between /etc/crontab and /etc/cron.d/at
because atrun(8) executes programs, which may rely on environment
currently set via /etc/crontab.

Noted by:	bdrewery (in an internal review)
MFC after:	2 months
Relnotes:	yes (may need to add environmental modifications to
		     /etc/cron.d/at)
Sponsored by:	Dell EMC Isilon
---
 etc/Makefile        |  1 +
 etc/cron.d/Makefile | 11 +++++++++++
 etc/cron.d/at       |  7 +++++++
 etc/crontab         |  2 --
 4 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 etc/cron.d/Makefile
 create mode 100644 etc/cron.d/at

diff --git a/etc/Makefile b/etc/Makefile
index 8bfeac2b6b96..4b117d5257de 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -8,6 +8,7 @@ FILESGROUPS=	FILES
 # No need as it is empty and just causes rebuilds since this file does so much.
 UPDATE_DEPENDFILE=	no
 SUBDIR=	\
+	cron.d \
 	newsyslog.conf.d \
 	syslog.d
 
diff --git a/etc/cron.d/Makefile b/etc/cron.d/Makefile
new file mode 100644
index 000000000000..3ddc5fb438ac
--- /dev/null
+++ b/etc/cron.d/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+.include <src.opts.mk>
+
+.if ${MK_AT} != "no"
+FILES+=		at
+.endif
+
+BINDIR=		/etc/cron.d
+
+.include <bsd.prog.mk>
diff --git a/etc/cron.d/at b/etc/cron.d/at
new file mode 100644
index 000000000000..61fe5a8431ae
--- /dev/null
+++ b/etc/cron.d/at
@@ -0,0 +1,7 @@
+# $FreeBSD$
+#
+SHELL=/bin/sh
+PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
+
+# See crontab(5) for field format.
+*/5	*	*	*	*	root	/usr/libexec/atrun
diff --git a/etc/crontab b/etc/crontab
index e1e6e8862c8d..a3248dabeb52 100644
--- a/etc/crontab
+++ b/etc/crontab
@@ -7,8 +7,6 @@ PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
 #
 #minute	hour	mday	month	wday	who	command
 #
-*/5	*	*	*	*	root	/usr/libexec/atrun
-#
 # Save some entropy so that /dev/random can re-seed on boot.
 */11	*	*	*	*	operator /usr/libexec/save-entropy
 #

From cb8106ba7799e792dd1b8a12593cbb8b1ac412d2 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 06:27:37 +0000
Subject: [PATCH 35/73] Revert r318441: the commit message was incoherent

---
 etc/Makefile        |  1 -
 etc/cron.d/Makefile | 11 -----------
 etc/cron.d/at       |  7 -------
 etc/crontab         |  2 ++
 4 files changed, 2 insertions(+), 19 deletions(-)
 delete mode 100644 etc/cron.d/Makefile
 delete mode 100644 etc/cron.d/at

diff --git a/etc/Makefile b/etc/Makefile
index 4b117d5257de..8bfeac2b6b96 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -8,7 +8,6 @@ FILESGROUPS=	FILES
 # No need as it is empty and just causes rebuilds since this file does so much.
 UPDATE_DEPENDFILE=	no
 SUBDIR=	\
-	cron.d \
 	newsyslog.conf.d \
 	syslog.d
 
diff --git a/etc/cron.d/Makefile b/etc/cron.d/Makefile
deleted file mode 100644
index 3ddc5fb438ac..000000000000
--- a/etc/cron.d/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-# $FreeBSD$
-
-.include <src.opts.mk>
-
-.if ${MK_AT} != "no"
-FILES+=		at
-.endif
-
-BINDIR=		/etc/cron.d
-
-.include <bsd.prog.mk>
diff --git a/etc/cron.d/at b/etc/cron.d/at
deleted file mode 100644
index 61fe5a8431ae..000000000000
--- a/etc/cron.d/at
+++ /dev/null
@@ -1,7 +0,0 @@
-# $FreeBSD$
-#
-SHELL=/bin/sh
-PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
-
-# See crontab(5) for field format.
-*/5	*	*	*	*	root	/usr/libexec/atrun
diff --git a/etc/crontab b/etc/crontab
index a3248dabeb52..e1e6e8862c8d 100644
--- a/etc/crontab
+++ b/etc/crontab
@@ -7,6 +7,8 @@ PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
 #
 #minute	hour	mday	month	wday	who	command
 #
+*/5	*	*	*	*	root	/usr/libexec/atrun
+#
 # Save some entropy so that /dev/random can re-seed on boot.
 */11	*	*	*	*	operator /usr/libexec/save-entropy
 #

From 043b080e33511355bd797c4408b05971eb352503 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 May 2017 06:33:55 +0000
Subject: [PATCH 36/73] Conditionally handle the crontab entry for atrun(8)

The default crontab prior to this commit assumes atrun(8) is always
present, which isn't true if MK_AT == no. Move atrun(8) execution
from /etc/crontab to /etc/cron.d/at, and base /etc/cron.d/at's installation
on MK_AT. cron(8) will detect /etc/cron.d/at's presence when the configuration
is loaded and run atrun every 5 minutes like it would prior to this commit.

SHELL and PATH are duplicated between /etc/crontab and /etc/cron.d/at
because atrun(8) executes programs, which may rely on environment
set in the current default /etc/crontab.

Noted by:	bdrewery (in an internal review)
MFC after:	2 months
Relnotes:	yes (may need to add environmental modifications to
		     /etc/cron.d/at)
Sponsored by:	Dell EMC Isilon
---
 etc/Makefile        |  1 +
 etc/cron.d/Makefile | 11 +++++++++++
 etc/cron.d/at       |  7 +++++++
 etc/crontab         |  2 --
 4 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 etc/cron.d/Makefile
 create mode 100644 etc/cron.d/at

diff --git a/etc/Makefile b/etc/Makefile
index 8bfeac2b6b96..4b117d5257de 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -8,6 +8,7 @@ FILESGROUPS=	FILES
 # No need as it is empty and just causes rebuilds since this file does so much.
 UPDATE_DEPENDFILE=	no
 SUBDIR=	\
+	cron.d \
 	newsyslog.conf.d \
 	syslog.d
 
diff --git a/etc/cron.d/Makefile b/etc/cron.d/Makefile
new file mode 100644
index 000000000000..3ddc5fb438ac
--- /dev/null
+++ b/etc/cron.d/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+.include <src.opts.mk>
+
+.if ${MK_AT} != "no"
+FILES+=		at
+.endif
+
+BINDIR=		/etc/cron.d
+
+.include <bsd.prog.mk>
diff --git a/etc/cron.d/at b/etc/cron.d/at
new file mode 100644
index 000000000000..61fe5a8431ae
--- /dev/null
+++ b/etc/cron.d/at
@@ -0,0 +1,7 @@
+# $FreeBSD$
+#
+SHELL=/bin/sh
+PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
+
+# See crontab(5) for field format.
+*/5	*	*	*	*	root	/usr/libexec/atrun
diff --git a/etc/crontab b/etc/crontab
index e1e6e8862c8d..a3248dabeb52 100644
--- a/etc/crontab
+++ b/etc/crontab
@@ -7,8 +7,6 @@ PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin
 #
 #minute	hour	mday	month	wday	who	command
 #
-*/5	*	*	*	*	root	/usr/libexec/atrun
-#
 # Save some entropy so that /dev/random can re-seed on boot.
 */11	*	*	*	*	operator /usr/libexec/save-entropy
 #

From 6635c8ed2f59c6c6613911eeefd6e4b9de212d45 Mon Sep 17 00:00:00 2001
From: Edward Tomasz Napierala <trasz@FreeBSD.org>
Date: Thu, 18 May 2017 08:25:07 +0000
Subject: [PATCH 37/73] Fix typo.

MFC after:	2 weeks
---
 sys/geom/geom_vfs.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/geom/geom_vfs.c b/sys/geom/geom_vfs.c
index c59cc6a569e2..c48214438a42 100644
--- a/sys/geom/geom_vfs.c
+++ b/sys/geom/geom_vfs.c
@@ -168,7 +168,7 @@ g_vfs_strategy(struct bufobj *bo, struct buf *bp)
 	sc = cp->geom->softc;
 
 	/*
-	 * If the provider has orphaned us, just return EXIO.
+	 * If the provider has orphaned us, just return ENXIO.
 	 */
 	mtx_lock(&sc->sc_mtx);
 	if (sc->sc_orphaned) {

From 591986a8daa6c81b4d146e999eb7fe29c8fd2890 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Thu, 18 May 2017 09:31:30 +0000
Subject: [PATCH 38/73] Fix style [1], add static keyword before static
 function definition.

Noted by:	bapt [1]
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
---
 libexec/rtld-elf/rtld.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 7025acb96dfb..1fe9ad409205 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -5378,7 +5378,8 @@ parse_integer(const char *str)
 	return (n);
 }
 
-void print_usage(const char *argv0)
+static void
+print_usage(const char *argv0)
 {
 
 	rtld_printf("Usage: %s [-h] [-f <FD>] [--] <binary> [<args>]\n"

From ce9600b12e00c541023a4c658a6c5728a125eb5f Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Thu, 18 May 2017 09:34:26 +0000
Subject: [PATCH 39/73] Update my copyright, note The FreeBSD Foundation
 involvement. While tweaking copyright block, switch to use __FBSDID for tag.

Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
---
 libexec/rtld-elf/rtld.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 1fe9ad409205..8a4bd820753e 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -1,10 +1,14 @@
 /*-
  * Copyright 1996, 1997, 1998, 1999, 2000 John D. Polstra.
  * Copyright 2003 Alexander Kabaev <kan@FreeBSD.ORG>.
- * Copyright 2009-2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * Copyright 2009-2013 Konstantin Belousov <kib@FreeBSD.ORG>.
  * Copyright 2012 John Marino <draco@marino.st>.
+ * Copyright 2014-2017 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -24,8 +28,6 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
 /*
@@ -34,6 +36,9 @@
  * John Polstra <jdp@polstra.com>.
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/mman.h>

From 49dd63a9bd7adeab1a4a1d82f2341091867fa34e Mon Sep 17 00:00:00 2001
From: Allan Jude <allanjude@FreeBSD.org>
Date: Thu, 18 May 2017 12:55:07 +0000
Subject: [PATCH 40/73] Explain the new fields in top(1) related to ZFS
 compressed ARC

Reviewed by:	bcr
X-MFC-with:	316314
Differential Revision:	https://reviews.freebsd.org/D10781
---
 usr.bin/top/top.local.1 | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/usr.bin/top/top.local.1 b/usr.bin/top/top.local.1
index 46a959f84c02..a77fb5fae257 100644
--- a/usr.bin/top/top.local.1
+++ b/usr.bin/top/top.local.1
@@ -2,9 +2,10 @@
 .SH "FreeBSD NOTES"
 
 .SH DESCRIPTION OF MEMORY
-Mem: 9220K Active, 1M Inact, 1M Laundry, 3284K Wired, 2M Buf, 932K Free
-ARC: 2048K Total, 342K MRU, 760K MFU, 272K Anon, 96K Header, 442K Other
-Swap: 91M Total, 79M Free, 13% Inuse, 80K In, 104K Out
+Mem: 61M Active, 86M Inact, 368K Laundry, 22G Wired, 102G Free
+ARC: 15G Total, 9303M MFU, 6155M MRU, 1464K Anon, 98M Header, 35M Other
+     15G Compressed, 27G Uncompressed, 1.75:1 Ratio, 174M Overhead
+Swap: 4096M Total, 532M Free, 13% Inuse, 80K In, 104K Out
 .TP
 .B K:
 Kilobyte
@@ -54,8 +55,20 @@ number of ARC bytes holding in flight data
 .B Header:
 number of ARC bytes holding headers
 .TP
-.B Other
+.B Other:
 miscellaneous ARC bytes
+.TP
+.B Compressed:
+bytes of memory used by ARC caches
+.TP
+.B Uncompressed:
+bytes of data stored in ARC caches before compression
+.TP
+.B Ratio:
+ratio of uncompressed data to total ARC size
+.TP
+.B Overhead:
+amount of overhead from ARC compression
 .SS Swap Stats
 .TP
 .B Total:

From fade31741ddd46cac640c2f053050ee60162734c Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Thu, 18 May 2017 13:49:53 +0000
Subject: [PATCH 41/73] Add tests for some cases in r318298.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The first test triggers the out of bounds read of the 'left' array. It
only fails when realpath.c is compiled with '-fsanitize=address'.

The other test checks for ENOENT when running into an empty
symlink. This matches NetBSD's realpath(3) semantics. Previously,
empty symlinks were treated like ".".

Submitted by:	Jan Kokemц╪ller <jan.kokemueller@gmail.com>
PR:	219154
MFC after:	2 weeks
---
 lib/libc/tests/gen/Makefile         |   1 +
 lib/libc/tests/gen/realpath2_test.c | 102 ++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 lib/libc/tests/gen/realpath2_test.c

diff --git a/lib/libc/tests/gen/Makefile b/lib/libc/tests/gen/Makefile
index 0c990f176692..7ec09259ae21 100644
--- a/lib/libc/tests/gen/Makefile
+++ b/lib/libc/tests/gen/Makefile
@@ -13,6 +13,7 @@ ATF_TESTS_C+=		popen_test
 ATF_TESTS_C+=		posix_spawn_test
 ATF_TESTS_C+=		wordexp_test
 ATF_TESTS_C+=		dlopen_empty_test
+ATF_TESTS_C+=		realpath2_test
 
 # TODO: t_closefrom, t_cpuset, t_fmtcheck, t_randomid,
 # TODO: t_siginfo (fixes require further inspection)
diff --git a/lib/libc/tests/gen/realpath2_test.c b/lib/libc/tests/gen/realpath2_test.c
new file mode 100644
index 000000000000..82ad7f870697
--- /dev/null
+++ b/lib/libc/tests/gen/realpath2_test.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2017 Jan Kokemüller
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+ATF_TC(realpath_buffer_overflow);
+ATF_TC_HEAD(realpath_buffer_overflow, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Test for out of bounds read from 'left' array "
+	    "(compile realpath.c with '-fsanitize=address')");
+}
+
+ATF_TC_BODY(realpath_buffer_overflow, tc)
+{
+	char path[MAXPATHLEN] = { 0 };
+	char resb[MAXPATHLEN] = { 0 };
+	size_t i;
+
+	path[0] = 'a';
+	path[1] = '/';
+	for (i = 2; i < sizeof(path) - 1; ++i) {
+		path[i] = 'a';
+	}
+
+	ATF_REQUIRE(realpath(path, resb) == NULL);
+}
+
+ATF_TC(realpath_empty_symlink);
+ATF_TC_HEAD(realpath_empty_symlink, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Test for correct behavior when encountering empty symlinks");
+}
+
+ATF_TC_BODY(realpath_empty_symlink, tc)
+{
+	char path[MAXPATHLEN] = { 0 };
+	char slnk[MAXPATHLEN] = { 0 };
+	char resb[MAXPATHLEN] = { 0 };
+	int fd;
+
+	(void)strlcat(slnk, "empty_symlink", sizeof(slnk));
+
+	ATF_REQUIRE(symlink("", slnk) == 0);
+
+	fd = open("aaa", O_RDONLY | O_CREAT, 0600);
+
+	ATF_REQUIRE(fd >= 0);
+	ATF_REQUIRE(close(fd) == 0);
+
+	(void)strlcat(path, "empty_symlink", sizeof(path));
+	(void)strlcat(path, "/aaa", sizeof(path));
+
+	ATF_REQUIRE_ERRNO(ENOENT, realpath(path, resb) == NULL);
+
+	ATF_REQUIRE(unlink("aaa") == 0);
+	ATF_REQUIRE(unlink(slnk) == 0);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, realpath_buffer_overflow);
+	ATF_TP_ADD_TC(tp, realpath_empty_symlink);
+
+	return atf_no_error();
+}

From 3afe6a68e0c9ed63a5101587e264657eb31102db Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Thu, 18 May 2017 14:05:29 +0000
Subject: [PATCH 42/73] makefs: clean up signedness warnings and bump WARNS to
 3

Reviewed by:	kib
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D10650
---
 usr.sbin/makefs/Makefile         |  2 +-
 usr.sbin/makefs/ffs.c            |  5 +++--
 usr.sbin/makefs/ffs/ffs_alloc.c  | 14 +++++++-------
 usr.sbin/makefs/ffs/ffs_balloc.c | 10 ++++++----
 usr.sbin/makefs/ffs/ffs_bswap.c  |  2 +-
 usr.sbin/makefs/mtree.c          |  3 ++-
 6 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/usr.sbin/makefs/Makefile b/usr.sbin/makefs/Makefile
index ac59e3c58a3d..94dd2d4c9687 100644
--- a/usr.sbin/makefs/Makefile
+++ b/usr.sbin/makefs/Makefile
@@ -14,7 +14,7 @@ SRCS=	cd9660.c ffs.c \
 	walk.c
 MAN=	makefs.8
 
-WARNS?=	2
+WARNS?=	3
 
 .include "${SRCDIR}/cd9660/Makefile.inc"
 .include "${SRCDIR}/ffs/Makefile.inc"
diff --git a/usr.sbin/makefs/ffs.c b/usr.sbin/makefs/ffs.c
index 49fadd3708e4..a6957edf5eff 100644
--- a/usr.sbin/makefs/ffs.c
+++ b/usr.sbin/makefs/ffs.c
@@ -1065,10 +1065,11 @@ ffs_write_inode(union dinode *dp, uint32_t ino, const fsinfo_t *fsopts)
 	struct ufs2_dinode *dp2, *dip;
 	struct cg	*cgp;
 	struct fs	*fs;
-	int		cg, cgino, i;
+	int		cg, cgino;
+	uint32_t	i;
 	daddr_t		d;
 	char		sbbuf[FFS_MAXBSIZE];
-	int32_t		initediblk;
+	uint32_t	initediblk;
 	ffs_opt_t	*ffs_opts = fsopts->fs_specific;
 
 	assert (dp != NULL);
diff --git a/usr.sbin/makefs/ffs/ffs_alloc.c b/usr.sbin/makefs/ffs/ffs_alloc.c
index fb1b09d66451..e899ea22afed 100644
--- a/usr.sbin/makefs/ffs/ffs_alloc.c
+++ b/usr.sbin/makefs/ffs/ffs_alloc.c
@@ -64,7 +64,7 @@ static int scanc(u_int, const u_char *, const u_char *, int);
 
 static daddr_t ffs_alloccg(struct inode *, int, daddr_t, int);
 static daddr_t ffs_alloccgblk(struct inode *, struct buf *, daddr_t);
-static daddr_t ffs_hashalloc(struct inode *, int, daddr_t, int,
+static daddr_t ffs_hashalloc(struct inode *, u_int, daddr_t, int,
 		     daddr_t (*)(struct inode *, int, daddr_t, int));
 static int32_t ffs_mapsearch(struct fs *, struct cg *, daddr_t, int);
 
@@ -152,8 +152,8 @@ daddr_t
 ffs_blkpref_ufs1(struct inode *ip, daddr_t lbn, int indx, int32_t *bap)
 {
 	struct fs *fs;
-	int cg;
-	int avgbfree, startcg;
+	u_int cg, startcg;
+	int avgbfree;
 
 	fs = ip->i_fs;
 	if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) {
@@ -191,8 +191,8 @@ daddr_t
 ffs_blkpref_ufs2(struct inode *ip, daddr_t lbn, int indx, int64_t *bap)
 {
 	struct fs *fs;
-	int cg;
-	int avgbfree, startcg;
+	u_int cg, startcg;
+	int avgbfree;
 
 	fs = ip->i_fs;
 	if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) {
@@ -240,12 +240,12 @@ ffs_blkpref_ufs2(struct inode *ip, daddr_t lbn, int indx, int64_t *bap)
  */
 /*VARARGS5*/
 static daddr_t
-ffs_hashalloc(struct inode *ip, int cg, daddr_t pref, int size,
+ffs_hashalloc(struct inode *ip, u_int cg, daddr_t pref, int size,
     daddr_t (*allocator)(struct inode *, int, daddr_t, int))
 {
 	struct fs *fs;
 	daddr_t result;
-	int i, icg = cg;
+	u_int i, icg = cg;
 
 	fs = ip->i_fs;
 	/*
diff --git a/usr.sbin/makefs/ffs/ffs_balloc.c b/usr.sbin/makefs/ffs/ffs_balloc.c
index 46a20ba99e78..7c6a038b6a88 100644
--- a/usr.sbin/makefs/ffs/ffs_balloc.c
+++ b/usr.sbin/makefs/ffs/ffs_balloc.c
@@ -123,7 +123,8 @@ ffs_balloc_ufs1(struct inode *ip, off_t offset, int bufsize, struct buf **bpp)
 
 	if (lbn < UFS_NDADDR) {
 		nb = ufs_rw32(ip->i_ffs1_db[lbn], needswap);
-		if (nb != 0 && ip->i_ffs1_size >= lblktosize(fs, lbn + 1)) {
+		if (nb != 0 && ip->i_ffs1_size >=
+		    (uint64_t)lblktosize(fs, lbn + 1)) {
 
 			/*
 			 * The block is an already-allocated direct block
@@ -178,7 +179,7 @@ ffs_balloc_ufs1(struct inode *ip, off_t offset, int bufsize, struct buf **bpp)
 			 * allocate a new block or fragment.
 			 */
 
-			if (ip->i_ffs1_size < lblktosize(fs, lbn + 1))
+			if (ip->i_ffs1_size < (uint64_t)lblktosize(fs, lbn + 1))
 				nsize = fragroundup(fs, size);
 			else
 				nsize = fs->fs_bsize;
@@ -373,7 +374,8 @@ ffs_balloc_ufs2(struct inode *ip, off_t offset, int bufsize, struct buf **bpp)
 
 	if (lbn < UFS_NDADDR) {
 		nb = ufs_rw64(ip->i_ffs2_db[lbn], needswap);
-		if (nb != 0 && ip->i_ffs2_size >= lblktosize(fs, lbn + 1)) {
+		if (nb != 0 && ip->i_ffs2_size >=
+		    (uint64_t)lblktosize(fs, lbn + 1)) {
 
 			/*
 			 * The block is an already-allocated direct block
@@ -428,7 +430,7 @@ ffs_balloc_ufs2(struct inode *ip, off_t offset, int bufsize, struct buf **bpp)
 			 * allocate a new block or fragment.
 			 */
 
-			if (ip->i_ffs2_size < lblktosize(fs, lbn + 1))
+			if (ip->i_ffs2_size < (uint64_t)lblktosize(fs, lbn + 1))
 				nsize = fragroundup(fs, size);
 			else
 				nsize = fs->fs_bsize;
diff --git a/usr.sbin/makefs/ffs/ffs_bswap.c b/usr.sbin/makefs/ffs/ffs_bswap.c
index 8cf8c50e5167..1f17e9bf9029 100644
--- a/usr.sbin/makefs/ffs/ffs_bswap.c
+++ b/usr.sbin/makefs/ffs/ffs_bswap.c
@@ -39,12 +39,12 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#define panic(x)	printf("%s\n", (x)), abort()
 #endif
 
 #include <ufs/ufs/dinode.h>
 #include "ffs/ufs_bswap.h"
 #include <ufs/ffs/fs.h>
+#include "ffs/ffs_extern.h"
 
 #define	fs_old_postbloff	fs_spare5[0]
 #define	fs_old_rotbloff		fs_spare5[1]
diff --git a/usr.sbin/makefs/mtree.c b/usr.sbin/makefs/mtree.c
index 76f1dac05613..0dcb45173123 100644
--- a/usr.sbin/makefs/mtree.c
+++ b/usr.sbin/makefs/mtree.c
@@ -505,7 +505,8 @@ read_mtree_keywords(FILE *fp, fsnode *node)
 	struct stat *st, sb;
 	intmax_t num;
 	u_long flset, flclr;
-	int error, istemp, type;
+	int error, istemp;
+	uint32_t type;
 
 	st = &node->inode->st;
 	do {

From d91e61179856ba5052a43a3d90451f1e75fa039a Mon Sep 17 00:00:00 2001
From: Emmanuel Vadot <manu@FreeBSD.org>
Date: Thu, 18 May 2017 14:19:06 +0000
Subject: [PATCH 43/73] makefs: Add soft-updates option

Add the ffs option to enable soft-updates.
The option is only processed is ufs2 has been selected.

Reviewed by:	emaste, bapt (earlier version), allanjude (earlier version)
Sponsored by:	Gandi.net
Differential Revision:	https://reviews.freebsd.org/D10773
---
 usr.sbin/makefs/ffs.c      | 3 +++
 usr.sbin/makefs/ffs.h      | 1 +
 usr.sbin/makefs/ffs/mkfs.c | 2 ++
 usr.sbin/makefs/makefs.8   | 4 +++-
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/usr.sbin/makefs/ffs.c b/usr.sbin/makefs/ffs.c
index a6957edf5eff..d775df9759a1 100644
--- a/usr.sbin/makefs/ffs.c
+++ b/usr.sbin/makefs/ffs.c
@@ -176,6 +176,8 @@ ffs_prep_opts(fsinfo_t *fsopts)
 	      0, 0, "Optimization (time|space)" },
 	    { 'l', "label", ffs_opts->label, OPT_STRARRAY,
 	      1, sizeof(ffs_opts->label), "UFS label" },
+	    { 's', "softupdates", &ffs_opts->softupdates, OPT_INT32,
+	      0, 1, "enable softupdates" },
 	    { .name = NULL }
 	};
 
@@ -190,6 +192,7 @@ ffs_prep_opts(fsinfo_t *fsopts)
 	ffs_opts->avgfilesize= -1;
 	ffs_opts->avgfpdir= -1;
 	ffs_opts->version = 1;
+	ffs_opts->softupdates = 0;
 
 	fsopts->fs_specific = ffs_opts;
 	fsopts->fs_options = copy_opts(ffs_options);
diff --git a/usr.sbin/makefs/ffs.h b/usr.sbin/makefs/ffs.h
index 2c1c5bbf2ff3..a3583b139e31 100644
--- a/usr.sbin/makefs/ffs.h
+++ b/usr.sbin/makefs/ffs.h
@@ -64,6 +64,7 @@ typedef struct {
 	int	version;	/* filesystem version (1 = FFS, 2 = UFS2) */
 	int	maxbsize;	/* maximum extent size */
 	int	maxblkspercg;	/* max # of blocks per cylinder group */
+	int	softupdates;	/* soft updates */
 		/* XXX: support `old' file systems ? */
 } ffs_opt_t;
 
diff --git a/usr.sbin/makefs/ffs/mkfs.c b/usr.sbin/makefs/ffs/mkfs.c
index 931007fb4f06..bf70f3d09a04 100644
--- a/usr.sbin/makefs/ffs/mkfs.c
+++ b/usr.sbin/makefs/ffs/mkfs.c
@@ -279,6 +279,8 @@ ffs_mkfs(const char *fsys, const fsinfo_t *fsopts, time_t tstamp)
 		sblock.fs_inopb = sblock.fs_bsize / sizeof(struct ufs2_dinode);
 		sblock.fs_maxsymlinklen = ((UFS_NDADDR + UFS_NIADDR) *
 		    sizeof (ufs2_daddr_t));
+		if (ffs_opts->softupdates == 1)
+			sblock.fs_flags |= FS_DOSOFTDEP;
 	}
 
 	sblock.fs_sblkno =
diff --git a/usr.sbin/makefs/makefs.8 b/usr.sbin/makefs/makefs.8
index 7a8e67e465d7..5ad56ab35d84 100644
--- a/usr.sbin/makefs/makefs.8
+++ b/usr.sbin/makefs/makefs.8
@@ -35,7 +35,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 6, 2017
+.Dd May 17, 2017
 .Dt MAKEFS 8
 .Os
 .Sh NAME
@@ -308,6 +308,8 @@ Maximum total number of blocks in a cylinder group.
 .It Sy version
 UFS version.
 1 for FFS (default), 2 for UFS2.
+.It Sy softupdates
+0 for disable (default), 1 for enable
 .El
 .Ss CD9660-specific options
 .Sy cd9660

From bd2969a00d614adfa6fe37b96bc15ca4cdcb3f3e Mon Sep 17 00:00:00 2001
From: Piotr Pawel Stefaniak <pstef@FreeBSD.org>
Date: Thu, 18 May 2017 17:15:58 +0000
Subject: [PATCH 44/73] =?UTF-8?q?indent(1):=20Support=20binary=20integer?=
 =?UTF-8?q?=20literals.=20This=20was=20done=20by=20Romain=20Tarti=C3=A8re?=
 =?UTF-8?q?=20for=20PR123553.=20I=20initially=20thought=20that=20it=20woul?=
 =?UTF-8?q?d=20break=20code=20like=20this:=20#define=20b00101010=20-1=20if?=
 =?UTF-8?q?=20(0=20b00101010)=20...?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

by joining 0 and b00101010 together. However, the real problem with that patch was that once it saw a 0, it assumed that the number was base 2, 8 or 16, ignoring base 10 floating point numbers. I fixed that.

I didn't copy the diagnostic part of the original patch as it seems out of scope of implementing binary integer literals formatting.

PR:		123553
Submitted by:	romain (original version)
Approved by:	pfg (mentor)
---
 usr.bin/indent/lexi.c                | 38 +++++++++++++++++++++++++---
 usr.bin/indent/tests/Makefile        |  2 ++
 usr.bin/indent/tests/binary.0        | 10 ++++++++
 usr.bin/indent/tests/binary.0.stdout | 12 +++++++++
 4 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 usr.bin/indent/tests/binary.0
 create mode 100644 usr.bin/indent/tests/binary.0.stdout

diff --git a/usr.bin/indent/lexi.c b/usr.bin/indent/lexi.c
index b9ff1de94dcf..6deb55ae6853 100644
--- a/usr.bin/indent/lexi.c
+++ b/usr.bin/indent/lexi.c
@@ -169,19 +169,47 @@ lexi(void)
 	struct templ *p;
 
 	if (isdigit(*buf_ptr) || (buf_ptr[0] == '.' && isdigit(buf_ptr[1]))) {
+	    enum base {
+		BASE_2, BASE_8, BASE_10, BASE_16
+	    };
 	    int         seendot = 0,
 	                seenexp = 0,
 			seensfx = 0;
-	    if (*buf_ptr == '0' &&
-		    (buf_ptr[1] == 'x' || buf_ptr[1] == 'X')) {
+	    enum base	in_base = BASE_10;
+
+	    if (*buf_ptr == '0') {
+		if (buf_ptr[1] == 'b' || buf_ptr[1] == 'B')
+		    in_base = BASE_2;
+		else if (buf_ptr[1] == 'x' || buf_ptr[1] == 'X')
+		    in_base = BASE_16;
+		else if (isdigit(buf_ptr[1]))
+		    in_base = BASE_8;
+	    }
+	    switch (in_base) {
+	    case BASE_2:
+		*e_token++ = *buf_ptr++;
+		*e_token++ = *buf_ptr++;
+		while (*buf_ptr == '0' || *buf_ptr == '1') {
+		    CHECK_SIZE_TOKEN;
+		    *e_token++ = *buf_ptr++;
+		}
+		break;
+	    case BASE_8:
+		*e_token++ = *buf_ptr++;
+		while (*buf_ptr >= '0' && *buf_ptr <= '8') {
+		    CHECK_SIZE_TOKEN;
+		    *e_token++ = *buf_ptr++;
+		}
+		break;
+	    case BASE_16:
 		*e_token++ = *buf_ptr++;
 		*e_token++ = *buf_ptr++;
 		while (isxdigit(*buf_ptr)) {
 		    CHECK_SIZE_TOKEN;
 		    *e_token++ = *buf_ptr++;
 		}
-	    }
-	    else
+		break;
+	    case BASE_10:
 		while (1) {
 		    if (*buf_ptr == '.') {
 			if (seendot)
@@ -204,6 +232,8 @@ lexi(void)
 			}
 		    }
 		}
+		break;
+	    }
 	    while (1) {
 		if (!(seensfx & 1) && (*buf_ptr == 'U' || *buf_ptr == 'u')) {
 		    CHECK_SIZE_TOKEN;
diff --git a/usr.bin/indent/tests/Makefile b/usr.bin/indent/tests/Makefile
index 1b6abb835828..01fa04aca755 100644
--- a/usr.bin/indent/tests/Makefile
+++ b/usr.bin/indent/tests/Makefile
@@ -2,6 +2,8 @@
 
 PACKAGE=	tests
 
+${PACKAGE}FILES+=	binary.0
+${PACKAGE}FILES+=	binary.0.stdout
 ${PACKAGE}FILES+=	comments.0
 ${PACKAGE}FILES+=	comments.0.stdout
 ${PACKAGE}FILES+=	declarations.0
diff --git a/usr.bin/indent/tests/binary.0 b/usr.bin/indent/tests/binary.0
new file mode 100644
index 000000000000..b2adc1fe6428
--- /dev/null
+++ b/usr.bin/indent/tests/binary.0
@@ -0,0 +1,10 @@
+/* $FreeBSD$ */
+#define b00101010 -1
+void t(void) {
+	unsigned a[] = {0b00101010, 0x00005678, 02, 17U};
+	float x[] = {.7f, 0.7f};
+	unsigned long ul[] = {0b00001111UL, 0x01010101UL, 02UL, 17UL};
+
+	if (0 b00101010)
+		return;
+}
diff --git a/usr.bin/indent/tests/binary.0.stdout b/usr.bin/indent/tests/binary.0.stdout
new file mode 100644
index 000000000000..a883ed9fcd06
--- /dev/null
+++ b/usr.bin/indent/tests/binary.0.stdout
@@ -0,0 +1,12 @@
+/* $FreeBSD$ */
+#define b00101010 -1
+void 
+t(void)
+{
+	unsigned	a[] = {0b00101010, 0x00005678, 02, 17U};
+	float		x[] = {.7f, 0.7f};
+	unsigned long	ul[] = {0b00001111UL, 0x01010101UL, 02UL, 17UL};
+
+	if (0 b00101010)
+		return;
+}

From 7092a907e537b88742dff6fdcb0a01b9d9b83f53 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Thu, 18 May 2017 17:55:33 +0000
Subject: [PATCH 45/73] makefs: drop WARNS back to 2

GCC warns about additional signed comparision issues compared to Clang.
Drop WARNS for now until the underlying issue is fixed.
---
 usr.sbin/makefs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/usr.sbin/makefs/Makefile b/usr.sbin/makefs/Makefile
index 94dd2d4c9687..ac59e3c58a3d 100644
--- a/usr.sbin/makefs/Makefile
+++ b/usr.sbin/makefs/Makefile
@@ -14,7 +14,7 @@ SRCS=	cd9660.c ffs.c \
 	walk.c
 MAN=	makefs.8
 
-WARNS?=	3
+WARNS?=	2
 
 .include "${SRCDIR}/cd9660/Makefile.inc"
 .include "${SRCDIR}/ffs/Makefile.inc"

From 3bd485f968c874b941d7430eb0232ea965252721 Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Thu, 18 May 2017 18:24:11 +0000
Subject: [PATCH 46/73] Avoid open-coding PRI_UNCHANGED.

MFC after:	1 week
---
 sys/kern/kern_fail.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/kern/kern_fail.c b/sys/kern/kern_fail.c
index 6f3115b3e15b..c53cb68cdd10 100644
--- a/sys/kern/kern_fail.c
+++ b/sys/kern/kern_fail.c
@@ -612,7 +612,7 @@ fail_point_eval_nontrivial(struct fail_point *fp, int *return_value)
 			break;
 
 		case FAIL_POINT_YIELD:
-			kern_yield(-1);
+			kern_yield(PRI_UNCHANGED);
 			break;
 
 		case FAIL_POINT_DELAY:

From 02fb845bbfb7e5dcfaa8d659c83fa5c9afcae2d3 Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Thu, 18 May 2017 18:35:14 +0000
Subject: [PATCH 47/73] Fix a few uses of kern_yield() in the TTM and the
 LinuxKPI.

kern_yield(0) effectively causes the calling thread to be rescheduled
immediately since it resets the thread's priority to the highest possible
value. This can cause livelocks when the pattern
"while (!trylock()) kern_yield(0);" is used since the thread holding the
lock may linger on the runqueue for the CPU on which the looping thread is
running.

MFC after:	1 week
---
 sys/compat/linuxkpi/common/src/linux_compat.c | 2 +-
 sys/dev/drm2/ttm/ttm_bo_vm.c                  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c
index ecea3a753192..02df4e325186 100644
--- a/sys/compat/linuxkpi/common/src/linux_compat.c
+++ b/sys/compat/linuxkpi/common/src/linux_compat.c
@@ -435,7 +435,7 @@ linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type,
 		err = vmap->vm_ops->fault(vmap, &vmf);
 
 		while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) {
-			kern_yield(0);
+			kern_yield(PRI_USER);
 			err = vmap->vm_ops->fault(vmap, &vmf);
 		}
 	}
diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c
index 60bf8e23c54d..6d8bb1129bcd 100644
--- a/sys/dev/drm2/ttm/ttm_bo_vm.c
+++ b/sys/dev/drm2/ttm/ttm_bo_vm.c
@@ -126,7 +126,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
 	ret = ttm_bo_reserve(bo, false, false, false, 0);
 	if (unlikely(ret != 0)) {
 		if (ret == -EBUSY) {
-			kern_yield(0);
+			kern_yield(PRI_USER);
 			goto reserve;
 		}
 	}
@@ -139,7 +139,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
 		case -EBUSY:
 		case -ERESTARTSYS:
 		case -EINTR:
-			kern_yield(0);
+			kern_yield(PRI_USER);
 			goto reserve;
 		default:
 			retval = VM_PAGER_ERROR;

From ed67acb7795cb098f087dc4b4c0faf845bf063de Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Thu, 18 May 2017 18:37:19 +0000
Subject: [PATCH 48/73] Don't bother enqueuing a page immediately before
 freeing it.

No functional change intended.

MFC after:	1 week
---
 sys/dev/drm2/ttm/ttm_page_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/dev/drm2/ttm/ttm_page_alloc.c b/sys/dev/drm2/ttm/ttm_page_alloc.c
index 2a2e9168fef1..fb6d18c81f2b 100644
--- a/sys/dev/drm2/ttm/ttm_page_alloc.c
+++ b/sys/dev/drm2/ttm/ttm_page_alloc.c
@@ -136,7 +136,7 @@ ttm_vm_page_free(vm_page_t m)
 	KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m));
 	m->flags &= ~PG_FICTITIOUS;
 	m->oflags |= VPO_UNMANAGED;
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 }
 

From 980e53a2990fe9ba766499bb30c420ecc36cdc59 Mon Sep 17 00:00:00 2001
From: Edward Tomasz Napierala <trasz@FreeBSD.org>
Date: Thu, 18 May 2017 19:42:19 +0000
Subject: [PATCH 49/73] Language fixes.

Submitted by:	wblock
MFC after:	2 weeks
---
 usr.bin/resizewin/resizewin.1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/usr.bin/resizewin/resizewin.1 b/usr.bin/resizewin/resizewin.1
index d6c11abdc515..70ad52eea49a 100644
--- a/usr.bin/resizewin/resizewin.1
+++ b/usr.bin/resizewin/resizewin.1
@@ -49,7 +49,7 @@ The following options are available:
 .Bl -tag -width ".Fl z"
 .It Fl z
 Do nothing unless the current kernel terminal size is zero.
-This is useful when run from user's profile (shell startup) scripts:
+This is useful when run from a user's profile (shell startup) scripts:
 querying the window size is required for serial lines, but not when
 logging in over the network, as protocols like TELNET or SSH already
 handle the terminal size by themselves.
@@ -72,7 +72,7 @@ commands to set environment variables.
 .Pp
 The terminal is assumed to be VT100/ANSI compatible.
 The VT100/ANSI escape sequences are supported by virtually all modern
-terminals; this include xterm, konsole, gnome-terminal, iTerm,
+terminals, including xterm, konsole, gnome-terminal, iTerm,
 Terminal.app, and PuTTY.
 .Sh SEE ALSO
 .Xr stty 1 ,

From de29cd0869e37df15a49283781c378181cdd0c93 Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Thu, 18 May 2017 21:44:14 +0000
Subject: [PATCH 50/73] sh: Ensure memout.bufsize matches allocated buffer, if
 it exists.

---
 bin/sh/eval.c   |  3 +--
 bin/sh/output.c | 11 ++++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/sh/eval.c b/bin/sh/eval.c
index f637cfab5fd9..9386b16f0b82 100644
--- a/bin/sh/eval.c
+++ b/bin/sh/eval.c
@@ -1081,8 +1081,6 @@ evalcommand(union node *cmd, int flags, struct backcmd *backcmd)
 		mode = (cmdentry.u.index == EXECCMD)? 0 : REDIR_PUSH;
 		if (flags == EV_BACKCMD) {
 			memout.nextc = memout.buf;
-			memout.bufend = memout.buf;
-			memout.bufsize = 64;
 			mode |= REDIR_BACKQ;
 		}
 		savecmdname = commandname;
@@ -1139,6 +1137,7 @@ evalcommand(union node *cmd, int flags, struct backcmd *backcmd)
 			memout.buf = NULL;
 			memout.nextc = NULL;
 			memout.bufend = NULL;
+			memout.bufsize = 64;
 		}
 		if (cmdentry.u.index != EXECCMD)
 			popredir();
diff --git a/bin/sh/output.c b/bin/sh/output.c
index 64d311f0c2d6..c3458272785e 100644
--- a/bin/sh/output.c
+++ b/bin/sh/output.c
@@ -73,7 +73,7 @@ static int doformat_wr(void *, const char *, int);
 
 struct output output = {NULL, NULL, NULL, OUTBUFSIZ, 1, 0};
 struct output errout = {NULL, NULL, NULL, 256, 2, 0};
-struct output memout = {NULL, NULL, NULL, 0, MEM_OUT, 0};
+struct output memout = {NULL, NULL, NULL, 64, MEM_OUT, 0};
 struct output *out1 = &output;
 struct output *out2 = &errout;
 
@@ -208,7 +208,7 @@ outbin(const void *data, size_t len, struct output *file)
 void
 emptyoutbuf(struct output *dest)
 {
-	int offset;
+	int offset, newsize;
 
 	if (dest->buf == NULL) {
 		INTOFF;
@@ -218,10 +218,11 @@ emptyoutbuf(struct output *dest)
 		INTON;
 	} else if (dest->fd == MEM_OUT) {
 		offset = dest->nextc - dest->buf;
+		newsize = dest->bufsize << 1;
 		INTOFF;
-		dest->bufsize <<= 1;
-		dest->buf = ckrealloc(dest->buf, dest->bufsize);
-		dest->bufend = dest->buf + dest->bufsize;
+		dest->buf = ckrealloc(dest->buf, newsize);
+		dest->bufsize = newsize;
+		dest->bufend = dest->buf + newsize;
 		dest->nextc = dest->buf + offset;
 		INTON;
 	} else {

From 29717eb02920eda35c810b444f7fa7b190155170 Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Thu, 18 May 2017 22:10:04 +0000
Subject: [PATCH 51/73] sh: Keep output buffer across builtins.

Allocating and deallocating repeatedly the 1024-byte buffer for stdout from
builtins costs CPU time for little or no benefit.

A simple loop containing builtins that write to a file descriptor, such as
  i=0; while [ "$i" -lt 1000000 ]; do printf .; i=$((i+1)); done >/dev/null
is over 10% faster in a simple benchmark on an amd64 virtual machine.
---
 bin/sh/output.c | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/bin/sh/output.c b/bin/sh/output.c
index c3458272785e..ced01d246a1c 100644
--- a/bin/sh/output.c
+++ b/bin/sh/output.c
@@ -254,14 +254,7 @@ flushout(struct output *dest)
 void
 freestdout(void)
 {
-	INTOFF;
-	if (output.buf) {
-		ckfree(output.buf);
-		output.nextc = NULL;
-		output.buf = NULL;
-		output.bufend = NULL;
-	}
-	INTON;
+	output.nextc = output.buf;
 }
 
 

From 9ebaf5f802a960c2dbd5ddca13a15b5bfb11dade Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Fri, 19 May 2017 00:25:09 +0000
Subject: [PATCH 52/73] Remove the EXFAIL annotation for tests which pass as of
 r309596.

Reported by:	bdrewery
Sponsored by:	Dell EMC Isilon
---
 cddl/usr.sbin/dtrace/tests/tools/exclude.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cddl/usr.sbin/dtrace/tests/tools/exclude.sh b/cddl/usr.sbin/dtrace/tests/tools/exclude.sh
index e7902423a7ff..bc2a060619ad 100755
--- a/cddl/usr.sbin/dtrace/tests/tools/exclude.sh
+++ b/cddl/usr.sbin/dtrace/tests/tools/exclude.sh
@@ -139,11 +139,6 @@ exclude EXFAIL common/pid/tst.newprobes.ksh
 exclude EXFAIL common/pid/tst.provregex2.ksh
 exclude EXFAIL common/pid/tst.provregex4.ksh
 
-# libproc doesn't properly handle probe sites that correspond to multiple
-# symbols.
-exclude EXFAIL common/pid/tst.weak1.d
-exclude EXFAIL common/pid/tst.weak2.d
-
 # This test checks for a leading tab on a line before #define. That is illegal
 # on Solaris, but the clang pre-processor on FreeBSD is happy with code like
 # that.

From 36fb8be63071a747d484a9adad9bed7f5c63141e Mon Sep 17 00:00:00 2001
From: Don Lewis <truckman@FreeBSD.org>
Date: Fri, 19 May 2017 01:23:06 +0000
Subject: [PATCH 53/73] The result of right shifting a negative signed value is
 implementation defined.  On machines without arithmetic shift instructions,
 zero bits may be shifted in from the left, giving a large positive result
 instead of the desired divide-by power-of-2.  Fix this by operating on the
 absolute value and compensating for the possible negation later.

Reverse the order of the underflow/overflow tests and the exponential
decay calculation to avoid the possibility of an erroneous overflow
detection if p is a sufficiently small non-negative value.  Also
check for negative values of prob before doing the exponential decay
to avoid another instance of of right shifting a negative value.

Tested by:	Rasool Al-Saadi <ralsaadi@swin.edu.au>
MFC after:	1 week
---
 sys/netpfil/ipfw/dn_aqm_pie.c      | 59 +++++++++++++++++++-----------
 sys/netpfil/ipfw/dn_sched_fq_pie.c | 59 +++++++++++++++++++-----------
 2 files changed, 76 insertions(+), 42 deletions(-)

diff --git a/sys/netpfil/ipfw/dn_aqm_pie.c b/sys/netpfil/ipfw/dn_aqm_pie.c
index 85062e258c9e..8259d0853df6 100644
--- a/sys/netpfil/ipfw/dn_aqm_pie.c
+++ b/sys/netpfil/ipfw/dn_aqm_pie.c
@@ -206,6 +206,7 @@ calculate_drop_prob(void *x)
 	int64_t p, prob, oldprob;
 	struct dn_aqm_pie_parms *pprms;
 	struct pie_status *pst = (struct pie_status *) x;
+	int p_isneg;
 
 	pprms = pst->parms;
 	prob = pst->drop_prob;
@@ -221,6 +222,12 @@ calculate_drop_prob(void *x)
 		((int64_t)pst->current_qdelay - (int64_t)pprms->qdelay_ref); 
 	p +=(int64_t) pprms->beta * 
 		((int64_t)pst->current_qdelay - (int64_t)pst->qdelay_old); 
+
+	/* take absolute value so right shift result is well defined */
+	p_isneg = p < 0;
+	if (p_isneg) {
+		p = -p;
+	}
 		
 	/* We PIE_MAX_PROB shift by 12-bits to increase the division precision */
 	p *= (PIE_MAX_PROB << 12) / AQM_TIME_1S;
@@ -243,37 +250,47 @@ calculate_drop_prob(void *x)
 
 	oldprob = prob;
 
-	/* Cap Drop adjustment */
-	if ((pprms->flags & PIE_CAPDROP_ENABLED) && prob >= PIE_MAX_PROB / 10
-		&& p > PIE_MAX_PROB / 50 ) 
+	if (p_isneg) {
+		prob = prob - p;
+
+		/* check for multiplication underflow */
+		if (prob > oldprob) {
+			prob= 0;
+			D("underflow");
+		}
+	} else {
+		/* Cap Drop adjustment */
+		if ((pprms->flags & PIE_CAPDROP_ENABLED) &&
+		    prob >= PIE_MAX_PROB / 10 &&
+		    p > PIE_MAX_PROB / 50 ) {
 			p = PIE_MAX_PROB / 50;
+		}
 
-	prob = prob + p;
+		prob = prob + p;
 
-	/* decay the drop probability exponentially */
-	if (pst->current_qdelay == 0 && pst->qdelay_old == 0)
-		/* 0.98 ~= 1- 1/64 */
-		prob = prob - (prob >> 6); 
-
-
-	/* check for multiplication overflow/underflow */
-	if (p>0) {
+		/* check for multiplication overflow */
 		if (prob<oldprob) {
 			D("overflow");
 			prob= PIE_MAX_PROB;
 		}
 	}
-	else
-		if (prob>oldprob) {
-			prob= 0;
-			D("underflow");
+
+	/*
+	 * decay the drop probability exponentially
+	 * and restrict it to range 0 to PIE_MAX_PROB
+	 */
+	if (prob < 0) {
+		prob = 0;
+	} else {
+		if (pst->current_qdelay == 0 && pst->qdelay_old == 0) {
+			/* 0.98 ~= 1- 1/64 */
+			prob = prob - (prob >> 6); 
 		}
 
-	/* make drop probability between 0 and PIE_MAX_PROB*/
-	if (prob < 0)
-		prob = 0;
-	else if (prob > PIE_MAX_PROB)
-		prob = PIE_MAX_PROB;
+		if (prob > PIE_MAX_PROB) {
+			prob = PIE_MAX_PROB;
+		}
+	}
 
 	pst->drop_prob = prob;
 	
diff --git a/sys/netpfil/ipfw/dn_sched_fq_pie.c b/sys/netpfil/ipfw/dn_sched_fq_pie.c
index bfcd6c5f2d3a..2ba16ab5fa6e 100644
--- a/sys/netpfil/ipfw/dn_sched_fq_pie.c
+++ b/sys/netpfil/ipfw/dn_sched_fq_pie.c
@@ -377,6 +377,7 @@ fq_calculate_drop_prob(void *x)
 	struct dn_aqm_pie_parms *pprms; 
 	int64_t p, prob, oldprob;
 	aqm_time_t now;
+	int p_isneg;
 
 	now = AQM_UNOW;
 	pprms = pst->parms;
@@ -393,6 +394,12 @@ fq_calculate_drop_prob(void *x)
 		((int64_t)pst->current_qdelay - (int64_t)pprms->qdelay_ref); 
 	p +=(int64_t) pprms->beta * 
 		((int64_t)pst->current_qdelay - (int64_t)pst->qdelay_old); 
+
+	/* take absolute value so right shift result is well defined */
+	p_isneg = p < 0;
+	if (p_isneg) {
+		p = -p;
+	}
 		
 	/* We PIE_MAX_PROB shift by 12-bits to increase the division precision  */
 	p *= (PIE_MAX_PROB << 12) / AQM_TIME_1S;
@@ -415,37 +422,47 @@ fq_calculate_drop_prob(void *x)
 
 	oldprob = prob;
 
-	/* Cap Drop adjustment */
-	if ((pprms->flags & PIE_CAPDROP_ENABLED) && prob >= PIE_MAX_PROB / 10
-		&& p > PIE_MAX_PROB / 50 ) 
+	if (p_isneg) {
+		prob = prob - p;
+
+		/* check for multiplication underflow */
+		if (prob > oldprob) {
+			prob= 0;
+			D("underflow");
+		}
+	} else {
+		/* Cap Drop adjustment */
+		if ((pprms->flags & PIE_CAPDROP_ENABLED) &&
+		    prob >= PIE_MAX_PROB / 10 &&
+		    p > PIE_MAX_PROB / 50 ) {
 			p = PIE_MAX_PROB / 50;
+		}
 
-	prob = prob + p;
+		prob = prob + p;
 
-	/* decay the drop probability exponentially */
-	if (pst->current_qdelay == 0 && pst->qdelay_old == 0)
-		/* 0.98 ~= 1- 1/64 */
-		prob = prob - (prob >> 6); 
-
-
-	/* check for multiplication over/under flow */
-	if (p>0) {
+		/* check for multiplication overflow */
 		if (prob<oldprob) {
 			D("overflow");
 			prob= PIE_MAX_PROB;
 		}
 	}
-	else
-		if (prob>oldprob) {
-			prob= 0;
-			D("underflow");
+
+	/*
+	 * decay the drop probability exponentially
+	 * and restrict it to range 0 to PIE_MAX_PROB
+	 */
+	if (prob < 0) {
+		prob = 0;
+	} else {
+		if (pst->current_qdelay == 0 && pst->qdelay_old == 0) {
+			/* 0.98 ~= 1- 1/64 */
+			prob = prob - (prob >> 6); 
 		}
 
-	/* make drop probability between 0 and PIE_MAX_PROB*/
-	if (prob < 0)
-		prob = 0;
-	else if (prob > PIE_MAX_PROB)
-		prob = PIE_MAX_PROB;
+		if (prob > PIE_MAX_PROB) {
+			prob = PIE_MAX_PROB;
+		}
+	}
 
 	pst->drop_prob = prob;
 	

From ae0669e39007a8b45521cbdddb1f166a78c0570a Mon Sep 17 00:00:00 2001
From: Sepherosa Ziehau <sephe@FreeBSD.org>
Date: Fri, 19 May 2017 01:42:31 +0000
Subject: [PATCH 54/73] net/vlan: Revert 305177

Miss read the parentheses.

Reported by:	oleg@
Reviewed by:	hps@
MFC after:	3 days
Sponsored by:	Microsoft
---
 sys/net/ethernet.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/net/ethernet.h b/sys/net/ethernet.h
index 5ec9d20e471d..bc5fa9cb33f7 100644
--- a/sys/net/ethernet.h
+++ b/sys/net/ethernet.h
@@ -92,7 +92,7 @@ struct ether_vlan_header {
 #define	EVL_PRIOFTAG(tag)	(((tag) >> 13) & 7)
 #define	EVL_CFIOFTAG(tag)	(((tag) >> 12) & 1)
 #define	EVL_MAKETAG(vlid, pri, cfi)					\
-	((((((pri) & 7) << 13) | ((cfi) & 1)) << 12) | ((vlid) & EVL_VLID_MASK))
+	((((((pri) & 7) << 1) | ((cfi) & 1)) << 12) | ((vlid) & EVL_VLID_MASK))
 
 /*
  *  NOTE: 0x0000-0x05DC (0..1500) are generally IEEE 802.3 length fields.

From a3f893fc61cd2f4f6e8a065ab579dc94490dce18 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Fri, 19 May 2017 04:44:14 +0000
Subject: [PATCH 55/73] Use size_t.

Inspired by:	OpenBSD src/lib/libc/stdlib/qsort.c,v 1.11
---
 lib/libc/stdlib/qsort.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/libc/stdlib/qsort.c b/lib/libc/stdlib/qsort.c
index 08816887cf79..1cea4d21b47a 100644
--- a/lib/libc/stdlib/qsort.c
+++ b/lib/libc/stdlib/qsort.c
@@ -41,7 +41,7 @@ typedef int		 cmp_t(void *, const void *, const void *);
 typedef int		 cmp_t(const void *, const void *);
 #endif
 static inline char	*med3(char *, char *, char *, cmp_t *, void *);
-static inline void	 swapfunc(char *, char *, int, int, int);
+static inline void	 swapfunc(char *, char *, size_t, int, int);
 
 #define	MIN(a, b)	((a) < (b) ? a : b)
 
@@ -49,7 +49,7 @@ static inline void	 swapfunc(char *, char *, int, int, int);
  * Qsort routine from Bentley & McIlroy's "Engineering a Sort Function".
  */
 #define	swapcode(TYPE, parmi, parmj, n) {		\
-	long i = (n) / sizeof (TYPE);			\
+	size_t i = (n) / sizeof (TYPE);			\
 	TYPE *pi = (TYPE *) (parmi);		\
 	TYPE *pj = (TYPE *) (parmj);		\
 	do { 						\
@@ -64,7 +64,7 @@ static inline void	 swapfunc(char *, char *, int, int, int);
 	es % sizeof(TYPE) ? 2 : es == sizeof(TYPE) ? 0 : 1;
 
 static inline void
-swapfunc( char *a, char *b, int n, int swaptype_long, int swaptype_int)
+swapfunc(char *a, char *b, size_t n, int swaptype_long, int swaptype_int)
 {
 	if (swaptype_long <= 1)
 		swapcode(long, a, b, n)

From ca1578f0c013f6e0e88142ffa647724d64a66fb3 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Fri, 19 May 2017 04:59:12 +0000
Subject: [PATCH 56/73] The current qsort(3) implementation ignores the sizes
 of partitions, and always perform recursion on the left partition, then use a
 tail call to handle the right partition.  In the worst case this could
 require O(N) levels of recursions.

Reduce the possible recursion level to log2(N) by always recursing on the
smaller partition instead.

Obtained from:	PostgreSQL 9d6077abf9d6efd992a59f05ef5aba981ea32096
---
 lib/libc/stdlib/qsort.c | 53 +++++++++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/lib/libc/stdlib/qsort.c b/lib/libc/stdlib/qsort.c
index 1cea4d21b47a..1ccc518d396f 100644
--- a/lib/libc/stdlib/qsort.c
+++ b/lib/libc/stdlib/qsort.c
@@ -117,7 +117,7 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
 #endif
 {
 	char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
-	size_t d, r;
+	size_t d1, d2;
 	int cmp_result;
 	int swaptype_long, swaptype_int, swap_cnt;
 
@@ -137,7 +137,8 @@ loop:	SWAPINIT(long, a, es);
 		pl = a;
 		pn = (char *)a + (n - 1) * es;
 		if (n > 40) {
-			d = (n / 8) * es;
+			size_t d = (n / 8) * es;
+
 			pl = med3(pl, pl + d, pl + 2 * d, cmp, thunk);
 			pm = med3(pm - d, pm, pm + d, cmp, thunk);
 			pn = med3(pn - 2 * d, pn - d, pn, cmp, thunk);
@@ -182,21 +183,43 @@ loop:	SWAPINIT(long, a, es);
 	}
 
 	pn = (char *)a + n * es;
-	r = MIN(pa - (char *)a, pb - pa);
-	vecswap(a, pb - r, r);
-	r = MIN(pd - pc, pn - pd - es);
-	vecswap(pb, pn - r, r);
-	if ((r = pb - pa) > es)
+	d1 = MIN(pa - (char *)a, pb - pa);
+	vecswap(a, pb - d1, d1);
+	d1 = MIN(pd - pc, pn - pd - es);
+	vecswap(pb, pn - d1, d1);
+
+	d1 = pb - pa;
+	d2 = pd - pc;
+	if (d1 <= d2) {
+		/* Recurse on left partition, then iterate on right partition */
+		if (d1 > es) {
 #ifdef I_AM_QSORT_R
-		qsort_r(a, r / es, es, thunk, cmp);
+			qsort_r(a, d1 / es, es, thunk, cmp);
 #else
-		qsort(a, r / es, es, cmp);
+			qsort(a, d1 / es, es, cmp);
 #endif
-	if ((r = pd - pc) > es) {
-		/* Iterate rather than recurse to save stack space */
-		a = pn - r;
-		n = r / es;
-		goto loop;
+		}
+		if (d2 > es) {
+			/* Iterate rather than recurse to save stack space */
+			/* qsort(pn - d2, d2 / es, es, cmp); */
+			a = pn - d2;
+			n = d2 / es;
+			goto loop;
+		}
+	} else {
+		/* Recurse on right partition, then iterate on left partition */
+		if (d2 > es) {
+#ifdef I_AM_QSORT_R
+			qsort_r(pn - d2, d2 / es, es, thunk, cmp);
+#else
+			qsort(pn - d2, d2 / es, es, cmp);
+#endif
+		}
+		if (d1 > es) {
+			/* Iterate rather than recurse to save stack space */
+			/* qsort(a, d1 / es, es, cmp); */
+			n = d1 / es;
+			goto loop;
+		}
 	}
-/*		qsort(pn - r, r / es, es, cmp);*/
 }

From c9bcbf38004eac1c27ff642224e9f54d351a9849 Mon Sep 17 00:00:00 2001
From: Alexander Motin <mav@FreeBSD.org>
Date: Fri, 19 May 2017 05:12:58 +0000
Subject: [PATCH 57/73] Fix time handling in cv_timedwait_hires().

pthread_cond_timedwait() receives absolute time, not relative.  Passing
wrong time there caused two threads of zdb to spin in a tight loop.

MFC after:	1 week
---
 cddl/contrib/opensolaris/lib/libzpool/common/kernel.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c b/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
index 4cff63e6a1ed..701f0b43bf84 100644
--- a/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
+++ b/cddl/contrib/opensolaris/lib/libzpool/common/kernel.c
@@ -368,7 +368,7 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
     int flag)
 {
 	int error;
-	timestruc_t ts;
+	timespec_t ts;
 	hrtime_t delta;
 
 	ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
@@ -381,8 +381,13 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 	if (delta <= 0)
 		return (-1);
 
-	ts.tv_sec = delta / NANOSEC;
-	ts.tv_nsec = delta % NANOSEC;
+	clock_gettime(CLOCK_REALTIME, &ts);
+	ts.tv_sec += delta / NANOSEC;
+	ts.tv_nsec += delta % NANOSEC;
+	if (ts.tv_nsec >= NANOSEC) {
+		ts.tv_sec++;
+		ts.tv_nsec -= NANOSEC;
+	}
 
 	ASSERT(mutex_owner(mp) == curthread);
 	mp->m_owner = NULL;

From da31cbbca0f3ce0826f8c0d1f71224b3a0264f37 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Fri, 19 May 2017 06:37:16 +0000
Subject: [PATCH 58/73] Sync qsort.c with userland r318515.

(Note that MIN macro is removed in favor of sys/param.h's version).

PR:		213922
---
 sys/libkern/qsort.c | 129 ++++++++++++++++++++++++++++----------------
 1 file changed, 82 insertions(+), 47 deletions(-)

diff --git a/sys/libkern/qsort.c b/sys/libkern/qsort.c
index 0600a3907c78..9e0db7f44a44 100644
--- a/sys/libkern/qsort.c
+++ b/sys/libkern/qsort.c
@@ -33,51 +33,57 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/libkern.h>
 
-#ifdef	I_AM_QSORT_R
-typedef int		cmp_t(void *, const void *, const void *);
+#ifdef I_AM_QSORT_R
+typedef int		 cmp_t(void *, const void *, const void *);
 #else
-typedef int		cmp_t(const void *, const void *);
+typedef int		 cmp_t(const void *, const void *);
 #endif
-static __inline char	*med3(char *, char *, char *, cmp_t *, void *);
-static __inline void	 swapfunc(char *, char *, int, int);
-
-#define min(a, b)	(a) < (b) ? (a) : (b)
+static inline char	*med3(char *, char *, char *, cmp_t *, void *);
+static inline void	 swapfunc(char *, char *, size_t, int, int);
 
 /*
  * Qsort routine from Bentley & McIlroy's "Engineering a Sort Function".
  */
-#define swapcode(TYPE, parmi, parmj, n) { 		\
-	long i = (n) / sizeof (TYPE); 			\
-	TYPE *pi = (TYPE *) (parmi); 			\
-	TYPE *pj = (TYPE *) (parmj); 			\
+#define	swapcode(TYPE, parmi, parmj, n) {		\
+	size_t i = (n) / sizeof (TYPE);			\
+	TYPE *pi = (TYPE *) (parmi);		\
+	TYPE *pj = (TYPE *) (parmj);		\
 	do { 						\
-		TYPE	t = *pi;			\
+		TYPE	t = *pi;		\
 		*pi++ = *pj;				\
 		*pj++ = t;				\
-        } while (--i > 0);				\
+	} while (--i > 0);				\
 }
 
-#define SWAPINIT(a, es) swaptype = ((char *)a - (char *)0) % sizeof(long) || \
-	es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1;
+#define	SWAPINIT(TYPE, a, es) swaptype_ ## TYPE =	\
+	((char *)a - (char *)0) % sizeof(TYPE) ||	\
+	es % sizeof(TYPE) ? 2 : es == sizeof(TYPE) ? 0 : 1;
 
-static __inline void
-swapfunc(char *a, char *b, int n, int swaptype)
+static inline void
+swapfunc(char *a, char *b, size_t n, int swaptype_long, int swaptype_int)
 {
-	if(swaptype <= 1)
+	if (swaptype_long <= 1)
 		swapcode(long, a, b, n)
+	else if (swaptype_int <= 1)
+		swapcode(int, a, b, n)
 	else
 		swapcode(char, a, b, n)
 }
 
-#define swap(a, b)					\
-	if (swaptype == 0) {				\
+#define	swap(a, b)					\
+	if (swaptype_long == 0) {			\
 		long t = *(long *)(a);			\
 		*(long *)(a) = *(long *)(b);		\
 		*(long *)(b) = t;			\
+	} else if (swaptype_int == 0) {			\
+		int t = *(int *)(a);			\
+		*(int *)(a) = *(int *)(b);		\
+		*(int *)(b) = t;			\
 	} else						\
-		swapfunc(a, b, es, swaptype)
+		swapfunc(a, b, es, swaptype_long, swaptype_int)
 
-#define vecswap(a, b, n) 	if ((n) > 0) swapfunc(a, b, n, swaptype)
+#define	vecswap(a, b, n)				\
+	if ((n) > 0) swapfunc(a, b, n, swaptype_long, swaptype_int)
 
 #ifdef I_AM_QSORT_R
 #define	CMP(t, x, y) (cmp((t), (x), (y)))
@@ -85,16 +91,16 @@ swapfunc(char *a, char *b, int n, int swaptype)
 #define	CMP(t, x, y) (cmp((x), (y)))
 #endif
 
-static __inline char *
+static inline char *
 med3(char *a, char *b, char *c, cmp_t *cmp, void *thunk
-#ifndef	I_AM_QSORT_R
+#ifndef I_AM_QSORT_R
 __unused
 #endif
 )
 {
 	return CMP(thunk, a, b) < 0 ?
 	       (CMP(thunk, b, c) < 0 ? b : (CMP(thunk, a, c) < 0 ? c : a ))
-              :(CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c ));
+	      :(CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c ));
 }
 
 #ifdef I_AM_QSORT_R
@@ -107,13 +113,17 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
 #endif
 {
 	char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
-	int d, r, swaptype, swap_cnt;
+	size_t d1, d2;
+	int cmp_result;
+	int swaptype_long, swaptype_int, swap_cnt;
 
-loop:	SWAPINIT(a, es);
+loop:	SWAPINIT(long, a, es);
+	SWAPINIT(int, a, es);
 	swap_cnt = 0;
 	if (n < 7) {
 		for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
-			for (pl = pm; pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
+			for (pl = pm; 
+			     pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
 			     pl -= es)
 				swap(pl, pl - es);
 		return;
@@ -123,7 +133,8 @@ loop:	SWAPINIT(a, es);
 		pl = a;
 		pn = (char *)a + (n - 1) * es;
 		if (n > 40) {
-			d = (n / 8) * es;
+			size_t d = (n / 8) * es;
+
 			pl = med3(pl, pl + d, pl + 2 * d, cmp, thunk);
 			pm = med3(pm - d, pm, pm + d, cmp, thunk);
 			pn = med3(pn - 2 * d, pn - d, pn, cmp, thunk);
@@ -135,16 +146,16 @@ loop:	SWAPINIT(a, es);
 
 	pc = pd = (char *)a + (n - 1) * es;
 	for (;;) {
-		while (pb <= pc && (r = CMP(thunk, pb, a)) <= 0) {
-			if (r == 0) {
+		while (pb <= pc && (cmp_result = CMP(thunk, pb, a)) <= 0) {
+			if (cmp_result == 0) {
 				swap_cnt = 1;
 				swap(pa, pb);
 				pa += es;
 			}
 			pb += es;
 		}
-		while (pb <= pc && (r = CMP(thunk, pc, a)) >= 0) {
-			if (r == 0) {
+		while (pb <= pc && (cmp_result = CMP(thunk, pc, a)) >= 0) {
+			if (cmp_result == 0) {
 				swap_cnt = 1;
 				swap(pc, pd);
 				pd -= es;
@@ -160,27 +171,51 @@ loop:	SWAPINIT(a, es);
 	}
 	if (swap_cnt == 0) {  /* Switch to insertion sort */
 		for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
-			for (pl = pm; pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
+			for (pl = pm; 
+			     pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
 			     pl -= es)
 				swap(pl, pl - es);
 		return;
 	}
 
 	pn = (char *)a + n * es;
-	r = min(pa - (char *)a, pb - pa);
-	vecswap(a, pb - r, r);
-	r = min(pd - pc, pn - pd - es);
-	vecswap(pb, pn - r, r);
-	if ((r = pb - pa) > es)
-#ifdef	I_AM_QSORT_R
-		qsort_r(a, r / es, es, thunk, cmp);
+	d1 = MIN(pa - (char *)a, pb - pa);
+	vecswap(a, pb - d1, d1);
+	d1 = MIN(pd - pc, pn - pd - es);
+	vecswap(pb, pn - d1, d1);
+
+	d1 = pb - pa;
+	d2 = pd - pc;
+	if (d1 <= d2) {
+		/* Recurse on left partition, then iterate on right partition */
+		if (d1 > es) {
+#ifdef I_AM_QSORT_R
+			qsort_r(a, d1 / es, es, thunk, cmp);
 #else
-		qsort(a, r / es, es, cmp);
+			qsort(a, d1 / es, es, cmp);
 #endif
-	if ((r = pd - pc) > es) {
-		/* Iterate rather than recurse to save stack space */
-		a = pn - r;
-		n = r / es;
-		goto loop;
+		}
+		if (d2 > es) {
+			/* Iterate rather than recurse to save stack space */
+			/* qsort(pn - d2, d2 / es, es, cmp); */
+			a = pn - d2;
+			n = d2 / es;
+			goto loop;
+		}
+	} else {
+		/* Recurse on right partition, then iterate on left partition */
+		if (d2 > es) {
+#ifdef I_AM_QSORT_R
+			qsort_r(pn - d2, d2 / es, es, thunk, cmp);
+#else
+			qsort(pn - d2, d2 / es, es, cmp);
+#endif
+		}
+		if (d1 > es) {
+			/* Iterate rather than recurse to save stack space */
+			/* qsort(a, d1 / es, es, cmp); */
+			n = d1 / es;
+			goto loop;
+		}
 	}
 }

From e5d27b37e39b4478c8df094a6ef50272f0c84988 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roger=20Pau=20Monn=C3=A9?= <royger@FreeBSD.org>
Date: Fri, 19 May 2017 08:11:15 +0000
Subject: [PATCH 59/73] xen/blkfront: correctly detach a disk with active users

Call disk_gone when the backend switches to the "Closing" state and blkfront
still has pending users. This allows the disk to be detached, and will call
into xbd_closing by itself when the geom layout cleanup has finished.

Reported by:		bapt
Tested by:		manu
Reviewed by:		bapt
Sponsored by:		Citrix Systems R&D
MFC after:		1 week
Differential revision:	https://reviews.freebsd.org/D10772
---
 sys/dev/xen/blkfront/blkfront.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/sys/dev/xen/blkfront/blkfront.c b/sys/dev/xen/blkfront/blkfront.c
index d76fc01a4bd3..3473b64f8e82 100644
--- a/sys/dev/xen/blkfront/blkfront.c
+++ b/sys/dev/xen/blkfront/blkfront.c
@@ -1578,11 +1578,14 @@ xbd_backend_changed(device_t dev, XenbusState backend_state)
 		break;
 
 	case XenbusStateClosing:
-		if (sc->xbd_users > 0)
-			xenbus_dev_error(dev, -EBUSY,
-			    "Device in use; refusing to close");
-		else
+		if (sc->xbd_users > 0) {
+			device_printf(dev, "detaching with pending users\n");
+			KASSERT(sc->xbd_disk != NULL,
+			    ("NULL disk with pending users\n"));
+			disk_gone(sc->xbd_disk);
+		} else {
 			xbd_closing(dev);
+		}
 		break;	
 	}
 }

From 0a6542a309036094e8118c269ad61c866b44671e Mon Sep 17 00:00:00 2001
From: Wojciech Macek <wma@FreeBSD.org>
Date: Fri, 19 May 2017 08:16:47 +0000
Subject: [PATCH 60/73] Improve busy-wait loop during switch phy access in
 e6000sw

Hitherto implementation of PHY polling resulted in a risk of an
endless loop and very high occupation of the SMI bus. Improve the
operation by limiting the polling tries and adding sleepable
pause.

Submitted by: Marcin Wojtas <mw@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10713
---
 sys/dev/etherswitch/e6000sw/e6000sw.c | 47 ++++++++++++++++++++++-----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/sys/dev/etherswitch/e6000sw/e6000sw.c b/sys/dev/etherswitch/e6000sw/e6000sw.c
index cfc20dbbf5f2..4ce3ae8bfbe1 100644
--- a/sys/dev/etherswitch/e6000sw/e6000sw.c
+++ b/sys/dev/etherswitch/e6000sw/e6000sw.c
@@ -431,13 +431,21 @@ e6000sw_attach(device_t dev)
 	return (err);
 }
 
-static __inline void
+static __inline int
 e6000sw_poll_done(e6000sw_softc_t *sc)
 {
+	int i;
 
-	while (e6000sw_readreg(sc, REG_GLOBAL2, PHY_CMD) &
-	    (1 << PHY_CMD_SMI_BUSY))
-		continue;
+	for (i = 0; i < 16; i++) {
+
+		if (!(e6000sw_readreg(sc, REG_GLOBAL2, PHY_CMD) &
+		    (1 << PHY_CMD_SMI_BUSY)))
+			return (0);
+
+		pause("e6000sw PHY poll", hz/1000);
+	}
+
+	return (ETIMEDOUT);
 }
 
 /*
@@ -449,6 +457,7 @@ e6000sw_readphy(device_t dev, int phy, int reg)
 {
 	e6000sw_softc_t *sc;
 	uint32_t val;
+	int err;
 
 	sc = device_get_softc(dev);
 	val = 0;
@@ -460,14 +469,25 @@ e6000sw_readphy(device_t dev, int phy, int reg)
 
 	E6000SW_LOCK_ASSERT(sc, SA_XLOCKED);
 
-	e6000sw_poll_done(sc);
+	err = e6000sw_poll_done(sc);
+	if (err != 0) {
+		device_printf(dev, "Timeout while waiting for switch\n");
+		return (err);
+	}
+
 	val |= 1 << PHY_CMD_SMI_BUSY;
 	val |= PHY_CMD_MODE_MDIO << PHY_CMD_MODE;
 	val |= PHY_CMD_OPCODE_READ << PHY_CMD_OPCODE;
 	val |= (reg << PHY_CMD_REG_ADDR) & PHY_CMD_REG_ADDR_MASK;
 	val |= (phy << PHY_CMD_DEV_ADDR) & PHY_CMD_DEV_ADDR_MASK;
 	e6000sw_writereg(sc, REG_GLOBAL2, SMI_PHY_CMD_REG, val);
-	e6000sw_poll_done(sc);
+
+	err = e6000sw_poll_done(sc);
+	if (err != 0) {
+		device_printf(dev, "Timeout while waiting for switch\n");
+		return (err);
+	}
+
 	val = e6000sw_readreg(sc, REG_GLOBAL2, SMI_PHY_DATA_REG)
 		& PHY_DATA_MASK;
 
@@ -479,6 +499,7 @@ e6000sw_writephy(device_t dev, int phy, int reg, int data)
 {
 	e6000sw_softc_t *sc;
 	uint32_t val;
+	int err;
 
 	sc = device_get_softc(dev);
 	val = 0;
@@ -490,7 +511,12 @@ e6000sw_writephy(device_t dev, int phy, int reg, int data)
 
 	E6000SW_LOCK_ASSERT(sc, SA_XLOCKED);
 
-	e6000sw_poll_done(sc);
+	err = e6000sw_poll_done(sc);
+	if (err != 0) {
+		device_printf(dev, "Timeout while waiting for switch\n");
+		return (err);
+	}
+
 	val |= PHY_CMD_MODE_MDIO << PHY_CMD_MODE;
 	val |= 1 << PHY_CMD_SMI_BUSY;
 	val |= PHY_CMD_OPCODE_WRITE << PHY_CMD_OPCODE;
@@ -499,7 +525,12 @@ e6000sw_writephy(device_t dev, int phy, int reg, int data)
 	e6000sw_writereg(sc, REG_GLOBAL2, SMI_PHY_DATA_REG,
 			 data & PHY_DATA_MASK);
 	e6000sw_writereg(sc, REG_GLOBAL2, SMI_PHY_CMD_REG, val);
-	e6000sw_poll_done(sc);
+
+	err = e6000sw_poll_done(sc);
+	if (err != 0) {
+		device_printf(dev, "Timeout while waiting for switch\n");
+		return (err);
+	}
 
 	return (0);
 }

From fcb93d74937ba64f0a5b0b054d08b948bdb9adf7 Mon Sep 17 00:00:00 2001
From: Wojciech Macek <wma@FreeBSD.org>
Date: Fri, 19 May 2017 08:19:39 +0000
Subject: [PATCH 61/73] Enable proper configuration of CESA MBUS windows

For all Marvell devices, MBUS windows configuration is done
in a common place. Only CESA was an exception, so move its
related code from driver to mv_common.c. This way it uses
same proper DRAM information, same as  all other interfaces
instead of parsing DT /memory node directly.

Submitted by: Marcin Wojtas <mw@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10723
---
 sys/arm/mv/mv_common.c | 68 +++++++++++++++++++++++++++++++++++++++++-
 sys/arm/mv/mvwin.h     |  5 ++++
 sys/dev/cesa/cesa.c    | 56 ----------------------------------
 sys/dev/cesa/cesa.h    |  5 ----
 4 files changed, 72 insertions(+), 62 deletions(-)

diff --git a/sys/arm/mv/mv_common.c b/sys/arm/mv/mv_common.c
index c711998fcdab..d90ebcce5c71 100644
--- a/sys/arm/mv/mv_common.c
+++ b/sys/arm/mv/mv_common.c
@@ -76,6 +76,7 @@ MALLOC_DEFINE(M_IDMA, "idma", "idma dma test memory");
 
 static int win_eth_can_remap(int i);
 
+static int decode_win_cesa_valid(void);
 static int decode_win_cpu_valid(void);
 static int decode_win_usb_valid(void);
 static int decode_win_usb3_valid(void);
@@ -91,6 +92,7 @@ static void decode_win_cpu_setup(void);
 #ifdef SOC_MV_ARMADAXP
 static int decode_win_sdram_fixup(void);
 #endif
+static void decode_win_cesa_setup(u_long);
 static void decode_win_usb_setup(u_long);
 static void decode_win_usb3_setup(u_long);
 static void decode_win_eth_setup(u_long);
@@ -101,6 +103,7 @@ static void decode_win_sdhci_setup(u_long);
 static void decode_win_idma_setup(u_long);
 static void decode_win_xor_setup(u_long);
 
+static void decode_win_cesa_dump(u_long);
 static void decode_win_usb_dump(u_long);
 static void decode_win_usb3_dump(u_long);
 static void decode_win_eth_dump(u_long base);
@@ -146,6 +149,7 @@ static struct soc_node_spec soc_nodes[] = {
 	{ "mrvl,sata", &decode_win_sata_setup, NULL },
 	{ "mrvl,xor", &decode_win_xor_setup, &decode_win_xor_dump },
 	{ "mrvl,idma", &decode_win_idma_setup, &decode_win_idma_dump },
+	{ "mrvl,cesa", &decode_win_cesa_setup, &decode_win_cesa_dump },
 	{ "mrvl,pcie", &decode_win_pcie_setup, NULL },
 	{ NULL, NULL, NULL },
 };
@@ -574,7 +578,7 @@ soc_decode_win(void)
 	    !decode_win_eth_valid() || !decode_win_idma_valid() ||
 	    !decode_win_pcie_valid() || !decode_win_sata_valid() ||
 	    !decode_win_xor_valid() || !decode_win_usb3_valid() ||
-	    !decode_win_sdhci_valid())
+	    !decode_win_sdhci_valid() || !decode_win_cesa_valid())
 		return (EINVAL);
 
 	decode_win_cpu_setup();
@@ -601,6 +605,11 @@ WIN_REG_IDX_WR(win_cpu, br, MV_WIN_CPU_BASE, MV_MBUS_BRIDGE_BASE)
 WIN_REG_IDX_WR(win_cpu, remap_l, MV_WIN_CPU_REMAP_LO, MV_MBUS_BRIDGE_BASE)
 WIN_REG_IDX_WR(win_cpu, remap_h, MV_WIN_CPU_REMAP_HI, MV_MBUS_BRIDGE_BASE)
 
+WIN_REG_BASE_IDX_RD(win_cesa, cr, MV_WIN_CESA_CTRL)
+WIN_REG_BASE_IDX_RD(win_cesa, br, MV_WIN_CESA_BASE)
+WIN_REG_BASE_IDX_WR(win_cesa, cr, MV_WIN_CESA_CTRL)
+WIN_REG_BASE_IDX_WR(win_cesa, br, MV_WIN_CESA_BASE)
+
 WIN_REG_BASE_IDX_RD(win_usb, cr, MV_WIN_USB_CTRL)
 WIN_REG_BASE_IDX_RD(win_usb, br, MV_WIN_USB_BASE)
 WIN_REG_BASE_IDX_WR(win_usb, cr, MV_WIN_USB_CTRL)
@@ -1070,6 +1079,63 @@ ddr_target(int i)
 	return (0);
 }
 
+/**************************************************************************
+ * CESA windows routines
+ **************************************************************************/
+static int
+decode_win_cesa_valid(void)
+{
+
+	return (decode_win_can_cover_ddr(MV_WIN_CESA_MAX));
+}
+
+static void
+decode_win_cesa_dump(u_long base)
+{
+	int i;
+
+	for (i = 0; i < MV_WIN_CESA_MAX; i++)
+		printf("CESA window#%d: c 0x%08x, b 0x%08x\n", i,
+		    win_cesa_cr_read(base, i), win_cesa_br_read(base, i));
+}
+
+/*
+ * Set CESA decode windows.
+ */
+static void
+decode_win_cesa_setup(u_long base)
+{
+	uint32_t br, cr;
+	int i, j;
+
+	for (i = 0; i < MV_WIN_CESA_MAX; i++) {
+		win_cesa_cr_write(base, i, 0);
+		win_cesa_br_write(base, i, 0);
+	}
+
+	/* Only access to active DRAM banks is required */
+	for (i = 0; i < MV_WIN_DDR_MAX; i++) {
+		if (ddr_is_active(i)) {
+			br = ddr_base(i);
+
+			cr = (((ddr_size(i) - 1) & 0xffff0000) |
+			    (ddr_attr(i) << IO_WIN_ATTR_SHIFT) |
+			    (ddr_target(i) << IO_WIN_TGT_SHIFT) |
+			    IO_WIN_ENA_MASK);
+
+			/* Set the first free CESA window */
+			for (j = 0; j < MV_WIN_CESA_MAX; j++) {
+				if (win_cesa_cr_read(base, j) & 0x1)
+					continue;
+
+				win_cesa_br_write(base, j, br);
+				win_cesa_cr_write(base, j, cr);
+				break;
+			}
+		}
+	}
+}
+
 /**************************************************************************
  * USB windows routines
  **************************************************************************/
diff --git a/sys/arm/mv/mvwin.h b/sys/arm/mv/mvwin.h
index 4c1062229eba..65a6eb9362d8 100644
--- a/sys/arm/mv/mvwin.h
+++ b/sys/arm/mv/mvwin.h
@@ -216,6 +216,11 @@
 #define MV_WIN_CESA_ATTR(eng_sel)	0
 #endif
 
+/* CESA TDMA address decoding registers */
+#define MV_WIN_CESA_CTRL(n)		(0x8 * (n) + 0xA04)
+#define MV_WIN_CESA_BASE(n)		(0x8 * (n) + 0xA00)
+#define MV_WIN_CESA_MAX			4
+
 #define MV_WIN_USB_CTRL(n)		(0x10 * (n) + 0x320)
 #define MV_WIN_USB_BASE(n)		(0x10 * (n) + 0x324)
 #define MV_WIN_USB_MAX			4
diff --git a/sys/dev/cesa/cesa.c b/sys/dev/cesa/cesa.c
index d60ca11b8a36..3b74d70adfce 100644
--- a/sys/dev/cesa/cesa.c
+++ b/sys/dev/cesa/cesa.c
@@ -69,7 +69,6 @@ __FBSDID("$FreeBSD$");
 #include "cryptodev_if.h"
 
 #include <arm/mv/mvreg.h>
-#include <arm/mv/mvwin.h>
 #include <arm/mv/mvvar.h>
 #include "cesa.h"
 
@@ -80,7 +79,6 @@ static void	cesa_intr(void *);
 static int	cesa_newsession(device_t, u_int32_t *, struct cryptoini *);
 static int	cesa_freesession(device_t, u_int64_t);
 static int	cesa_process(device_t, struct cryptop *, int);
-static int	decode_win_cesa_setup(struct cesa_softc *sc);
 
 static struct resource_spec cesa_res_spec[] = {
 	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
@@ -1085,13 +1083,6 @@ cesa_attach(device_t dev)
 		goto err0;
 	}
 
-	/* Setup CESA decoding windows */
-	error = decode_win_cesa_setup(sc);
-	if (error) {
-		device_printf(dev, "could not setup decoding windows\n");
-		goto err1;
-	}
-
 	/* Acquire SRAM base address */
 	error = cesa_setup_sram(sc);
 	if (error) {
@@ -1706,50 +1697,3 @@ cesa_process(device_t dev, struct cryptop *crp, int hint)
 
 	return (0);
 }
-
-/*
- * Set CESA TDMA decode windows.
- */
-static int
-decode_win_cesa_setup(struct cesa_softc *sc)
-{
-	struct mem_region availmem_regions[FDT_MEM_REGIONS];
-	int availmem_regions_sz;
-	uint32_t br, cr, i;
-
-	/* Grab physical memory regions information from DTS */
-	if (fdt_get_mem_regions(availmem_regions, &availmem_regions_sz,
-	    NULL) != 0)
-		return (ENXIO);
-
-	if (availmem_regions_sz > MV_WIN_CESA_MAX) {
-		device_printf(sc->sc_dev, "Too much memory regions, cannot "
-		    " set CESA windows to cover whole DRAM \n");
-		return (ENXIO);
-	}
-
-	/* Disable and clear all CESA windows */
-	for (i = 0; i < MV_WIN_CESA_MAX; i++) {
-		CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), 0);
-		CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), 0);
-	}
-
-	/* Fill CESA TDMA decoding windows with information acquired from DTS */
-	for (i = 0; i < availmem_regions_sz; i++) {
-		br = availmem_regions[i].mr_start;
-		cr = availmem_regions[i].mr_size;
-
-		/* Don't add entries with size lower than 64KB */
-		if (cr & 0xffff0000) {
-			cr = (((cr - 1) & 0xffff0000) |
-			(MV_WIN_DDR_ATTR(i) << MV_WIN_CPU_ATTR_SHIFT) |
-			    (MV_WIN_DDR_TARGET << MV_WIN_CPU_TARGET_SHIFT) |
-			    MV_WIN_CPU_ENABLE_BIT);
-			CESA_TDMA_WRITE(sc, MV_WIN_CESA_BASE(i), br);
-			CESA_TDMA_WRITE(sc, MV_WIN_CESA_CTRL(i), cr);
-		}
-	}
-
-	return (0);
-}
-
diff --git a/sys/dev/cesa/cesa.h b/sys/dev/cesa/cesa.h
index fbd7d9396c21..1b284decdb1f 100644
--- a/sys/dev/cesa/cesa.h
+++ b/sys/dev/cesa/cesa.h
@@ -350,11 +350,6 @@ struct cesa_chain_info {
 #define CESA_TDMA_EMR_BOTH_HIT		CESA_TDMA_ECR_BOTH_HIT
 #define CESA_TDMA_EMR_DATA_ERROR	CESA_TDMA_ECR_DATA_ERROR
 
-/*  CESA TDMA address decoding registers */
-#define MV_WIN_CESA_CTRL(n)		(0x8 * (n) + 0xA04)
-#define MV_WIN_CESA_BASE(n)		(0x8 * (n) + 0xA00)
-#define MV_WIN_CESA_MAX			4
-
 /* CESA SA registers definitions */
 #define CESA_SA_CMD			0x0E00
 #define CESA_SA_CMD_ACTVATE		(1 << 0)

From bf319173f298f79632aaff2013c09f4bb50914b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roger=20Pau=20Monn=C3=A9?= <royger@FreeBSD.org>
Date: Fri, 19 May 2017 08:19:51 +0000
Subject: [PATCH 62/73] xen/netfront: don't drop the ring RX lock with
 inconsistent ring state

Make sure the RX ring lock is only released when the state of the ring is
consistent, or else concurrent calls to xn_rxeof might get an inconsistent ring
state and thus some packets might be processed twice.

Note that this is not very common, and could only happen when an interrupt is
delivered while in xn_ifinit.

Reported by:	cperciva
Tested by:	cperciva
MFC after:	1 week
Sponsored by:	Citrix Systems R&D
---
 sys/dev/xen/netfront/netfront.c | 90 ++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 47 deletions(-)

diff --git a/sys/dev/xen/netfront/netfront.c b/sys/dev/xen/netfront/netfront.c
index 99ccdaaf5000..482b9e948fde 100644
--- a/sys/dev/xen/netfront/netfront.c
+++ b/sys/dev/xen/netfront/netfront.c
@@ -1161,17 +1161,18 @@ xn_rxeof(struct netfront_rxq *rxq)
 	struct mbufq mbufq_rxq, mbufq_errq;
 	int err, work_to_do;
 
+	XN_RX_LOCK_ASSERT(rxq);
+
+	if (!netfront_carrier_ok(np))
+		return;
+
+	/* XXX: there should be some sane limit. */
+	mbufq_init(&mbufq_errq, INT_MAX);
+	mbufq_init(&mbufq_rxq, INT_MAX);
+
+	ifp = np->xn_ifp;
+
 	do {
-		XN_RX_LOCK_ASSERT(rxq);
-		if (!netfront_carrier_ok(np))
-			return;
-
-		/* XXX: there should be some sane limit. */
-		mbufq_init(&mbufq_errq, INT_MAX);
-		mbufq_init(&mbufq_rxq, INT_MAX);
-
-		ifp = np->xn_ifp;
-
 		rp = rxq->ring.sring->rsp_prod;
 		rmb();	/* Ensure we see queued responses up to 'rp'. */
 
@@ -1191,7 +1192,7 @@ xn_rxeof(struct netfront_rxq *rxq)
 			}
 
 			m->m_pkthdr.rcvif = ifp;
-			if ( rx->flags & NETRXF_data_validated ) {
+			if (rx->flags & NETRXF_data_validated) {
 				/*
 				 * According to mbuf(9) the correct way to tell
 				 * the stack that the checksum of an inbound
@@ -1214,50 +1215,45 @@ xn_rxeof(struct netfront_rxq *rxq)
 			}
 
 			(void )mbufq_enqueue(&mbufq_rxq, m);
-			rxq->ring.rsp_cons = i;
-		}
-
-		mbufq_drain(&mbufq_errq);
-
-		/*
-		 * Process all the mbufs after the remapping is complete.
-		 * Break the mbuf chain first though.
-		 */
-		while ((m = mbufq_dequeue(&mbufq_rxq)) != NULL) {
-			if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
-
-			/* XXX: Do we really need to drop the rx lock? */
-			XN_RX_UNLOCK(rxq);
-#if (defined(INET) || defined(INET6))
-			/* Use LRO if possible */
-			if ((ifp->if_capenable & IFCAP_LRO) == 0 ||
-			    lro->lro_cnt == 0 || tcp_lro_rx(lro, m, 0)) {
-				/*
-				 * If LRO fails, pass up to the stack
-				 * directly.
-				 */
-				(*ifp->if_input)(ifp, m);
-			}
-#else
-			(*ifp->if_input)(ifp, m);
-#endif
-
-			XN_RX_LOCK(rxq);
 		}
 
 		rxq->ring.rsp_cons = i;
 
-#if (defined(INET) || defined(INET6))
-		/*
-		 * Flush any outstanding LRO work
-		 */
-		tcp_lro_flush_all(lro);
-#endif
-
 		xn_alloc_rx_buffers(rxq);
 
 		RING_FINAL_CHECK_FOR_RESPONSES(&rxq->ring, work_to_do);
 	} while (work_to_do);
+
+	XN_RX_UNLOCK(rxq);
+	mbufq_drain(&mbufq_errq);
+	/*
+	 * Process all the mbufs after the remapping is complete.
+	 * Break the mbuf chain first though.
+	 */
+	while ((m = mbufq_dequeue(&mbufq_rxq)) != NULL) {
+		if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
+#if (defined(INET) || defined(INET6))
+		/* Use LRO if possible */
+		if ((ifp->if_capenable & IFCAP_LRO) == 0 ||
+		    lro->lro_cnt == 0 || tcp_lro_rx(lro, m, 0)) {
+			/*
+			 * If LRO fails, pass up to the stack
+			 * directly.
+			 */
+			(*ifp->if_input)(ifp, m);
+		}
+#else
+		(*ifp->if_input)(ifp, m);
+#endif
+	}
+
+#if (defined(INET) || defined(INET6))
+	/*
+	 * Flush any outstanding LRO work
+	 */
+	tcp_lro_flush_all(lro);
+#endif
+	XN_RX_LOCK(rxq);
 }
 
 static void

From 2dd020069e3d868047357044d87dfe509df1643e Mon Sep 17 00:00:00 2001
From: Wojciech Macek <wma@FreeBSD.org>
Date: Fri, 19 May 2017 08:24:23 +0000
Subject: [PATCH 63/73] Poll PHY status using internal e6000sw registers

e6000sw family automatically reflects PHY status in each port's registers.
Therefore it is not necessary to do a full PHY polling squence, which
results in much quicker operation and much less significant usage of
the SMI bus.

Care must be taken that the resulting ifmedia_active is identical to
what the PHY will compute, or gratuitous link status changes will
occur whenever the PHYs update function is called.

This patch implements above improvement. On the occasion set a pointer to
the proc structure to be part of software context instead of being
a global variable.

Submitted by: Marcin Wojtas <mw@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: loos
Differential revision: https://reviews.freebsd.org/D10714
---
 sys/dev/etherswitch/e6000sw/e6000sw.c    | 51 +++++++++++++++++++++---
 sys/dev/etherswitch/e6000sw/e6000swreg.h |  8 ++++
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/sys/dev/etherswitch/e6000sw/e6000sw.c b/sys/dev/etherswitch/e6000sw/e6000sw.c
index 4ce3ae8bfbe1..5884a23c5516 100644
--- a/sys/dev/etherswitch/e6000sw/e6000sw.c
+++ b/sys/dev/etherswitch/e6000sw/e6000sw.c
@@ -89,7 +89,7 @@ typedef struct e6000sw_softc {
 	char			*ifname[E6000SW_MAX_PORTS];
 	device_t		miibus[E6000SW_MAX_PORTS];
 	struct mii_data		*mii[E6000SW_MAX_PORTS];
-	struct callout		tick_callout;
+	struct proc		*kproc;
 
 	uint32_t		cpuports_mask;
 	uint32_t		fixed_mask;
@@ -149,8 +149,6 @@ static __inline int e6000sw_is_phyport(e6000sw_softc_t *sc, int port);
 static __inline struct mii_data *e6000sw_miiforphy(e6000sw_softc_t *sc,
     unsigned int phy);
 
-static struct proc *e6000sw_kproc;
-
 static device_method_t e6000sw_methods[] = {
 	/* device interface */
 	DEVMETHOD(device_identify,		e6000sw_identify),
@@ -419,8 +417,7 @@ e6000sw_attach(device_t dev)
 	bus_generic_probe(dev);
 	bus_generic_attach(dev);
 
-	kproc_create(e6000sw_tick, sc, &e6000sw_kproc, 0, 0,
-	    "e6000sw tick kproc");
+	kproc_create(e6000sw_tick, sc, &sc->kproc, 0, 0, "e6000sw tick kproc");
 
 	return (0);
 
@@ -1010,23 +1007,65 @@ e6000sw_get_pvid(e6000sw_softc_t *sc, int port, int *pvid)
 	return (0);
 }
 
+/*
+ * Convert port status to ifmedia.
+ */
+static void
+e6000sw_update_ifmedia(uint16_t portstatus, u_int *media_status, u_int *media_active)
+{
+	*media_active = IFM_ETHER;
+	*media_status = IFM_AVALID;
+
+	if ((portstatus & PORT_STATUS_LINK_MASK) != 0)
+		*media_status |= IFM_ACTIVE;
+	else {
+		*media_active |= IFM_NONE;
+		return;
+	}
+
+	switch (portstatus & PORT_STATUS_SPEED_MASK) {
+	case PORT_STATUS_SPEED_10:
+		*media_active |= IFM_10_T;
+		break;
+	case PORT_STATUS_SPEED_100:
+		*media_active |= IFM_100_TX;
+		break;
+	case PORT_STATUS_SPEED_1000:
+		*media_active |= IFM_1000_T;
+		break;
+	}
+
+	if ((portstatus & PORT_STATUS_DUPLEX_MASK) == 0)
+		*media_active |= IFM_FDX;
+	else
+		*media_active |= IFM_HDX;
+}
+
 static void
 e6000sw_tick (void *arg)
 {
 	e6000sw_softc_t *sc;
 	struct mii_softc *miisc;
+	uint16_t portstatus;
 	int port;
 
 	sc = arg;
 
 	E6000SW_LOCK_ASSERT(sc, SA_UNLOCKED);
+
 	for (;;) {
 		E6000SW_LOCK(sc);
 		for (port = 0; port < sc->num_ports; port++) {
 			/* Tick only on PHY ports */
 			if (!e6000sw_is_phyport(sc, port))
 				continue;
-			mii_tick(sc->mii[port]);
+
+			portstatus = e6000sw_readreg(sc, REG_PORT(port), PORT_STATUS);
+
+			e6000sw_update_ifmedia(portstatus,
+			    &sc->mii[port]->mii_media_status,
+			    &sc->mii[port]->mii_media_active);
+
 			LIST_FOREACH(miisc, &sc->mii[port]->mii_phys, mii_list) {
 				if (IFM_INST(sc->mii[port]->mii_media.ifm_cur->ifm_media)
 				    != miisc->mii_inst)
diff --git a/sys/dev/etherswitch/e6000sw/e6000swreg.h b/sys/dev/etherswitch/e6000sw/e6000swreg.h
index b3a053366d45..d8206eac4519 100644
--- a/sys/dev/etherswitch/e6000sw/e6000swreg.h
+++ b/sys/dev/etherswitch/e6000sw/e6000swreg.h
@@ -55,6 +55,14 @@ struct atu_opt {
  * Per-Port Switch Registers
  */
 #define PORT_STATUS			0x0
+#define	PORT_STATUS_SPEED_MASK		0x300
+#define	PORT_STATUS_SPEED_10		0
+#define	PORT_STATUS_SPEED_100		1
+#define	PORT_STATUS_SPEED_1000		2
+#define	PORT_STATUS_DUPLEX_MASK		(1 << 10)
+#define	PORT_STATUS_LINK_MASK		(1 << 11)
+#define	PORT_STATUS_PHY_DETECT_MASK	(1 << 12)
+
 #define PSC_CONTROL			0x1
 #define SWITCH_ID			0x3
 #define PORT_CONTROL			0x4

From c7a65ae3fe7ffc97af114efb07c7e167192bf057 Mon Sep 17 00:00:00 2001
From: Wojciech Macek <wma@FreeBSD.org>
Date: Fri, 19 May 2017 08:25:40 +0000
Subject: [PATCH 64/73] Fix MPIC mask/unmask

Before the fix for single interrupt, both percpu and non-percpu routes
were enabled/disable at the same time.

Submitted by:  Marcin Wojtas <mw@semihalf.com
Obtained from: Semihalf
Sponsored by:  Stormshield, Netgate
Reviewed by:   loos
Differential revision: https://reviews.freebsd.org/D10716
---
 sys/arm/mv/mpic.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/sys/arm/mv/mpic.c b/sys/arm/mv/mpic.c
index 70c17ae3faa2..49e4bd49da68 100644
--- a/sys/arm/mv/mpic.c
+++ b/sys/arm/mv/mpic.c
@@ -148,6 +148,7 @@ static void	mpic_unmask_irq(uintptr_t nb);
 static void	mpic_mask_irq(uintptr_t nb);
 static void	mpic_mask_irq_err(uintptr_t nb);
 static void	mpic_unmask_irq_err(uintptr_t nb);
+static boolean_t mpic_irq_is_percpu(uintptr_t);
 #ifdef INTRNG
 static int	mpic_intr(void *arg);
 #endif
@@ -474,14 +475,24 @@ mpic_mask_irq_err(uintptr_t nb)
 	MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ERR_MASK, mask);
 }
 
+static boolean_t
+mpic_irq_is_percpu(uintptr_t nb)
+{
+	if (nb < MPIC_PPI)
+		return TRUE;
+
+	return FALSE;
+}
+
 static void
 mpic_unmask_irq(uintptr_t nb)
 {
 
-	if (nb < ERR_IRQ) {
-		MPIC_WRITE(mv_mpic_sc, MPIC_ISE, nb);
+	if (mpic_irq_is_percpu(nb))
 		MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, nb);
-	} else if (nb < MSI_IRQ)
+	else if (nb < ERR_IRQ)
+		MPIC_WRITE(mv_mpic_sc, MPIC_ISE, nb);
+	else if (nb < MSI_IRQ)
 		mpic_unmask_irq_err(nb);
 
 	if (nb == 0)
@@ -492,10 +503,11 @@ static void
 mpic_mask_irq(uintptr_t nb)
 {
 
-	if (nb < ERR_IRQ) {
-		MPIC_WRITE(mv_mpic_sc, MPIC_ICE, nb);
+	if (mpic_irq_is_percpu(nb))
 		MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ISM, nb);
-	} else if (nb < MSI_IRQ)
+	else if (nb < ERR_IRQ)
+		MPIC_WRITE(mv_mpic_sc, MPIC_ICE, nb);
+	else if (nb < MSI_IRQ)
 		mpic_mask_irq_err(nb);
 }
 

From 22a752b482e357c78745b310f6bb916346f05721 Mon Sep 17 00:00:00 2001
From: Wojciech Macek <wma@FreeBSD.org>
Date: Fri, 19 May 2017 08:26:41 +0000
Subject: [PATCH 65/73] Fix boot up on ARMADA38X uniprocessor variant

Marvell Armada 380 is a uni-processor variant of the 38x SoC
family. A function platform_mp_setmaxid() was setting a hardcoded
value, which caused boot fail on A380. Fix this by relying on
the CPU count obtained from device tree nodes.

Submitted by:  Marcin Wojtas <mw@semihalf.com
Obtained from: Semihalf
Sponsored by:  Netgate
Reviewed by:   loos
Differential revision: https://reviews.freebsd.org/D10783
---
 sys/arm/mv/armada38x/armada38x_mp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/arm/mv/armada38x/armada38x_mp.c b/sys/arm/mv/armada38x/armada38x_mp.c
index 8dc404fcfeb0..ab6550233a18 100644
--- a/sys/arm/mv/armada38x/armada38x_mp.c
+++ b/sys/arm/mv/armada38x/armada38x_mp.c
@@ -127,7 +127,7 @@ platform_mp_setmaxid(void)
 
 	/* Armada38x family supports maximum 2 cores */
 	mp_ncpus = platform_cnt_cpus();
-	mp_maxid = 1;
+	mp_maxid = mp_ncpus - 1;
 }
 
 void

From d196c9ee16ff7416f68e5e1c6529d4344baef236 Mon Sep 17 00:00:00 2001
From: Don Lewis <truckman@FreeBSD.org>
Date: Fri, 19 May 2017 08:38:03 +0000
Subject: [PATCH 66/73] Fix the queue delay estimation in PIE/FQ-PIE when the
 timestamp (TS) method is used.  When packet timestamp is used, the
 "current_qdelay" keeps storing the last queue delay value calculated in the
 dequeue function.  Therefore, when a burst of packets arrives followed by a
 pause, the "current_qdelay" will store a high value caused by the burst and
 stick to that value during the pause because the queue delay measurement is
 done inside the dequeue function.  This causes the drop probability
 calculation function to calculate high drop probability value instead of zero
 and prevents the burst allowance mechanism from working properly.  Fix this
 problem by resetting "current_qdelay" inside the drop probability calculation
 function when the queue length is zero and TS option is used.

Submitted by:	Rasool Al-Saadi <ralsaadi@swin.edu.au>
MFC after:	1 week
---
 sys/netpfil/ipfw/dn_aqm_pie.c      | 11 ++++++++---
 sys/netpfil/ipfw/dn_sched_fq_pie.c | 11 ++++++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/sys/netpfil/ipfw/dn_aqm_pie.c b/sys/netpfil/ipfw/dn_aqm_pie.c
index 8259d0853df6..c306a4caac9d 100644
--- a/sys/netpfil/ipfw/dn_aqm_pie.c
+++ b/sys/netpfil/ipfw/dn_aqm_pie.c
@@ -211,11 +211,16 @@ calculate_drop_prob(void *x)
 	pprms = pst->parms;
 	prob = pst->drop_prob;
 
-	/* calculate current qdelay */
-	if (pprms->flags & PIE_DEPRATEEST_ENABLED) {
+	/* calculate current qdelay using DRE method.
+	 * If TS is used and no data in the queue, reset current_qdelay
+	 * as it stays at last value during dequeue process. 
+	*/
+	if (pprms->flags & PIE_DEPRATEEST_ENABLED)
 		pst->current_qdelay = ((uint64_t)pst->pq->ni.len_bytes *
 			pst->avg_dq_time) >> PIE_DQ_THRESHOLD_BITS;
-	}
+	else 
+		if (!pst->pq->ni.len_bytes)
+			 pst->current_qdelay = 0;
 
 	/* calculate drop probability */
 	p = (int64_t)pprms->alpha * 
diff --git a/sys/netpfil/ipfw/dn_sched_fq_pie.c b/sys/netpfil/ipfw/dn_sched_fq_pie.c
index 2ba16ab5fa6e..5ee40069fa42 100644
--- a/sys/netpfil/ipfw/dn_sched_fq_pie.c
+++ b/sys/netpfil/ipfw/dn_sched_fq_pie.c
@@ -383,11 +383,16 @@ fq_calculate_drop_prob(void *x)
 	pprms = pst->parms;
 	prob = pst->drop_prob;
 
-	/* calculate current qdelay */
-	if (pprms->flags & PIE_DEPRATEEST_ENABLED) {
+	/* calculate current qdelay using DRE method.
+	 * If TS is used and no data in the queue, reset current_qdelay
+	 * as it stays at last value during dequeue process.
+	*/
+	if (pprms->flags & PIE_DEPRATEEST_ENABLED)
 		pst->current_qdelay = ((uint64_t)q->stats.len_bytes  * pst->avg_dq_time)
 			>> PIE_DQ_THRESHOLD_BITS;
-	}
+	else
+		if (!q->stats.len_bytes)
+			pst->current_qdelay = 0;
 
 	/* calculate drop probability */
 	p = (int64_t)pprms->alpha * 

From d304b9ec4aaeaaab16f9544f1758c06a227771d1 Mon Sep 17 00:00:00 2001
From: Michal Meloun <mmel@FreeBSD.org>
Date: Fri, 19 May 2017 11:45:14 +0000
Subject: [PATCH 67/73] Increase maximum text segment size. LLVM binaries are
 huge...

MFC after:	3 days
---
 sys/arm/include/vmparam.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h
index c28b28ebfd73..061ccabe9d2c 100644
--- a/sys/arm/include/vmparam.h
+++ b/sys/arm/include/vmparam.h
@@ -42,7 +42,7 @@
  * Virtual memory related constants, all in bytes
  */
 #ifndef	MAXTSIZ
-#define	MAXTSIZ		(64UL*1024*1024)	/* max text size */
+#define	MAXTSIZ		(256UL*1024*1024)	/* max text size */
 #endif
 #ifndef	DFLDSIZ
 #define	DFLDSIZ		(128UL*1024*1024)	/* initial data size limit */

From fe3ca95c9d82b1544b2cded067796a75b21e30e2 Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Fri, 19 May 2017 12:22:48 +0000
Subject: [PATCH 68/73] mlx4: Use the CQ quota for SRIOV when creating
 completion EQs

When creating EQs to handle CQ completion events for the PF or for
VFs, we create enough EQE entries to handle completions for the max
number of CQs that can use that EQ.

When SRIOV is activated, the max number of CQs a VF (or the PF) can
obtain is its CQ quota (determined by the Hypervisor resource
tracker).  Therefore, when creating an EQ, the number of EQE entries
that the VF should request for that EQ is the CQ quota value (and not
the total number of CQs available in the firmware).

Under SRIOV, the PF, also must use its CQ quota, because the resource
tracker also controls how many CQs the PF can obtain.

Using the firmware total CQs instead of the CQ quota when creating EQs
resulted wasting MTT entries, due to allocating more EQEs than were
needed.

MFC after:		3 days
Sponsored by:		Mellanox Technologies
---
 sys/dev/mlx4/mlx4_core/mlx4_eq.c   | 6 ++----
 sys/dev/mlx4/mlx4_core/mlx4_main.c | 3 ++-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/sys/dev/mlx4/mlx4_core/mlx4_eq.c b/sys/dev/mlx4/mlx4_core/mlx4_eq.c
index b0f002442f62..4765ea1d2437 100644
--- a/sys/dev/mlx4/mlx4_core/mlx4_eq.c
+++ b/sys/dev/mlx4/mlx4_core/mlx4_eq.c
@@ -1169,8 +1169,7 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
 	}
 
 	for (i = 0; i < dev->caps.num_comp_vectors; ++i) {
-		err = mlx4_create_eq(dev, dev->caps.num_cqs -
-					  dev->caps.reserved_cqs +
+		err = mlx4_create_eq(dev, dev->quotas.cq +
 					  MLX4_NUM_SPARE_EQE,
 				     (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
 				     &priv->eq_table.eq[i]);
@@ -1190,8 +1189,7 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
 	for (i = dev->caps.num_comp_vectors + 1;
 	      i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) {
 
-		err = mlx4_create_eq(dev, dev->caps.num_cqs -
-					  dev->caps.reserved_cqs +
+		err = mlx4_create_eq(dev, dev->quotas.cq +
 					  MLX4_NUM_SPARE_EQE,
 				     (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
 				     &priv->eq_table.eq[i]);
diff --git a/sys/dev/mlx4/mlx4_core/mlx4_main.c b/sys/dev/mlx4/mlx4_core/mlx4_main.c
index 9bef78ac50cd..3f452d37dbc5 100644
--- a/sys/dev/mlx4/mlx4_core/mlx4_main.c
+++ b/sys/dev/mlx4/mlx4_core/mlx4_main.c
@@ -3446,6 +3446,8 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data)
 			goto err_free_eq;
 	}
 
+	mlx4_init_quotas(dev);
+
 	err = mlx4_setup_hca(dev);
 	if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) &&
 	    !mlx4_is_mfunc(dev)) {
@@ -3459,7 +3461,6 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data)
 	if (err)
 		goto err_steer;
 
-	mlx4_init_quotas(dev);
 	mlx4_init_hca_info(dev);
 
 	for (port = 1; port <= dev->caps.num_ports; port++) {

From d2335a57f43de255dc8b4d2d328fca11f89dce1f Mon Sep 17 00:00:00 2001
From: Eric van Gyzen <vangyzen@FreeBSD.org>
Date: Fri, 19 May 2017 13:04:05 +0000
Subject: [PATCH 69/73] libthr: fix warnings at WARNS=6

Fix warnings about the following when WARNS=6 (which I will commit soon):

- casting away const
- no previous 'extern' declaration for non-static variable
- others as explained by #pragmas and comments
- unused parameters

The last is the only functional change.

Reviewed by:	kib
MFC after:	3 days
Sponsored by:	Dell EMC
Differential Revision:	https://reviews.freebsd.org/D10808
---
 lib/libthr/thread/thr_attr.c    |  2 +-
 lib/libthr/thread/thr_exit.c    | 11 ++++++-----
 lib/libthr/thread/thr_sig.c     |  2 +-
 lib/libthr/thread/thr_spec.c    |  2 +-
 lib/libthr/thread/thr_stack.c   | 17 +++++++++++++++++
 lib/libthr/thread/thr_symbols.c |  4 ++++
 lib/libthr/thread/thr_umtx.c    |  2 +-
 lib/libthr/thread/thr_umtx.h    |  4 ++--
 8 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/lib/libthr/thread/thr_attr.c b/lib/libthr/thread/thr_attr.c
index ff9eb2d5a39d..a79048325104 100644
--- a/lib/libthr/thread/thr_attr.c
+++ b/lib/libthr/thread/thr_attr.c
@@ -607,7 +607,7 @@ _pthread_attr_setaffinity_np(pthread_attr_t *pattr, size_t cpusetsize,
 			/* Kernel checks invalid bits, we check it here too. */
 			size_t i;
 			for (i = kern_size; i < cpusetsize; ++i) {
-				if (((char *)cpusetp)[i])
+				if (((const char *)cpusetp)[i])
 					return (EINVAL);
 			}
 		}
diff --git a/lib/libthr/thread/thr_exit.c b/lib/libthr/thread/thr_exit.c
index 53904c6feaf2..decc2f2983b7 100644
--- a/lib/libthr/thread/thr_exit.c
+++ b/lib/libthr/thread/thr_exit.c
@@ -119,7 +119,8 @@ _Unwind_GetCFA(struct _Unwind_Context *context)
 #endif /* PIC */
 
 static void
-thread_unwind_cleanup(_Unwind_Reason_Code code, struct _Unwind_Exception *e)
+thread_unwind_cleanup(_Unwind_Reason_Code code __unused,
+    struct _Unwind_Exception *e __unused)
 {
 	/*
 	 * Specification said that _Unwind_Resume should not be used here,
@@ -130,10 +131,10 @@ thread_unwind_cleanup(_Unwind_Reason_Code code, struct _Unwind_Exception *e)
 }
 
 static _Unwind_Reason_Code
-thread_unwind_stop(int version, _Unwind_Action actions,
-	int64_t exc_class,
-	struct _Unwind_Exception *exc_obj,
-	struct _Unwind_Context *context, void *stop_parameter)
+thread_unwind_stop(int version __unused, _Unwind_Action actions,
+	int64_t exc_class __unused,
+	struct _Unwind_Exception *exc_obj __unused,
+	struct _Unwind_Context *context, void *stop_parameter __unused)
 {
 	struct pthread *curthread = _get_curthread();
 	struct pthread_cleanup *cur;
diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index 824153952d89..3329a82159b6 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -441,7 +441,7 @@ _thr_signal_init(int dlopened)
 }
 
 void
-_thr_sigact_unload(struct dl_phdr_info *phdr_info)
+_thr_sigact_unload(struct dl_phdr_info *phdr_info __unused)
 {
 #if 0
 	struct pthread *curthread = _get_curthread();
diff --git a/lib/libthr/thread/thr_spec.c b/lib/libthr/thread/thr_spec.c
index 46d4ddf30bc6..303e34e4afe0 100644
--- a/lib/libthr/thread/thr_spec.c
+++ b/lib/libthr/thread/thr_spec.c
@@ -42,7 +42,7 @@ __FBSDID("$FreeBSD$");
 
 #include "thr_private.h"
 
-struct pthread_key _thread_keytable[PTHREAD_KEYS_MAX];
+static struct pthread_key _thread_keytable[PTHREAD_KEYS_MAX];
 
 __weak_reference(_pthread_key_create, pthread_key_create);
 __weak_reference(_pthread_key_delete, pthread_key_delete);
diff --git a/lib/libthr/thread/thr_stack.c b/lib/libthr/thread/thr_stack.c
index 3510a74ab43f..19bb2903f3c3 100644
--- a/lib/libthr/thread/thr_stack.c
+++ b/lib/libthr/thread/thr_stack.c
@@ -290,6 +290,19 @@ _thr_stack_alloc(struct pthread_attr *attr)
 		return (-1);
 }
 
+/*
+ * Disable this warning from clang:
+ *
+ * cast from 'char *' to
+ *    'struct stack *' increases required alignment from 1 to 8
+ *    [-Werror,-Wcast-align]
+ *                 spare_stack = (struct stack *)
+ */
+#ifdef __clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-align"
+#endif
+
 /* This function must be called with _thread_list_lock held. */
 void
 _thr_stack_free(struct pthread_attr *attr)
@@ -316,3 +329,7 @@ _thr_stack_free(struct pthread_attr *attr)
 		attr->stackaddr_attr = NULL;
 	}
 }
+
+#ifdef __clang__
+#pragma GCC diagnostic pop
+#endif
diff --git a/lib/libthr/thread/thr_symbols.c b/lib/libthr/thread/thr_symbols.c
index 9eef8db1a0a6..0c9ccb8788ec 100644
--- a/lib/libthr/thread/thr_symbols.c
+++ b/lib/libthr/thread/thr_symbols.c
@@ -37,6 +37,10 @@ __FBSDID("$FreeBSD$");
 
 #include "thr_private.h"
 
+#ifdef __clang__
+#pragma GCC diagnostic ignored "-Wmissing-variable-declarations"
+#endif
+
 /* A collection of symbols needed by debugger */
 
 /* int _libthr_debug */
diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c
index cd2b101e0770..085e04e22dfb 100644
--- a/lib/libthr/thread/thr_umtx.c
+++ b/lib/libthr/thread/thr_umtx.c
@@ -168,7 +168,7 @@ __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 }
 
 int
-__thr_umutex_unlock(struct umutex *mtx, uint32_t id)
+__thr_umutex_unlock(struct umutex *mtx)
 {
 
 	return (_umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0));
diff --git a/lib/libthr/thread/thr_umtx.h b/lib/libthr/thread/thr_umtx.h
index fff8729d589b..5dae304528da 100644
--- a/lib/libthr/thread/thr_umtx.h
+++ b/lib/libthr/thread/thr_umtx.h
@@ -44,7 +44,7 @@ int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 	const struct timespec *timeout) __hidden;
-int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden;
+int __thr_umutex_unlock(struct umutex *mtx) __hidden;
 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
 	uint32_t *oldceiling) __hidden;
@@ -155,7 +155,7 @@ _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer)
 		if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ?
 		    UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))
 			return (0);
-		return (__thr_umutex_unlock(mtx, id));
+		return (__thr_umutex_unlock(mtx));
 	}
 
 	do {

From 56ba774ebcfa4ae0cc37553bedc925cd552fa656 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Fri, 19 May 2017 17:04:01 +0000
Subject: [PATCH 70/73] Install {cron.d,newsyslog.conf.d,syslog.d} via `make
 distribution`, not `make install`

I incorrectly started this pattern in r277541 with the opensm newsyslog.conf.d file,
and continued using it in r318441 and r318443.

This will fix the files being handled improperly via installworld, preventing tools like
etcupdate, mergemaster, etc from functioning properly when comparing the installed
contents on a system vs the contents in a source tree when doing merges.

PR:		219404
Submitted by:	Dan McGregor <dan.mcgregor@usask.ca>
MFC after:	2 weeks
MFC with:	r277541, r318441, r318443
Sponsored by:	Dell EMC Isilon
---
 etc/Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/etc/Makefile b/etc/Makefile
index 4b117d5257de..1ff7f1cd4b83 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -7,10 +7,6 @@ FILESGROUPS=	FILES
 
 # No need as it is empty and just causes rebuilds since this file does so much.
 UPDATE_DEPENDFILE=	no
-SUBDIR=	\
-	cron.d \
-	newsyslog.conf.d \
-	syslog.d
 
 .if ${MK_SENDMAIL} != "no"
 SUBDIR+=sendmail
@@ -254,9 +250,11 @@ distribution:
 .if ${MK_CASPER} != "no"
 	${_+_}cd ${.CURDIR}/casper; ${MAKE} install
 .endif
+	${_+_}cd ${.CURDIR}/cron.d; ${MAKE} install
 	${_+_}cd ${.CURDIR}/defaults; ${MAKE} install
 	${_+_}cd ${.CURDIR}/devd; ${MAKE} install
 	${_+_}cd ${.CURDIR}/gss; ${MAKE} install
+	${_+_}cd ${.CURDIR}/newsyslog.conf.d; ${MAKE} install
 .if ${MK_NTP} != "no"
 	${_+_}cd ${.CURDIR}/ntp; ${MAKE} install
 .endif
@@ -266,6 +264,7 @@ distribution:
 .endif
 	${_+_}cd ${.CURDIR}/rc.d; ${MAKE} install
 	${_+_}cd ${SRCTOP}/share/termcap; ${MAKE} etc-termcap
+	${_+_}cd ${.CURDIR}/syslog.d; ${MAKE} install
 	${_+_}cd ${SRCTOP}/usr.sbin/rmt; ${MAKE} etc-rmt
 	${_+_}cd ${.CURDIR}/pam.d; ${MAKE} install
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 0444 \

From 99429157e8615dc3b7f11afbe3ed92de7476a5db Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Fri, 19 May 2017 17:14:29 +0000
Subject: [PATCH 71/73] sys/fs/tmpfs/vnd_test: make md(4) allocation dynamic

The previous logic was flawed in the sense that it assumed that /dev/md3
was always available. This was a caveat I noted in r306038, that I hadn't
gotten around to solving before now.

Cache the device for the mountpoint after executing mdmfs, then use the
cached value in basic_cleanup(..) when unmounting/disconnecting the md(4)
device.

Apply sed expressions to use reuse logic in the NetBSD code that could
also be applied to FreeBSD, just with different tools.

Differential Revision:	D10766
MFC after:	1 week
Reviewed by:	bdrewery
Sponsored by:	Dell EMC Isilon
---
 contrib/netbsd-tests/fs/tmpfs/t_vnd.sh | 33 +++++++++++++-------------
 tests/sys/fs/tmpfs/Makefile            |  3 +++
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/contrib/netbsd-tests/fs/tmpfs/t_vnd.sh b/contrib/netbsd-tests/fs/tmpfs/t_vnd.sh
index 1f42f622617e..0929b55d1852 100755
--- a/contrib/netbsd-tests/fs/tmpfs/t_vnd.sh
+++ b/contrib/netbsd-tests/fs/tmpfs/t_vnd.sh
@@ -28,6 +28,10 @@
 # Verifies that vnd works with files stored in tmpfs.
 #
 
+# Begin FreeBSD
+MD_DEVICE_FILE=md.device
+# End FreeBSD
+
 atf_test_case basic cleanup
 basic_head() {
 	atf_set "descr" "Verifies that vnd works with files stored in tmpfs"
@@ -41,7 +45,10 @@ basic_body() {
 	# Begin FreeBSD
 	if true; then
 		atf_check -s eq:0 -o empty -e empty mkdir mnt
-		atf_check -s eq:0 -o empty -e empty mdmfs -F disk.img md3 mnt
+		atf_check -s eq:0 -o empty -e empty mdmfs -F disk.img md mnt
+		md_dev=$(df mnt | awk 'NR != 1 { print $1 }' | xargs basename)
+		atf_check test -c /dev/$md_dev # Sanity check
+		echo -n $md_dev > $TMPDIR/$MD_DEVICE_FILE
 	else
 	# End FreeBSD
 	atf_check -s eq:0 -o empty -e empty vndconfig /dev/vnd3 disk.img
@@ -67,31 +74,23 @@ basic_body() {
 	done
 
 	atf_check -s eq:0 -o empty -e empty umount mnt
-	# Begin FreeBSD
-	if true; then
-		atf_check -s eq:0 -o empty -e empty mdconfig -d -u 3
-	else
-	# End FreeBSD
 	atf_check -s eq:0 -o empty -e empty vndconfig -u /dev/vnd3
-	# Begin FreeBSD
-	fi
-	# End FreeBSD
 
 	test_unmount
 	touch done
 }
 basic_cleanup() {
+	# Begin FreeBSD
+	if md_dev=$(cat $TMPDIR/$MD_DEVICE_FILE); then
+		echo "Will try disconnecting $md_dev"
+	else
+		echo "$MD_DEVICE_FILE doesn't exist in $TMPDIR; returning early"
+		return 0
+	fi
+	# End FreeBSD
 	if [ ! -f done ]; then
 		umount mnt 2>/dev/null 1>&2
-		# Begin FreeBSD
-		if true; then
-			[ ! -c /dev/md3 ] || mdconfig -d -u 3
-		else
-		# End FreeBSD
 		vndconfig -u /dev/vnd3 2>/dev/null 1>&2
-		# Begin FreeBSD
-		fi
-		# End FreeBSD
 	fi
 }
 
diff --git a/tests/sys/fs/tmpfs/Makefile b/tests/sys/fs/tmpfs/Makefile
index 388d9115c5c8..78c1e10e6792 100644
--- a/tests/sys/fs/tmpfs/Makefile
+++ b/tests/sys/fs/tmpfs/Makefile
@@ -54,6 +54,9 @@ ATF_TESTS_SH_SED_mount_test=		\
 ATF_TESTS_SH_SED_readdir_test=		-e 's,mknod fifo p,mkfifo fifo,g'
 ATF_TESTS_SH_SED_sizes_test=		-e 's,-o -s,-o size=,g'
 ATF_TESTS_SH_SED_statvfs_test=		-e 's,-o -s,-o size=,g'
+ATF_TESTS_SH_SED_vnd_test=		\
+					-e 's,vndconfig -u /dev/vnd3,mdconfig -d -u $$md_dev,g' \
+					-e 's,/dev/vnd3,/dev/$$md_dev,g'
 ATF_TESTS_SH_SED_vnode_leak_test=	-e 's,-o -s,-o size=,g'
 
 .include <netbsd-tests.test.mk>

From 23c53312086edd30a25c4569eac2861a70a4fdde Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Fri, 19 May 2017 18:13:41 +0000
Subject: [PATCH 72/73] msdosfs: use C99 types

General cleanup, for diff reduction with NetBSD and future use by FAT
support in makefs.

Submitted by:	Siva Mahadevan <smahadevan@freebsdfoundation.org>
Obtained from:	NetBSD
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D10821
---
 sys/fs/msdosfs/bootsect.h       |  18 ++---
 sys/fs/msdosfs/bpb.h            | 136 ++++++++++++++++----------------
 sys/fs/msdosfs/denode.h         |  10 +--
 sys/fs/msdosfs/direntry.h       |  42 +++++-----
 sys/fs/msdosfs/msdosfs_conv.c   |  40 +++++-----
 sys/fs/msdosfs/msdosfs_fat.c    |   4 +-
 sys/fs/msdosfs/msdosfs_lookup.c |   8 +-
 sys/fs/msdosfs/msdosfs_vfsops.c |   2 +-
 sys/fs/msdosfs/msdosfsmount.h   |   6 +-
 9 files changed, 133 insertions(+), 133 deletions(-)

diff --git a/sys/fs/msdosfs/bootsect.h b/sys/fs/msdosfs/bootsect.h
index ebd60a064167..5a3ff56b8508 100644
--- a/sys/fs/msdosfs/bootsect.h
+++ b/sys/fs/msdosfs/bootsect.h
@@ -25,13 +25,13 @@
  * first sector of a partitioned hard disk.
  */
 struct bootsector33 {
-	u_int8_t	bsJump[3];		/* jump inst E9xxxx or EBxx90 */
+	uint8_t		bsJump[3];		/* jump inst E9xxxx or EBxx90 */
 	int8_t		bsOemName[8];		/* OEM name and version */
 	int8_t		bsBPB[19];		/* BIOS parameter block */
 	int8_t		bsDriveNumber;		/* drive number (0x80) */
 	int8_t		bsBootCode[479];	/* pad so struct is 512b */
-	u_int8_t	bsBootSectSig0;
-	u_int8_t	bsBootSectSig1;
+	uint8_t		bsBootSectSig0;
+	uint8_t		bsBootSectSig1;
 #define	BOOTSIG0	0x55
 #define	BOOTSIG1	0xaa
 };
@@ -47,25 +47,25 @@ struct extboot {
 };
 
 struct bootsector50 {
-	u_int8_t	bsJump[3];		/* jump inst E9xxxx or EBxx90 */
+	uint8_t		bsJump[3];		/* jump inst E9xxxx or EBxx90 */
 	int8_t		bsOemName[8];		/* OEM name and version */
 	int8_t		bsBPB[25];		/* BIOS parameter block */
 	int8_t		bsExt[26];		/* Bootsector Extension */
 	int8_t		bsBootCode[448];	/* pad so structure is 512b */
-	u_int8_t	bsBootSectSig0;
-	u_int8_t	bsBootSectSig1;
+	uint8_t		bsBootSectSig0;
+	uint8_t		bsBootSectSig1;
 #define	BOOTSIG0	0x55
 #define	BOOTSIG1	0xaa
 };
 
 struct bootsector710 {
-	u_int8_t	bsJump[3];		/* jump inst E9xxxx or EBxx90 */
+	uint8_t		bsJump[3];		/* jump inst E9xxxx or EBxx90 */
 	int8_t		bsOEMName[8];		/* OEM name and version */
 	int8_t		bsBPB[53];		/* BIOS parameter block */
 	int8_t		bsExt[26];		/* Bootsector Extension */
 	int8_t		bsBootCode[420];	/* pad so structure is 512b */
-	u_int8_t	bsBootSectSig0;
-	u_int8_t	bsBootSectSig1;
+	uint8_t		bsBootSectSig0;
+	uint8_t		bsBootSectSig1;
 #define	BOOTSIG0	0x55
 #define	BOOTSIG1	0xaa
 };
diff --git a/sys/fs/msdosfs/bpb.h b/sys/fs/msdosfs/bpb.h
index ce20f53d411d..682974fb780c 100644
--- a/sys/fs/msdosfs/bpb.h
+++ b/sys/fs/msdosfs/bpb.h
@@ -24,17 +24,17 @@
  * BIOS Parameter Block (BPB) for DOS 3.3
  */
 struct bpb33 {
-	u_int16_t	bpbBytesPerSec;	/* bytes per sector */
-	u_int8_t	bpbSecPerClust;	/* sectors per cluster */
-	u_int16_t	bpbResSectors;	/* number of reserved sectors */
-	u_int8_t	bpbFATs;	/* number of FATs */
-	u_int16_t	bpbRootDirEnts;	/* number of root directory entries */
-	u_int16_t	bpbSectors;	/* total number of sectors */
-	u_int8_t	bpbMedia;	/* media descriptor */
-	u_int16_t	bpbFATsecs;	/* number of sectors per FAT */
-	u_int16_t	bpbSecPerTrack;	/* sectors per track */
-	u_int16_t	bpbHeads;	/* number of heads */
-	u_int16_t	bpbHiddenSecs;	/* number of hidden sectors */
+	uint16_t	bpbBytesPerSec;	/* bytes per sector */
+	uint8_t		bpbSecPerClust;	/* sectors per cluster */
+	uint16_t	bpbResSectors;	/* number of reserved sectors */
+	uint8_t		bpbFATs;	/* number of FATs */
+	uint16_t	bpbRootDirEnts;	/* number of root directory entries */
+	uint16_t	bpbSectors;	/* total number of sectors */
+	uint8_t		bpbMedia;	/* media descriptor */
+	uint16_t	bpbFATsecs;	/* number of sectors per FAT */
+	uint16_t	bpbSecPerTrack;	/* sectors per track */
+	uint16_t	bpbHeads;	/* number of heads */
+	uint16_t	bpbHiddenSecs;	/* number of hidden sectors */
 };
 
 /*
@@ -42,46 +42,46 @@ struct bpb33 {
  * and bpbHugeSectors is not in the 3.3 bpb.
  */
 struct bpb50 {
-	u_int16_t	bpbBytesPerSec;	/* bytes per sector */
-	u_int8_t	bpbSecPerClust;	/* sectors per cluster */
-	u_int16_t	bpbResSectors;	/* number of reserved sectors */
-	u_int8_t	bpbFATs;	/* number of FATs */
-	u_int16_t	bpbRootDirEnts;	/* number of root directory entries */
-	u_int16_t	bpbSectors;	/* total number of sectors */
-	u_int8_t	bpbMedia;	/* media descriptor */
-	u_int16_t	bpbFATsecs;	/* number of sectors per FAT */
-	u_int16_t	bpbSecPerTrack;	/* sectors per track */
-	u_int16_t	bpbHeads;	/* number of heads */
-	u_int32_t	bpbHiddenSecs;	/* # of hidden sectors */
-	u_int32_t	bpbHugeSectors;	/* # of sectors if bpbSectors == 0 */
+	uint16_t	bpbBytesPerSec;	/* bytes per sector */
+	uint8_t		bpbSecPerClust;	/* sectors per cluster */
+	uint16_t	bpbResSectors;	/* number of reserved sectors */
+	uint8_t		bpbFATs;	/* number of FATs */
+	uint16_t	bpbRootDirEnts;	/* number of root directory entries */
+	uint16_t	bpbSectors;	/* total number of sectors */
+	uint8_t		bpbMedia;	/* media descriptor */
+	uint16_t	bpbFATsecs;	/* number of sectors per FAT */
+	uint16_t	bpbSecPerTrack;	/* sectors per track */
+	uint16_t	bpbHeads;	/* number of heads */
+	uint32_t	bpbHiddenSecs;	/* # of hidden sectors */
+	uint32_t	bpbHugeSectors;	/* # of sectors if bpbSectors == 0 */
 };
 
 /*
  * BPB for DOS 7.10 (FAT32).  This one has a few extensions to bpb50.
  */
 struct bpb710 {
-	u_int16_t	bpbBytesPerSec;	/* bytes per sector */
-	u_int8_t	bpbSecPerClust;	/* sectors per cluster */
-	u_int16_t	bpbResSectors;	/* number of reserved sectors */
-	u_int8_t	bpbFATs;	/* number of FATs */
-	u_int16_t	bpbRootDirEnts;	/* number of root directory entries */
-	u_int16_t	bpbSectors;	/* total number of sectors */
-	u_int8_t	bpbMedia;	/* media descriptor */
-	u_int16_t	bpbFATsecs;	/* number of sectors per FAT */
-	u_int16_t	bpbSecPerTrack;	/* sectors per track */
-	u_int16_t	bpbHeads;	/* number of heads */
-	u_int32_t	bpbHiddenSecs;	/* # of hidden sectors */
-	u_int32_t	bpbHugeSectors;	/* # of sectors if bpbSectors == 0 */
-	u_int32_t	bpbBigFATsecs;	/* like bpbFATsecs for FAT32 */
-	u_int16_t	bpbExtFlags;	/* extended flags: */
+	uint16_t	bpbBytesPerSec;	/* bytes per sector */
+	uint8_t		bpbSecPerClust;	/* sectors per cluster */
+	uint16_t	bpbResSectors;	/* number of reserved sectors */
+	uint8_t		bpbFATs;	/* number of FATs */
+	uint16_t	bpbRootDirEnts;	/* number of root directory entries */
+	uint16_t	bpbSectors;	/* total number of sectors */
+	uint8_t		bpbMedia;	/* media descriptor */
+	uint16_t	bpbFATsecs;	/* number of sectors per FAT */
+	uint16_t	bpbSecPerTrack;	/* sectors per track */
+	uint16_t	bpbHeads;	/* number of heads */
+	uint32_t	bpbHiddenSecs;	/* # of hidden sectors */
+	uint32_t	bpbHugeSectors;	/* # of sectors if bpbSectors == 0 */
+	uint32_t	bpbBigFATsecs;	/* like bpbFATsecs for FAT32 */
+	uint16_t	bpbExtFlags;	/* extended flags: */
 #define	FATNUM		0xf		/* mask for numbering active FAT */
 #define	FATMIRROR	0x80		/* FAT is mirrored (like it always was) */
-	u_int16_t	bpbFSVers;	/* filesystem version */
+	uint16_t	bpbFSVers;	/* filesystem version */
 #define	FSVERS		0		/* currently only 0 is understood */
-	u_int32_t	bpbRootClust;	/* start cluster for root directory */
-	u_int16_t	bpbFSInfo;	/* filesystem info structure sector */
-	u_int16_t	bpbBackup;	/* backup boot sector */
-	u_int8_t	bpbReserved[12]; /* reserved for future expansion */
+	uint32_t	bpbRootClust;	/* start cluster for root directory */
+	uint16_t	bpbFSInfo;	/* filesystem info structure sector */
+	uint16_t	bpbBackup;	/* backup boot sector */
+	uint8_t		bpbReserved[12]; /* reserved for future expansion */
 };
 
 /*
@@ -138,37 +138,37 @@ struct byte_bpb50 {
  * BPB for DOS 7.10 (FAT32).  This one has a few extensions to bpb50.
  */
 struct byte_bpb710 {
-	u_int8_t bpbBytesPerSec[2];	/* bytes per sector */
-	u_int8_t bpbSecPerClust;	/* sectors per cluster */
-	u_int8_t bpbResSectors[2];	/* number of reserved sectors */
-	u_int8_t bpbFATs;		/* number of FATs */
-	u_int8_t bpbRootDirEnts[2];	/* number of root directory entries */
-	u_int8_t bpbSectors[2];		/* total number of sectors */
-	u_int8_t bpbMedia;		/* media descriptor */
-	u_int8_t bpbFATsecs[2];		/* number of sectors per FAT */
-	u_int8_t bpbSecPerTrack[2];	/* sectors per track */
-	u_int8_t bpbHeads[2];		/* number of heads */
-	u_int8_t bpbHiddenSecs[4];	/* # of hidden sectors */
-	u_int8_t bpbHugeSectors[4];	/* # of sectors if bpbSectors == 0 */
-	u_int8_t bpbBigFATsecs[4];	/* like bpbFATsecs for FAT32 */
-	u_int8_t bpbExtFlags[2];	/* extended flags: */
-	u_int8_t bpbFSVers[2];		/* filesystem version */
-	u_int8_t bpbRootClust[4];	/* start cluster for root directory */
-	u_int8_t bpbFSInfo[2];		/* filesystem info structure sector */
-	u_int8_t bpbBackup[2];		/* backup boot sector */
-	u_int8_t bpbReserved[12];	/* reserved for future expansion */
+	uint8_t bpbBytesPerSec[2];	/* bytes per sector */
+	uint8_t bpbSecPerClust;		/* sectors per cluster */
+	uint8_t bpbResSectors[2];	/* number of reserved sectors */
+	uint8_t bpbFATs;		/* number of FATs */
+	uint8_t bpbRootDirEnts[2];	/* number of root directory entries */
+	uint8_t bpbSectors[2];		/* total number of sectors */
+	uint8_t bpbMedia;		/* media descriptor */
+	uint8_t bpbFATsecs[2];		/* number of sectors per FAT */
+	uint8_t bpbSecPerTrack[2];	/* sectors per track */
+	uint8_t bpbHeads[2];		/* number of heads */
+	uint8_t bpbHiddenSecs[4];	/* # of hidden sectors */
+	uint8_t bpbHugeSectors[4];	/* # of sectors if bpbSectors == 0 */
+	uint8_t bpbBigFATsecs[4];	/* like bpbFATsecs for FAT32 */
+	uint8_t bpbExtFlags[2];		/* extended flags: */
+	uint8_t bpbFSVers[2];		/* filesystem version */
+	uint8_t bpbRootClust[4];	/* start cluster for root directory */
+	uint8_t bpbFSInfo[2];		/* filesystem info structure sector */
+	uint8_t bpbBackup[2];		/* backup boot sector */
+	uint8_t bpbReserved[12];	/* reserved for future expansion */
 };
 
 /*
  * FAT32 FSInfo block.
  */
 struct fsinfo {
-	u_int8_t fsisig1[4];
-	u_int8_t fsifill1[480];
-	u_int8_t fsisig2[4];
-	u_int8_t fsinfree[4];
-	u_int8_t fsinxtfree[4];
-	u_int8_t fsifill2[12];
-	u_int8_t fsisig3[4];
+	uint8_t fsisig1[4];
+	uint8_t fsifill1[480];
+	uint8_t fsisig2[4];
+	uint8_t fsinfree[4];
+	uint8_t fsinxtfree[4];
+	uint8_t fsifill2[12];
+	uint8_t fsisig3[4];
 };
 #endif /* !_FS_MSDOSFS_BPB_H_ */
diff --git a/sys/fs/msdosfs/denode.h b/sys/fs/msdosfs/denode.h
index 6c467244dd30..215cb96a5184 100644
--- a/sys/fs/msdosfs/denode.h
+++ b/sys/fs/msdosfs/denode.h
@@ -158,7 +158,7 @@ struct denode {
 	u_long de_FileSize;	/* size of file in bytes */
 	struct fatcache de_fc[FC_SIZE];	/* fat cache */
 	u_quad_t de_modrev;	/* Revision level for lease. */
-	u_int64_t de_inode;	/* Inode number (really byte offset of direntry) */
+	uint64_t de_inode;	/* Inode number (really byte offset of direntry) */
 };
 
 /*
@@ -224,7 +224,7 @@ struct denode {
 		break;							\
 	}								\
 	if ((dep)->de_flag & DE_ACCESS) {				\
-		u_int16_t adate;					\
+		uint16_t adate;						\
 									\
 		timespec2fattime((acc), 0, &adate, NULL, NULL);		\
 		if (adate != (dep)->de_ADate) {				\
@@ -247,10 +247,10 @@ struct defid {
 	u_short defid_len;	/* length of structure */
 	u_short defid_pad;	/* force long alignment */
 
-	u_int32_t defid_dirclust; /* cluster this dir entry came from */
-	u_int32_t defid_dirofs;	/* offset of entry within the cluster */
+	uint32_t defid_dirclust; /* cluster this dir entry came from */
+	uint32_t defid_dirofs;	/* offset of entry within the cluster */
 #if 0
-	u_int32_t defid_gen;	/* generation number */
+	uint32_t defid_gen;	/* generation number */
 #endif
 };
 
diff --git a/sys/fs/msdosfs/direntry.h b/sys/fs/msdosfs/direntry.h
index facec3095dd3..6ff18eb4a9ca 100644
--- a/sys/fs/msdosfs/direntry.h
+++ b/sys/fs/msdosfs/direntry.h
@@ -54,11 +54,11 @@
  * Structure of a dos directory entry.
  */
 struct direntry {
-	u_int8_t	deName[11];	/* filename, blank filled */
+	uint8_t		deName[11];	/* filename, blank filled */
 #define	SLOT_EMPTY	0x00		/* slot has never been used */
 #define	SLOT_E5		0x05		/* the real value is 0xe5 */
 #define	SLOT_DELETED	0xe5		/* file in this slot deleted */
-	u_int8_t	deAttributes;	/* file attributes */
+	uint8_t		deAttributes;	/* file attributes */
 #define	ATTR_NORMAL	0x00		/* normal file */
 #define	ATTR_READONLY	0x01		/* file is readonly */
 #define	ATTR_HIDDEN	0x02		/* file is hidden */
@@ -66,35 +66,35 @@ struct direntry {
 #define	ATTR_VOLUME	0x08		/* entry is a volume label */
 #define	ATTR_DIRECTORY	0x10		/* entry is a directory name */
 #define	ATTR_ARCHIVE	0x20		/* file is new or modified */
-	u_int8_t	deLowerCase;	/* NT VFAT lower case flags */
+	uint8_t		deLowerCase;	/* NT VFAT lower case flags */
 #define	LCASE_BASE	0x08		/* filename base in lower case */
 #define	LCASE_EXT	0x10		/* filename extension in lower case */
-	u_int8_t	deCHundredth;	/* hundredth of seconds in CTime */
-	u_int8_t	deCTime[2];	/* create time */
-	u_int8_t	deCDate[2];	/* create date */
-	u_int8_t	deADate[2];	/* access date */
-	u_int8_t	deHighClust[2];	/* high bytes of cluster number */
-	u_int8_t	deMTime[2];	/* last update time */
-	u_int8_t	deMDate[2];	/* last update date */
-	u_int8_t	deStartCluster[2]; /* starting cluster of file */
-	u_int8_t	deFileSize[4];	/* size of file in bytes */
+	uint8_t		deCHundredth;	/* hundredth of seconds in CTime */
+	uint8_t		deCTime[2];	/* create time */
+	uint8_t		deCDate[2];	/* create date */
+	uint8_t		deADate[2];	/* access date */
+	uint8_t		deHighClust[2];	/* high bytes of cluster number */
+	uint8_t		deMTime[2];	/* last update time */
+	uint8_t		deMDate[2];	/* last update date */
+	uint8_t		deStartCluster[2]; /* starting cluster of file */
+	uint8_t		deFileSize[4];	/* size of file in bytes */
 };
 
 /*
  * Structure of a Win95 long name directory entry
  */
 struct winentry {
-	u_int8_t	weCnt;
+	uint8_t		weCnt;
 #define	WIN_LAST	0x40
 #define	WIN_CNT		0x3f
-	u_int8_t	wePart1[10];
-	u_int8_t	weAttributes;
+	uint8_t		wePart1[10];
+	uint8_t		weAttributes;
 #define	ATTR_WIN95	0x0f
-	u_int8_t	weReserved1;
-	u_int8_t	weChksum;
-	u_int8_t	wePart2[12];
-	u_int16_t	weReserved2;
-	u_int8_t	wePart3[4];
+	uint8_t		weReserved1;
+	uint8_t		weChksum;
+	uint8_t		wePart2[12];
+	uint16_t	weReserved2;
+	uint8_t		wePart3[4];
 };
 #define	WIN_CHARS	13	/* Number of chars per winentry */
 
@@ -156,7 +156,7 @@ int	winChkName(struct mbnambuf *nbp, const u_char *un, size_t unlen,
 	    int chksum, struct msdosfsmount *pmp);
 int	win2unixfn(struct mbnambuf *nbp, struct winentry *wep, int chksum,
 	    struct msdosfsmount *pmp);
-u_int8_t winChksum(u_int8_t *name);
+uint8_t winChksum(uint8_t *name);
 int	winSlotCnt(const u_char *un, size_t unlen, struct msdosfsmount *pmp);
 size_t	winLenFixup(const u_char *un, size_t unlen);
 #endif	/* _KERNEL */
diff --git a/sys/fs/msdosfs/msdosfs_conv.c b/sys/fs/msdosfs/msdosfs_conv.c
index 574417b79361..770b126b9477 100644
--- a/sys/fs/msdosfs/msdosfs_conv.c
+++ b/sys/fs/msdosfs/msdosfs_conv.c
@@ -62,9 +62,9 @@ extern struct iconv_functions *msdosfs_iconv;
 
 static int mbsadjpos(const char **, size_t, size_t, int, int, void *handle);
 static u_char * dos2unixchr(u_char *, const u_char **, size_t *, int, struct msdosfsmount *);
-static u_int16_t unix2doschr(const u_char **, size_t *, struct msdosfsmount *);
-static u_char * win2unixchr(u_char *, u_int16_t, struct msdosfsmount *);
-static u_int16_t unix2winchr(const u_char **, size_t *, int, struct msdosfsmount *);
+static uint16_t unix2doschr(const u_char **, size_t *, struct msdosfsmount *);
+static u_char * win2unixchr(u_char *, uint16_t, struct msdosfsmount *);
+static uint16_t unix2winchr(const u_char **, size_t *, int, struct msdosfsmount *);
 
 /*
  * 0 - character disallowed in long file name.
@@ -303,7 +303,7 @@ unix2dosfn(const u_char *un, u_char dn[12], size_t unlen, u_int gen,
 	int conv = 1;
 	const u_char *cp, *dp, *dp1;
 	u_char gentext[6], *wcp;
-	u_int16_t c;
+	uint16_t c;
 
 	/*
 	 * Fill the dos filename string with blanks. These are DOS's pad
@@ -518,9 +518,9 @@ int
 unix2winfn(const u_char *un, size_t unlen, struct winentry *wep, int cnt,
     int chksum, struct msdosfsmount *pmp)
 {
-	u_int8_t *wcp;
+	uint8_t *wcp;
 	int i, end;
-	u_int16_t code;
+	uint16_t code;
 
 	/*
 	 * Drop trailing blanks and dots
@@ -536,7 +536,7 @@ unix2winfn(const u_char *un, size_t unlen, struct winentry *wep, int cnt,
 	/*
 	 * Initialize winentry to some useful default
 	 */
-	for (wcp = (u_int8_t *)wep, i = sizeof(*wep); --i >= 0; *wcp++ = 0xff);
+	for (wcp = (uint8_t *)wep, i = sizeof(*wep); --i >= 0; *wcp++ = 0xff);
 	wep->weCnt = cnt;
 	wep->weAttributes = ATTR_WIN95;
 	wep->weReserved1 = 0;
@@ -583,7 +583,7 @@ winChkName(struct mbnambuf *nbp, const u_char *un, size_t unlen, int chksum,
     struct msdosfsmount *pmp)
 {
 	size_t len;
-	u_int16_t c1, c2;
+	uint16_t c1, c2;
 	u_char *np;
 	struct dirent dirbuf;
 
@@ -630,9 +630,9 @@ win2unixfn(struct mbnambuf *nbp, struct winentry *wep, int chksum,
     struct msdosfsmount *pmp)
 {
 	u_char *c, tmpbuf[5];
-	u_int8_t *cp;
-	u_int8_t *np, name[WIN_CHARS * 3 + 1];
-	u_int16_t code;
+	uint8_t *cp;
+	uint8_t *np, name[WIN_CHARS * 3 + 1];
+	uint16_t code;
 	int i;
 
 	if ((wep->weCnt&WIN_CNT) > howmany(WIN_MAXLEN, WIN_CHARS)
@@ -722,11 +722,11 @@ win2unixfn(struct mbnambuf *nbp, struct winentry *wep, int chksum,
 /*
  * Compute the unrolled checksum of a DOS filename for Win95 LFN use.
  */
-u_int8_t
-winChksum(u_int8_t *name)
+uint8_t
+winChksum(uint8_t *name)
 {
 	int i;
-	u_int8_t s;
+	uint8_t s;
 
 	for (s = 0, i = 11; --i >= 0; s += *name++)
 		s = (s << 7)|(s >> 1);
@@ -838,12 +838,12 @@ dos2unixchr(u_char *outbuf, const u_char **instr, size_t *ilen, int lower, struc
 /*
  * Convert Local char to DOS char
  */
-static u_int16_t
+static uint16_t
 unix2doschr(const u_char **instr, size_t *ilen, struct msdosfsmount *pmp)
 {
 	u_char c;
 	char *up, *outp, unicode[3], outbuf[3];
-	u_int16_t wc;
+	uint16_t wc;
 	size_t len, ucslen, unixlen, olen;
 
 	if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) {
@@ -909,14 +909,14 @@ unix2doschr(const u_char **instr, size_t *ilen, struct msdosfsmount *pmp)
 	c = *(*instr)++;
 	c = l2u[c];
 	c = unix2dos[c];
-	return ((u_int16_t)c);
+	return ((uint16_t)c);
 }
 
 /*
  * Convert Windows char to Local char
  */
 static u_char *
-win2unixchr(u_char *outbuf, u_int16_t wc, struct msdosfsmount *pmp)
+win2unixchr(u_char *outbuf, uint16_t wc, struct msdosfsmount *pmp)
 {
 	u_char *inp, *outp, inbuf[3];
 	size_t ilen, olen, len;
@@ -951,11 +951,11 @@ win2unixchr(u_char *outbuf, u_int16_t wc, struct msdosfsmount *pmp)
 /*
  * Convert Local char to Windows char
  */
-static u_int16_t
+static uint16_t
 unix2winchr(const u_char **instr, size_t *ilen, int lower, struct msdosfsmount *pmp)
 {
 	u_char *outp, outbuf[3];
-	u_int16_t wc;
+	uint16_t wc;
 	size_t olen;
 
 	if (*ilen == 0)
diff --git a/sys/fs/msdosfs/msdosfs_fat.c b/sys/fs/msdosfs/msdosfs_fat.c
index 4bceeba537d1..cdb4c68e5f26 100644
--- a/sys/fs/msdosfs/msdosfs_fat.c
+++ b/sys/fs/msdosfs/msdosfs_fat.c
@@ -338,9 +338,9 @@ updatefats(struct msdosfsmount *pmp, struct buf *bp, u_long fatbn)
 			bcopy(bp->b_data, bpn->b_data, bp->b_bcount);
 			/* Force the clean bit on in the other copies. */
 			if (cleanfat == 16)
-				((u_int8_t *)bpn->b_data)[3] |= 0x80;
+				((uint8_t *)bpn->b_data)[3] |= 0x80;
 			else if (cleanfat == 32)
-				((u_int8_t *)bpn->b_data)[7] |= 0x08;
+				((uint8_t *)bpn->b_data)[7] |= 0x08;
 			if (pmp->pm_mountp->mnt_flag & MNT_SYNCHRONOUS)
 				bwrite(bpn);
 			else
diff --git a/sys/fs/msdosfs/msdosfs_lookup.c b/sys/fs/msdosfs/msdosfs_lookup.c
index 405085f32d2f..e68ce69c3aa5 100644
--- a/sys/fs/msdosfs/msdosfs_lookup.c
+++ b/sys/fs/msdosfs/msdosfs_lookup.c
@@ -62,7 +62,7 @@
 #include <fs/msdosfs/msdosfsmount.h>
 
 static int msdosfs_lookup_(struct vnode *vdp, struct vnode **vpp,
-    struct componentname *cnp, u_int64_t *inum);
+    struct componentname *cnp, uint64_t *inum);
 
 int
 msdosfs_lookup(struct vop_cachedlookup_args *ap)
@@ -110,7 +110,7 @@ msdosfs_deget_dotdot(struct mount *mp, void *arg, int lkflags,
  */
 static int
 msdosfs_lookup_(struct vnode *vdp, struct vnode **vpp,
-    struct componentname *cnp, u_int64_t *dd_inum)
+    struct componentname *cnp, uint64_t *dd_inum)
 {
 	struct mbnambuf nb;
 	daddr_t bn;
@@ -135,7 +135,7 @@ msdosfs_lookup_(struct vnode *vdp, struct vnode **vpp,
 	int flags = cnp->cn_flags;
 	int nameiop = cnp->cn_nameiop;
 	int unlen;
-	u_int64_t inode1;
+	uint64_t inode1;
 
 	int wincnt = 1;
 	int chksum = -1, chksum_ok;
@@ -656,7 +656,7 @@ createde(struct denode *dep, struct denode *ddep, struct denode **depp,
 	 * Now write the Win95 long name
 	 */
 	if (ddep->de_fndcnt > 0) {
-		u_int8_t chksum = winChksum(ndep->deName);
+		uint8_t chksum = winChksum(ndep->deName);
 		const u_char *un = (const u_char *)cnp->cn_nameptr;
 		int unlen = cnp->cn_namelen;
 		int cnt = 1;
diff --git a/sys/fs/msdosfs/msdosfs_vfsops.c b/sys/fs/msdosfs/msdosfs_vfsops.c
index 4c122bfb5942..3c2962b29648 100644
--- a/sys/fs/msdosfs/msdosfs_vfsops.c
+++ b/sys/fs/msdosfs/msdosfs_vfsops.c
@@ -398,7 +398,7 @@ mountmsdosfs(struct vnode *devvp, struct mount *mp)
 	struct byte_bpb33 *b33;
 	struct byte_bpb50 *b50;
 	struct byte_bpb710 *b710;
-	u_int8_t SecPerClust;
+	uint8_t SecPerClust;
 	u_long clusters;
 	int ronly, error;
 	struct g_consumer *cp;
diff --git a/sys/fs/msdosfs/msdosfsmount.h b/sys/fs/msdosfs/msdosfsmount.h
index 9446a3e2a867..ef23caa27789 100644
--- a/sys/fs/msdosfs/msdosfsmount.h
+++ b/sys/fs/msdosfs/msdosfsmount.h
@@ -96,7 +96,7 @@ struct msdosfsmount {
 	u_long pm_fatblocksize;	/* size of fat blocks in bytes */
 	u_long pm_fatblocksec;	/* size of fat blocks in sectors */
 	u_long pm_fatsize;	/* size of fat in bytes */
-	u_int32_t pm_fatmask;	/* mask to use for fat numbers */
+	uint32_t pm_fatmask;	/* mask to use for fat numbers */
 	u_long pm_fsinfo;	/* fsinfo block number */
 	u_long pm_nxtfree;	/* next place to search for a free cluster */
 	u_int pm_fatmult;	/* these 2 values are used in fat */
@@ -108,7 +108,7 @@ struct msdosfsmount {
 	void *pm_w2u;	/* Unicode->Local iconv handle */
 	void *pm_u2d;	/* Unicode->DOS iconv handle */
 	void *pm_d2u;	/* DOS->Local iconv handle */
-	u_int32_t pm_nfileno;	/* next 32-bit fileno */
+	uint32_t pm_nfileno;	/* next 32-bit fileno */
 	RB_HEAD(msdosfs_filenotree, msdosfs_fileno)
 	    pm_filenos; /* 64<->32-bit fileno mapping */
 	struct lock pm_fatlock;	/* lockmgr protecting allocations and rb tree */
@@ -240,7 +240,7 @@ struct msdosfs_args {
 	mode_t	mask;		/* file mask to be applied for msdosfs perms */
 	int	flags;		/* see below */
 	int	unused1;	/* unused, was version number */
-	u_int16_t unused2[128];	/* no longer used, was Local->Unicode table */
+	uint16_t unused2[128];	/* no longer used, was Local->Unicode table */
 	char	*cs_win;	/* Windows(Unicode) Charset */
 	char	*cs_dos;	/* DOS Charset */
 	char	*cs_local;	/* Local Charset */

From 6dce7e9908a12ec9f2c236e33fce1ec08f9c0e60 Mon Sep 17 00:00:00 2001
From: Maxim Konovalov <maxim@FreeBSD.org>
Date: Fri, 19 May 2017 20:02:32 +0000
Subject: [PATCH 73/73] o Missed flag restored.

PR:		219395
Submitted by:	Tiwei Bie
---
 usr.sbin/devctl/devctl.8 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/usr.sbin/devctl/devctl.8 b/usr.sbin/devctl/devctl.8
index 67b870960fe4..4974ee9c7685 100644
--- a/usr.sbin/devctl/devctl.8
+++ b/usr.sbin/devctl/devctl.8
@@ -156,7 +156,7 @@ the device will not be changed.
 Rescan a bus device checking for devices that have been added or
 removed.
 .It Xo Cm delete
-.Op Fl
+.Op Fl f
 .Ar device
 .Xc
 Delete the device from the device tree.