Constrain IPv6 routes to single FIBs when net.add_addr_allfibs=0

sys/netinet6/icmp6.c
	Use the interface's FIB for source address selection in ICMPv6 error
	responses.

sys/netinet6/in6.c
	In in6_newaddrmsg, announce arrival of local addresses on the
	interface's FIB only.  In in6_lltable_rtcheck, use a per-fib ND6
	cache instead of a single cache.

sys/netinet6/in6_src.c
	In in6_selectsrc, use the caller's fib instead of the default fib.
	In in6_selectsrc_socket, remove a superfluous check.

sys/netinet6/nd6.c
	In nd6_lle_event, use the interface's fib for routing socket
	messages.  In nd6_is_new_addr_neighbor, check all FIBs when trying
	to determine whether an address is a neighbor.  Also, simplify the
	code for point to point interfaces.

sys/netinet6/nd6.h
sys/netinet6/nd6.c
sys/netinet6/nd6_rtr.c
	Make defrouter_select fib-aware, and make all of its callers pass in
	the interface fib.

sys/netinet6/nd6_nbr.c
	When inputting a Neighbor Solicitation packet, consider the
	interface fib instead of the default fib for DAD.  Output NS and
	Neighbor Advertisement packets on the correct fib.

sys/netinet6/nd6_rtr.c
	Allow installing the same host route on different interfaces in
	different FIBs.  If rt_add_addr_allfibs=0, only install or delete
	the prefix route on the interface fib.

tests/sys/netinet/fibs_test.sh
	Clear some expected failures, but add a skip for the newly revealed
	BUG217871.

PR:		196361
Submitted by:	Erick Turnquist <jhujhiti@adjectivism.org>
Reported by:	Jason Healy <jhealy@logn.net>
Reviewed by:	asomers
MFC after:	3 weeks
Sponsored by:	Spectra Logic Corp
Differential Revision:	https://reviews.freebsd.org/D9451

sys/netinet6/icmp6.c

@@ -2147,7 +2147,7 @@ icmp6_reflect(struct mbuf *m, size_t off)
 		 * source address of the erroneous packet.
 		 */
 		in6_splitscope(&ip6->ip6_src, &dst6, &scopeid);
-		error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+		error = in6_selectsrc_addr(M_GETFIB(m), &dst6,
 		    scopeid, NULL, &src6, &hlim);
 
 		if (error) {
@@ -2289,7 +2289,7 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	uint32_t scopeid;
 
 	in6_splitscope(&reddst6, &kdst, &scopeid);
-	if (fib6_lookup_nh_basic(RT_DEFAULT_FIB, &kdst, scopeid, 0, 0,&nh6)==0){
+	if (fib6_lookup_nh_basic(ifp->if_fib, &kdst, scopeid, 0, 0,&nh6)==0){
 		if ((nh6.nh_flags & NHF_GATEWAY) == 0) {
 			nd6log((LOG_ERR,
 			    "ICMP6 redirect rejected; no route "

sys/netinet6/in6.c

@@ -159,6 +159,7 @@ in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
 	struct sockaddr_dl gateway;
 	struct sockaddr_in6 mask, addr;
 	struct rtentry rt;
+	int fibnum;
 
 	/*
 	 * initialize for rtmsg generation
@@ -176,8 +177,9 @@ in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
 	rt.rt_flags = RTF_HOST | RTF_STATIC;
 	if (cmd == RTM_ADD)
 		rt.rt_flags |= RTF_UP;
-	/* Announce arrival of local address to all FIBs. */
-	rt_newaddrmsg(cmd, &ia->ia_ifa, 0, &rt);
+	fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS : ia62ifa(ia)->ifa_ifp->if_fib;
+	/* Announce arrival of local address to this FIB. */
+	rt_newaddrmsg_fib(cmd, &ia->ia_ifa, 0, &rt, fibnum);
 }
 
 int
@@ -2117,15 +2119,15 @@ in6_lltable_rtcheck(struct ifnet *ifp,
 	uint32_t scopeid;
 	int error;
 	char ip6buf[INET6_ADDRSTRLEN];
+	int fibnum;
 
 	KASSERT(l3addr->sa_family == AF_INET6,
 	    ("sin_family %d", l3addr->sa_family));
 
-	/* Our local addresses are always only installed on the default FIB. */
-
 	sin6 = (const struct sockaddr_in6 *)l3addr;
 	in6_splitscope(&sin6->sin6_addr, &dst, &scopeid);
-	error = fib6_lookup_nh_basic(RT_DEFAULT_FIB, &dst, scopeid, 0, 0, &nh6);
+	fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib;
+	error = fib6_lookup_nh_basic(fibnum, &dst, scopeid, 0, 0, &nh6);
 	if (error != 0 || (nh6.nh_flags & NHF_GATEWAY) || nh6.nh_ifp != ifp) {
 		struct ifaddr *ifa;
 		/*

sys/netinet6/in6_src.c

@@ -297,7 +297,7 @@ in6_selectsrc(uint32_t fibnum, struct sockaddr_in6 *dstsock,
 	 */
 	/* get the outgoing interface */
 	if ((error = in6_selectif(dstsock, opts, mopts, &ifp, oifp,
-	    (inp != NULL) ? inp->inp_inc.inc_fibnum : RT_DEFAULT_FIB)) != 0)
+	    (inp != NULL) ? inp->inp_inc.inc_fibnum : fibnum)) != 0)
 		return (error);
 
 #ifdef DIAGNOSTIC
@@ -563,7 +563,7 @@ in6_selectsrc_socket(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts,
 	uint32_t fibnum;
 	int error;
 
-	fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : RT_DEFAULT_FIB;
+	fibnum = inp->inp_inc.inc_fibnum;
 	retifp = NULL;
 
 	error = in6_selectsrc(fibnum, dstsock, opts, inp, cred, &retifp, srcp);

sys/netinet6/nd6.c

@@ -157,6 +157,7 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt)
 	struct sockaddr_dl gw;
 	struct ifnet *ifp;
 	int type;
+	int fibnum;
 
 	LLE_WLOCK_ASSERT(lle);
 
@@ -194,8 +195,9 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt)
 	rtinfo.rti_info[RTAX_DST] = (struct sockaddr *)&dst;
 	rtinfo.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&gw;
 	rtinfo.rti_addrs = RTA_DST | RTA_GATEWAY;
+	fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS : ifp->if_fib;
 	rt_missmsg_fib(type, &rtinfo, RTF_HOST | RTF_LLDATA | (
-	    type == RTM_ADD ? RTF_UP: 0), 0, RT_DEFAULT_FIB);
+	    type == RTM_ADD ? RTF_UP: 0), 0, fibnum);
 }
 
 /*
@@ -1200,7 +1202,7 @@ nd6_purge(struct ifnet *ifp)
 
 	if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) {
 		/* Refresh default router list. */
-		defrouter_select();
+		defrouter_select_fib(ifp->if_fib);
 	}
 }
 
@@ -1253,7 +1255,7 @@ static int
 nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 {
 	struct nd_prefix *pr;
-	struct ifaddr *dstaddr;
+	struct ifaddr *ifa;
 	struct rt_addrinfo info;
 	struct sockaddr_in6 rt_key;
 	const struct sockaddr *dst6;
@@ -1287,9 +1289,6 @@ nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 	bzero(&info, sizeof(info));
 	info.rti_info[RTAX_DST] = (struct sockaddr *)&rt_key;
 
-	/* Always use the default FIB here. XXME - why? */
-	fibnum = RT_DEFAULT_FIB;
-
 	/*
 	 * If the address matches one of our addresses,
 	 * it should be a neighbor.
@@ -1303,19 +1302,31 @@ nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 			continue;
 
 		if ((pr->ndpr_stateflags & NDPRF_ONLINK) == 0) {
-			/* Always use the default FIB here. */
 			dst6 = (const struct sockaddr *)&pr->ndpr_prefix;
 
+			/*
+			 * We only need to check all FIBs if add_addr_allfibs
+			 * is unset. If set, checking any FIB will suffice.
+			 */
+			fibnum = V_rt_add_addr_allfibs ? rt_numfibs - 1 : 0;
+			for (; fibnum < rt_numfibs; fibnum++) {
 				genid = V_nd6_list_genid;
 				ND6_RUNLOCK();
 
-			/* Restore length field before retrying lookup */
+				/*
+				 * Restore length field before
+				 * retrying lookup
+				 */
 				rt_key.sin6_len = sizeof(rt_key);
-			error = rib_lookup_info(fibnum, dst6, 0, 0, &info);
+				error = rib_lookup_info(fibnum, dst6, 0, 0,
+						        &info);
 
 				ND6_RLOCK();
 				if (genid != V_nd6_list_genid)
 					goto restart;
+				if (error == 0)
+					break;
+			}
 			if (error != 0)
 				continue;
 
@@ -1346,13 +1357,18 @@ nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 	 * If the address is assigned on the node of the other side of
 	 * a p2p interface, the address should be a neighbor.
 	 */
-	dstaddr = ifa_ifwithdstaddr((const struct sockaddr *)addr, RT_ALL_FIBS);
-	if (dstaddr != NULL) {
-		if (dstaddr->ifa_ifp == ifp) {
-			ifa_free(dstaddr);
-			return (1);
+	if (ifp->if_flags & IFF_POINTOPOINT) {
+		IF_ADDR_RLOCK(ifp);
+		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+			if (ifa->ifa_addr->sa_family != addr->sin6_family)
+				continue;
+			if (ifa->ifa_dstaddr != NULL &&
+			    sa_equal(addr, ifa->ifa_dstaddr)) {
+				IF_ADDR_RUNLOCK(ifp);
+				return 1;
 			}
-		ifa_free(dstaddr);
+		}
+		IF_ADDR_RUNLOCK(ifp);
 	}
 
 	/*
@@ -1485,7 +1501,7 @@ nd6_free(struct llentry **lnp, int gc)
 			/*
 			 * We need to unlock to avoid a LOR with rt6_flush() with the
 			 * rnh and for the calls to pfxlist_onlink_check() and
-			 * defrouter_select() in the block further down for calls
+			 * defrouter_select_fib() in the block further down for calls
 			 * into nd6_lookup().  We still hold a ref.
 			 */
 			LLE_WUNLOCK(ln);
@@ -1500,7 +1516,7 @@ nd6_free(struct llentry **lnp, int gc)
 
 		if (dr) {
 			/*
-			 * Since defrouter_select() does not affect the
+			 * Since defrouter_select_fib() does not affect the
 			 * on-link determination and MIP6 needs the check
 			 * before the default router selection, we perform
 			 * the check now.
@@ -1510,7 +1526,7 @@ nd6_free(struct llentry **lnp, int gc)
 			/*
 			 * Refresh default router list.
 			 */
-			defrouter_select();
+			defrouter_select_fib(dr->ifp->if_fib);
 		}
 
 		/*
@@ -2104,11 +2120,11 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	 * Question: can we restrict the first condition to the "is_newentry"
 	 * case?
 	 * XXX: when we hear an RA from a new router with the link-layer
-	 * address option, defrouter_select() is called twice, since
+	 * address option, defrouter_select_fib() is called twice, since
 	 * defrtrlist_update called the function as well.  However, I believe
 	 * we can compromise the overhead, since it only happens the first
 	 * time.
-	 * XXX: although defrouter_select() should not have a bad effect
+	 * XXX: although defrouter_select_fib() should not have a bad effect
 	 * for those are not autoconfigured hosts, we explicitly avoid such
 	 * cases for safety.
 	 */
@@ -2117,7 +2133,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 		/*
 		 * guaranteed recursion
 		 */
-		defrouter_select();
+		defrouter_select_fib(ifp->if_fib);
 	}
 }
 

sys/netinet6/nd6.h

@@ -469,6 +469,7 @@ void nd6_dad_stop(struct ifaddr *);
 void nd6_rs_input(struct mbuf *, int, int);
 void nd6_ra_input(struct mbuf *, int, int);
 void defrouter_reset(void);
+void defrouter_select_fib(int fibnum);
 void defrouter_select(void);
 void defrouter_ref(struct nd_defrouter *);
 void defrouter_rele(struct nd_defrouter *);

sys/netinet6/nd6_nbr.c

@@ -262,8 +262,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		bzero(&info, sizeof(info));
 		info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&rt_gateway;
 
-		/* Always use the default FIB. */
-		if (rib_lookup_info(RT_DEFAULT_FIB, (struct sockaddr *)&dst6,
+		if (rib_lookup_info(ifp->if_fib, (struct sockaddr *)&dst6,
 		    0, 0, &info) == 0) {
 			if ((info.rti_flags & RTF_ANNOUNCE) != 0 &&
 			    rt_gateway.sdl_family == AF_LINK) {
@@ -485,7 +484,7 @@ nd6_ns_output_fib(struct ifnet *ifp, const struct in6_addr *saddr6,
 			uint32_t scopeid;
 
 			in6_splitscope(&ip6->ip6_dst, &dst6, &scopeid);
-			error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+			error = in6_selectsrc_addr(fibnum, &dst6,
 			    scopeid, ifp, &src6, NULL);
 			if (error) {
 				char ip6buf[INET6_ADDRSTRLEN];
@@ -982,7 +981,7 @@ nd6_na_output_fib(struct ifnet *ifp, const struct in6_addr *daddr6_0,
 	 * Select a source whose scope is the same as that of the dest.
 	 */
 	in6_splitscope(&daddr6, &dst6, &scopeid);
-	error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+	error = in6_selectsrc_addr(fibnum, &dst6,
 	    scopeid, ifp, &src6, NULL);
 	if (error) {
 		char ip6buf[INET6_ADDRSTRLEN];

sys/netinet6/nd6_rtr.c

@@ -500,7 +500,7 @@ defrouter_addreq(struct nd_defrouter *new)
 
 	error = in6_rtrequest(RTM_ADD, (struct sockaddr *)&def,
 	    (struct sockaddr *)&gate, (struct sockaddr *)&mask,
-	    RTF_GATEWAY, &newrt, RT_DEFAULT_FIB);
+	    RTF_GATEWAY, &newrt, new->ifp->if_fib);
 	if (newrt) {
 		nd6_rtmsg(RTM_ADD, newrt); /* tell user process */
 		RTFREE(newrt);
@@ -551,8 +551,8 @@ defrouter_rele(struct nd_defrouter *dr)
 
 /*
  * Remove the default route for a given router.
- * This is just a subroutine function for defrouter_select(), and should
- * not be called from anywhere else.
+ * This is just a subroutine function for defrouter_select_fib(), and
+ * should not be called from anywhere else.
  */
 static void
 defrouter_delreq(struct nd_defrouter *dr)
@@ -571,7 +571,7 @@ defrouter_delreq(struct nd_defrouter *dr)
 
 	in6_rtrequest(RTM_DELETE, (struct sockaddr *)&def,
 	    (struct sockaddr *)&gate,
-	    (struct sockaddr *)&mask, RTF_GATEWAY, &oldrt, RT_DEFAULT_FIB);
+	    (struct sockaddr *)&mask, RTF_GATEWAY, &oldrt, dr->ifp->if_fib);
 	if (oldrt) {
 		nd6_rtmsg(RTM_DELETE, oldrt);
 		RTFREE(oldrt);
@@ -698,11 +698,11 @@ defrouter_del(struct nd_defrouter *dr)
 
 	/*
 	 * If the router is the primary one, choose a new one.
-	 * Note that defrouter_select() will remove the current gateway
-	 * from the routing table.
+	 * Note that defrouter_select_fib() will remove the current
+         * gateway from the routing table.
 	 */
 	if (deldr)
-		defrouter_select();
+		defrouter_select_fib(deldr->ifp->if_fib);
 
 	/*
 	 * Release the list reference.
@@ -730,13 +730,23 @@ defrouter_del(struct nd_defrouter *dr)
  * even when the multipath routing is available, because we're not sure about
  * the benefits for stub hosts comparing to the risk of making the code
  * complicated and the possibility of introducing bugs.
+ *
+ * We maintain a single list of routers for multiple FIBs, only considering one
+ * at a time based on the receiving interface's FIB. If @fibnum is RT_ALL_FIBS,
+ * we do the whole thing multiple times.
  */
 void
-defrouter_select(void)
+defrouter_select_fib(int fibnum)
 {
 	struct nd_defrouter *dr, *selected_dr, *installed_dr;
 	struct llentry *ln = NULL;
 
+	if (fibnum == RT_ALL_FIBS) {
+		for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+			defrouter_select_fib(fibnum);
+		}
+	}
+
 	ND6_RLOCK();
 	/*
 	 * Let's handle easy case (3) first:
@@ -755,7 +765,7 @@ defrouter_select(void)
 	selected_dr = installed_dr = NULL;
 	TAILQ_FOREACH(dr, &V_nd_defrouter, dr_entry) {
 		IF_AFDATA_RLOCK(dr->ifp);
-		if (selected_dr == NULL &&
+		if (selected_dr == NULL && dr->ifp->if_fib == fibnum &&
 		    (ln = nd6_lookup(&dr->rtaddr, 0, dr->ifp)) &&
 		    ND6_IS_LLINFO_PROBREACH(ln)) {
 			selected_dr = dr;
@@ -767,14 +777,17 @@ defrouter_select(void)
 			ln = NULL;
 		}
 
-		if (dr->installed) {
+		if (dr->installed && dr->ifp->if_fib == fibnum) {
 			if (installed_dr == NULL) {
 				installed_dr = dr;
 				defrouter_ref(installed_dr);
 			} else {
-				/* this should not happen.  warn for diagnosis. */
-				log(LOG_ERR,
-		    "defrouter_select: more than one router is installed\n");
+				/*
+				 * this should not happen.
+				 * warn for diagnosis.
+				 */
+				log(LOG_ERR, "defrouter_select_fib: more than "
+				             "one router is installed\n");
 			}
 		}
 	}
@@ -789,14 +802,24 @@ defrouter_select(void)
 	if (selected_dr == NULL) {
 		if (installed_dr == NULL ||
 		    TAILQ_NEXT(installed_dr, dr_entry) == NULL)
-			selected_dr = TAILQ_FIRST(&V_nd_defrouter);
+			dr = TAILQ_FIRST(&V_nd_defrouter);
 		else
-			selected_dr = TAILQ_NEXT(installed_dr, dr_entry);
+			dr = TAILQ_NEXT(installed_dr, dr_entry);
+
+		/* Ensure we select a router for this FIB. */
+		TAILQ_FOREACH_FROM(dr, &V_nd_defrouter, dr_entry) {
+			if (dr->ifp->if_fib == fibnum) {
+				selected_dr = dr;
 				defrouter_ref(selected_dr);
+				break;
+			}
+		}
 	} else if (installed_dr != NULL) {
 		IF_AFDATA_RLOCK(installed_dr->ifp);
-		if ((ln = nd6_lookup(&installed_dr->rtaddr, 0, installed_dr->ifp)) &&
+		if ((ln = nd6_lookup(&installed_dr->rtaddr, 0,
+		                     installed_dr->ifp)) &&
 		    ND6_IS_LLINFO_PROBREACH(ln) &&
+		    installed_dr->ifp->if_fib == fibnum &&
 		    rtpref(selected_dr) <= rtpref(installed_dr)) {
 			defrouter_rele(selected_dr);
 			selected_dr = installed_dr;
@@ -808,20 +831,32 @@ defrouter_select(void)
 	ND6_RUNLOCK();
 
 	/*
-	 * If the selected router is different than the installed one,
-	 * remove the installed router and install the selected one.
-	 * Note that the selected router is never NULL here.
+	 * If we selected a router for this FIB and it's different
+	 * than the installed one, remove the installed router and
+	 * install the selected one in its place.
 	 */
 	if (installed_dr != selected_dr) {
 		if (installed_dr != NULL) {
 			defrouter_delreq(installed_dr);
 			defrouter_rele(installed_dr);
 		}
+		if (selected_dr != NULL)
 			defrouter_addreq(selected_dr);
 	}
+	if (selected_dr != NULL)
 		defrouter_rele(selected_dr);
 }
 
+/*
+ * Maintain old KPI for default router selection.
+ * If unspecified, we can re-select routers for all FIBs.
+ */
+void
+defrouter_select(void)
+{
+	defrouter_select_fib(RT_ALL_FIBS);
+}
+
 /*
  * for default router selection
  * regards router-preference field as a 2-bit signed integer
@@ -942,7 +977,7 @@ defrtrlist_update(struct nd_defrouter *new)
 	V_nd6_list_genid++;
 	ND6_WUNLOCK();
 
-	defrouter_select();
+	defrouter_select_fib(new->ifp->if_fib);
 
 	return (n);
 }
@@ -1731,7 +1766,7 @@ nd6_prefix_onlink_rtrequest(struct nd_prefix *pr, struct ifaddr *ifa)
 	struct rtentry *rt;
 	struct sockaddr_in6 mask6;
 	u_long rtflags;
-	int error, a_failure, fibnum;
+	int error, a_failure, fibnum, maxfib;
 
 	/*
 	 * in6_ifinit() sets nd6_rtrequest to ifa_rtrequest for all ifaddrs.
@@ -1742,8 +1777,15 @@ nd6_prefix_onlink_rtrequest(struct nd_prefix *pr, struct ifaddr *ifa)
 	mask6.sin6_addr = pr->ndpr_mask;
 	rtflags = (ifa->ifa_flags & ~IFA_RTSELF) | RTF_UP;
 
+	if(V_rt_add_addr_allfibs) {
+		fibnum = 0;
+		maxfib = rt_numfibs;
+	} else {
+		fibnum = ifa->ifa_ifp->if_fib;
+		maxfib = fibnum + 1;
+	}
 	a_failure = 0;
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+	for (; fibnum < maxfib; fibnum++) {
 
 		rt = NULL;
 		error = in6_rtrequest(RTM_ADD,
@@ -1831,6 +1873,10 @@ nd6_prefix_onlink(struct nd_prefix *pr)
 		if ((opr->ndpr_stateflags & NDPRF_ONLINK) == 0)
 			continue;
 
+		if (!V_rt_add_addr_allfibs &&
+		    opr->ndpr_ifp->if_fib != pr->ndpr_ifp->if_fib)
+			continue;
+
 		if (opr->ndpr_plen == pr->ndpr_plen &&
 		    in6_are_prefix_equal(&pr->ndpr_prefix.sin6_addr,
 		    &opr->ndpr_prefix.sin6_addr, pr->ndpr_plen)) {
@@ -1891,7 +1937,7 @@ nd6_prefix_offlink(struct nd_prefix *pr)
 	struct rtentry *rt;
 	char ip6buf[INET6_ADDRSTRLEN];
 	uint64_t genid;
-	int fibnum, a_failure;
+	int fibnum, maxfib, a_failure;
 
 	ND6_ONLINK_LOCK_ASSERT();
 	ND6_UNLOCK_ASSERT();
@@ -1909,8 +1955,16 @@ nd6_prefix_offlink(struct nd_prefix *pr)
 	mask6.sin6_len = sizeof(sa6);
 	bcopy(&pr->ndpr_mask, &mask6.sin6_addr, sizeof(struct in6_addr));
 
+	if (V_rt_add_addr_allfibs) {
+		fibnum = 0;
+		maxfib = rt_numfibs;
+	} else {
+		fibnum = ifp->if_fib;
+		maxfib = fibnum + 1;
+	}
+
 	a_failure = 0;
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+	for (; fibnum < maxfib; fibnum++) {
 		rt = NULL;
 		error = in6_rtrequest(RTM_DELETE, (struct sockaddr *)&sa6, NULL,
 		    (struct sockaddr *)&mask6, 0, &rt, fibnum);

tests/sys/netinet/fibs_test.sh

@@ -163,7 +163,6 @@ loopback_and_network_routes_on_nondefault_fib_inet6_head()
 
 loopback_and_network_routes_on_nondefault_fib_inet6_body()
 {
-	atf_expect_fail "PR196361 IPv6 network routes don't respect net.add_addr_allfibs=0"
 	# Configure the TAP interface to use a nonrouteable RFC3849
 	# address and a non-default fib
 	ADDR="2001:db8::2"
@@ -452,6 +451,7 @@ slaac_on_nondefault_fib6_head()
 }
 slaac_on_nondefault_fib6_body()
 {
+	atf_skip "BUG217871 SLAAC on a newly created epair sometimes fails to add routes"
 	# Configure the epair interfaces to use nonrouteable RFC3849
 	# addresses and non-default FIBs
 	ADDR="2001:db8::2"
@@ -459,8 +459,6 @@ slaac_on_nondefault_fib6_body()
 	SUBNET="2001:db8:"
 	MASK="64"
 
-	atf_expect_fail "PR196361 IPv6 network routes don't respect net.add_addr_allfibs=0"
-
 	# Check system configuration
 	if [ 0 != `sysctl -n net.add_addr_allfibs` ]; then
 		atf_skip "This test requires net.add_addr_allfibs=0"
@@ -478,8 +476,8 @@ slaac_on_nondefault_fib6_body()
 	# Configure epair interfaces
 	get_epair
 	setup_iface "$EPAIRA" "$FIB0" inet6 ${ADDR} ${MASK}
-	echo ifconfig "$EPAIRB" up inet6 fib $FIB1 -ifdisabled accept_rtadv
-	ifconfig "$EPAIRB" inet6 -ifdisabled accept_rtadv fib $FIB1 up
+	echo setfib $FIB1 ifconfig "$EPAIRB" inet6 -ifdisabled accept_rtadv fib $FIB1 up
+	setfib $FIB1 ifconfig "$EPAIRB" inet6 -ifdisabled accept_rtadv fib $FIB1 up
 	rtadvd -p rtadvd.pid -C rtadvd.sock -c /dev/null "$EPAIRA"
 	rtsol "$EPAIRB"
 
@@ -514,7 +512,7 @@ slaac_on_nondefault_fib6_cleanup()
 	cleanup_ifaces
 	if [ -f "rtadvd.pid" ]; then
 		pkill -F rtadvd.pid
-		rm rtadvd.pid
+		rm -f rtadvd.pid
 	fi
 	if [ -f "rfc6204w3.state" ] ; then
 		sysctl "net.inet6.ip6.rfc6204w3"=`cat "rfc6204w3.state"`
@@ -685,8 +683,6 @@ udp_dontroute6_body()
 	TARGET="2001:db8::100"
 	SRCDIR=`atf_get_srcdir`
 
-	atf_expect_fail "PR196361 IPv6 network routes don't respect net.add_addr_allfibs=0"
-
 	# Check system configuration
 	if [ 0 != `sysctl -n net.add_addr_allfibs` ]; then
 		atf_skip "This test requires net.add_addr_allfibs=0"
@@ -767,7 +763,7 @@ get_epair()
 	local EPAIRD
 
 	if EPAIRD=`ifconfig epair create`; then
-		# Record the TAP device so we can clean it up later
+		# Record the epair device so we can clean it up later
 		echo ${EPAIRD} >> "ifaces_to_cleanup"
 		EPAIRA=${EPAIRD}
 		EPAIRB=${EPAIRD%a}b