Switch to using a script to dynamically spin up the netgraph bridge for jails.

ansible/environments/laptop/host_vars/odofreebsd

@@ -10,7 +10,6 @@ pflog_conf:
 network_rc: "odofreebsd_network.conf"
 rc_conf: "odofreebsd_rc.conf"
 loader_conf: "odofreebsd_loader.conf"
-netgraph_config: "setup_netgraph_odo"
 install_graphics: true
 graphics_driver: "intel"
 cputype: "intel"

ansible/roles/firewall/files/odofreebsd_pf.conf

@@ -1,6 +1,8 @@
-ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-jail_net_v4 = "10.193.223.0/24"
-full_nat_v4 = "10.213.177.0/24"
+ext_if = "{ wlan0 }"
+not_ext_if = "{ !wlan0 }"
+jail_nat_v4 = "{ 10.193.223.0/24 10.213.177.0/24 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.193.223.0/24 !10.213.177.0/24 !10.215.1.0/24 }"
+dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgf wgh drmario colo }"
@@ -14,16 +16,13 @@ udp_pass_in = "{ 53 51820 }"
 set skip on lo
 
 # redirections
-nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
-nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0)
-
-rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
-rdr on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
 # filtering
 block log all
-pass quick on $ext_if tagged ALLOWED
 pass out on $ext_if
+pass in on jail_nat
 
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
@@ -37,9 +36,3 @@ pass on $ext_if proto icmp6 all
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
-
-pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
-
-pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-pass out on host_uplink0 proto tcp from any to any port 8081
-pass on host_uplink1

ansible/roles/jail/files/jail_netgraph_bridge.bash

@@ -0,0 +1,120 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    if [ "$1" = "start" ]; then
+        shift 1
+        start_jail "${@}"
+    elif [ "$1" = "stop" ]; then
+        shift 1
+        stop_jail "${@}"
+    else
+        >&2 echo "Unrecognized command"
+        exit 1
+    fi
+}
+
+function start_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+    ip_range="$3"
+
+    assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+
+    bridge_link_name=$(detect_available_link "${bridge_name}")
+    ngctl -d -f - <<EOF
+mkpeer ${bridge_name}: eiface $bridge_link_name ether
+name ${bridge_name}:$bridge_link_name $jail_interface_name
+EOF
+    ifconfig $(ngctl msg "${jail_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${jail_interface_name}" up
+}
+
+function stop_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+
+    if ng_exists "${jail_interface_name}:"; then
+        wait_for_interface_to_exist "${jail_interface_name}" 120
+        ngctl shutdown "${jail_interface_name}:"
+    fi
+
+    if ng_exists "${bridge_name}:"; then
+        num_remaining_hooks=$(ng_bridge_get_num_hooks "${bridge_name}:")
+        if [ $num_remaining_hooks -eq 1 ]; then
+            ngctl shutdown "${bridge_name}:"
+            ngctl shutdown "${host_interface_name}:"
+        fi
+    fi
+}
+
+function assert_bridge {
+    host_interface_name="$1"
+    bridge_name="$2"
+    ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function ng_bridge_get_num_hooks {
+    ngctl show "${1}" | grep -oE 'Num hooks: [0-9]+' | sed 's/Num hooks: //g'
+}
+
+function detect_available_link {
+    bridge_name="$1"
+    linknum=1
+    while true; do
+        link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        (>&2 echo "$link_name failed on $bridge_name")
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function wait_for_interface_to_exist {
+    # Wait for a vnet interface to exist again as a jail is shutting
+    # down. If you delete the netgraph node before the interface
+    # device exists, then the interface device will persist in a
+    # broken state.
+    ifname="$1"
+    max_wait_seconds="$2"
+    start=$(date +%s)
+    while true; do
+        now=$(date +%s)
+        if [ $((now - start)) -gt $max_wait_seconds ]; then
+            (>&2 echo "Waited for at least $max_wait_seconds seconds but the interface $ifname did not appear.")
+            return 1;
+        fi
+        if ifconfig "$ifname" >/dev/null 2>&1; then
+            return 0;
+        fi
+        sleep 2
+    done
+}
+
+main "${@}"

ansible/roles/jail/files/jails/nat_dhcp.conf

@@ -1,9 +1,12 @@
 nat_dhcp {
-    path = "/jail/main/jails/nat_dhcp";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "host_link3";
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
     devfs_ruleset = 14;
-    mount.devfs; # To expose tun device
+    mount.devfs;
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";

ansible/roles/jail/files/jails/sample.conf

@@ -0,0 +1,14 @@
+sample {
+    path = "/jail/main/jails/${name}";
+    vnet;
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
+    devfs_ruleset = 14;
+    mount.devfs;
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

ansible/roles/jail/files/setup_netgraph_odo

@@ -1,87 +0,0 @@
-#!/usr/local/bin/bash
-
-cleanup() {
-    ngctl shutdown host_link2:
-    ngctl shutdown host_uplink0:
-    ngctl shutdown host_bridge0:
-    ngctl shutdown wg_link2:
-    ngctl shutdown wg_uplink0:
-    ngctl shutdown wg_bridge0:
-    ngctl shutdown host_link3:
-    ngctl shutdown host_uplink1:
-    ngctl shutdown host_bridge1:
-}
-
-setup_netgraph_start() {
-    cleanup
-
-    # Create a bridge for jails that only speak wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink0: bridge ether link0
-name host_uplink0:ether host_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge0: eiface link2 ether
-name host_bridge0:link2 host_link2
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
-
-    # Create internal bridge for jails that are forced through wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook wg_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_uplink0: bridge ether link0
-name wg_uplink0:ether wg_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_bridge0: eiface link2 ether
-name wg_bridge0:link2 wg_link2
-EOF
-
-    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
-    # Create a bridge for jails given full access to NAT
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink1: bridge ether link0
-name host_uplink1:ether host_bridge1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge1: eiface link2 ether
-name host_bridge1:link2 host_link3
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink1:' getifname | grep Args | cut -d '"' -f 2) name host_uplink1 10.213.177.1/24 up
-    ifconfig $(ngctl msg 'host_bridge1:link2' getifname | grep Args | cut -d '"' -f 2) name host_link3
-
-}
-
-setup_netgraph_stop() {
-    cleanup
-}
-
-if [ "$1" = "start" ]; then
-    setup_netgraph_start
-elif [ "$1" = "stop" ]; then
-    setup_netgraph_stop
-else
-    >&2 echo "Unrecognized command"
-fi

ansible/roles/jail/tasks/freebsd.yaml

@@ -125,6 +125,17 @@
     - src: "{{ netgraph_config }}"
       dest: /usr/local/bin/setup_netgraph
 
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: jail_netgraph_bridge.bash
+      dest: /usr/local/bin/jail_netgraph_bridge
+
 - name: Enable setup_netgraph
   when: netgraph_config is defined
   community.general.sysrc: