Merge branch 'dynamic_netgraph'

ansible/environments/laptop/host_vars/odofreebsd

@@ -10,7 +10,6 @@ pflog_conf:
 network_rc: "odofreebsd_network.conf"
 rc_conf: "odofreebsd_rc.conf"
 loader_conf: "odofreebsd_loader.conf"
-netgraph_config: "setup_netgraph_odo"
 install_graphics: true
 graphics_driver: "intel"
 cputype: "intel"

ansible/roles/base/tasks/freebsd.yaml

@@ -4,6 +4,7 @@
       - pstree
       - gsed
       - gmake
+      - rust-coreutils
     state: present
 
 - name: See if the alacritty termcap has been added

ansible/roles/base/tasks/linux.yaml

@@ -6,6 +6,7 @@
       - linux-firmware
       - bind # dig
       - man-db
+      - uutils-coreutils
     state: present
 
 - name: Start pkgfile update service

ansible/roles/bhyve/files/arch.conf

@@ -19,7 +19,7 @@ disk0_dev="sparse-zvol"
 virt_random="yes"  # virtio-rnd
 
 # Creates a link to host_bridge1's link3 hook to the vmlink hook on a type socket
-bhyve_options="-s 2:0,virtio-net,netgraph,path=host_bridge1:,peerhook=link3"
+bhyve_options="-s 2:0,virtio-net,netgraph,path=bridge_jail_nat:,peerhook=link90"
 
 # Share a host directory to the guest via 9pfs.
 #

ansible/roles/bhyve/files/bhyve_netgraph_bridge.bash

@@ -0,0 +1,142 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    if [ "$1" = "create-disk" ]; then
+        shift 1
+        create_disk "${@}"
+    elif [ "$1" = "start" ]; then
+        shift 1
+        start_vm "${@}"
+    else
+        >&2 echo "Unrecognized command"
+        exit 1
+    fi
+}
+
+function create_disk {
+    zfs_path="$1"
+    mount_path="$2"
+    gigabytes="$3"
+    zfs create -o "mountpoint=$mount_path" "$zfs_path"
+    cp /usr/local/share/edk2-bhyve/BHYVE_UEFI_VARS.fd "${mount_path}/"
+    tee "${mount_path}/settings" <<EOF
+CPU_CORES=1
+MEMORY=1G
+EOF
+    zfs create -s "-V${gigabytes}G" -o volmode=dev "$zfs_path/disk0"
+}
+
+function start_vm {
+    name="$1"
+    zfs_path="$2"
+    mount_path="$3"
+    host_interface_name="$4"
+    bridge_name="bridge_${host_interface_name}"
+    ip_range="$5"
+
+    mac_address=$(calculate_mac_address "$name")
+
+    assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+    bridge_link_name=$(detect_available_link "${bridge_name}")
+
+
+    CPU_CORES=1
+    MEMORY=1G
+    if [ -e "${mount_path}/settings" ]; then
+        source "${mount_path}/settings"
+    fi
+    # -H release the CPU when guest issues HLT instruction. Otherwise 100% of core will be consumed.
+         # -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+             # -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080,wait \
+    while true; do
+        set -x
+        set +e
+        bhyve \
+            -D \
+            -c $CPU_CORES \
+            -m $MEMORY \
+            -H \
+            -s 0,hostbridge \
+            -s "4,nvme,/dev/zvol/${zfs_path}/disk0" \
+            -s "2:0,virtio-net,netgraph,path=${bridge_name}:,peerhook=${bridge_link_name},mac=${mac_address}" \
+            -s 3,ahci-cd,/vm/.iso/archlinux-2023.04.01-x86_64.iso \
+            -s 29,fbuf,tcp=0.0.0.0:5900,w=1920,h=1080 \
+            -s 30,xhci,tablet \
+            -s 31,lpc -l com1,stdio \
+            -l "bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,${mount_path}/BHYVE_UEFI_VARS.fd" \
+            "$name"
+        exit_code=$?
+        set -e
+        set +x
+        if [ $exit_code -eq 0 ]; then
+            echo "Rebooting."
+        elif [ $exit_code -eq 1 ]; then
+            echo "Powered off."
+            break
+        elif [ $exit_code -eq 2 ]; then
+            echo "Halted."
+            break
+        elif [ $exit_code -eq 3 ]; then
+            echo "Triple fault."
+            break
+        elif [ $exit_code -eq 4 ]; then
+            echo "Exited due to an error."
+            break
+        fi
+    done
+
+    bhyvectl "--vm=$name" --destroy
+    echo "Destroyed bhyve vm."
+}
+
+function detect_available_link {
+    bridge_name="$1"
+    linknum=1
+    while true; do
+        link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function assert_bridge {
+    host_interface_name="$1"
+    bridge_name="$2"
+    ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function calculate_mac_address {
+    name="$1"
+    source=$(md5 -r -s "$name" | awk '{print $1}')
+    echo "${source:0:2}:${source:2:2}:${source:4:2}:${source:6:2}:${source:8:2}:${source:10:2}"
+}
+
+
+main "${@}"

ansible/roles/bhyve/tasks/freebsd.yaml

@@ -13,6 +13,17 @@
       - bhyve-firmware # For UEFI
     state: present
 
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: bhyve_netgraph_bridge.bash
+      dest: /usr/local/bin/bhyve_netgraph_bridge
+
 - name: Create zfs dataset
   zfs:
     name: "{{ bhyve_dataset }}"

ansible/roles/firewall/files/homeserver_pf.conf

@@ -1,6 +1,7 @@
 ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
-jail_net_v4 = "10.193.223.0/24"
+not_ext_if = "{ !igb0 !igb1 !ix0 !ix1 !wlan0 }"
-full_nat_v4 = "10.213.177.0/24"
+jail_nat_v4 = "{ 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgh wgf }"
@@ -15,23 +16,29 @@ unifi_ports = "{ 8443 3478 10001 8080 1900 8843 8880 6789 5514 }"
 set skip on lo
 
 # redirections
-nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
-nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
-rdr pass on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+# cloak
-rdr pass on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+nat pass on $ext_if inet from 10.215.2.0/24 to !10.215.2.0/24 -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.2.1 port 53 -> 1.1.1.1 port 53
 
-nat pass on host_uplink0 inet proto tcp from any to any port 8081 tag ALLOWED -> (host_uplink0)
+rdr pass on $ext_if inet proto tcp from any to any port 8081 -> 10.215.2.2 port 8081
-rdr pass on $ext_if inet proto tcp from any to any port 8081 tag ALLOWED -> 10.193.223.20 port 8081
 
 # Forward ports for unifi controller
-rdr pass on $ext_if inet proto tcp from any to any port 65022 tag ALLOWED -> 10.213.177.8 port 22
+rdr pass on $ext_if inet proto tcp from any to any port 65022 -> 10.213.177.8 port 22
-rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports tag ALLOWED -> 10.213.177.8
+rdr pass on $ext_if inet proto {udp, tcp} from any to any port $unifi_ports -> 10.213.177.8
 
 # filtering
 block log all
 pass out on $ext_if
 
+pass in on jail_nat
+# Allow traffic from my machine to the jails/virtual machines
+pass out on jail_nat from $jail_nat_v4
+
+pass in on restricted_nat proto {udp, tcp} from any to any port { 53 51820 }
+
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
@@ -44,9 +51,3 @@ pass on $ext_if proto icmp6 all
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
-
-pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
-
-pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-pass out on host_uplink0 proto tcp from any to any port 8081
-pass on host_uplink1

ansible/roles/firewall/files/odofreebsd_pf.conf

@@ -1,6 +1,8 @@
-ext_if = "{ igb0 igb1 ix0 ix1 wlan0 }"
+ext_if = "{ wlan0 }"
-jail_net_v4 = "10.193.223.0/24"
+not_ext_if = "{ !wlan0 }"
-full_nat_v4 = "10.213.177.0/24"
+jail_nat_v4 = "{ 10.215.1.0/24 }"
+not_jail_nat_v4 = "{ any, !10.215.1.0/24 }"
+dns_redirect = "{ 10.193.223.1 10.213.177.1 10.215.1.1 }"
 
 dhcp = "{ bootpc, bootps }"
 allow = "{ wgf wgh drmario colo }"
@@ -14,22 +16,26 @@ udp_pass_in = "{ 53 51820 }"
 set skip on lo
 
 # redirections
-nat on $ext_if inet from $jail_net_v4 to { any, !$jail_net_v4 } tag ALLOWED -> (wlan0)
+nat pass on $ext_if inet from $jail_nat_v4 to $not_jail_nat_v4 -> (wlan0)
-nat on $ext_if inet from $full_nat_v4 to { any, !$full_nat_v4 } tag ALLOWED -> (wlan0)
+rdr pass on $not_ext_if proto {tcp, udp} from any to 10.215.1.1 port 53 -> 1.1.1.1 port 53
 
-rdr on host_uplink0 inet proto {tcp, udp} from any to 10.193.223.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+# Redirect jaeger ports to virtual machine.
-rdr on host_uplink1 inet proto {tcp, udp} from any to 10.213.177.1 port 53 tag ALLOWED -> 1.1.1.1 port 53
+# nat pass on lo inet from 127.0.0.0/24 to 127.0.0.0/24 port {6831 6832 16686 14268} -> (jail_nat)
+# rdr pass proto {tcp, udp} from jail_nat:network to 127.0.0.0/24 port {6831 6832 16686 14268} -> 10.215.1.201
 
 # filtering
 block log all
-pass quick on $ext_if tagged ALLOWED
 pass out on $ext_if
 
+pass in on jail_nat
+# Allow traffic from my machine to the jails/virtual machines
+pass out on jail_nat from $jail_nat_v4
+
 # We pass on the interfaces listed in allow rather than skipping on
 # them because changes to pass rules will update when running a
 # `service pf reload` but interfaces that we `skip` will not update (I
 # forget if its from adding, removing, or both. TODO: test to figure
-# it out)
+# it out). Also skipped interfaces are not subject to nat/rdr rules.
 pass quick on $allow
 
 pass on $ext_if proto icmp all
@@ -37,9 +43,3 @@ pass on $ext_if proto icmp6 all
 
 pass in on $ext_if proto tcp to any port $tcp_pass_in
 pass in on $ext_if proto udp to any port $udp_pass_in
-
-pass quick on $ext_if proto udp from any port $dhcp to any port $dhcp
-
-pass in on host_uplink0 proto udp from any to any port { 53 51820 }
-pass out on host_uplink0 proto tcp from any to any port 8081
-pass on host_uplink1

ansible/roles/jail/files/jail_netgraph_bridge.bash

@@ -0,0 +1,120 @@
+#!/usr/local/bin/bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    if [ "$1" = "start" ]; then
+        shift 1
+        start_jail "${@}"
+    elif [ "$1" = "stop" ]; then
+        shift 1
+        stop_jail "${@}"
+    else
+        >&2 echo "Unrecognized command"
+        exit 1
+    fi
+}
+
+function start_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+    ip_range="$3"
+
+    assert_bridge "$host_interface_name" "$bridge_name" "$ip_range"
+
+    bridge_link_name=$(detect_available_link "${bridge_name}")
+    ngctl -d -f - <<EOF
+mkpeer ${bridge_name}: eiface $bridge_link_name ether
+name ${bridge_name}:$bridge_link_name $jail_interface_name
+EOF
+    ifconfig $(ngctl msg "${jail_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${jail_interface_name}" up
+}
+
+function stop_jail {
+    host_interface_name="$1"
+    bridge_name="bridge_${host_interface_name}"
+    jail_interface_name="$2"
+
+    if ng_exists "${jail_interface_name}:"; then
+        wait_for_interface_to_exist "${jail_interface_name}" 120
+        ngctl shutdown "${jail_interface_name}:"
+    fi
+
+    if ng_exists "${bridge_name}:"; then
+        num_remaining_hooks=$(ng_bridge_get_num_hooks "${bridge_name}:")
+        if [ $num_remaining_hooks -eq 1 ]; then
+            ngctl shutdown "${bridge_name}:"
+            ngctl shutdown "${host_interface_name}:"
+        fi
+    fi
+}
+
+function assert_bridge {
+    host_interface_name="$1"
+    bridge_name="$2"
+    ip_range="$3"
+
+    if ! ng_exists "${bridge_name}:"; then
+        ngctl -d -f - <<EOF
+mkpeer . eiface hook ether
+name .:hook $host_interface_name
+EOF
+        ngctl -d -f - <<EOF
+mkpeer ${host_interface_name}: bridge ether link0
+name ${host_interface_name}:ether $bridge_name
+EOF
+        ifconfig $(ngctl msg "${host_interface_name}:" getifname | grep Args | cut -d '"' -f 2) name "${host_interface_name}" "$ip_range" up
+    fi
+}
+
+function ng_exists {
+    ngctl status "${1}" >/dev/null 2>&1
+}
+
+function ng_bridge_get_num_hooks {
+    ngctl show "${1}" | grep -oE 'Num hooks: [0-9]+' | sed 's/Num hooks: //g'
+}
+
+function detect_available_link {
+    bridge_name="$1"
+    linknum=1
+    while true; do
+        link_name="link${linknum}"
+        if ! ng_exists "${bridge_name}:${link_name}"; then
+            echo "$link_name"
+            return
+        fi
+        (>&2 echo "$link_name failed on $bridge_name")
+        linknum=$((linknum + 1))
+        if [ "$linknum" -gt 90 ]; then
+            (>&2 echo "No available links on bridge $bridge_name")
+            exit 1
+        fi
+    done
+}
+
+function wait_for_interface_to_exist {
+    # Wait for a vnet interface to exist again as a jail is shutting
+    # down. If you delete the netgraph node before the interface
+    # device exists, then the interface device will persist in a
+    # broken state.
+    ifname="$1"
+    max_wait_seconds="$2"
+    start=$(date +%s)
+    while true; do
+        now=$(date +%s)
+        if [ $((now - start)) -gt $max_wait_seconds ]; then
+            (>&2 echo "Waited for at least $max_wait_seconds seconds but the interface $ifname did not appear.")
+            return 1;
+        fi
+        if ifconfig "$ifname" >/dev/null 2>&1; then
+            return 0;
+        fi
+        sleep 2
+    done
+}
+
+main "${@}"

ansible/roles/jail/files/jails/cloak.conf

@@ -1,8 +1,11 @@
 cloak {
-    path = "/jail/main/jails/cloak";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "host_link2";
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start restricted_nat jail${name} 10.215.2.1/24";
-    vnet.interface += "wg_uplink0";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop restricted_nat jail${name}";
+    vnet.interface += "jail${name}";
+    vnet.interface += "cloak";
+
     devfs_ruleset = 13;
     mount.devfs; # To expose tun device
 

ansible/roles/jail/files/jails/dagger.conf

@@ -1,7 +1,7 @@
 dagger {
-    path = "/jail/main/jails/dagger";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "wg_link2";
+    vnet.interface += "dagger";
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";

ansible/roles/jail/files/jails/nat_dhcp.conf

@@ -1,9 +1,12 @@
 nat_dhcp {
-    path = "/jail/main/jails/nat_dhcp";
+    path = "/jail/main/jails/${name}";
     vnet;
-    vnet.interface += "host_link3";
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
     devfs_ruleset = 14;
-    mount.devfs; # To expose tun device
+    mount.devfs;
 
     exec.start += "/bin/sh /etc/rc";
     exec.stop = "/bin/sh /etc/rc.shutdown jail";

ansible/roles/jail/files/jails/sample.conf

@@ -0,0 +1,14 @@
+sample {
+    path = "/jail/main/jails/${name}";
+    vnet;
+    exec.prestart += "/usr/local/bin/jail_netgraph_bridge start jail_nat jail${name} 10.215.1.1/24";
+    exec.poststop += "/usr/local/bin/jail_netgraph_bridge stop jail_nat jail${name}";
+    vnet.interface += "jail${name}";
+
+    devfs_ruleset = 14;
+    mount.devfs;
+
+    exec.start += "/bin/sh /etc/rc";
+    exec.stop = "/bin/sh /etc/rc.shutdown jail";
+    exec.consolelog = "/var/log/jail_${name}_console.log";
+}

ansible/roles/jail/files/setup_netgraph_homeserver

@@ -1,77 +1,13 @@
 #!/usr/local/bin/bash
 
 cleanup() {
-    ngctl shutdown host_link2:
+    /usr/local/bin/jail_netgraph_bridge stop cloak dagger
-    ngctl shutdown host_uplink0:
-    ngctl shutdown host_bridge0:
-    ngctl shutdown wg_link2:
-    ngctl shutdown wg_uplink0:
-    ngctl shutdown wg_bridge0:
-    ngctl shutdown host_link3:
-    ngctl shutdown host_uplink1:
-    ngctl shutdown host_bridge1:
 }
 
 setup_netgraph_start() {
     cleanup
 
-    # Create a bridge for jails that only speak wireguard
+    /usr/local/bin/jail_netgraph_bridge start cloak dagger 192.168.1.0/24
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink0: bridge ether link0
-name host_uplink0:ether host_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge0: eiface link2 ether
-name host_bridge0:link2 host_link2
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
-
-    # Create internal bridge for jails that are forced through wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook wg_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_uplink0: bridge ether link0
-name wg_uplink0:ether wg_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_bridge0: eiface link2 ether
-name wg_bridge0:link2 wg_link2
-EOF
-
-    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
-    # Create a bridge for jails given full access to NAT
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink1: bridge ether link0
-name host_uplink1:ether host_bridge1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge1: eiface link2 ether
-name host_bridge1:link2 host_link3
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink1:' getifname | grep Args | cut -d '"' -f 2) name host_uplink1 10.213.177.1/24 up
-    ifconfig $(ngctl msg 'host_bridge1:link2' getifname | grep Args | cut -d '"' -f 2) name host_link3
-
 }
 
 setup_netgraph_stop() {

ansible/roles/jail/files/setup_netgraph_odo

@@ -1,87 +0,0 @@
-#!/usr/local/bin/bash
-
-cleanup() {
-    ngctl shutdown host_link2:
-    ngctl shutdown host_uplink0:
-    ngctl shutdown host_bridge0:
-    ngctl shutdown wg_link2:
-    ngctl shutdown wg_uplink0:
-    ngctl shutdown wg_bridge0:
-    ngctl shutdown host_link3:
-    ngctl shutdown host_uplink1:
-    ngctl shutdown host_bridge1:
-}
-
-setup_netgraph_start() {
-    cleanup
-
-    # Create a bridge for jails that only speak wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink0: bridge ether link0
-name host_uplink0:ether host_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge0: eiface link2 ether
-name host_bridge0:link2 host_link2
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink0:' getifname | grep Args | cut -d '"' -f 2) name host_uplink0 10.193.223.1/24 up
-    ifconfig $(ngctl msg 'host_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name host_link2
-
-    # Create internal bridge for jails that are forced through wireguard
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook wg_uplink0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_uplink0: bridge ether link0
-name wg_uplink0:ether wg_bridge0
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer wg_bridge0: eiface link2 ether
-name wg_bridge0:link2 wg_link2
-EOF
-
-    ifconfig $(ngctl msg 'wg_uplink0:' getifname | grep Args | cut -d '"' -f 2) name wg_uplink0 10.241.199.1/24 up
-    ifconfig $(ngctl msg 'wg_bridge0:link2' getifname | grep Args | cut -d '"' -f 2) name wg_link2
-
-    # Create a bridge for jails given full access to NAT
-    ngctl -d -f - <<EOF
-mkpeer . eiface hook ether
-name .:hook host_uplink1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_uplink1: bridge ether link0
-name host_uplink1:ether host_bridge1
-EOF
-
-    ngctl -d -f - <<EOF
-mkpeer host_bridge1: eiface link2 ether
-name host_bridge1:link2 host_link3
-EOF
-
-    ifconfig $(ngctl msg 'host_uplink1:' getifname | grep Args | cut -d '"' -f 2) name host_uplink1 10.213.177.1/24 up
-    ifconfig $(ngctl msg 'host_bridge1:link2' getifname | grep Args | cut -d '"' -f 2) name host_link3
-
-}
-
-setup_netgraph_stop() {
-    cleanup
-}
-
-if [ "$1" = "start" ]; then
-    setup_netgraph_start
-elif [ "$1" = "stop" ]; then
-    setup_netgraph_stop
-else
-    >&2 echo "Unrecognized command"
-fi

ansible/roles/jail/tasks/freebsd.yaml

@@ -125,6 +125,17 @@
     - src: "{{ netgraph_config }}"
       dest: /usr/local/bin/setup_netgraph
 
+- name: Install scripts
+  copy:
+    src: "files/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+    owner: root
+    group: wheel
+  loop:
+    - src: jail_netgraph_bridge.bash
+      dest: /usr/local/bin/jail_netgraph_bridge
+
 - name: Enable setup_netgraph
   when: netgraph_config is defined
   community.general.sysrc:

ansible/roles/jail_nat_dhcp/files/dhcpd.conf

@@ -1,13 +0,0 @@
-# option definitions common to all supported networks...
-option domain-name "home.arpa";
-# option domain-name-servers ns1.home.arpa;
-option subnet-mask 255.255.255.0;
-default-lease-time 600;
-max-lease-time 7200;
-
-subnet 10.213.177.0 netmask 255.255.255.0 {
-  range 10.213.177.10 10.213.177.250;
-  option broadcast-address 10.213.177.255;
-  option routers 10.213.177.1;
-  option domain-name-servers 10.213.177.1;
-}

ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf

@@ -0,0 +1,31 @@
+{
+    "Dhcp4": {
+        "interfaces-config": {
+            "interfaces": [ "jailnat_dhcp" ]
+        },
+        "subnet4": [
+            {
+                "subnet": "10.215.1.0/24",
+                "pools": [ { "pool": "10.215.1.10-10.215.1.200" } ],
+                "option-data": [
+                    {
+                        "name": "routers",
+                        "data": "10.215.1.1"
+                    }
+                ],
+                "reservations": [
+                    {
+                        "hw-address": "d6:19:4c:68:fc:c7",
+                        "ip-address": "10.215.1.201"
+                    }
+                ]
+            }
+        ],
+        "option-data": [
+            {
+                "name": "domain-name-servers",
+                "data": "10.215.1.1"
+            }
+        ]
+    }
+}

ansible/roles/jail_nat_dhcp/files/rc.conf

@@ -1,3 +1,3 @@
-ifconfig_host_link3="inet 10.213.177.254 netmask 255.255.255.0"
+ifconfig_jailnat_dhcp="inet 10.215.1.254 netmask 255.255.255.0"
-defaultrouter="10.213.177.1"
+defaultrouter="10.215.1.1"
-dhcpd_enable="YES"
+kea_enable="YES"

ansible/roles/jail_nat_dhcp/files/resolv.conf

@@ -1,2 +1,2 @@
 search home.arpa
-nameserver 10.213.177.1
+nameserver 10.215.1.1

ansible/roles/jail_nat_dhcp/tasks/freebsd.yaml

@@ -1,7 +1,7 @@
 - name: Install packages
   package:
     name:
-      - dhcpd
+      - kea
     state: present
 
 - name: Install Configuration
@@ -14,7 +14,7 @@
   loop:
     - src: rc.conf
       dest: /etc/rc.conf
-    - src: dhcpd.conf
+    - src: kea-dhcp4.conf
-      dest: /usr/local/etc/dhcpd.conf
+      dest: /usr/local/etc/kea/kea-dhcp4.conf
     - src: resolv.conf
       dest: /etc/resolv.conf

ansible/roles/package_manager/tasks/freebsd.yaml

@@ -8,6 +8,12 @@
   loop:
     - /usr/local/etc/pkg
     - /usr/local/etc/pkg/repos
+    
+- name: Install packages
+  package:
+    name:
+      - pkg-provides
+    state: present
 
 - name: Install Configuration
   register: changed_config
@@ -26,9 +32,3 @@
 # - name: Replace all packages with packages from new repo
 #   command: pkg upgrade -f -y
 #   when: changed_config.changed
-
-- name: Install packages
-  package:
-    name:
-      - pkg-provides
-    state: present